1
1
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-10-07 13:52:39 +02:00

Compare commits

...

1 Commits

Author SHA1 Message Date
Viktor Lofgren
58e6f141ce (crawler) Reduce congestion throttle go-rate 2025-04-17 20:36:58 +02:00

View File

@@ -53,7 +53,7 @@ public class CrawlerRetreiver implements AutoCloseable {
private final CrawlerRevisitor crawlerRevisitor; private final CrawlerRevisitor crawlerRevisitor;
private static final CrawlerConnectionThrottle connectionThrottle = new CrawlerConnectionThrottle( private static final CrawlerConnectionThrottle connectionThrottle = new CrawlerConnectionThrottle(
Duration.ofSeconds(1) // pace the connections to avoid network congestion by waiting 1 second between establishing them Duration.ofMillis(50) // pace the connections to avoid network congestion at startup
); );
int errorCount = 0; int errorCount = 0;