(crawler) Adjust crawl order to get a better mixture of domains

2025-10-05 21:22:39 +02:00 · 2025-03-27 11:12:48 +01:00
1 changed files with 1 additions and 1 deletions
--- a/code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java
+++ b/code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java
@@ -319,7 +319,7 @@ public class CrawlerMain extends ProcessMainClass {
            randomOrder.put(spec.domain, r.nextInt());
        }

-        return Comparator.comparing((CrawlSpecRecord spec) -> topDomainCounts.getOrDefault(EdgeDomain.getTopDomain(spec.domain), 0))
+        return Comparator.comparing((CrawlSpecRecord spec) -> topDomainCounts.getOrDefault(EdgeDomain.getTopDomain(spec.domain), 0) >= 8)
                .reversed()
                .thenComparing(spec -> randomOrder.get(spec.domain))
                .thenComparing(Record::hashCode); // non-deterministic tie-breaker to