(crawler) Further rearrange crawl order

Limit crawl order preferrence to edu domains, to avoid hitting stuff like medium and wordpress with shotgun requests.

build.gradle

@@ -43,12 +43,11 @@ subprojects.forEach {it ->
 }
 
 ext {
-    jvmVersion=23
-    dockerImageBase='container-registry.oracle.com/graalvm/jdk:23'
+    jvmVersion = 24
+    dockerImageBase='container-registry.oracle.com/graalvm/jdk:24'
     dockerImageTag='latest'
     dockerImageRegistry='marginalia'
     jibVersion = '3.4.4'
-
 }
 
 idea {

code/common/model/java/nu/marginalia/model/EdgeDomain.java

@@ -14,7 +14,7 @@ public class EdgeDomain implements Serializable {
     @Nonnull
     public final String topDomain;
 
-    public EdgeDomain(String host) {
+    public EdgeDomain(@Nonnull String host) {
         Objects.requireNonNull(host, "domain name must not be null");
 
         host = host.toLowerCase();
@@ -61,6 +61,10 @@ public class EdgeDomain implements Serializable {
         this.topDomain = topDomain;
     }
 
+    public static String getTopDomain(String host) {
+        return new EdgeDomain(host).topDomain;
+    }
+
     private boolean looksLikeGovTld(String host) {
         if (host.length() < 8)
             return false;
@@ -116,24 +120,6 @@ public class EdgeDomain implements Serializable {
         return topDomain.substring(0, cutPoint).toLowerCase();
     }
 
-    public String getLongDomainKey() {
-        StringBuilder ret = new StringBuilder();
-
-        int cutPoint = topDomain.indexOf('.');
-        if (cutPoint < 0) {
-            ret.append(topDomain);
-        } else {
-            ret.append(topDomain, 0, cutPoint);
-        }
-
-        if (!subDomain.isEmpty() && !"www".equals(subDomain)) {
-            ret.append(":");
-            ret.append(subDomain);
-        }
-
-        return ret.toString().toLowerCase();
-    }
-
     /** If possible, try to provide an alias domain,
      * i.e. a domain name that is very likely to link to this one
      * */

code/common/service/test/nu/marginalia/service/discovery/ZkServiceRegistryTest.java

@@ -25,7 +25,7 @@ import static org.mockito.Mockito.when;
 class ZkServiceRegistryTest {
     private static final int ZOOKEEPER_PORT = 2181;
     private static final GenericContainer<?> zookeeper =
-            new GenericContainer<>("zookeeper:3.8.0")
+            new GenericContainer<>("zookeeper:3.8")
                     .withExposedPorts(ZOOKEEPER_PORT);
 
     List<ZkServiceRegistry> registries = new ArrayList<>();

code/libraries/blocking-thread-pool/java/nu/marginalia/util/SimpleBlockingThreadPool.java

@@ -23,16 +23,33 @@ public class SimpleBlockingThreadPool {
     private final Logger logger = LoggerFactory.getLogger(SimpleBlockingThreadPool.class);
 
     public SimpleBlockingThreadPool(String name, int poolSize, int queueSize) {
+        this(name, poolSize, queueSize, ThreadType.PLATFORM);
+    }
+
+    public SimpleBlockingThreadPool(String name, int poolSize, int queueSize, ThreadType threadType) {
         tasks = new ArrayBlockingQueue<>(queueSize);
 
         for (int i = 0; i < poolSize; i++) {
-            Thread worker = new Thread(this::worker, name  + "[" + i + "]");
-            worker.setDaemon(true);
-            worker.start();
+
+            Thread.Builder threadBuilder = switch (threadType) {
+                case VIRTUAL -> Thread.ofVirtual();
+                case PLATFORM -> Thread.ofPlatform().daemon(true);
+            };
+
+            Thread worker = threadBuilder
+                    .name(name  + "[" + i + "]")
+                    .start(this::worker);
+
             workers.add(worker);
         }
 
     }
+
+    public enum ThreadType {
+        VIRTUAL,
+        PLATFORM
+    }
+
     public void submit(Task task) throws InterruptedException {
         tasks.put(task);
     }

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -41,10 +41,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.security.Security;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -108,7 +105,8 @@ public class CrawlerMain extends ProcessMainClass {
 
         pool = new SimpleBlockingThreadPool("CrawlerPool",
                 Integer.getInteger("crawler.poolSize", 256),
-                1);
+                1,
+                SimpleBlockingThreadPool.ThreadType.VIRTUAL);
 
 
         // Wait for the blacklist to be loaded before starting the crawl
@@ -224,10 +222,7 @@ public class CrawlerMain extends ProcessMainClass {
 
         logger.info("Loaded {} domains", crawlSpecRecords.size());
 
-        // Shuffle the domains to ensure we get a good mix of domains in each crawl,
-        // so that e.g. the big domains don't get all crawled at once, or we end up
-        // crawling the same server in parallel from different subdomains...
-        Collections.shuffle(crawlSpecRecords);
+        crawlSpecRecords.sort(crawlSpecArrangement(crawlSpecRecords));
 
         // First a validation run to ensure the file is all good to parse
         if (crawlSpecRecords.isEmpty()) {
@@ -306,6 +301,28 @@ public class CrawlerMain extends ProcessMainClass {
         }
     }
 
+    /** Create a comparator that sorts the crawl specs in a way that is beneficial for the crawl,
+     * we want to enqueue domains that tend ro be large and have common top domains first,
+     * but otherwise have a random order.
+     * <p></p>
+     * Note, we can't use hash codes for randomization as it is not desirable to have the same order
+     * every time the process is restarted (and CrawlSpecRecord is a record, which defines equals and
+     * hashcode based on the fields).
+     * */
+    private Comparator<CrawlSpecRecord> crawlSpecArrangement(List<CrawlSpecRecord> records) {
+        Random r = new Random();
+        Map<String, Integer> randomOrder = new HashMap<>(records.size());
+
+        for (var spec : records) {
+            randomOrder.put(spec.domain, r.nextInt());
+        }
+
+        return Comparator.comparing((CrawlSpecRecord spec) -> spec.domain.contains(".edu"))
+                .reversed()
+                .thenComparing(spec -> randomOrder.get(spec.domain))
+                .thenComparing(Record::hashCode); // non-deterministic tie-breaker to
+    }
+
     /** Submit a task for execution if it can be run, returns true if it was submitted
      * or if it can be discarded */
     private boolean trySubmitDeferredTask(CrawlTask task) {

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -30,6 +30,7 @@ import java.net.http.HttpTimeoutException;
 import java.time.Duration;
 import java.util.*;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
 import java.util.zip.GZIPInputStream;
 
 
@@ -60,7 +61,7 @@ public class HttpFetcherImpl implements HttpFetcher {
                 .cookieHandler(cookies)
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .connectTimeout(Duration.ofSeconds(8))
-                .executor(Executors.newCachedThreadPool())
+                .executor(Executors.newVirtualThreadPerTaskExecutor())
                 .build();
     }
 
@@ -116,7 +117,7 @@ public class HttpFetcherImpl implements HttpFetcher {
 
         for (int tries = 0;; tries++) {
             try {
-                var rsp = client.send(head, HttpResponse.BodyHandlers.discarding());
+                var rsp = SendLock.wrapSend(client, head, HttpResponse.BodyHandlers.discarding());
                 EdgeUrl rspUri = new EdgeUrl(rsp.uri());
 
                 if (!Objects.equals(rspUri.domain, url.domain)) {
@@ -153,7 +154,7 @@ public class HttpFetcherImpl implements HttpFetcher {
                     .timeout(requestTimeout)
                     ;
 
-                var rsp = client.send(headBuilder.build(), HttpResponse.BodyHandlers.discarding());
+                var rsp = SendLock.wrapSend(client, headBuilder.build(), HttpResponse.BodyHandlers.discarding());
                 var headers = rsp.headers();
 
                 var contentTypeHeader = headers.firstValue("Content-Type").orElse(null);
@@ -229,21 +230,24 @@ public class HttpFetcherImpl implements HttpFetcher {
 
         contentTags.paint(getBuilder);
 
-        HttpFetchResult result = warcRecorder.fetch(client, getBuilder.build());
+        try (var sl = new SendLock()) {
+            HttpFetchResult result = warcRecorder.fetch(client, getBuilder.build());
 
-        if (result instanceof HttpFetchResult.ResultOk ok) {
-            if (ok.statusCode() == 429) {
-                throw new RateLimitException(Objects.requireNonNullElse(ok.header("Retry-After"), "1"));
-            }
-            if (ok.statusCode() == 304) {
-                return new HttpFetchResult.Result304Raw();
-            }
-            if (ok.statusCode() == 200) {
-                return ok;
+            if (result instanceof HttpFetchResult.ResultOk ok) {
+                if (ok.statusCode() == 429) {
+                    throw new RateLimitException(Objects.requireNonNullElse(ok.header("Retry-After"), "1"));
+                }
+                if (ok.statusCode() == 304) {
+                    return new HttpFetchResult.Result304Raw();
+                }
+                if (ok.statusCode() == 200) {
+                    return ok;
+                }
             }
+
+            return result;
         }
 
-        return result;
     }
 
     @Override
@@ -317,22 +321,28 @@ public class HttpFetcherImpl implements HttpFetcher {
                 .timeout(requestTimeout)
                 .build();
 
-        var response = client.send(getRequest, HttpResponse.BodyHandlers.ofInputStream());
-        if (response.statusCode() != 200) {
-            return new SitemapResult.SitemapError();
-        }
-
-        try (InputStream inputStream = response.body()) {
-
-            InputStream parserStream;
-            if (sitemapUrl.path.endsWith(".gz")) {
-                parserStream = new GZIPInputStream(inputStream);
-            }
-            else {
-                parserStream = inputStream;
+        try (var sl = new SendLock()) {
+            var response = client.send(getRequest, HttpResponse.BodyHandlers.ofInputStream());
+            if (response.statusCode() != 200) {
+                return new SitemapResult.SitemapError();
+            }
+
+            Document parsedSitemap;
+
+            try (InputStream inputStream = response.body()) {
+                InputStream parserStream;
+                if (sitemapUrl.path.endsWith(".gz")) {
+                    parserStream = new GZIPInputStream(inputStream);
+                } else {
+                    parserStream = inputStream;
+                }
+
+                parsedSitemap = Jsoup.parse(parserStream, "UTF-8", sitemapUrl.toString(), Parser.xmlParser());
+            }
+            finally {
+                sl.close();
             }
 
-            Document parsedSitemap = Jsoup.parse(parserStream, "UTF-8", sitemapUrl.toString(), Parser.xmlParser());
             if (parsedSitemap.childrenSize() == 0) {
                 return new SitemapResult.SitemapError();
             }
@@ -386,7 +396,7 @@ public class HttpFetcherImpl implements HttpFetcher {
     }
 
     private Optional<SimpleRobotRules> fetchAndParseRobotsTxt(EdgeUrl url, WarcRecorder recorder) {
-        try {
+        try (var sl = new SendLock()) {
             var getRequest = HttpRequest.newBuilder()
                     .GET()
                     .uri(url.asURI())
@@ -429,5 +439,30 @@ public class HttpFetcherImpl implements HttpFetcher {
             }
         }
     }
+
+}
+
+class SendLock implements AutoCloseable {
+
+    private static final Semaphore maxConcurrentRequests = new Semaphore(Integer.getInteger("crawler.maxConcurrentRequests", 512));
+    boolean closed = false;
+
+    public SendLock() {
+        maxConcurrentRequests.acquireUninterruptibly();
+    }
+
+    public static <T> HttpResponse<T> wrapSend(HttpClient client, HttpRequest request, HttpResponse.BodyHandler<T> handler) throws IOException, InterruptedException {
+        try (var lock = new SendLock()) {
+            return client.send(request, handler);
+        }
+    }
+
+    @Override
+    public void close() {
+        if (!closed) {
+            maxConcurrentRequests.release();
+            closed = true;
+        }
+    }
 }
 

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java

@@ -3,6 +3,7 @@ package nu.marginalia.crawl.retreival;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 
 import java.time.Duration;
+import java.util.concurrent.ThreadLocalRandom;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
@@ -53,12 +54,13 @@ public class CrawlDelayTimer {
     public void waitFetchDelay(long spentTime) {
         long sleepTime = delayTime;
 
+        long jitter = ThreadLocalRandom.current().nextLong(0, 150);
         try {
             if (sleepTime >= 1) {
                 if (spentTime > sleepTime)
                     return;
 
-                Thread.sleep(min(sleepTime - spentTime, 5000));
+                Thread.sleep(min(sleepTime - spentTime, 5000) + jitter);
             } else {
                 // When no crawl delay is specified, lean toward twice the fetch+process time,
                 // within sane limits. This means slower servers get slower crawling, and faster
@@ -71,17 +73,17 @@ public class CrawlDelayTimer {
                 if (spentTime > sleepTime)
                     return;
 
-                Thread.sleep(sleepTime - spentTime);
+                Thread.sleep(sleepTime - spentTime + jitter);
             }
 
             if (slowDown) {
                 // Additional delay when the server is signalling it wants slower requests
-                Thread.sleep(DEFAULT_CRAWL_DELAY_MIN_MS);
+                Thread.sleep(DEFAULT_CRAWL_DELAY_MIN_MS + jitter);
             }
         }
         catch (InterruptedException e) {
             Thread.currentThread().interrupt();
-            throw new RuntimeException();
+            throw new RuntimeException("Interrupted", e);
         }
     }
 }

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java

@@ -108,6 +108,11 @@ public class CrawlerRetreiver implements AutoCloseable {
                     DomainStateDb.SummaryRecord summaryRecord = sniffRootDocument(probedUrl, delayTimer);
                     domainStateDb.save(summaryRecord);
 
+                    if (Thread.interrupted()) {
+                        // There's a small chance we're interrupted during the sniffing portion
+                        throw new InterruptedException();
+                    }
+
                     // Play back the old crawl data (if present) and fetch the documents comparing etags and last-modified
                     if (crawlerRevisitor.recrawl(oldCrawlData, robotsRules, delayTimer) > 0) {
                         // If we have reference data, we will always grow the crawl depth a bit
@@ -140,7 +145,6 @@ public class CrawlerRetreiver implements AutoCloseable {
                             CrawlDelayTimer delayTimer,
                             DomainLinks domainLinks) {
 
-
         // Add external links to the crawl frontier
         crawlFrontier.addAllToQueue(domainLinks.getUrls(rootUrl.proto));
 
@@ -289,6 +293,10 @@ public class CrawlerRetreiver implements AutoCloseable {
         }
         catch (Exception ex) {
             logger.error("Error configuring link filter", ex);
+            if (Thread.interrupted()) {
+                Thread.currentThread().interrupt();
+                return DomainStateDb.SummaryRecord.forError(domain, "Crawler Interrupted", ex.getMessage());
+            }
         }
         finally {
             crawlFrontier.addVisited(rootUrl);

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java

@@ -46,6 +46,10 @@ public class CrawlerRevisitor {
                 break;
             }
 
+            if (Thread.interrupted()) {
+                throw new InterruptedException();
+            }
+
             var urlMaybe = EdgeUrl.parse(doc.url);
             if (urlMaybe.isEmpty())
                 continue;

code/services-application/search-service/java/nu/marginalia/search/SearchService.java

@@ -82,17 +82,17 @@ public class SearchService extends JoobyService {
         jooby.get("/site/https://*", this::handleSiteUrlRedirect);
         jooby.get("/site/http://*", this::handleSiteUrlRedirect);
 
+        String emptySvg = "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>";
         jooby.get("/site/{domain}/favicon", ctx -> {
             String domain = ctx.path("domain").value();
             logger.info("Finding icon for domain {}", domain);
-            domainQueries.getDomainId(new EdgeDomain(domain));
             try {
                 DbDomainQueries.DomainIdWithNode domainIdWithNode = domainQueries.getDomainIdWithNode(new EdgeDomain(domain));
                 var faviconMaybe = faviconClient.getFavicon(domain, domainIdWithNode.nodeAffinity());
 
                 if (faviconMaybe.isEmpty()) {
-                    ctx.setResponseCode(404);
-                    return "";
+                    ctx.setResponseType(MediaType.valueOf("image/svg+xml"));
+                    return emptySvg;
                 } else {
                     var favicon = faviconMaybe.get();
 
@@ -102,7 +102,8 @@ public class SearchService extends JoobyService {
                 }
             }
             catch (NoSuchElementException ex) {
-                ctx.setResponseCode(404);
+                ctx.setResponseType(MediaType.valueOf("image/svg+xml"));
+                return emptySvg;
             }
             return "";
         });

code/services-application/search-service/resources/jte/explore/main.jte

@@ -10,7 +10,7 @@
 
 @template.part.head(title = "Marginalia Search - Explore")
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans ">
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans ">
 
 @template.part.navbar(navbar = navbar)
 
@@ -23,7 +23,7 @@
 </header>
 
 <div class="max-w-[1400px] mx-auto flex flex-col gap-1 place-items-center">
-    <div class="border dark:border-gray-600 bg-white dark:bg-gray-800 dark:text-gray-100 my-4 p-3 rounded overflow-hidden flex flex-col space-y-4">
+    <div class="border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 dark:text-gray-100 my-4 p-3 rounded overflow-hidden flex flex-col space-y-4">
         @if (results.hasFocusDomain())
             <div class="flex space-x-1">
                 <span>Showing websites similar to <a class="font-mono text-liteblue dark:text-blue-200" href="/site/${results.focusDomain()}"><i class="fas fa-globe"></i> <span class="underline">${results.focusDomain()}</span></a></span>
@@ -36,7 +36,7 @@
     </div>
     <div class="grid-cols-1 gap-4 sm:grid sm:grid-cols-1 md:grid-cols-3 xl:grid-cols-4 mx-auto sm:p-4">
         @for (BrowseResult result : results.results())
-            <div class="bg-white border dark:border-gray-600 dark:bg-gray-800 rounded overflow-hidden">
+            <div class="bg-white border border-gray-300 dark:border-gray-600 dark:bg-gray-800 rounded overflow-hidden">
                 <div class="bg-margeblue text-white p-2 flex space-x-4 text-sm">
                     <span class="break-words">${result.displayDomain()}</span>
                     <div class="grow"></div>

code/services-application/search-service/resources/jte/part/testenvHead.jte

@@ -9,6 +9,7 @@
                     nicotine: '#f8f8ee',
                     margeblue: '#3e5f6f',
                     liteblue: '#0066cc',
+                    bgblue: '#e5e9eb',
                 },
                 screens: {
                     'coarsepointer': {

code/services-application/search-service/resources/jte/serp/dict-lookup.jte

@@ -15,7 +15,7 @@
 
 @template.part.head(title = "Marginalia Search - " +  parameters.query())
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 @template.part.navbar(navbar = navbar)
 
 <div>

code/services-application/search-service/resources/jte/serp/error.jte

@@ -9,7 +9,7 @@
 
 @template.part.head(title = "Marginalia Search - Error")
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 
 @template.part.navbar(navbar = navbar)
 

code/services-application/search-service/resources/jte/serp/main.jte

@@ -11,7 +11,7 @@
 
 @template.part.head(title = "Marginalia Search - " +  results.getQuery())
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 @template.part.navbar(navbar = navbar)
 
 <div>

code/services-application/search-service/resources/jte/serp/part/sidebar.jte

@@ -13,7 +13,7 @@
                 @for (List<SearchFilters.Filter> filterGroup : filters.getFilterGroups())
                     @for (SearchFilters.Filter filter : filterGroup)
                         <label class="flex items-center">
-                            <button title="${filter.displayName}" onclick="document.location='$unsafe{filter.url}'" class="flex-1 py-2 pl-2 rounded flex space-x-2 dark:has-[:checked]:bg-gray-950 has-[:checked]:bg-gray-100 has-[:checked]:text-slate-900 dark:has-[:checked]:text-slate-100 hover:bg-gray-50 dark:hover:bg-gray-950 bg-white dark:bg-gray-900 dark:border dark:border-gray-600 text-margeblue dark:text-slate-200 outline-1 active:outline">
+                            <button title="${filter.displayName}" onclick="document.location='$unsafe{filter.url}'" class="flex-1 py-2 pl-2 rounded flex space-x-2 dark:has-[:checked]:bg-gray-950 has-[:checked]:bg-gray-300 has-[:checked]:text-slate-900 dark:has-[:checked]:text-slate-100 hover:bg-gray-50 dark:hover:bg-gray-950 bg-white dark:bg-gray-900 dark:border dark:border-gray-600 text-margeblue dark:text-slate-200 outline-1 active:outline">
                                 @if (filter.current)
                                     <input type="checkbox" checked class="sr-only" aria-checked="true" />
                                 @else
@@ -38,7 +38,7 @@
             <div class="space-y-2">
                 @for (SearchFilters.SearchOption option : filters.searchOptions())
                     <label class="flex items-center">
-                            <button title="${option.name()}" onclick="document.location='$unsafe{option.getUrl()}'" class="flex-1 py-2 pl-2 rounded flex space-x-2 dark:has-[:checked]:bg-gray-950 has-[:checked]:bg-gray-100 has-[:checked]:text-slate-900 dark:has-[:checked]:text-slate-100 hover:bg-gray-50 dark:hover:bg-gray-950 bg-white dark:bg-gray-900 dark:border dark:border-gray-600 text-margeblue dark:text-slate-200 outline-1 active:outline">
+                            <button title="${option.name()}" onclick="document.location='$unsafe{option.getUrl()}'" class="flex-1 py-2 pl-2 rounded flex space-x-2 dark:has-[:checked]:bg-gray-950 has-[:checked]:bg-gray-300 has-[:checked]:text-slate-900 dark:has-[:checked]:text-slate-100 hover:bg-gray-50 dark:hover:bg-gray-950 bg-white dark:bg-gray-900 dark:border dark:border-gray-600 text-margeblue dark:text-slate-200 outline-1 active:outline">
                                 @if (option.isSet())
                                     <input type="checkbox" checked class="sr-only" aria-checked="true" />
                                 @else

code/services-application/search-service/resources/jte/serp/start.jte

@@ -15,7 +15,7 @@
 
 @template.part.head(title = "Marginalia Search", allowIndexing = true)
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 
 @template.part.navbar(navbar = navbar)
 
@@ -32,18 +32,14 @@
 
 @if (model.news().isEmpty())
 <div class="max-w-7xl mx-auto flex flex-col space-y-4 fill-w">
-    <div class="border dark:border-gray-600 dark:bg-gray-800 bg-white rounded p-2 m-4 ">
+    <div class="border border-gray-300 border-gray-100 dark:border-gray-600 dark:bg-gray-800 bg-white rounded p-2 m-4 ">
         <div class="text-slate-700 dark:text-white text-sm p-4">
-            <div class="fas fa-gift mr-1 text-margeblue dark:text-slate-200"></div>
-            This is the new design and home of Marginalia Search.
-            You can read about what this entails <a href="https://about.marginalia-search.com/article/redesign/"  class="underline text-liteblue dark:text-blue-200">here</a>.
-        <p class="my-4"></p>
-        The old version of Marginalia Search remains available at
-        <a href="https://old-search.marginalia.nu/" class="underline text-liteblue dark:text-blue-200">https://old-search.marginalia.nu/</a>.
+        The old version of Marginalia Search remains available
+        <a href="https://old-search.marginalia.nu/" class="underline text-liteblue dark:text-blue-200">here</a>.
         </div>
     </div>
     <div class="mx-auto flex flex-col sm:flex-row my-4 sm:space-x-2 space-y-2 sm:space-y-0 w-full md:w-auto px-2">
-    <div class="flex flex-col border dark:border-gray-600 rounded overflow-hidden dark:bg-gray-800 bg-white p-6 space-y-3">
+    <div class="flex flex-col border border-gray-300 dark:border-gray-600 rounded overflow-hidden dark:bg-gray-800 bg-white p-6 space-y-3">
         <div><i class="fas fa-sailboat mx-2 text-margeblue dark:text-slate-200"></i>Explore the Web</div>
         <ul class="list-disc ml-6 text-slate-700 dark:text-white text-xs leading-5">
             <li>Prioritizes non-commercial content</li>
@@ -52,7 +48,7 @@
         </ul>
     </div>
 
-    <div class="flex flex-col border dark:border-gray-600 rounded overflow-hidden dark:bg-gray-800 bg-white p-6  space-y-3 ">
+    <div class="flex flex-col border border-gray-300 dark:border-gray-600 rounded overflow-hidden dark:bg-gray-800 bg-white p-6  space-y-3 ">
         <div><i class="fas fa-hand-holding-hand mx-2 text-margeblue dark:text-slate-200"></i>Open Source</div>
         <ul class="list-disc ml-6 text-slate-700 dark:text-white text-xs leading-5">
            <li>Custom index and crawler software</li>
@@ -65,7 +61,7 @@
         </div>
     </div>
 
-    <div class="flex flex-col border dark:border-gray-600 rounded overflow-hidden dark:bg-gray-800 bg-white p-6  space-y-3 ">
+    <div class="flex flex-col border border-gray-300 dark:border-gray-600 rounded overflow-hidden dark:bg-gray-800 bg-white p-6  space-y-3 ">
         <div><i class="fas fa-lock mx-2 text-margeblue dark:text-slate-200"></i> Privacy by default</div>
         <ul class="list-disc ml-6 text-slate-700 dark:text-white text-xs leading-5">
             <li>Filter out tracking and adtech</li>

code/services-application/search-service/resources/jte/serp/unit-conversion.jte

@@ -13,7 +13,7 @@
 
 @template.part.head(title = "Marginalia Search - " +  parameters.query())
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 @template.part.navbar(navbar = navbar)
 
 <div>

code/services-application/search-service/resources/jte/siteinfo/crosstalk.jte

@@ -11,7 +11,7 @@
 
 @template.part.head(title = "Marginalia Search - " + model.domainA() + "/" + model.domainB())
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 
 @template.part.navbar(navbar = navbar)
 

code/services-application/search-service/resources/jte/siteinfo/main.jte

@@ -9,7 +9,7 @@
 
 @template.part.head(title = "Marginalia Search - " + model.domain())
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 
 @template.part.navbar(navbar = navbar)
 

code/services-application/search-service/resources/jte/siteinfo/part/linkedDomains.jte

@@ -7,7 +7,7 @@
 
 @if (!list.isEmpty())
 
-    <div class="bg-white dark:bg-gray-800 shadow-sm rounded overflow-hidden border dark:border-gray-600">
+    <div class="bg-white dark:bg-gray-800 shadow-sm rounded overflow-hidden border border-gray-300 dark:border-gray-600">
         <div class="px-4 py-2 bg-margeblue text-white border-b border-gray-200 dark:border-gray-600 flex place-items-baseline">
             <h2 class="text-md">${title}</h2>
             <div class="grow"></div>

code/services-application/search-service/resources/jte/siteinfo/start.jte

@@ -9,11 +9,11 @@
 
 @template.part.head(title = "Marginalia Search - Site Viewer")
 
-<body class="min-h-screen bg-slate-100 dark:bg-gray-900 dark:text-white font-sans " >
+<body class="min-h-screen bg-bgblue dark:bg-gray-900 dark:text-white font-sans " >
 
 @template.part.navbar(navbar = navbar)
 
-<header class="border-b border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 shadow-md">
+<header class="border-b border-gray-300 border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 shadow-md">
     <div class="max-w-[1400px] mx-auto px-4 py-4">
         <h1 class="text-base md:text-xl mr-2 md:mr-8 font-serif">View Site Information</h1>
     </div>
@@ -22,7 +22,7 @@
 <div class="max-w-[1000px] mx-auto flex gap-4 flex-col md:flex-row place-items-center md:place-items-start p-4">
 
 
-    <div class="border dark:border-gray-600 rounded md:my-4 overflow-hidden bg-white dark:bg-gray-800 flex flex-col space-y-2 flex-1">
+    <div class="border border-gray-300 dark:border-gray-600 rounded md:my-4 overflow-hidden bg-white dark:bg-gray-800 flex flex-col space-y-2 flex-1">
         <div class="bg-margeblue text-white p-2 text-sm mb-2">View Site Information</div>
 
         <p class="mx-4">This utility lets you explore what the search engine knows about the web,
@@ -45,7 +45,7 @@
     </div>
 
     @if (!model.domains().isEmpty())
-    <div class="border dark:border-gray-600 rounded md:my-4 overflow-hidden w-full md:w-auto">
+    <div class="border border-gray-300 dark:border-gray-600 rounded md:my-4 overflow-hidden w-full md:w-auto">
         <div class="bg-margeblue text-white p-2 text-sm">Recently Discovered Domains</div>
 
 

code/services-application/search-service/resources/jte/siteinfo/view/backlinks.jte

@@ -8,17 +8,17 @@
 <div class="flex flex-col space-y-4 my-4  w-full">
 
 @if (backlinks.results().isEmpty())
-<div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm ">
+<div class="border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm ">
     The search engine isn't aware of any backlinks to ${backlinks.domain()}!
 </div>
 @else
-<div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm">
+<div class="border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm">
     Showing documents linking to ${backlinks.domain()}
 </div>
 @endif
 
 @for (GroupedUrlDetails group : backlinks.results())
-<div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden mx-4">
+<div class="border dark:border-gray-600 border-gray-300 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden mx-4">
     <div class="flex space-x-2 flex-row place-items-baseline bg-margeblue text-white p-2  text-md">
         <span class="fas fa-globe"></span>
         <a href="/site/${group.domain().toString()}">${group.domain().toString()}</a>

code/services-application/search-service/resources/jte/siteinfo/view/docs.jte

@@ -9,17 +9,17 @@
 <div class="flex flex-col space-y-4 my-4">
 
 @if (docs.results().isEmpty())
-    <div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm">
+    <div class="border border-gray-300  dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm">
         The search engine doesn't index any documents from ${docs.domain()}
     </div>
 @else
-    <div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm">
+    <div class="border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden p-4 mx-4 text-gray-800 text-sm">
         Showing documents from ${docs.domain()}
     </div>
 @endif
 
 @for (UrlDetails details : docs.results())
-<div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden mx-4">
+<div class="border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-800 dark:text-white flex flex-col overflow-hidden mx-4">
     <div class="flex grow justify-between items-start p-4">
         <div class="flex-1">
             <h2 class="text-xl text-gray-800 dark:text-white font-serif mr-4">

code/services-application/search-service/resources/jte/siteinfo/view/overview.jte

@@ -8,7 +8,7 @@
 <!-- Main content -->
 
 <div class="flex-1 p-4 space-y-4 mx-auto w-full md:w-auto">
-    <div class="flex border dark:border-gray-600 rounded bg-white dark:bg-gray-800 flex-col space-y-4 pb-4 overflow-hidden md:max-w-lg" >
+    <div class="flex border border-gray-300  dark:border-gray-600 rounded bg-white dark:bg-gray-800 flex-col space-y-4 pb-4 overflow-hidden md:max-w-lg" >
         <div class="flex place-items-center space-x-2 p-2 text-md border-b dark:border-gray-600 bg-margeblue text-white">
             <img src="/site/${siteInfo.domain()}/favicon" style="width: 16px; height: 16px; vertical-align: center">
             <span>${siteInfo.domain()}</span>

code/services-application/search-service/resources/jte/siteinfo/view/reportDomain.jte

@@ -4,7 +4,7 @@
 @param ReportDomain reportDomain
 
 <div class="flex-col mx-auto">
-<div class="max-w-2xl mx-auto bg-white dark:bg-gray-800 border dark:border-gray-600 rounded overflow-auto shadow-sm my-4 space-y-4 w-full">
+<div class="max-w-2xl mx-auto bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded overflow-auto shadow-sm my-4 space-y-4 w-full">
     <div class="px-4 py-2 bg-margeblue text-white border-b border-gray-200 dark:border-gray-800">
         <h2 class="text-md">Report Domain Issue</h2>
     </div>

code/services-application/search-service/tailwind/tailwind.config.js

@@ -9,6 +9,7 @@ module.exports = {
         nicotine: '#f8f8ee',
         margeblue: '#3e5f6f',
         liteblue: '#0066cc',
+        bgblue: '#e5e9eb',
       },
       screens: {
         'coarsepointer': {