(sample) Ensure we flush the log before adding it to the tar file

We now sometimes get 206 when good.

build.gradle

@@ -5,7 +5,7 @@ plugins {
 
     // This is a workaround for a bug in the Jib plugin that causes it to stall randomly
     // https://github.com/GoogleContainerTools/jib/issues/3347
-    id 'com.google.cloud.tools.jib' version '3.4.4' apply(false)
+    id 'com.google.cloud.tools.jib' version '3.4.5' apply(false)
 }
 
 group 'marginalia'
@@ -47,7 +47,7 @@ ext {
     dockerImageBase='container-registry.oracle.com/graalvm/jdk:24'
     dockerImageTag='latest'
     dockerImageRegistry='marginalia'
-    jibVersion = '3.4.4'
+    jibVersion = '3.4.5'
 }
 
 idea {

code/common/model/java/nu/marginalia/model/EdgeUrl.java

@@ -1,16 +1,14 @@
 package nu.marginalia.model;
 
 import nu.marginalia.util.QueryParams;
+import org.apache.commons.lang3.StringUtils;
 
 import javax.annotation.Nullable;
 import java.io.Serializable;
-import java.net.MalformedURLException;
+import java.net.*;
-import java.net.URI;
+import java.nio.charset.StandardCharsets;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.regex.Pattern;
 
 public class EdgeUrl implements Serializable {
     public final String proto;
@@ -33,7 +31,7 @@ public class EdgeUrl implements Serializable {
 
     private static URI parseURI(String url) throws URISyntaxException {
         try {
-            return new URI(urlencodeFixer(url));
+            return EdgeUriFactory.parseURILenient(url);
         } catch (URISyntaxException ex) {
             throw new URISyntaxException("Failed to parse URI '" + url + "'", ex.getMessage());
         }
@@ -51,58 +49,6 @@ public class EdgeUrl implements Serializable {
         }
     }
 
-    private static Pattern badCharPattern = Pattern.compile("[ \t\n\"<>\\[\\]()',|]");
-
-    /* Java's URI parser is a bit too strict in throwing exceptions when there's an error.
-
-       Here on the Internet, standards are like the picture on the box of the frozen pizza,
-       and what you get is more like what's on the inside, we try to patch things instead,
-       just give it a best-effort attempt att cleaning out broken or unnecessary constructions
-       like bad or missing URLEncoding
-     */
-    public static String urlencodeFixer(String url) throws URISyntaxException {
-        var s = new StringBuilder();
-        String goodChars = "&.?:/-;+$#";
-        String hexChars = "0123456789abcdefABCDEF";
-
-        int pathIdx = findPathIdx(url);
-        if (pathIdx < 0) { // url looks like http://marginalia.nu
-            return url + "/";
-        }
-        s.append(url, 0, pathIdx);
-
-        // We don't want the fragment, and multiple fragments breaks the Java URIParser for some reason
-        int end = url.indexOf("#");
-        if (end < 0) end = url.length();
-
-        for (int i = pathIdx; i < end; i++) {
-            int c = url.charAt(i);
-
-            if (goodChars.indexOf(c) >= 0 || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
-                s.appendCodePoint(c);
-            } else if (c == '%' && i + 2 < end) {
-                int cn = url.charAt(i + 1);
-                int cnn = url.charAt(i + 2);
-                if (hexChars.indexOf(cn) >= 0 && hexChars.indexOf(cnn) >= 0) {
-                    s.appendCodePoint(c);
-                } else {
-                    s.append("%25");
-                }
-            } else {
-                s.append(String.format("%%%02X", c));
-            }
-        }
-
-        return s.toString();
-    }
-
-    private static int findPathIdx(String url) throws URISyntaxException {
-        int colonIdx = url.indexOf(':');
-        if (colonIdx < 0 || colonIdx + 2 >= url.length()) {
-            throw new URISyntaxException(url, "Lacking protocol");
-        }
-        return url.indexOf('/', colonIdx + 2);
-    }
 
     public EdgeUrl(URI URI) {
         try {
@@ -166,11 +112,32 @@ public class EdgeUrl implements Serializable {
             sb.append(port);
         }
 
+        EdgeUriFactory.urlencodePath(sb, path);
+
+        if (param != null) {
+            EdgeUriFactory.urlencodeQuery(sb, param);
+        }
+
+        return sb.toString();
+    }
+
+
+    public String toDisplayString() {
+        StringBuilder sb = new StringBuilder(256);
+
+        sb.append(proto);
+        sb.append("://");
+        sb.append(domain);
+
+        if (port != null) {
+            sb.append(':');
+            sb.append(port);
+        }
+
         sb.append(path);
 
         if (param != null) {
-            sb.append('?');
+            sb.append('?').append(param);
-            sb.append(param);
         }
 
         return sb.toString();
@@ -247,3 +214,244 @@ public class EdgeUrl implements Serializable {
     }
 
 }
+
+class EdgeUriFactory {
+    public static URI parseURILenient(String url) throws URISyntaxException {
+
+        if (shouldOmitUrlencodeRepair(url)) {
+            try {
+                return new URI(url);
+            }
+            catch (URISyntaxException ex) {
+                // ignore and run the lenient parser
+            }
+        }
+
+        var s = new StringBuilder(url.length()+8);
+
+        int pathIdx = findPathIdx(url);
+        if (pathIdx < 0) { // url looks like http://marginalia.nu
+            return new URI(url + "/");
+        }
+        s.append(url, 0, pathIdx);
+
+        // We don't want the fragment, and multiple fragments breaks the Java URIParser for some reason
+        int end = url.indexOf("#");
+        if (end < 0) end = url.length();
+
+        int queryIdx = url.indexOf('?');
+        if (queryIdx < 0) queryIdx = end;
+
+        urlencodePath(s, url.substring(pathIdx, queryIdx));
+        if (queryIdx < end) {
+            urlencodeQuery(s, url.substring(queryIdx + 1, end));
+        }
+        return new URI(s.toString());
+    }
+
+    /** Break apart the path element of an URI into its components, and then
+     * urlencode any component that needs it, and recombine it into a single
+     * path element again.
+     */
+    public static void urlencodePath(StringBuilder sb, String path) {
+        if (path == null || path.isEmpty()) {
+            return;
+        }
+
+        String[] pathParts = StringUtils.split(path, '/');
+        if (pathParts.length == 0) {
+            sb.append('/');
+            return;
+        }
+
+        boolean shouldUrlEncode = false;
+        for (String pathPart : pathParts) {
+            if (pathPart.isEmpty()) continue;
+
+            if (needsUrlEncode(pathPart)) {
+                shouldUrlEncode = true;
+                break;
+            }
+        }
+
+        for (String pathPart : pathParts) {
+            if (pathPart.isEmpty()) continue;
+
+            if (shouldUrlEncode) {
+                sb.append('/');
+                sb.append(URLEncoder.encode(pathPart, StandardCharsets.UTF_8).replace("+", "%20"));
+            } else {
+                sb.append('/');
+                sb.append(pathPart);
+            }
+        }
+
+        if (path.endsWith("/")) {
+            sb.append('/');
+        }
+
+    }
+
+    /** Break apart the query element of a URI into its components, and then
+     * urlencode any component that needs it, and recombine it into a single
+     * query element again.
+     */
+    public static void urlencodeQuery(StringBuilder sb, String param) {
+        if (param == null || param.isEmpty()) {
+            return;
+        }
+
+        String[] queryParts = StringUtils.split(param, '&');
+
+        boolean shouldUrlEncode = false;
+        for (String queryPart : queryParts) {
+            if (queryPart.isEmpty()) continue;
+
+            if (needsUrlEncode(queryPart)) {
+                shouldUrlEncode = true;
+                break;
+            }
+        }
+
+        boolean first = true;
+        for (String queryPart : queryParts) {
+            if (queryPart.isEmpty()) continue;
+
+            if (first) {
+                sb.append('?');
+                first = false;
+            } else {
+                sb.append('&');
+            }
+
+            if (shouldUrlEncode) {
+                int idx = queryPart.indexOf('=');
+                if (idx < 0) {
+                    sb.append(URLEncoder.encode(queryPart, StandardCharsets.UTF_8));
+                } else {
+                    sb.append(URLEncoder.encode(queryPart.substring(0, idx), StandardCharsets.UTF_8));
+                    sb.append('=');
+                    sb.append(URLEncoder.encode(queryPart.substring(idx + 1), StandardCharsets.UTF_8));
+                }
+            } else {
+                sb.append(queryPart);
+            }
+        }
+    }
+
+    /** Test if the url element needs URL encoding.
+     * <p></p>
+     * Note we may have been given an already encoded path element,
+     * so we include % and + in the list of good characters
+     */
+    static boolean needsUrlEncode(String urlElement) {
+        for (int i = 0; i < urlElement.length(); i++) {
+            char c = urlElement.charAt(i);
+
+            if (isUrlSafe(c)) continue;
+            if ("+".indexOf(c) >= 0) continue;
+            if (c == '%' && i + 2 < urlElement.length()) {
+                char c1 = urlElement.charAt(i + 1);
+                char c2 = urlElement.charAt(i + 2);
+                if (isHexDigit(c1) && isHexDigit(c2)) {
+                    i += 2;
+                    continue;
+                }
+            }
+
+            return true;
+        }
+
+        return false;
+    }
+
+
+    static boolean isUrlSafe(int c) {
+        if (c >= 'a' && c <= 'z') return true;
+        if (c >= 'A' && c <= 'Z') return true;
+        if (c >= '0' && c <= '9') return true;
+        if (c == '-' || c == '_' || c == '.' || c == '~') return true;
+
+        return false;
+    }
+
+    /** Test if the URL is a valid URL that does not need to be
+     * urlencoded.
+     * <p></p>
+     * This is a very simple heuristic test that does not guarantee
+     * that the URL is valid, but it will identify cases where we
+     * are fairly certain that the URL does not need encoding,
+     * so we can skip a bunch of allocations and string operations
+     * that would otherwise be needed to fix the URL.
+     */
+    static boolean shouldOmitUrlencodeRepair(String url) {
+        int idx = 0;
+        final int len = url.length();
+
+        // Validate the scheme
+        while (idx < len - 2) {
+            char c = url.charAt(idx++);
+            if (c == ':') break;
+            if (!isAsciiAlphabetic(c)) return false;
+        }
+        if (url.charAt(idx++) != '/') return false;
+        if (url.charAt(idx++) != '/') return false;
+
+        // Validate the authority
+        while (idx < len) {
+            char c = url.charAt(idx++);
+            if (c == '/') break;
+            if (c == ':') continue;
+            if (c == '@') continue;
+            if (!isUrlSafe(c)) return false;
+        }
+
+        // Validate the path
+        if (idx >= len) return true;
+
+        while (idx < len) {
+            char c = url.charAt(idx++);
+            if (c == '?') break;
+            if (c == '/') continue;
+            if (c == '#') return true;
+            if (!isUrlSafe(c)) return false;
+        }
+
+        if (idx >= len) return true;
+
+        // Validate the query
+        while (idx < len) {
+            char c = url.charAt(idx++);
+            if (c == '&') continue;
+            if (c == '=') continue;
+            if (c == '#') return true;
+            if (!isUrlSafe(c)) return false;
+        }
+
+        return true;
+    }
+
+
+    private static boolean isAsciiAlphabetic(int c) {
+        return (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    private static boolean isHexDigit(int c) {
+        return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    /** Find the index of the path element in a URL.
+     * <p></p>
+     * The path element starts after the scheme and authority part of the URL,
+     * which is everything up to and including the first slash after the colon.
+     */
+    private static int findPathIdx(String url) throws URISyntaxException {
+        int colonIdx = url.indexOf(':');
+        if (colonIdx < 0 || colonIdx + 3 >= url.length()) {
+            throw new URISyntaxException(url, "Lacking scheme");
+        }
+        return url.indexOf('/', colonIdx + 3);
+    }
+
+
+}

code/common/model/test/nu/marginalia/model/EdgeUrlTest.java

@@ -1,6 +1,6 @@
 package nu.marginalia.model;
 
-import nu.marginalia.model.EdgeUrl;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.net.URISyntaxException;
@@ -21,25 +21,70 @@ class EdgeUrlTest {
                 new EdgeUrl("https://memex.marginalia.nu/#here")
         );
     }
+
     @Test
-    public void testParam() throws URISyntaxException {
+    void testUriFromString() throws URISyntaxException {
-        System.out.println(new EdgeUrl("https://memex.marginalia.nu/index.php?id=1").toString());
+        // We test these URLs several times as we perform URLEncode-fixing both when parsing the URL and when
-        System.out.println(new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").toString());
+        // converting it back to a string, we want to ensure there is no changes along the way.
-    }
+
-    @Test
+        Assertions.assertEquals("/", EdgeUriFactory.parseURILenient("https://www.example.com/").getPath());
-    void urlencodeFixer() throws URISyntaxException {
+        Assertions.assertEquals("https://www.example.com/", EdgeUriFactory.parseURILenient("https://www.example.com/").toString());
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/#heredoc"));
+        Assertions.assertEquals("https://www.example.com/", new EdgeUrl("https://www.example.com/").toString());
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/%-sign"));
+
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/%22-sign"));
+        Assertions.assertEquals("/", EdgeUriFactory.parseURILenient("https://www.example.com/#heredoc").getPath());
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/\n \"huh\""));
+        Assertions.assertEquals("https://www.example.com/", EdgeUriFactory.parseURILenient("https://www.example.com/#heredoc").toString());
+        Assertions.assertEquals("https://www.example.com/", new EdgeUrl("https://www.example.com/#heredoc").toString());
+
+        Assertions.assertEquals("/trailingslash/", EdgeUriFactory.parseURILenient("https://www.example.com/trailingslash/").getPath());
+        Assertions.assertEquals("https://www.example.com/trailingslash/", EdgeUriFactory.parseURILenient("https://www.example.com/trailingslash/").toString());
+        Assertions.assertEquals("https://www.example.com/trailingslash/", new EdgeUrl("https://www.example.com/trailingslash/").toString());
+
+        Assertions.assertEquals("/%-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%25-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%25-sign", new EdgeUrl("https://www.example.com/%-sign").toString());
+
+        Assertions.assertEquals("/%-sign/\"-sign", EdgeUriFactory.parseURILenient("https://www.example.com//%-sign/\"-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%25-sign/%22-sign", EdgeUriFactory.parseURILenient("https://www.example.com//%-sign/\"-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%25-sign/%22-sign", new EdgeUrl("https://www.example.com//%-sign/\"-sign").toString());
+
+        Assertions.assertEquals("/\"-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%22-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%22-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%22-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%22-sign", new EdgeUrl("https://www.example.com/%22-sign").toString());
+
+        Assertions.assertEquals("/\n \"huh\"", EdgeUriFactory.parseURILenient("https://www.example.com/\n \"huh\"").getPath());
+        Assertions.assertEquals("https://www.example.com/%0A%20%22huh%22", EdgeUriFactory.parseURILenient("https://www.example.com/\n \"huh\"").toString());
+        Assertions.assertEquals("https://www.example.com/%0A%20%22huh%22", new EdgeUrl("https://www.example.com/\n \"huh\"").toString());
+
+        Assertions.assertEquals("/wiki/Sámi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").getPath());
+        Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").toString());
+        Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", new EdgeUrl("https://en.wikipedia.org/wiki/Sámi").toString());
+
+        Assertions.assertEquals("https://www.prijatelji-zivotinja.hr/index.en.php?id=2301k", new EdgeUrl("https://www.prijatelji-zivotinja.hr/index.en.php?id=2301k").toString());
     }
 
     @Test
     void testParms() throws URISyntaxException {
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?id=123"));
+        Assertions.assertEquals("id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").param);
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?t=123"));
+        Assertions.assertEquals("https://search.marginalia.nu/?id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").toString());
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?v=123"));
+
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?m=123"));
+        Assertions.assertEquals("t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").param);
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?follow=123"));
+        Assertions.assertEquals("https://search.marginalia.nu/?t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").toString());
+
+        Assertions.assertEquals("v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").toString());
+
+        Assertions.assertEquals("id=1", new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/showthread.php?id=1",
+                new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").toString());
+
+
+        Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/sh%C3%B6wthr%C3%ABad.php?id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").toString());
+
+        Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/sh%C3%B6wthr%C3%ABad.php?id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").toString());
+
+        Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?m=123").param);
+        Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?follow=123").param);
     }
 }

code/common/service/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java

@@ -59,16 +59,13 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo
      */
     @Override
     public void progress(String step, int stepProgress, int stepCount) {
+        int lastProgress = this.progress;
         this.step = step;
-
-
-        // off by one since we calculate the progress based on the number of steps,
-        // and Enum.ordinal() is zero-based (so the 5th step in a 5 step task is 4, not 5; resulting in the
-        // final progress being 80% and not 100%)
-
         this.progress = (int) Math.round(100. * stepProgress / (double) stepCount);
 
-        logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        if (this.progress / 10 != lastProgress / 10) {
+            logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        }
     }
 
     /** Wrap a collection to provide heartbeat progress updates as it's iterated through */

code/common/service/java/nu/marginalia/service/control/ServiceAdHocTaskHeartbeatImpl.java

@@ -57,16 +57,13 @@ public class ServiceAdHocTaskHeartbeatImpl implements AutoCloseable, ServiceAdHo
      */
     @Override
     public void progress(String step, int stepProgress, int stepCount) {
+        int lastProgress = this.progress;
         this.step = step;
-
-
-        // off by one since we calculate the progress based on the number of steps,
-        // and Enum.ordinal() is zero-based (so the 5th step in a 5 step task is 4, not 5; resulting in the
-        // final progress being 80% and not 100%)
-
         this.progress = (int) Math.round(100. * stepProgress / (double) stepCount);
 
-        logger.info("ServiceTask {} progress: {}%", taskBase, progress);
+        if (this.progress / 10 != lastProgress / 10) {
+            logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        }
     }
 
     public void shutDown() {

code/common/service/java/nu/marginalia/service/server/JoobyService.java

@@ -122,6 +122,11 @@ public class JoobyService {
         // single digit percentage difference since HTML already compresses very well with level = 1.
         options.setCompressionLevel(1);
 
+        // Set a cap on the number of worker threads, as Jooby's default value does not seem to consider
+        // multi-tenant servers with high thread counts, and spins up an exorbitant number of threads in that
+        // scenario
+        options.setWorkerThreads(Math.min(128, options.getWorkerThreads()));
+
 
         jooby.setServerOptions(options);
 

code/execution/api/java/nu/marginalia/executor/client/ExecutorExportClient.java

@@ -48,12 +48,13 @@ public class ExecutorExportClient {
         return msgId;
     }
 
-    public void exportSampleData(int node, FileStorageId fid, int size, String name) {
+    public void exportSampleData(int node, FileStorageId fid, int size, String ctFilter, String name) {
         channelPool.call(ExecutorExportApiBlockingStub::exportSampleData)
                 .forNode(node)
                 .run(RpcExportSampleData.newBuilder()
                         .setFileStorageId(fid.id())
                         .setSize(size)
+                        .setCtFilter(ctFilter)
                         .setName(name)
                         .build());
     }

code/execution/api/src/main/protobuf/executor-api.proto

@@ -100,6 +100,7 @@ message RpcExportSampleData {
   int64 fileStorageId = 1;
   int32 size = 2;
   string name = 3;
+  string ctFilter = 4;
 }
 message RpcDownloadSampleData {
   string sampleSet = 1;

code/execution/java/nu/marginalia/actor/task/DownloadSampleActor.java

@@ -8,6 +8,7 @@ import nu.marginalia.actor.state.ActorResumeBehavior;
 import nu.marginalia.actor.state.ActorStep;
 import nu.marginalia.actor.state.Resume;
 import nu.marginalia.service.control.ServiceEventLog;
+import nu.marginalia.service.control.ServiceHeartbeat;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorage;
 import nu.marginalia.storage.model.FileStorageId;
@@ -19,6 +20,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
@@ -32,6 +34,7 @@ public class DownloadSampleActor extends RecordActorPrototype {
 
     private final FileStorageService storageService;
     private final ServiceEventLog eventLog;
+    private final ServiceHeartbeat heartbeat;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resume(behavior = ActorResumeBehavior.ERROR)
@@ -66,15 +69,39 @@ public class DownloadSampleActor extends RecordActorPrototype {
 
                 Files.deleteIfExists(Path.of(tarFileName));
 
-                try (var is = new BufferedInputStream(new URI(downloadURI).toURL().openStream());
+                HttpURLConnection urlConnection = (HttpURLConnection) new URI(downloadURI).toURL().openConnection();
-                     var os = new BufferedOutputStream(Files.newOutputStream(Path.of(tarFileName), StandardOpenOption.CREATE))) {
+
-                    is.transferTo(os);
+                try (var hb = heartbeat.createServiceAdHocTaskHeartbeat("Downloading sample")) {
+                    long size = urlConnection.getContentLengthLong();
+                    byte[] buffer = new byte[8192];
+
+                    try (var is = new BufferedInputStream(urlConnection.getInputStream());
+                         var os = new BufferedOutputStream(Files.newOutputStream(Path.of(tarFileName), StandardOpenOption.CREATE))) {
+                        long copiedSize = 0;
+
+                        while (copiedSize < size) {
+                            int read = is.read(buffer);
+
+                            if (read < 0) // We've been promised a file of length 'size'
+                                throw new IOException("Unexpected end of stream");
+
+                            os.write(buffer, 0, read);
+                            copiedSize += read;
+
+                            // Update progress bar
+                            hb.progress(String.format("%d MB", copiedSize / 1024 / 1024), (int) (copiedSize / 1024), (int) (size / 1024));
+                        }
+                    }
+
                 }
                 catch (Exception ex) {
                     eventLog.logEvent(DownloadSampleActor.class, "Error downloading sample");
                     logger.error("Error downloading sample", ex);
                     yield new Error();
                 }
+                finally {
+                    urlConnection.disconnect();
+                }
 
                 eventLog.logEvent(DownloadSampleActor.class, "Download complete");
                 yield new Extract(fileStorageId, tarFileName);
@@ -170,11 +197,12 @@ public class DownloadSampleActor extends RecordActorPrototype {
     @Inject
     public DownloadSampleActor(Gson gson,
                                FileStorageService storageService,
-                               ServiceEventLog eventLog)
+                               ServiceEventLog eventLog, ServiceHeartbeat heartbeat)
     {
         super(gson);
         this.storageService = storageService;
         this.eventLog = eventLog;
+        this.heartbeat = heartbeat;
     }
 
 }

code/execution/java/nu/marginalia/actor/task/ExportSampleDataActor.java

@@ -26,32 +26,32 @@ public class ExportSampleDataActor extends RecordActorPrototype {
     private final MqOutbox exportTasksOutbox;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    public record Export(FileStorageId crawlId, int size, String name) implements ActorStep {}
+    public record Export(FileStorageId crawlId, int size, String ctFilter, String name) implements ActorStep {}
-    public record Run(FileStorageId crawlId, FileStorageId destId, int size, String name, long msgId) implements ActorStep {
+    public record Run(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name, long msgId) implements ActorStep {
-        public Run(FileStorageId crawlId, FileStorageId destId, int size, String name) {
+        public Run(FileStorageId crawlId, FileStorageId destId, int size, String name, String ctFilter) {
-            this(crawlId, destId, size, name, -1);
+            this(crawlId, destId, size, name, ctFilter,-1);
         }
     }
 
     @Override
     public ActorStep transition(ActorStep self) throws Exception {
         return switch(self) {
-            case Export(FileStorageId crawlId, int size, String name) -> {
+            case Export(FileStorageId crawlId, int size, String ctFilter, String name) -> {
                 var storage = storageService.allocateStorage(FileStorageType.EXPORT,
                         "crawl-sample-export",
                         "Crawl Data Sample " + name + "/" + size + " " + LocalDateTime.now()
                 );
 
                 if (storage == null) yield new Error("Bad storage id");
-                yield new Run(crawlId, storage.id(), size, name);
+                yield new Run(crawlId, storage.id(), size, ctFilter, name);
             }
-            case Run(FileStorageId crawlId, FileStorageId destId, int size, String name, long msgId) when msgId < 0 -> {
+            case Run(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name, long msgId) when msgId < 0 -> {
                 storageService.setFileStorageState(destId, FileStorageState.NEW);
 
-                long newMsgId = exportTasksOutbox.sendAsync(ExportTaskRequest.sampleData(crawlId, destId, size, name));
+                long newMsgId = exportTasksOutbox.sendAsync(ExportTaskRequest.sampleData(crawlId, destId, ctFilter, size, name));
-                yield new Run(crawlId, destId, size, name, newMsgId);
+                yield new Run(crawlId, destId, size, ctFilter, name, newMsgId);
             }
-            case Run(_, FileStorageId destId, _, _, long msgId) -> {
+            case Run(_, FileStorageId destId, _, _, _, long msgId) -> {
                 var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
@@ -70,7 +70,7 @@ public class ExportSampleDataActor extends RecordActorPrototype {
 
     @Override
     public String describe() {
-        return "Export RSS/Atom feeds from crawl data";
+        return "Export sample crawl data";
     }
 
     @Inject

code/execution/java/nu/marginalia/execution/ExecutorExportGrpcService.java

@@ -49,6 +49,7 @@ public class ExecutorExportGrpcService
                     new ExportSampleDataActor.Export(
                             FileStorageId.of(request.getFileStorageId()),
                             request.getSize(),
+                            request.getCtFilter(),
                             request.getName()
                     )
             );

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedFetcherService.java

@@ -229,13 +229,15 @@ public class FeedFetcherService {
                     .timeout(Duration.ofSeconds(15))
                     ;
 
-            if (ifModifiedSinceDate != null) {
+            // Set the If-Modified-Since or If-None-Match headers if we have them
+            // though since there are certain idiosyncrasies in server implementations,
+            // we avoid setting both at the same time as that may turn a 304 into a 200.
+            if (ifNoneMatchTag != null) {
+                requestBuilder.header("If-None-Match", ifNoneMatchTag);
+            } else if (ifModifiedSinceDate != null) {
                 requestBuilder.header("If-Modified-Since", ifModifiedSinceDate);
             }
 
-            if (ifNoneMatchTag != null) {
-                requestBuilder.header("If-None-Match", ifNoneMatchTag);
-            }
 
             HttpRequest getRequest = requestBuilder.build();
 

code/processes/crawling-process/build.gradle

@@ -67,8 +67,6 @@ dependencies {
     testImplementation libs.mockito
     testImplementation libs.wiremock
 
-
-
     testImplementation project(':code:processes:test-data')
 }
 

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -42,8 +42,8 @@ import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.security.Security;
 import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -66,7 +66,8 @@ public class CrawlerMain extends ProcessMainClass {
     private final DomainLocks domainLocks = new DomainLocks();
 
     private final Map<String, CrawlTask> pendingCrawlTasks = new ConcurrentHashMap<>();
-    private final ArrayBlockingQueue<CrawlTask> retryQueue = new ArrayBlockingQueue<>(64);
+
+    private final LinkedBlockingQueue<CrawlTask> retryQueue = new LinkedBlockingQueue<>();
 
     private final AtomicInteger tasksDone = new AtomicInteger(0);
     private final HttpFetcherImpl fetcher;
@@ -263,17 +264,16 @@ public class CrawlerMain extends ProcessMainClass {
                 if (workLog.isJobFinished(crawlSpec.domain))
                     continue;
 
-                var task = new CrawlTask(
+                var task = new CrawlTask(crawlSpec, anchorTagsSource, outputDir, warcArchiver, domainStateDb, workLog);
-                        crawlSpec,
-                        anchorTagsSource,
-                        outputDir,
-                        warcArchiver,
-                        domainStateDb,
-                        workLog);
 
                 // Try to run immediately, to avoid unnecessarily keeping the entire work set in RAM
                 if (!trySubmitDeferredTask(task)) {
-                    // Otherwise add to the taskList for deferred execution
+
+                    // Drain the retry queue to the taskList, and try to submit any tasks that are in the retry queue
+                    retryQueue.drainTo(taskList);
+                    taskList.removeIf(this::trySubmitDeferredTask);
+
+                    // Then add this new task to the retry queue
                     taskList.add(task);
                 }
             }
@@ -289,10 +289,13 @@ public class CrawlerMain extends ProcessMainClass {
 
                 if (hasTasks || hasRetryTasks || hasRunningTasks) {
                     retryQueue.drainTo(taskList);
+
+                    // Try to submit any tasks that are in the retry queue (this will block if the pool is full)
                     taskList.removeIf(this::trySubmitDeferredTask);
+
                     // Add a small pause here to avoid busy looping toward the end of the execution cycle when
                     // we might have no new viable tasks to run for hours on end
-                    TimeUnit.MILLISECONDS.sleep(50);
+                    TimeUnit.MILLISECONDS.sleep(5);
                 } else {
                     // We have no tasks to run, and no tasks in the retry queue
                     // but we wait a bit to see if any new tasks come in via the retry queue
@@ -430,7 +433,7 @@ public class CrawlerMain extends ProcessMainClass {
         /** Best effort indicator whether we could start this now without getting stuck in
          * DomainLocks purgatory */
         public boolean canRun() {
-            return domainLocks.canLock(new EdgeDomain(domain));
+            return domainLocks.isLockableHint(new EdgeDomain(domain));
         }
 
         @Override
@@ -445,14 +448,15 @@ public class CrawlerMain extends ProcessMainClass {
             // We don't have a lock, so we can't run this task
             // we return to avoid blocking the pool for too long
             if (lock.isEmpty()) {
-                // Sleep a moment to avoid busy looping via the retry queue
+                pendingCrawlTasks.remove(domain);
-                Thread.sleep(10);
+                retryQueue.put(this);
-                retryQueue.add(this);
                 return;
             }
             DomainLocks.DomainLock domainLock = lock.get();
 
             try (domainLock) {
+                Thread.currentThread().setName("crawling:" + domain);
+
                 Path newWarcFile = CrawlerOutputFile.createWarcPath(outputDir, id, domain, CrawlerOutputFile.WarcFileVersion.LIVE);
                 Path tempFile = CrawlerOutputFile.createWarcPath(outputDir, id, domain, CrawlerOutputFile.WarcFileVersion.TEMP);
                 Path slopFile = CrawlerOutputFile.createSlopPath(outputDir, id, domain);
@@ -484,7 +488,7 @@ public class CrawlerMain extends ProcessMainClass {
                     // (mostly a case when migrating from legacy->warc)
                     reference.delete();
 
-                    // Convert the WARC file to Parquet
+                    // Convert the WARC file to Slop
                     SlopCrawlDataRecord
                             .convertWarc(domain, userAgent, newWarcFile, slopFile);
 

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/ContentTags.java

@@ -19,11 +19,13 @@ public record ContentTags(String etag, String lastMod) {
     /** Paints the tags onto the request builder. */
     public void paint(HttpGet request) {
 
+        // Paint the ETag header if present,
+        // otherwise paint the Last-Modified header
+        // (but not both at the same time due to some servers not liking it)
+
         if (etag != null) {
             request.addHeader("If-None-Match", etag);
-        }
+        } else if (lastMod != null) {
-
-        if (lastMod != null) {
             request.addHeader("If-Modified-Since", lastMod);
         }
     }

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -51,8 +51,10 @@ import javax.net.ssl.SSLException;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.net.URISyntaxException;
+import java.net.UnknownHostException;
 import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
@@ -393,25 +395,31 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
             if (probeType == HttpFetcher.ProbeType.FULL) {
                 try {
                     var probeResult = probeContentType(url, cookies, timer, contentTags);
-                    logger.info(crawlerAuditMarker, "Probe result {} for {}", probeResult.getClass().getSimpleName(), url);
+
                     switch (probeResult) {
                         case HttpFetcher.ContentTypeProbeResult.NoOp():
                             break; //
                         case HttpFetcher.ContentTypeProbeResult.Ok(EdgeUrl resolvedUrl):
+                            logger.info(crawlerAuditMarker, "Probe result OK for {}", url);
                             url = resolvedUrl; // If we were redirected while probing, use the final URL for fetching
                             break;
                         case ContentTypeProbeResult.BadContentType badContentType:
                             warcRecorder.flagAsFailedContentTypeProbe(url, badContentType.contentType(), badContentType.statusCode());
+                            logger.info(crawlerAuditMarker, "Probe result Bad ContenType ({}) for {}", badContentType.contentType(), url);
                             return new HttpFetchResult.ResultNone();
                         case ContentTypeProbeResult.BadContentType.Timeout(Exception ex):
+                            logger.info(crawlerAuditMarker, "Probe result Timeout for {}", url);
                             warcRecorder.flagAsTimeout(url);
                             return new HttpFetchResult.ResultException(ex);
                         case ContentTypeProbeResult.Exception(Exception ex):
+                            logger.info(crawlerAuditMarker, "Probe result Exception({}) for {}", ex.getClass().getSimpleName(), url);
                             warcRecorder.flagAsError(url, ex);
                             return new HttpFetchResult.ResultException(ex);
                         case ContentTypeProbeResult.HttpError httpError:
+                            logger.info(crawlerAuditMarker, "Probe result HTTP Error ({}) for {}", httpError.statusCode(), url);
                             return new HttpFetchResult.ResultException(new HttpException("HTTP status code " + httpError.statusCode() + ": " + httpError.message()));
                         case ContentTypeProbeResult.Redirect redirect:
+                            logger.info(crawlerAuditMarker, "Probe result redirect for {} -> {}", url, redirect.location());
                             return new HttpFetchResult.ResultRedirect(redirect.location());
                     }
                 } catch (Exception ex) {
@@ -430,27 +438,32 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
             contentTags.paint(request);
 
             try (var sl = new SendLock()) {
+                Instant start = Instant.now();
                 HttpFetchResult result = warcRecorder.fetch(client, cookies, request);
 
+                Duration fetchDuration = Duration.between(start, Instant.now());
+
                 if (result instanceof HttpFetchResult.ResultOk ok) {
                     if (ok.statusCode() == 304) {
-                        return new HttpFetchResult.Result304Raw();
+                        result = new HttpFetchResult.Result304Raw();
                     }
                 }
 
                 switch (result) {
-                    case HttpFetchResult.ResultOk ok -> logger.info(crawlerAuditMarker, "Fetch result OK {} for {}", ok.statusCode(), url);
+                    case HttpFetchResult.ResultOk ok -> logger.info(crawlerAuditMarker, "Fetch result OK {} for {} ({} ms)", ok.statusCode(), url, fetchDuration.toMillis());
                     case HttpFetchResult.ResultRedirect redirect -> logger.info(crawlerAuditMarker, "Fetch result redirect: {}  for {}", redirect.url(), url);
-                    case HttpFetchResult.ResultNone none -> logger.info(crawlerAuditMarker, "Fetch result none  for {}", url);
+                    case HttpFetchResult.ResultNone none -> logger.info(crawlerAuditMarker, "Fetch result none for {}", url);
-                    case HttpFetchResult.ResultException ex -> logger.error(crawlerAuditMarker, "Fetch result exception for " + url + ": {}", ex.ex());
+                    case HttpFetchResult.ResultException ex -> logger.error(crawlerAuditMarker, "Fetch result exception for {}", url, ex.ex());
                     case HttpFetchResult.Result304Raw raw -> logger.info(crawlerAuditMarker, "Fetch result: 304 Raw for {}", url);
                     case HttpFetchResult.Result304ReplacedWithReference ref -> logger.info(crawlerAuditMarker, "Fetch result: 304 With reference for {}", url);
                 }
+
                 return result;
             }
         }
         catch (Exception ex) {
-            ex.printStackTrace();
+            logger.error(crawlerAuditMarker, "Fetch result exception for {}", url, ex);
+
             return new HttpFetchResult.ResultException(ex);
         }
 
@@ -623,14 +636,12 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
 
     @Override
     public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context) {
-        if (exception instanceof SocketTimeoutException) { // Timeouts are not recoverable
+        return switch (exception) {
-            return false;
+            case SocketTimeoutException ste -> false;
-        }
+            case SSLException ssle -> false;
-        if (exception instanceof SSLException) { // SSL exceptions are unlikely to be recoverable
+            case UnknownHostException uhe -> false;
-            return false;
+            default -> executionCount <= 3;
-        }
+        };
-
-        return executionCount <= 3;
     }
 
     @Override

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcInputBuffer.java

@@ -57,6 +57,7 @@ public abstract class WarcInputBuffer implements AutoCloseable {
             return new ErrorBuffer();
         }
 
+        Instant start = Instant.now();
         InputStream is = null;
         try {
             is = entity.getContent();
@@ -71,8 +72,25 @@ public abstract class WarcInputBuffer implements AutoCloseable {
             }
         }
         finally {
+            // We're required to consume the stream to avoid leaking connections,
+            // but we also don't want to get stuck on slow or malicious connections
+            // forever, so we set a time limit on this phase and call abort() if it's exceeded.
             try {
-                is.skip(Long.MAX_VALUE);
+                while (is != null) {
+                    // Consume some data
+                    if (is.skip(65536) == 0) {
+                        // Note that skip may return 0 if the stream is empty
+                        // or for other unspecified reasons, so we need to check
+                        // with read() as well to determine if the stream is done
+                        if (is.read() == -1)
+                            is = null;
+                    }
+                    // Check if the time limit has been exceeded
+                    else if (Duration.between(start, Instant.now()).compareTo(timeLimit) > 0) {
+                        request.abort();
+                        is = null;
+                    }
+                }
             }
             catch (IOException e) {
                 // Ignore the exception

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcRecorder.java

@@ -41,7 +41,7 @@ public class WarcRecorder implements AutoCloseable {
     static final int MAX_TIME = 30_000;
 
     /** Maximum (decompressed) size we'll save */
-    static final int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 10 * 1024 * 1024);
+    static final int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 32 * 1024 * 1024);
 
     private final WarcWriter writer;
     private final Path warcFile;

code/processes/crawling-process/java/nu/marginalia/crawl/logic/DomainLocks.java

@@ -20,16 +20,17 @@ public class DomainLocks {
      * and may be held by another thread.  The caller is responsible for locking and  releasing the lock.
      */
     public DomainLock lockDomain(EdgeDomain domain) throws InterruptedException {
-        var ret = new DomainLock(domain.toString(),
+        var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits);
-                locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits));
+
-        ret.lock();
+        sem.acquire();
-        return ret;
+
+        return new DomainLock(sem);
     }
 
     public Optional<DomainLock> tryLockDomain(EdgeDomain domain) {
         var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits);
         if (sem.tryAcquire(1)) {
-            return Optional.of(new DomainLock(domain.toString(), sem));
+            return Optional.of(new DomainLock(sem));
         }
         else {
             // We don't have a lock, so we return an empty optional
@@ -42,23 +43,27 @@ public class DomainLocks {
             return new Semaphore(16);
         if (topDomain.equals("blogspot.com"))
             return new Semaphore(8);
-
+        if (topDomain.equals("tumblr.com"))
+            return new Semaphore(8);
         if (topDomain.equals("neocities.org"))
-            return new Semaphore(4);
+            return new Semaphore(8);
         if (topDomain.equals("github.io"))
-            return new Semaphore(4);
+            return new Semaphore(8);
 
+        // Substack really dislikes broad-scale crawlers, so we need to be careful
+        // to not get blocked.
         if (topDomain.equals("substack.com")) {
             return new Semaphore(1);
         }
-        if (topDomain.endsWith(".edu")) {
-            return new Semaphore(1);
-        }
 
         return new Semaphore(2);
     }
 
-    public boolean canLock(EdgeDomain domain) {
+    /** Returns true if the domain is lockable, i.e. if it is not already locked by another thread.
+     * (this is just a hint, and does not guarantee that the domain is actually lockable any time
+     * after this method returns true)
+     */
+    public boolean isLockableHint(EdgeDomain domain) {
         Semaphore sem = locks.get(domain.topDomain.toLowerCase());
         if (null == sem)
             return true;
@@ -67,25 +72,16 @@ public class DomainLocks {
     }
 
     public static class DomainLock implements AutoCloseable {
-        private final String domainName;
         private final Semaphore semaphore;
 
-        DomainLock(String domainName, Semaphore semaphore) {
+        DomainLock(Semaphore semaphore) {
-            this.domainName = domainName;
             this.semaphore = semaphore;
         }
 
-        // This method is called to lock the domain.  It will block until the lock is available.
-        private void lock() throws InterruptedException {
-            Thread.currentThread().setName("crawling:" + domainName + " [await domain lock]");
-            semaphore.acquire();
-            Thread.currentThread().setName("crawling:" + domainName);
-        }
-
         @Override
         public void close() throws Exception {
             semaphore.release();
-            Thread.currentThread().setName("crawling:" + domainName + " [wrapping up]");
+            Thread.currentThread().setName("[idle]");
         }
     }
 }

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java

@@ -74,7 +74,7 @@ public class CrawlerRevisitor {
 
             // If the reference document is empty or the HTTP status is not 200, we'll skip it since it's
             // unlikely to produce anything meaningful for us.
-            if (doc.httpStatus != 200)
+            if (doc.httpStatus != 200 && doc.httpStatus != 206)
                 continue;
             if (!doc.hasBody())
                 continue;

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java

@@ -58,7 +58,7 @@ public record DocumentWithReference(
         if (null == doc)
             return ContentTags.empty();
 
-        if (doc.documentBodyBytes.length == 0 || doc.httpStatus != 200)
+        if (doc.documentBodyBytes.length == 0 || (doc.httpStatus != 200 && doc.httpStatus != 206))
             return ContentTags.empty();
 
         String lastmod = doc.getLastModified();

code/processes/crawling-process/model/java/nu/marginalia/ContentTypes.java

@@ -1,22 +1,30 @@
 package nu.marginalia;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.util.Set;
 
 public class ContentTypes {
     public static final Set<String> acceptedContentTypes = Set.of("application/xhtml+xml",
             "application/xhtml",
             "text/html",
+            "application/pdf",
             "image/x-icon",
             "text/plain");
 
     public static boolean isAccepted(String contentTypeHeader) {
-        String lcHeader = contentTypeHeader.toLowerCase();
+        String lcHeader = StringUtils.substringBefore(contentTypeHeader.toLowerCase(), ';');
         for (var type : acceptedContentTypes) {
-            if (lcHeader.startsWith(type)) {
+            if (lcHeader.equals(type)) {
                 return true;
             }
         }
         return false;
     }
 
+    public static boolean isBinary(String contentTypeHeader) {
+        String lcHeader = StringUtils.substringBefore(contentTypeHeader.toLowerCase(), ';');
+        return lcHeader.startsWith("application/pdf");
+    }
+
 }

code/processes/crawling-process/model/java/nu/marginalia/io/crawldata/format/SlopSerializableCrawlDataStream.java

@@ -37,8 +37,12 @@ public class SlopSerializableCrawlDataStream implements AutoCloseable, Serializa
             public boolean filter(String url, int status, String contentType) {
                 String ctLc = contentType.toLowerCase();
 
+                // Permit all plain text content types
                 if (ctLc.startsWith("text/"))
                     return true;
+                // PDF
+                else if (ctLc.startsWith("application/pdf"))
+                    return true;
                 else if (ctLc.startsWith("x-marginalia/"))
                     return true;
 

code/processes/crawling-process/model/java/nu/marginalia/model/body/ContentTypeLogic.java

@@ -10,7 +10,7 @@ import java.util.regex.Pattern;
 
 public class ContentTypeLogic {
 
-    private static final Predicate<String> probableHtmlPattern = Pattern.compile("^.*\\.(htm|html|php|txt|md)$").asMatchPredicate();
+    private static final Predicate<String> probableGoodPattern = Pattern.compile("^.*\\.(htm|html|php|txt|md|pdf)$").asMatchPredicate();
     private static final Predicate<String> probableBinaryPattern = Pattern.compile("^.*\\.[a-z]+$").asMatchPredicate();
     private static final Set<String> blockedContentTypes = Set.of("text/css", "text/javascript");
     private static final List<String> acceptedContentTypePrefixes = List.of(
@@ -22,6 +22,7 @@ public class ContentTypeLogic {
             "application/rss+xml",
             "application/x-rss+xml",
             "application/rdf+xml",
+            "application/pdf",
             "x-rss+xml"
     );
     private boolean allowAllContentTypes = false;
@@ -34,7 +35,7 @@ public class ContentTypeLogic {
     public boolean isUrlLikeBinary(EdgeUrl url) {
         String pathLowerCase = url.path.toLowerCase();
 
-        if (probableHtmlPattern.test(pathLowerCase))
+        if (probableGoodPattern.test(pathLowerCase))
             return false;
 
         return probableBinaryPattern.test(pathLowerCase);

code/processes/crawling-process/model/java/nu/marginalia/slop/SlopCrawlDataRecord.java

@@ -216,6 +216,11 @@ public record SlopCrawlDataRecord(String domain,
             return false;
         }
 
+        // If the format is binary, we don't want to translate it if the response is truncated
+        if (response.truncated() != WarcTruncationReason.NOT_TRUNCATED && ContentTypes.isBinary(contentType)) {
+            return false;
+        }
+
         return true;
     }
 
@@ -272,7 +277,8 @@ public record SlopCrawlDataRecord(String domain,
         try (var table = new SlopTable(path)) {
             ShortColumn.Reader statusReader = statusColumn.open(table);
             while (statusReader.hasRemaining()) {
-                if (statusReader.get() == 200) {
+                int status = statusReader.get();
+                if (status == 200 || status == 206) {
                     cnt++;
                 }
             }

code/processes/crawling-process/test/nu/marginalia/crawl/fetcher/HttpFetcherImplFetchTest.java

@@ -40,6 +40,8 @@ class HttpFetcherImplFetchTest {
     private static EdgeUrl badHttpStatusUrl;
     private static EdgeUrl keepAliveUrl;
 
+    private static EdgeUrl pdfUrl;
+
     @BeforeAll
     public static void setupAll() throws URISyntaxException {
         wireMockServer =
@@ -133,6 +135,13 @@ class HttpFetcherImplFetchTest {
                         ));
 
 
+        pdfUrl = new EdgeUrl("http://localhost:18089/test.pdf");
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(pdfUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "application/pdf")
+                        .withStatus(200)
+                        .withBody("Hello World")));
+
         wireMockServer.start();
 
     }
@@ -352,6 +361,14 @@ class HttpFetcherImplFetchTest {
         Assertions.assertTrue(result.isOk());
     }
 
+    @Test
+    public void testPdf() {
+        var result = fetcher.fetchContent(pdfUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+    }
+
 
     private List<WarcRecord> getWarcRecords() throws IOException {
         List<WarcRecord> records = new ArrayList<>();

code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java

@@ -4,9 +4,9 @@ import nu.marginalia.UserAgent;
 import nu.marginalia.crawl.fetcher.ContentTags;
 import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
+import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.model.EdgeUrl;
-import nu.marginalia.parquet.crawldata.CrawledDocumentParquetRecordFileReader;
+import nu.marginalia.slop.SlopCrawlDataRecord;
-import nu.marginalia.parquet.crawldata.CrawledDocumentParquetRecordFileWriter;
 import org.apache.hc.client5.http.classic.HttpClient;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.impl.classic.HttpClients;
@@ -24,13 +24,14 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 class WarcRecorderTest {
     Path fileNameWarc;
-    Path fileNameParquet;
+    Path fileNameSlop;
     WarcRecorder client;
 
     HttpClient httpClient;
@@ -39,7 +40,7 @@ class WarcRecorderTest {
         httpClient = HttpClients.createDefault();
 
         fileNameWarc = Files.createTempFile("test", ".warc");
-        fileNameParquet = Files.createTempFile("test", ".parquet");
+        fileNameSlop = Files.createTempFile("test", ".slop.zip");
 
         client = new WarcRecorder(fileNameWarc);
     }
@@ -159,17 +160,28 @@ class WarcRecorderTest {
 
         client.fetch(httpClient, new DomainCookies(), request3);
 
-        CrawledDocumentParquetRecordFileWriter.convertWarc(
+        HttpGet request4 = new HttpGet("https://downloads.marginalia.nu/test.pdf");
+        request4.addHeader("User-agent", "test.marginalia.nu");
+        request4.addHeader("Accept-Encoding", "gzip");
+
+        client.fetch(httpClient, new DomainCookies(), request4);
+
+        SlopCrawlDataRecord.convertWarc(
                 "www.marginalia.nu",
                 new UserAgent("test", "test"),
                 fileNameWarc,
-                fileNameParquet);
+                fileNameSlop);
 
-        var urls = CrawledDocumentParquetRecordFileReader.stream(fileNameParquet).map(doc -> doc.url).toList();
+        List<String> urls;
-        assertEquals(2, urls.size());
+        try (var stream = SerializableCrawlDataStream.openDataStream(fileNameSlop)) {
+            urls = stream.docsAsList().stream().map(doc -> doc.url.toString()).toList();
+        }
+
+        assertEquals(3, urls.size());
         assertEquals("https://www.marginalia.nu/", urls.get(0));
         assertEquals("https://www.marginalia.nu/log/", urls.get(1));
         // sanic.jpg gets filtered out for its bad mime type
+        assertEquals("https://downloads.marginalia.nu/test.pdf", urls.get(2));
 
     }
 

code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java

@@ -117,6 +117,86 @@ class CrawlerRetreiverTest {
         }
     }
 
+
+    @Test
+    public void testWarcOutputPDF() throws IOException {
+        var specs = CrawlerMain.CrawlSpecRecord
+                .builder()
+                .crawlDepth(5)
+                .domain("www.marginalia.nu")
+                .urls(List.of("https://www.marginalia.nu/junk/test.pdf"))
+                .build();
+        Path tempFile = null;
+        Path slopFile = null;
+        try {
+            tempFile = Files.createTempFile("crawling-process", "warc");
+            slopFile = Files.createTempFile("crawling-process", ".slop.zip");
+
+            doCrawl(tempFile, specs);
+
+            Set<String> requests = new HashSet<>();
+            Set<String> responses = new HashSet<>();
+
+            // Inspect the WARC file
+            try (var reader = new WarcReader(tempFile)) {
+                reader.forEach(record -> {
+                    if (record instanceof WarcRequest req) {
+                        requests.add(req.target());
+                        System.out.println(req.type() + ":" + req.target());
+                    }
+                    else if (record instanceof WarcResponse rsp) {
+                        responses.add(rsp.target());
+                        System.out.println(rsp.type() + ":" + rsp.target());
+                    }
+                    else {
+                        System.out.println(record.type());
+                    }
+                });
+            }
+
+            assertTrue(requests.contains("https://www.marginalia.nu/junk/test.pdf"));
+            assertEquals(requests, responses);
+
+            // Convert the WARC file to a Slop file
+            SlopCrawlDataRecord
+                    .convertWarc("www.marginalia.nu", new UserAgent("test.marginalia.nu", "test.marginalia.nu"), tempFile, slopFile);
+
+            CrawledDomain domain = null;
+            Map<String, CrawledDocument> documents = new HashMap<>();
+
+            // Extract the contents of the Slop file
+            try (var stream = SerializableCrawlDataStream.openDataStream(slopFile)) {
+                while (stream.hasNext()) {
+                    var doc = stream.next();
+                    if (doc instanceof CrawledDomain dr) {
+                        assertNull(domain);
+                        domain = dr;
+                    }
+                    else if (doc instanceof CrawledDocument dc) {
+                        System.out.println(dc.url + "\t" + dc.crawlerStatus + "\t" + dc.httpStatus);
+                        documents.put(dc.url, dc);
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+            // Verify we have a PDF in the Slop file
+            assertNotNull(domain);
+            var pdfDoc = documents.get("https://www.marginalia.nu/junk/test.pdf");
+            assertNotNull(pdfDoc);
+            assertEquals("https://www.marginalia.nu/junk/test.pdf", pdfDoc.url);
+            assertEquals(206, pdfDoc.httpStatus);
+            assertTrue(pdfDoc.documentBodyBytes.length > 100);
+        }
+        finally {
+            if (tempFile != null)
+                Files.deleteIfExists(tempFile);
+            if (slopFile != null)
+                Files.deleteIfExists(slopFile);
+        }
+    }
+
     @Test
     public void testWarcOutputNoKnownUrls() throws IOException {
         var specs = CrawlerMain.CrawlSpecRecord

code/processes/export-task-process/build.gradle

@@ -53,6 +53,8 @@ dependencies {
     implementation libs.commons.compress
     implementation libs.commons.codec
     implementation libs.jsoup
+    implementation libs.slop
+    implementation libs.jwarc
 
 
 

code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java

@@ -1,13 +1,18 @@
 package nu.marginalia.extractor;
 
 import com.google.inject.Inject;
+import nu.marginalia.process.control.ProcessHeartbeat;
 import nu.marginalia.process.log.WorkLog;
 import nu.marginalia.process.log.WorkLogEntry;
+import nu.marginalia.slop.SlopCrawlDataRecord;
+import nu.marginalia.slop.SlopTablePacker;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorage;
 import nu.marginalia.storage.model.FileStorageId;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -16,18 +21,19 @@ import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.List;
 
 public class SampleDataExporter {
     private final FileStorageService storageService;
+    private final ProcessHeartbeat processHeartbeat;
 
     @Inject
-    public SampleDataExporter(FileStorageService storageService) {
+    public SampleDataExporter(FileStorageService storageService, ProcessHeartbeat processHeartbeat) {
         this.storageService = storageService;
+        this.processHeartbeat = processHeartbeat;
     }
-    public void export(FileStorageId crawlId, FileStorageId destId, int size, String name) throws SQLException, IOException {
+
+    public void export(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name) throws SQLException, IOException {
         FileStorage destStorage = storageService.getStorage(destId);
         Path inputDir = storageService.getStorage(crawlId).asPath();
 
@@ -54,11 +60,6 @@ public class SampleDataExporter {
 
         Path newCrawlerLogFile = Files.createTempFile(destStorage.asPath(), "crawler", ".log",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
-        try (var bw = Files.newBufferedWriter(newCrawlerLogFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
-            for (var item : entriesAll) {
-                bw.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
-            }
-        }
 
         Path newManifestJsonFile = Files.createTempFile(destStorage.asPath(), "manifest", ".json",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
@@ -67,14 +68,38 @@ public class SampleDataExporter {
         var tmpTarFile = Files.createTempFile(destStorage.asPath(), "data", ".tar",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
 
-        try (var stream = new TarArchiveOutputStream(Files.newOutputStream(tmpTarFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) {
+        try (var stream = new TarArchiveOutputStream(Files.newOutputStream(tmpTarFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING));
-            for (var item : entriesAll) {
+             var logWriter = Files.newBufferedWriter(newCrawlerLogFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+             var hb = processHeartbeat.createAdHocTaskHeartbeat("Generating Sample")
+        ) {
+            for (var item : hb.wrap("Scanning", entriesAll)) {
                 Path crawlDataPath = inputDir.resolve(item.relPath());
                 if (!Files.exists(crawlDataPath)) continue;
 
-                addFileToTar(stream, crawlDataPath, item.relPath());
+                if (StringUtils.isBlank(ctFilter)) {
+                    addFileToTar(stream, crawlDataPath, item.relPath());
+                    logWriter.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
+                }
+                else /* filter != null */ {
+                    Path filteredData = null;
+                    try {
+                        filteredData = filterEntries(crawlDataPath, ctFilter);
+                        addFileToTar(stream, filteredData, item.relPath());
+                        logWriter.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
+                    }
+                    catch (NoSuchElementException ex) {
+                        // Ignore
+                    }
+                    finally {
+                        if (filteredData != null) {
+                            Files.deleteIfExists(filteredData);
+                        }
+                    }
+                }
             }
 
+            logWriter.flush();
+
             addFileToTar(stream, newCrawlerLogFile, "crawler.log");
             addFileToTar(stream, newManifestJsonFile, "marginalia-manifest.json");
         }
@@ -86,6 +111,48 @@ public class SampleDataExporter {
         Files.move(tmpTarFile, destStorage.asPath().resolve("crawl-data.tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
     }
 
+    /** Filters the entries in the crawl data file based on the content type. */
+    private Path filterEntries(Path crawlDataPath, String contentTypeFilter) throws IOException, NoSuchElementException {
+        Path tempDir = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered");
+        Path tempFile = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered.slop.zip");
+
+        // We may have debris from a previous run, so let's clean it up
+        if (Files.isDirectory(tempDir)) {
+            FileUtils.deleteDirectory(tempDir.toFile());
+        }
+        Files.createDirectory(tempDir);
+
+        try (var writer = new SlopCrawlDataRecord.Writer(tempDir);
+             var reader = new SlopCrawlDataRecord.FilteringReader(crawlDataPath) {
+                 @Override
+                 public boolean filter(String url, int status, String contentType) {
+                     return Objects.equals(StringUtils.substringBefore(contentType, ';'), contentTypeFilter)
+                                || contentType.startsWith("x-marginalia/"); // metadata records
+                 }
+             }
+        ) {
+            boolean wroteEntry = false;
+            while (reader.hasRemaining()) {
+                var entry = reader.get();
+                writer.write(entry);
+
+                wroteEntry = wroteEntry || Objects.equals(StringUtils.substringBefore(entry.contentType(), ';'), contentTypeFilter);
+            }
+
+            if (!wroteEntry) {
+                throw new NoSuchElementException("No relevant entries");
+            }
+
+            SlopTablePacker.packToSlopZip(tempDir, tempFile);
+        }
+        finally {
+            FileUtils.deleteDirectory(tempDir.toFile());
+        }
+
+
+        return tempFile;
+    }
+
     private void addFileToTar(TarArchiveOutputStream outputStream, Path file, String fileName) throws IOException {
         var entry = outputStream.createArchiveEntry(file.toFile(), fileName);
         entry.setSize(Files.size(file));

code/processes/export-task-process/java/nu/marginalia/task/ExportTasksMain.java

@@ -92,7 +92,7 @@ public class ExportTasksMain extends ProcessMainClass {
                     termFrequencyExporter.export(request.crawlId, request.destId);
                     break;
                 case SAMPLE_DATA:
-                    sampleDataExporter.export(request.crawlId, request.destId, request.size, request.name);
+                    sampleDataExporter.export(request.crawlId, request.destId, request.size, request.ctFilter, request.name);
                     break;
                 case ADJACENCIES:
                     websiteAdjacenciesCalculator.export();

code/processes/process-mq-api/java/nu/marginalia/mqapi/tasks/ExportTaskRequest.java

@@ -16,6 +16,7 @@ public class ExportTaskRequest {
     public FileStorageId destId;
     public int size;
     public String name;
+    public String ctFilter;
 
     public ExportTaskRequest(Task task) {
         this.task = task;
@@ -42,12 +43,13 @@ public class ExportTaskRequest {
         return request;
     }
 
-    public static ExportTaskRequest sampleData(FileStorageId crawlId, FileStorageId destId, int size, String name) {
+    public static ExportTaskRequest sampleData(FileStorageId crawlId, FileStorageId destId, String ctFilter, int size, String name) {
         ExportTaskRequest request = new ExportTaskRequest(Task.SAMPLE_DATA);
         request.crawlId = crawlId;
         request.destId = destId;
         request.size = size;
         request.name = name;
+        request.ctFilter = ctFilter;
         return request;
     }
 

code/services-application/api-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-application/dating-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/explorer-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/search-service-legacy/build.gradle

@@ -5,7 +5,7 @@ plugins {
     id 'application'
     id 'jvm-test-suite'
 
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/search-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
     id 'application'
     id 'jvm-test-suite'
     id 'gg.jte.gradle' version '3.1.15'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/search-service/java/nu/marginalia/search/model/UrlDetails.java

@@ -180,7 +180,7 @@ public class UrlDetails implements Comparable<UrlDetails> {
      * semantically meaningful codepoints into entity codes */
     public String displayUrl() {
         StringBuilder sb = new StringBuilder();
-        String urlStr = url.toString();
+        String urlStr = url.toDisplayString();
         for (int i = 0; i < urlStr.length(); i++) {
             char c = urlStr.charAt(i);
 

code/services-application/search-service/resources/jte/part/head.jte

@@ -26,4 +26,10 @@
 
     <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Marginalia">
 
-</head>
+</head>
+<noscript>
+    <h1>Users of text-based browsers</h1>
+    <p>Consider using the old interface at <a href="https://old-search.marginalia.nu/">https://old-search.marginalia.nu/</a>,
+    as it uses fewer modern CSS tricks, and should work better than the new UI.  It's functionally nearly identical, but just renders it using a different layout.</p>
+    <hr>
+</noscript>

code/services-application/search-service/resources/jte/part/warmup.jte

@@ -1,9 +1,16 @@
 This is a bit of a hack!
 
 This class exists to let tailwind we're using these classes even though they aren't visible in the code,
-as we sometimes generate classes from Java code!
+as we sometimes generate classes from Java code or javascript!
 
 <i class="text-blue-800 bg-blue-50 dark:text-blue-200 dark:bg-blue-950"></i>
 <i class="text-green-800 bg-green-50 dark:text-green-200 dark:bg-green-950"></i>
 <i class="text-purple-800 bg-purple-50 dark:text-purple-200 dark:bg-purple-950"></i>
 <i class="text-blue-950 bg-gray-100 dark:text-blue-50 dark:bg-gray-900"></i>
+<span class="hover:bg-gray-300 "></span>
+
+<label class="suggestion group block relative">
+    <input type="radio" name="suggestion" class="peer hidden" checked>
+    <div class="px-4 py-2 cursor-pointer dark:peer-checked:bg-gray-700 dark:hover:bg-gray-700 peer-checked:bg-gray-300 hover:bg-gray-300 w-full">
+    </div>
+</label>

code/services-application/search-service/resources/jte/serp/error.jte

@@ -26,7 +26,7 @@
 
     <!-- Main content -->
     <main class="flex-1 p-4 max-w-2xl space-y-4">
-        <div class="border dark:border-gray-600 rounded bg-white text-black dark:bg-gray-800 dark:text-white text-m p-4">
+        <div class="border border-gray-300 dark:border-gray-600 rounded bg-white text-black dark:bg-gray-800 dark:text-white text-m p-4">
             <div class="flex space-x-3  place-items-baseline">
                 <i class="fa fa-circle-exclamation text-red-800"></i>
                 <div class="grow">${model.errorTitle()}</div>

code/services-application/search-service/resources/jte/serp/part/footerHowto.jte

@@ -80,10 +80,6 @@
             <tr><td>rank&gt;50</td><td>The ranking of the website is at least 50 in a span of 1 - 255</td></tr>
             <tr><td>rank&lt;50</td><td>The ranking of the website is at most 50 in a span of 1 - 255</td></tr>
 
-            <tr><td>count&gt;10</td><td> The search term must appear in at least 10 results form the domain</td></tr>
-            <tr><td>count&lt;10</td><td> The search term must appear in at most 10 results from the domain</td></tr>
-
-
             <tr><td>format:html5</td><td>Filter documents using the HTML5 standard. This is typically modern websites.</td></tr>
             <tr><td>format:xhtml</td><td>Filter documents using the XHTML standard</td></tr>
             <tr><td>format:html123</td><td>Filter documents using the HTML standards 1, 2, and 3. This is typically very old websites. </td></tr>

code/services-application/search-service/resources/jte/serp/part/searchform.jte

@@ -7,13 +7,13 @@
 
 <form class="flex-1 max-w-2xl" action="/search">
     <div class="flex">
-        @if (query.isBlank())
+        @if (query != null && query.isBlank())
         <%-- Add autofocus if the query is blank --%>
             <input type="text"
                    class="shadow-inner flex-1 dark:bg-black dark:text-gray-100 bg-gray-50 border dark:border-gray-600 border-gray-300 text-gray-900 text-sm rounded-sm block w-full p-2.5"
                    value="${query}"
                    autofocus
-                   placeholder="Search..."
+                   placeholder="Search the web!"
                    autocomplete="off"
                    name="query"
                    id="searchInput" />
@@ -21,13 +21,13 @@
             <input type="text"
                    class="shadow-inner flex-1 dark:bg-black dark:text-gray-100 bg-gray-50 border dark:border-gray-600 border-gray-300 text-gray-900 text-sm rounded-sm block w-full p-2.5"
                    value="${query}"
-                   placeholder="Search..."
+                   placeholder="Search the web!"
                    autocomplete="off"
                    name="query"
                    id="searchInput" />
         @endif
 
-        <div id="searchSuggestions" class="text-sm absolute top-2 mt-10 w-96 bg-white dark:bg-black border dark:border-gray-600 border-gray-200 rounded-lg shadow-lg hidden"></div>
+        <div aria-hidden="true" id="searchSuggestions" class="text-sm absolute top-3 mt-10 w-96 bg-white dark:bg-black border dark:border-gray-600 border-gray-300 rounded-lg shadow-lg hidden"></div>
 
         <button class="px-4 py-2 bg-margeblue text-white ml-2 rounded whitespace-nowrap active:text-slate-200">
             <i class="fas fa-search text-sm sm:mr-3"></i>

code/services-application/search-service/resources/static/js/typeahead.js

@@ -43,13 +43,13 @@ function displaySuggestions(suggestions) {
     }
 
     suggestionsContainer.innerHTML = suggestions.map((suggestion, index) => `
-                <div 
+        <label class="suggestion group block relative">
-                    class="suggestion px-4 py-2 cursor-pointer hover:bg-gray-100 ${index === selectedIndex ? 'bg-blue-50' : ''}"
+            <input type="radio" name="suggestion" class="peer hidden" ${index === selectedIndex ? 'checked' : ''}>
-                    data-index="${index}"
+            <div class="px-4 py-2 cursor-pointer dark:peer-checked:bg-gray-700 dark:hover:bg-gray-700 peer-checked:bg-gray-300 hover:bg-gray-300 w-full" data-index="${index}">
-                >
+                ${suggestion}
-                    ${suggestion}
+            </div>
-                </div>
+        </label>
-            `).join('');
+    `).join('');
 
     suggestionsContainer.classList.remove('hidden');
 

code/services-application/status-service/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'java'
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-application/status-service/java/nu/marginalia/status/StatusModule.java

@@ -20,6 +20,6 @@ public class StatusModule extends AbstractModule {
         bind(String.class)
                 .annotatedWith(Names.named("searchEngineTestQuery"))
                 .toInstance(System.getProperty("status-service.public-query",
-                        "https://search.marginalia.nu/search?query=plato&ref=marginalia-automatic-metrics"));
+                        "https://marginalia-search.com/search?query=plato&ref=marginalia-automatic-metrics"));
     }
 }

code/services-core/assistant-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantModule.java

@@ -10,7 +10,8 @@ import static com.google.inject.name.Names.named;
 
 public class AssistantModule extends AbstractModule {
     public void configure() {
-        bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(WmsaHome.getHomePath().resolve("data/suggestions.txt"));
+        bind(Path.class).annotatedWith(named("suggestions-file1")).toInstance(WmsaHome.getHomePath().resolve("data/suggestions2.txt.gz"));
+        bind(Path.class).annotatedWith(named("suggestions-file2")).toInstance(WmsaHome.getHomePath().resolve("data/suggestions3.txt.gz"));
 
         bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
     }

code/services-core/assistant-service/java/nu/marginalia/assistant/suggest/PrefixSearchStructure.java

@@ -0,0 +1,465 @@
+package nu.marginalia.assistant.suggest;
+
+import gnu.trove.list.array.TIntArrayList;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.*;
+
+/** Unhinged data structure for fast prefix searching.
+ */
+public class PrefixSearchStructure {
+    // Core data structures
+    private final HashMap<String, TIntArrayList> prefixIndex;     // Short prefix index (up to 8 chars)
+    private final HashMap<String, TIntArrayList> longPrefixIndex; // Long prefix index (9-16 chars)
+    private final ArrayList<String> words;                        // All words by ID
+    private final TIntArrayList wordScores;                       // Scores for all words
+
+    // Configuration
+    private static final int SHORT_PREFIX_LENGTH = 8;
+    private static final int MAX_INDEXED_PREFIX_LENGTH = 16;
+
+    public int size() {
+        return words.size();
+    }
+
+    // For sorting efficiency
+    private static class WordScorePair {
+        final String word;
+        final int score;
+
+        WordScorePair(String word, int score) {
+            this.word = word;
+            this.score = score;
+        }
+    }
+
+    /**
+     * Creates a new PrefixTrie for typeahead search.
+     */
+    public PrefixSearchStructure() {
+        prefixIndex = new HashMap<>(1024);
+        longPrefixIndex = new HashMap<>(1024);
+        words = new ArrayList<>(1024);
+        wordScores = new TIntArrayList(1024);
+    }
+
+    /**
+     * Adds a prefix to the index.
+     */
+    private void indexPrefix(String word, int wordId) {
+        // Index short prefixes
+        for (int i = 1; i <= Math.min(word.length(), SHORT_PREFIX_LENGTH); i++) {
+            String prefix = word.substring(0, i);
+            TIntArrayList wordIds = prefixIndex.computeIfAbsent(
+                    prefix, k -> new TIntArrayList(16));
+            wordIds.add(wordId);
+        }
+
+        // Index longer prefixes
+        for (int i = SHORT_PREFIX_LENGTH + 1; i <= Math.min(word.length(), MAX_INDEXED_PREFIX_LENGTH); i++) {
+            String prefix = word.substring(0, i);
+            TIntArrayList wordIds = longPrefixIndex.computeIfAbsent(
+                    prefix, k -> new TIntArrayList(8));
+            wordIds.add(wordId);
+        }
+
+        // If the word contains spaces, also index by each term for multi-word queries
+        if (word.contains(" ")) {
+            String[] terms = word.split("\\s+");
+            for (String term : terms) {
+                if (term.length() >= 2) {
+                    for (int i = 1; i <= Math.min(term.length(), SHORT_PREFIX_LENGTH); i++) {
+                        String termPrefix = "t:" + term.substring(0, i);
+                        TIntArrayList wordIds = prefixIndex.computeIfAbsent(
+                                termPrefix, k -> new TIntArrayList(16));
+                        wordIds.add(wordId);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Inserts a word with its associated score.
+     */
+    public void insert(String word, int score) {
+        if (word == null || word.isEmpty()) {
+            return;
+        }
+
+        // Add to the word list and index
+        int wordId = words.size();
+        words.add(word);
+        wordScores.add(score);
+        indexPrefix(word, wordId);
+    }
+
+    /**
+     * Returns the top k completions for a given prefix.
+     */
+    public List<ScoredSuggestion> getTopCompletions(String prefix, int k) {
+        if (prefix == null || prefix.isEmpty()) {
+            // Return top k words by score
+            return getTopKWords(k);
+        }
+
+        // Check if this is a term search (t:) - for searching within multi-word items
+        boolean isTermSearch = false;
+        if (prefix.startsWith("t:") && prefix.length() > 2) {
+            isTermSearch = true;
+            prefix = prefix.substring(2);
+        }
+
+        // 1. Fast path for short prefixes
+        if (prefix.length() <= SHORT_PREFIX_LENGTH) {
+            String lookupPrefix = isTermSearch ? "t:" + prefix : prefix;
+            TIntArrayList wordIds = prefixIndex.get(lookupPrefix);
+            if (wordIds != null) {
+                return getTopKFromWordIds(wordIds, k);
+            }
+        }
+
+        // 2. Fast path for long prefixes (truncate to MAX_INDEXED_PREFIX_LENGTH)
+        if (prefix.length() > SHORT_PREFIX_LENGTH) {
+            // Try exact match in longPrefixIndex first
+            if (prefix.length() <= MAX_INDEXED_PREFIX_LENGTH) {
+                TIntArrayList wordIds = longPrefixIndex.get(prefix);
+                if (wordIds != null) {
+                    return getTopKFromWordIds(wordIds, k);
+                }
+            }
+
+            // If prefix is longer than MAX_INDEXED_PREFIX_LENGTH, truncate and filter
+            if (prefix.length() > MAX_INDEXED_PREFIX_LENGTH) {
+                String truncatedPrefix = prefix.substring(0, MAX_INDEXED_PREFIX_LENGTH);
+                TIntArrayList candidateIds = longPrefixIndex.get(truncatedPrefix);
+                if (candidateIds != null) {
+                    // Filter candidates by the full prefix
+                    return getFilteredTopKFromWordIds(candidateIds, prefix, k);
+                }
+            }
+        }
+
+        // 3. Optimized fallback for long prefixes - use prefix tree for segments
+        List<ScoredSuggestion> results = new ArrayList<>();
+
+        // Handle multi-segment queries by finding candidates from first 8 chars
+        if (prefix.length() > SHORT_PREFIX_LENGTH) {
+            String shortPrefix = prefix.substring(0, Math.min(prefix.length(), SHORT_PREFIX_LENGTH));
+            TIntArrayList candidates = prefixIndex.get(shortPrefix);
+
+            if (candidates != null) {
+                return getFilteredTopKFromWordIds(candidates, prefix, k);
+            }
+        }
+
+        // 4. Last resort - optimized binary search in sorted segments
+        return findByBinarySearchPrefix(prefix, k);
+    }
+
+    /**
+     * Helper to get the top k words by score.
+     */
+    private List<ScoredSuggestion> getTopKWords(int k) {
+        // Create pairs of (score, wordId)
+        int[][] pairs = new int[words.size()][2];
+        for (int i = 0; i < words.size(); i++) {
+            pairs[i][0] = wordScores.get(i);
+            pairs[i][1] = i;
+        }
+
+        // Sort by score (descending)
+        Arrays.sort(pairs, (a, b) -> Integer.compare(b[0], a[0]));
+
+        // Take top k
+        List<ScoredSuggestion> results = new ArrayList<>();
+        for (int i = 0; i < Math.min(k, pairs.length); i++) {
+            String word = words.get(pairs[i][1]);
+            int score = pairs[i][0];
+            results.add(new ScoredSuggestion(word, score));
+        }
+
+        return results;
+    }
+
+    /**
+     * Helper to get the top k words from a list of word IDs.
+     */
+    private List<ScoredSuggestion> getTopKFromWordIds(TIntArrayList wordIds, int k) {
+        if (wordIds == null || wordIds.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        // For small lists, avoid sorting
+        if (wordIds.size() <= k) {
+            List<ScoredSuggestion> results = new ArrayList<>(wordIds.size());
+            int[] ids = wordIds.toArray();
+            for (int wordId : ids) {
+                if (wordId >= 0 && wordId < words.size()) {
+                    results.add(new ScoredSuggestion(words.get(wordId), wordScores.get(wordId)));
+                }
+            }
+            results.sort((a, b) -> Integer.compare(b.getScore(), a.getScore()));
+            return results;
+        }
+
+        // For larger lists, use an array-based approach for better performance
+        // Find top k without full sorting
+        int[] topScores = new int[k];
+        int[] topWordIds = new int[k];
+        int[] ids = wordIds.toArray();
+
+        // Initialize with first k elements
+        int filledCount = Math.min(k, ids.length);
+        for (int i = 0; i < filledCount; i++) {
+            int wordId = ids[i];
+            if (wordId >= 0 && wordId < words.size()) {
+                topWordIds[i] = wordId;
+                topScores[i] = wordScores.get(wordId);
+            }
+        }
+
+        // Sort initial elements
+        for (int i = 0; i < filledCount; i++) {
+            for (int j = i + 1; j < filledCount; j++) {
+                if (topScores[j] > topScores[i]) {
+                    // Swap scores
+                    int tempScore = topScores[i];
+                    topScores[i] = topScores[j];
+                    topScores[j] = tempScore;
+
+                    // Swap word IDs
+                    int tempId = topWordIds[i];
+                    topWordIds[i] = topWordIds[j];
+                    topWordIds[j] = tempId;
+                }
+            }
+        }
+
+        // Process remaining elements
+        int minScore = filledCount > 0 ? topScores[filledCount - 1] : Integer.MIN_VALUE;
+
+        for (int i = k; i < ids.length; i++) {
+            int wordId = ids[i];
+            if (wordId >= 0 && wordId < words.size()) {
+                int score = wordScores.get(wordId);
+
+                if (score > minScore) {
+                    // Replace the lowest element
+                    topScores[filledCount - 1] = score;
+                    topWordIds[filledCount - 1] = wordId;
+
+                    // Bubble up the new element
+                    for (int j = filledCount - 1; j > 0; j--) {
+                        if (topScores[j] > topScores[j - 1]) {
+                            // Swap scores
+                            int tempScore = topScores[j];
+                            topScores[j] = topScores[j - 1];
+                            topScores[j - 1] = tempScore;
+
+                            // Swap word IDs
+                            int tempId = topWordIds[j];
+                            topWordIds[j] = topWordIds[j - 1];
+                            topWordIds[j - 1] = tempId;
+                        } else {
+                            break;
+                        }
+                    }
+
+                    // Update min score
+                    minScore = topScores[filledCount - 1];
+                }
+            }
+        }
+
+        // Create result list
+        List<ScoredSuggestion> results = new ArrayList<>(filledCount);
+        for (int i = 0; i < filledCount; i++) {
+            results.add(new ScoredSuggestion(words.get(topWordIds[i]), topScores[i]));
+        }
+
+        return results;
+    }
+
+    /**
+     * Use binary search on sorted word segments to efficiently find matches.
+     */
+    private List<ScoredSuggestion> findByBinarySearchPrefix(String prefix, int k) {
+        // If we have a lot of words, use an optimized segment approach
+        if (words.size() > 1000) {
+            // Divide words into segments for better locality
+            int segmentSize = 1000;
+            int numSegments = (words.size() + segmentSize - 1) / segmentSize;
+
+            // Find matches using binary search within each segment
+            List<WordScorePair> allMatches = new ArrayList<>();
+            for (int segment = 0; segment < numSegments; segment++) {
+                int start = segment * segmentSize;
+                int end = Math.min(start + segmentSize, words.size());
+
+                // Binary search for first potential match
+                int pos = Collections.binarySearch(
+                        words.subList(start, end),
+                        prefix,
+                        (a, b) -> a.compareTo(b)
+                );
+
+                if (pos < 0) {
+                    pos = -pos - 1;
+                }
+
+                // Collect all matches
+                for (int i = start + pos; i < end && i < words.size(); i++) {
+                    String word = words.get(i);
+                    if (word.startsWith(prefix)) {
+                        allMatches.add(new WordScorePair(word, wordScores.get(i)));
+                    } else if (word.compareTo(prefix) > 0) {
+                        break; // Past potential matches
+                    }
+                }
+            }
+
+            // Sort by score and take top k
+            allMatches.sort((a, b) -> Integer.compare(b.score, a.score));
+            List<ScoredSuggestion> results = new ArrayList<>(Math.min(k, allMatches.size()));
+            for (int i = 0; i < Math.min(k, allMatches.size()); i++) {
+                WordScorePair pair = allMatches.get(i);
+                results.add(new ScoredSuggestion(pair.word, pair.score));
+            }
+            return results;
+        }
+
+        // Fallback for small dictionaries - linear scan but optimized
+        return simpleSearchFallback(prefix, k);
+    }
+
+    /**
+     * Optimized linear scan - only used for small dictionaries.
+     */
+    private List<ScoredSuggestion> simpleSearchFallback(String prefix, int k) {
+        // Use primitive arrays for better cache locality
+        int[] matchScores = new int[Math.min(words.size(), 100)]; // Assume we won't find more than 100 matches
+        String[] matchWords = new String[matchScores.length];
+        int matchCount = 0;
+
+        for (int i = 0; i < words.size() && matchCount < matchScores.length; i++) {
+            String word = words.get(i);
+            if (word.startsWith(prefix)) {
+                matchWords[matchCount] = word;
+                matchScores[matchCount] = wordScores.get(i);
+                matchCount++;
+            }
+        }
+
+        // Sort matches by score (in-place for small arrays)
+        for (int i = 0; i < matchCount; i++) {
+            for (int j = i + 1; j < matchCount; j++) {
+                if (matchScores[j] > matchScores[i]) {
+                    // Swap scores
+                    int tempScore = matchScores[i];
+                    matchScores[i] = matchScores[j];
+                    matchScores[j] = tempScore;
+
+                    // Swap words
+                    String tempWord = matchWords[i];
+                    matchWords[i] = matchWords[j];
+                    matchWords[j] = tempWord;
+                }
+            }
+        }
+
+        // Create results
+        List<ScoredSuggestion> results = new ArrayList<>(Math.min(k, matchCount));
+        for (int i = 0; i < Math.min(k, matchCount); i++) {
+            results.add(new ScoredSuggestion(matchWords[i], matchScores[i]));
+        }
+
+        return results;
+    }
+
+    /**
+     * Get top k words from candidate IDs, filtering by the full prefix.
+     */
+    private List<ScoredSuggestion> getFilteredTopKFromWordIds(TIntArrayList wordIds, String fullPrefix, int k) {
+        if (wordIds == null || wordIds.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        // Make primitive arrays for better performance
+        String[] matchWords = new String[Math.min(wordIds.size(), 1000)];
+        int[] matchScores = new int[matchWords.length];
+        int matchCount = 0;
+
+        int[] ids = wordIds.toArray();
+        for (int i = 0; i < ids.length && matchCount < matchWords.length; i++) {
+            int wordId = ids[i];
+            if (wordId >= 0 && wordId < words.size()) {
+                String word = words.get(wordId);
+                if (word.startsWith(fullPrefix)) {
+                    matchWords[matchCount] = word;
+                    matchScores[matchCount] = wordScores.get(wordId);
+                    matchCount++;
+                }
+            }
+        }
+
+        // Sort by score (efficient insertion sort for small k)
+        for (int i = 0; i < Math.min(matchCount, k); i++) {
+            int maxPos = i;
+            for (int j = i + 1; j < matchCount; j++) {
+                if (matchScores[j] > matchScores[maxPos]) {
+                    maxPos = j;
+                }
+            }
+            if (maxPos != i) {
+                // Swap
+                int tempScore = matchScores[i];
+                matchScores[i] = matchScores[maxPos];
+                matchScores[maxPos] = tempScore;
+
+                String tempWord = matchWords[i];
+                matchWords[i] = matchWords[maxPos];
+                matchWords[maxPos] = tempWord;
+            }
+        }
+
+        // Create result list (only up to k elements)
+        List<ScoredSuggestion> results = new ArrayList<>(Math.min(k, matchCount));
+        for (int i = 0; i < Math.min(k, matchCount); i++) {
+            results.add(new ScoredSuggestion(matchWords[i], matchScores[i]));
+        }
+
+        return results;
+    }
+
+    /**
+     * Class representing a suggested completion.
+     */
+    public static class ScoredSuggestion implements Comparable<ScoredSuggestion> {
+        private final String word;
+        private final int score;
+
+        public ScoredSuggestion(String word, int score) {
+            this.word = word;
+            this.score = score;
+        }
+
+        public String getWord() {
+            return word;
+        }
+
+        public int getScore() {
+            return score;
+        }
+
+        @Override
+        public String toString() {
+            return word + " (" + score + ")";
+        }
+
+        @Override
+        public int compareTo(@NotNull PrefixSearchStructure.ScoredSuggestion o) {
+            return Integer.compare(this.score, o.score);
+        }
+    }
+}

code/services-core/assistant-service/java/nu/marginalia/assistant/suggest/Suggestions.java

@@ -2,74 +2,89 @@ package nu.marginalia.assistant.suggest;
 
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
-import nu.marginalia.functions.math.dict.SpellChecker;
-import nu.marginalia.term_frequency_dict.TermFrequencyDict;
-import nu.marginalia.model.crawl.HtmlFeature;
-import org.apache.commons.collections4.trie.PatriciaTrie;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.*;
-import java.util.function.Supplier;
+import java.util.zip.GZIPInputStream;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public class Suggestions {
-    private PatriciaTrie<String> suggestionsTrie = null;
+    List<PrefixSearchStructure> searchStructures = new ArrayList<>();
-    private TermFrequencyDict termFrequencyDict = null;
+
-    private volatile boolean ready = false;
+    private volatile boolean ready = false;
-    private final SpellChecker spellChecker;
 
-    private static final Pattern suggestionPattern = Pattern.compile("^[a-zA-Z0-9]+( [a-zA-Z0-9]+)*$");
     private static final Logger logger = LoggerFactory.getLogger(Suggestions.class);
 
     private static final int MIN_SUGGEST_LENGTH = 3;
     @Inject
-    public Suggestions(@Named("suggestions-file") Path suggestionsFile,
+    public Suggestions(@Named("suggestions-file1") Path suggestionsFile1,
-                       SpellChecker spellChecker,
+                       @Named("suggestions-file2") Path suggestionsFile2
-                       TermFrequencyDict dict
                        ) {
-        this.spellChecker = spellChecker;
 
         Thread.ofPlatform().start(() -> {
-            suggestionsTrie = loadSuggestions(suggestionsFile);
+            searchStructures.add(loadSuggestions(suggestionsFile1));
-            termFrequencyDict = dict;
+            searchStructures.add(loadSuggestions(suggestionsFile2));
             ready = true;
-            logger.info("Loaded {} suggestions", suggestionsTrie.size());
+            logger.info("Loaded suggestions");
         });
     }
 
-    private static PatriciaTrie<String> loadSuggestions(Path file) {
+    private static PrefixSearchStructure loadSuggestions(Path file) {
+        PrefixSearchStructure ret = new PrefixSearchStructure();
+
         if (!Files.exists(file)) {
             logger.error("Suggestions file {} absent, loading empty suggestions db", file);
-            return new PatriciaTrie<>();
+            return ret;
         }
-        try (var lines = Files.lines(file)) {
-            var ret = new PatriciaTrie<String>();
 
-            lines.filter(suggestionPattern.asPredicate())
+        try (var scanner = new Scanner(new GZIPInputStream(new BufferedInputStream(Files.newInputStream(file, StandardOpenOption.READ))))) {
-                    .filter(line -> line.length()<32)
+            while (scanner.hasNextLine()) {
-                    .map(String::toLowerCase)
+                String line = scanner.nextLine().trim();
-                    .forEach(w -> ret.put(w, w));
+                String[] parts = StringUtils.split(line, " ,", 2);
+                if (parts.length != 2) {
+                    logger.warn("Invalid suggestion line: {}", line);
+                    continue;
+                }
+                int cnt = Integer.parseInt(parts[0]);
+                if (cnt > 1) {
+                    String word = parts[1];
 
-            // Add special keywords to the suggestions
+                    // Remove quotes and trailing periods if this is a CSV
-            for (var feature : HtmlFeature.values()) {
+                    if (word.startsWith("\"") && word.endsWith("\"")) {
-                String keyword = feature.getKeyword();
+                        word = word.substring(1, word.length() - 1);
+                    }
 
-                ret.put(keyword, keyword);
+                    // Remove trailing periods
-                ret.put("-" + keyword, "-" + keyword);
+                    while (word.endsWith(".")) {
+                        word = word.substring(0, word.length() - 1);
+                    }
+
+                    // Remove junk items we may have gotten from link extraction
+                    if (word.startsWith("click here"))
+                        continue;
+                    if (word.contains("new window"))
+                        continue;
+                    if (word.contains("click to"))
+                        continue;
+                    if (word.startsWith("share "))
+                        continue;
+
+                    if (word.length() > 3) {
+                        ret.insert(word, cnt);
+                    }
+                }
             }
-
             return ret;
         }
         catch (IOException ex) {
             logger.error("Failed to load suggestions file", ex);
-            return new PatriciaTrie<>();
+            return new PrefixSearchStructure();
         }
     }
 
@@ -83,96 +98,36 @@ public class Suggestions {
 
         searchWord = StringUtils.stripStart(searchWord.toLowerCase(), " ");
 
-        return Stream.of(
+        return getSuggestionsForKeyword(count, searchWord);
-                    new SuggestionStream("", getSuggestionsForKeyword(count, searchWord)),
-                    suggestionsForLastWord(count, searchWord),
-                    spellCheckStream(searchWord)
-                )
-                .flatMap(SuggestionsStreamable::stream)
-                .limit(count)
-                .collect(Collectors.toList());
     }
 
-    private SuggestionsStreamable suggestionsForLastWord(int count, String searchWord) {
+    public List<String> getSuggestionsForKeyword(int count, String prefix) {
-        int sp = searchWord.lastIndexOf(' ');
-
-        if (sp < 0) {
-            return Stream::empty;
-        }
-
-        String prefixString = searchWord.substring(0, sp+1);
-        String suggestString = searchWord.substring(sp+1);
-
-        return new SuggestionStream(prefixString, getSuggestionsForKeyword(count, suggestString));
-
-    }
-
-    private SuggestionsStreamable spellCheckStream(String word) {
-        int start = word.lastIndexOf(' ');
-        String prefix;
-        String corrWord;
-
-        if (start < 0) {
-            corrWord = word;
-            prefix = "";
-        }
-        else {
-            prefix = word.substring(0, start + 1);
-            corrWord = word.substring(start + 1);
-        }
-
-        if (corrWord.length() >= MIN_SUGGEST_LENGTH) {
-            Supplier<Stream<String>> suggestionsLazyEval = () -> spellChecker.correct(corrWord).stream();
-            return new SuggestionStream(prefix, Stream.of(suggestionsLazyEval).flatMap(Supplier::get));
-        }
-        else {
-            return Stream::empty;
-        }
-    }
-
-
-    public Stream<String> getSuggestionsForKeyword(int count, String prefix) {
         if (!ready)
-            return Stream.empty();
+            return List.of();
 
         if (prefix.length() < MIN_SUGGEST_LENGTH) {
-            return Stream.empty();
+            return List.of();
         }
 
-        var start = suggestionsTrie.select(prefix);
+        List<PrefixSearchStructure.ScoredSuggestion> resultsAll = new ArrayList<>();
 
-        if (start == null) {
+        for (var searchStructure : searchStructures) {
-            return Stream.empty();
+            resultsAll.addAll(searchStructure.getTopCompletions(prefix, count));
+        }
+        resultsAll.sort(Comparator.reverseOrder());
+        List<String> ret = new ArrayList<>(count);
+
+        Set<String> seen = new HashSet<>();
+        for (var result : resultsAll) {
+            if (seen.add(result.getWord())) {
+                ret.add(result.getWord());
+            }
+            if (ret.size() >= count) {
+                break;
+            }
         }
 
-        if (!start.getKey().startsWith(prefix)) {
+        return ret;
-            return Stream.empty();
-        }
-
-        SuggestionsValueCalculator sv = new SuggestionsValueCalculator();
-
-        return Stream.iterate(start.getKey(), Objects::nonNull, suggestionsTrie::nextKey)
-                .takeWhile(s -> s.startsWith(prefix))
-                .limit(256)
-                .sorted(Comparator.comparing(sv::get).thenComparing(String::length).thenComparing(Comparator.naturalOrder()))
-                .limit(count);
     }
 
-    private record SuggestionStream(String prefix, Stream<String> suggestionStream) implements SuggestionsStreamable {
-        public Stream<String> stream() {
-            return suggestionStream.map(s -> prefix + s);
-        }
-    }
-
-    interface SuggestionsStreamable { Stream<String> stream(); }
-
-    private class SuggestionsValueCalculator {
-
-        private final Map<String, Long> hashCache = new HashMap<>(512);
-
-        public int get(String s) {
-            long hash = hashCache.computeIfAbsent(s, TermFrequencyDict::getStringHash);
-            return -termFrequencyDict.getTermFreqHash(hash);
-        }
-    }
 }

code/services-core/control-service/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'java'
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-core/control-service/java/nu/marginalia/control/ControlMain.java

@@ -59,9 +59,14 @@ public class ControlMain extends MainClass {
             download(adblockFile, new URI("https://downloads.marginalia.nu/data/adblock.txt"));
         }
 
-        Path suggestionsFile = dataPath.resolve("suggestions.txt");
+        Path suggestionsFile = dataPath.resolve("suggestions2.txt.gz");
         if (!Files.exists(suggestionsFile)) {
-            downloadGzipped(suggestionsFile, new URI("https://downloads.marginalia.nu/data/suggestions.txt.gz"));
+            download(suggestionsFile, new URI("https://downloads.marginalia.nu/data/suggestions2.txt.gz"));
+        }
+
+        Path altSuggestionsFile = dataPath.resolve("suggestions3.txt.gz");
+        if (!Files.exists(altSuggestionsFile)) {
+            download(altSuggestionsFile, new URI("https://downloads.marginalia.nu/data/suggestions3.txt.gz"));
         }
 
         Path asnRawData = dataPath.resolve("asn-data-raw-table");

code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java

@@ -321,9 +321,10 @@ public class ControlNodeActionsService {
     private Object exportSampleData(Request req, Response rsp) {
         FileStorageId source = parseSourceFileStorageId(req.queryParams("source"));
         int size = Integer.parseInt(req.queryParams("size"));
+        String ctFilter = req.queryParams("ctFilter");
         String name = req.queryParams("name");
 
-        exportClient.exportSampleData(Integer.parseInt(req.params("id")), source, size, name);
+        exportClient.exportSampleData(Integer.parseInt(req.params("id")), source, size, ctFilter, name);
 
         return "";
     }

code/services-core/control-service/resources/templates/control/node/actions/partial-download-sample-data.hdb

@@ -24,25 +24,25 @@ This is a sample of real crawl data.  It is intended for demo, testing and devel
     <tr>
         <td><input id="sample-s" value="sample-s" name="sample" class="form-check-input" type="radio"></td>
         <td><label for="sample-s">Small</label></td>
-        <td>1000 Domains. About 2 GB. </td>
+        <td>1000 Domains. About 1 GB. </td>
     </tr>
 
     <tr>
         <td><input id="sample-m" value="sample-m" name="sample" class="form-check-input" type="radio"></td>
         <td><label for="sample-m">Medium</label></td>
-        <td>2000 Domains. About 6 GB. Recommended.</td>
+        <td>2000 Domains. About 2 GB. Recommended.</td>
     </tr>
 
     <tr>
         <td><input id="sample-l" value="sample-l" name="sample" class="form-check-input" type="radio"></td>
         <td><label for="sample-l">Large</label></td>
-        <td>5000 Domains.  About 20 GB.</td>
+        <td>5000 Domains.  About 7 GB.</td>
     </tr>
 
     <tr>
         <td><input id="sample-xl" value="sample-xl" name="sample" class="form-check-input" type="radio"></td>
         <td><label for="sample-xl">Huge</label></td>
-        <td>50,000 Domains.  Around 180 GB.  Primarily intended for pre-production like testing environments.
+        <td>50,000 Domains.  Around 80 GB.  Primarily intended for pre-production like testing environments.
             Expect hours of processing time. </td>
     </tr>
 </table>

code/services-core/control-service/resources/templates/control/node/actions/partial-export-sample-data.hdb

@@ -35,6 +35,11 @@
         <div><input type="text" name="size" id="size" pattern="\d+" /></div>
         <small class="text-muted">How many domains to include in the sample set</small>
     </div>
+    <div class="mb-3">
+        <label for="ctFilter">Content Type Filter</label>
+        <div><input type="text" name="ctFilter" id="ctFilter" /></div>
+        <small class="text-muted">If set, includes only documents with the specified content type value</small>
+    </div>
     <div class="mb-3">
         <label for="name">Name</label>
         <div><input type="text" name="name" id="name" /></div>

code/services-core/executor-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-core/index-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-core/query-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/tools/screenshot-capture-tool/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

deploy.txt

@@ -1,4 +1,9 @@
-## This is a token file for automatic deployment
+## This is a token file for triggering automatic deployment when no commit is made.
 
 2025-01-08:  Deploy executor.
-2025-01-07:  Deploy executor.
+2025-01-07:  Deploy executor.
+2025-04-24:  Deploy executor.
+2025-04-24:  Deploy assistant.
+2025-05-04:  Deploy qs, search and api-services.
+2025-05-05:  Deploy executor partition 4.
+2025-05-05:  Deploy control.

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

tools/deployment/deployment.py

@@ -314,6 +314,13 @@ if __name__ == '__main__':
             deploy_tier=0,
             groups={"all", "core"}
         ),
+        'status': ServiceConfig(
+            gradle_target=':code:services-application:status-service:docker',
+            docker_name='status-service',
+            instances=None,
+            deploy_tier=4,
+            groups={"all"}
+        ),
         'query': ServiceConfig(
             gradle_target=':code:services-core:query-service:docker',
             docker_name='query-service',