(url) Fix urlencoding issues with certain symbols

We also need to apply the fix when performing toString() on the EdgeUrl, the URI class will URLDecode the input.

The change also alters the parseURI method to only run the URLEncode-fixer during parsing if URI doesn't throw an exception.  This bad path is obviously going to be slower, but realistically, most URLs are valid, so it's probably a significant optimization to do it like this.

build.gradle

@@ -5,7 +5,7 @@ plugins {
 
     // This is a workaround for a bug in the Jib plugin that causes it to stall randomly
     // https://github.com/GoogleContainerTools/jib/issues/3347
-    id 'com.google.cloud.tools.jib' version '3.4.4' apply(false)
+    id 'com.google.cloud.tools.jib' version '3.4.5' apply(false)
 }
 
 group 'marginalia'
@@ -47,7 +47,7 @@ ext {
     dockerImageBase='container-registry.oracle.com/graalvm/jdk:24'
     dockerImageTag='latest'
     dockerImageRegistry='marginalia'
-    jibVersion = '3.4.4'
+    jibVersion = '3.4.5'
 }
 
 idea {

code/common/model/java/nu/marginalia/model/EdgeUrl.java

@@ -1,16 +1,14 @@
 package nu.marginalia.model;
 
 import nu.marginalia.util.QueryParams;
+import org.apache.commons.lang3.StringUtils;
 
 import javax.annotation.Nullable;
 import java.io.Serializable;
-import java.net.MalformedURLException;
+import java.net.*;
-import java.net.URI;
+import java.nio.charset.StandardCharsets;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.regex.Pattern;
 
 public class EdgeUrl implements Serializable {
     public final String proto;
@@ -33,9 +31,21 @@ public class EdgeUrl implements Serializable {
 
     private static URI parseURI(String url) throws URISyntaxException {
         try {
-            return new URI(urlencodeFixer(url));
+            return new URI(url);
-        } catch (URISyntaxException ex) {
+        } catch (URISyntaxException _) {
-            throw new URISyntaxException("Failed to parse URI '" + url + "'", ex.getMessage());
+            try {
+                /* Java's URI parser is a bit too strict in throwing exceptions when there's an error.
+
+                   Here on the Internet, standards are like the picture on the box of the frozen pizza,
+                   and what you get is more like what's on the inside, we try to patch things instead,
+                   just give it a best-effort attempt att cleaning out broken or unnecessary constructions
+                   like bad or missing URLEncoding
+                 */
+                return EdgeUriFactory.parseURILenient(url);
+            }
+            catch (URISyntaxException ex2) {
+                throw new URISyntaxException("Failed to parse URI '" + url + "'", ex2.getMessage());
+            }
         }
     }
 
@@ -51,58 +61,6 @@ public class EdgeUrl implements Serializable {
         }
     }
 
-    private static Pattern badCharPattern = Pattern.compile("[ \t\n\"<>\\[\\]()',|]");
-
-    /* Java's URI parser is a bit too strict in throwing exceptions when there's an error.
-
-       Here on the Internet, standards are like the picture on the box of the frozen pizza,
-       and what you get is more like what's on the inside, we try to patch things instead,
-       just give it a best-effort attempt att cleaning out broken or unnecessary constructions
-       like bad or missing URLEncoding
-     */
-    public static String urlencodeFixer(String url) throws URISyntaxException {
-        var s = new StringBuilder();
-        String goodChars = "&.?:/-;+$#";
-        String hexChars = "0123456789abcdefABCDEF";
-
-        int pathIdx = findPathIdx(url);
-        if (pathIdx < 0) { // url looks like http://marginalia.nu
-            return url + "/";
-        }
-        s.append(url, 0, pathIdx);
-
-        // We don't want the fragment, and multiple fragments breaks the Java URIParser for some reason
-        int end = url.indexOf("#");
-        if (end < 0) end = url.length();
-
-        for (int i = pathIdx; i < end; i++) {
-            int c = url.charAt(i);
-
-            if (goodChars.indexOf(c) >= 0 || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
-                s.appendCodePoint(c);
-            } else if (c == '%' && i + 2 < end) {
-                int cn = url.charAt(i + 1);
-                int cnn = url.charAt(i + 2);
-                if (hexChars.indexOf(cn) >= 0 && hexChars.indexOf(cnn) >= 0) {
-                    s.appendCodePoint(c);
-                } else {
-                    s.append("%25");
-                }
-            } else {
-                s.append(String.format("%%%02X", c));
-            }
-        }
-
-        return s.toString();
-    }
-
-    private static int findPathIdx(String url) throws URISyntaxException {
-        int colonIdx = url.indexOf(':');
-        if (colonIdx < 0 || colonIdx + 2 >= url.length()) {
-            throw new URISyntaxException(url, "Lacking protocol");
-        }
-        return url.indexOf('/', colonIdx + 2);
-    }
 
     public EdgeUrl(URI URI) {
         try {
@@ -166,11 +124,10 @@ public class EdgeUrl implements Serializable {
             sb.append(port);
         }
 
-        sb.append(path);
+        EdgeUriFactory.urlencodePath(sb, path);
 
         if (param != null) {
-            sb.append('?');
+            EdgeUriFactory.urlencodeQuery(sb, param);
-            sb.append(param);
         }
 
         return sb.toString();
@@ -247,3 +204,138 @@ public class EdgeUrl implements Serializable {
     }
 
 }
+
+class EdgeUriFactory {
+    public static URI parseURILenient(String url) throws URISyntaxException {
+        var s = new StringBuilder();
+
+        int pathIdx = findPathIdx(url);
+        if (pathIdx < 0) { // url looks like http://marginalia.nu
+            return new URI(url + "/");
+        }
+        s.append(url, 0, pathIdx);
+
+        // We don't want the fragment, and multiple fragments breaks the Java URIParser for some reason
+        int end = url.indexOf("#");
+        if (end < 0) end = url.length();
+
+        int queryIdx = url.indexOf('?');
+        if (queryIdx < 0) queryIdx = end;
+
+        urlencodePath(s, url.substring(pathIdx, queryIdx));
+        if (queryIdx < end) {
+            urlencodeQuery(s, url.substring(queryIdx + 1, end));
+        }
+        return new URI(s.toString());
+    }
+
+    /** Break apart the path element of an URI into its components, and then
+     * urlencode any component that needs it, and recombine it into a single
+     * path element again.
+     */
+    public static void urlencodePath(StringBuilder sb, String path) {
+        if (path == null || path.isEmpty()) {
+            return;
+        }
+
+        String[] pathParts = StringUtils.split(path, '/');
+        if (pathParts.length == 0) {
+            sb.append('/');
+            return;
+        }
+
+        for (String pathPart : pathParts) {
+            if (pathPart.isEmpty()) continue;
+
+            if (needsUrlEncode(pathPart)) {
+                sb.append('/');
+                sb.append(URLEncoder.encode(pathPart, StandardCharsets.UTF_8).replace("+", "%20"));
+            } else {
+                sb.append('/');
+                sb.append(pathPart);
+            }
+        }
+
+        if (path.endsWith("/")) {
+            sb.append('/');
+        }
+
+    }
+
+    /** Break apart the query element of a URI into its components, and then
+     * urlencode any component that needs it, and recombine it into a single
+     * query element again.
+     */
+    public static void urlencodeQuery(StringBuilder sb, String param) {
+        if (param == null || param.isEmpty()) {
+            return;
+        }
+
+        String[] pathParts = StringUtils.split(param, '&');
+        boolean first = true;
+        for (String queryPart : pathParts) {
+            if (queryPart.isEmpty()) continue;
+
+            if (first) {
+                sb.append('?');
+                first = false;
+            } else {
+                sb.append('&');
+            }
+
+            if (needsUrlEncode(queryPart)) {
+                sb.append(URLEncoder.encode(queryPart, StandardCharsets.UTF_8));
+            } else {
+                sb.append(queryPart);
+            }
+        }
+    }
+
+
+    /** Test if the url element needs URL encoding.
+     * <p></p>
+     * Note we may have been given an already encoded path element,
+     * so we include % and + in the list of good characters
+     */
+    static boolean needsUrlEncode(String urlElement) {
+        for (int i = 0; i < urlElement.length(); i++) {
+            char c = urlElement.charAt(i);
+
+            if (c >= 'a' && c <= 'z') continue;
+            if (c >= 'A' && c <= 'Z') continue;
+            if (c >= '0' && c <= '9') continue;
+            if ("-_.~+?=&".indexOf(c) >= 0) continue;
+            if (c == '%' && i + 2 < urlElement.length()) {
+                char c1 = urlElement.charAt(i + 1);
+                char c2 = urlElement.charAt(i + 2);
+                if (isHexDigit(c1) && isHexDigit(c2)) {
+                    i += 2;
+                    continue;
+                }
+            }
+
+            return true;
+        }
+
+        return false;
+    }
+
+    private static boolean isHexDigit(char c) {
+        return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    /** Find the index of the path element in a URL.
+     * <p></p>
+     * The path element starts after the scheme and authority part of the URL,
+     * which is everything up to and including the first slash after the colon.
+     */
+    private static int findPathIdx(String url) throws URISyntaxException {
+        int colonIdx = url.indexOf(':');
+        if (colonIdx < 0 || colonIdx + 3 >= url.length()) {
+            throw new URISyntaxException(url, "Lacking scheme");
+        }
+        return url.indexOf('/', colonIdx + 3);
+    }
+
+
+}

code/common/model/test/nu/marginalia/model/EdgeUrlTest.java

@@ -1,6 +1,6 @@
 package nu.marginalia.model;
 
-import nu.marginalia.model.EdgeUrl;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.net.URISyntaxException;
@@ -21,25 +21,50 @@ class EdgeUrlTest {
                 new EdgeUrl("https://memex.marginalia.nu/#here")
         );
     }
+
     @Test
-    public void testParam() throws URISyntaxException {
+    void testUriFromString() throws URISyntaxException {
-        System.out.println(new EdgeUrl("https://memex.marginalia.nu/index.php?id=1").toString());
+        // We test these URLs several times as we perform URLEncode-fixing both when parsing the URL and when
-        System.out.println(new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").toString());
+        // converting it back to a string, we want to ensure there is no changes along the way.
-    }
+
-    @Test
+        Assertions.assertEquals("/", EdgeUriFactory.parseURILenient("https://www.example.com/").getPath());
-    void urlencodeFixer() throws URISyntaxException {
+        Assertions.assertEquals("https://www.example.com/", EdgeUriFactory.parseURILenient("https://www.example.com/").toString());
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/#heredoc"));
+        Assertions.assertEquals("https://www.example.com/", new EdgeUrl("https://www.example.com/").toString());
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/%-sign"));
+
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/%22-sign"));
+        Assertions.assertEquals("/", EdgeUriFactory.parseURILenient("https://www.example.com/#heredoc").getPath());
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/\n \"huh\""));
+        Assertions.assertEquals("https://www.example.com/", EdgeUriFactory.parseURILenient("https://www.example.com/#heredoc").toString());
+        Assertions.assertEquals("https://www.example.com/", new EdgeUrl("https://www.example.com/#heredoc").toString());
+
+        Assertions.assertEquals("/trailingslash/", EdgeUriFactory.parseURILenient("https://www.example.com/trailingslash/").getPath());
+        Assertions.assertEquals("https://www.example.com/trailingslash/", EdgeUriFactory.parseURILenient("https://www.example.com/trailingslash/").toString());
+        Assertions.assertEquals("https://www.example.com/trailingslash/", new EdgeUrl("https://www.example.com/trailingslash/").toString());
+
+        Assertions.assertEquals("/%-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%25-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%25-sign", new EdgeUrl("https://www.example.com/%-sign").toString());
+
+        Assertions.assertEquals("/\"-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%22-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%22-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%22-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%22-sign", new EdgeUrl("https://www.example.com/%22-sign").toString());
+
+        Assertions.assertEquals("/\n \"huh\"", EdgeUriFactory.parseURILenient("https://www.example.com/\n \"huh\"").getPath());
+        Assertions.assertEquals("https://www.example.com/%0A%20%22huh%22", EdgeUriFactory.parseURILenient("https://www.example.com/\n \"huh\"").toString());
+        Assertions.assertEquals("https://www.example.com/%0A%20%22huh%22", new EdgeUrl("https://www.example.com/\n \"huh\"").toString());
+
+        Assertions.assertEquals("/wiki/Sámi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").getPath());
+        Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").toString());
+        Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", new EdgeUrl("https://en.wikipedia.org/wiki/Sámi").toString());
     }
 
     @Test
     void testParms() throws URISyntaxException {
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?id=123"));
+        Assertions.assertEquals("id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").param);
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?t=123"));
+        Assertions.assertEquals("t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").param);
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?v=123"));
+        Assertions.assertEquals("v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").param);
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?m=123"));
+        Assertions.assertEquals("id=1", new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").param);
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?follow=123"));
+        Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").param);
+        Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").param);
+        Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?m=123").param);
+        Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?follow=123").param);
     }
 }

code/common/service/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java

@@ -59,16 +59,13 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo
      */
     @Override
     public void progress(String step, int stepProgress, int stepCount) {
+        int lastProgress = this.progress;
         this.step = step;
-
-
-        // off by one since we calculate the progress based on the number of steps,
-        // and Enum.ordinal() is zero-based (so the 5th step in a 5 step task is 4, not 5; resulting in the
-        // final progress being 80% and not 100%)
-
         this.progress = (int) Math.round(100. * stepProgress / (double) stepCount);
 
-        logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        if (this.progress / 10 != lastProgress / 10) {
+            logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        }
     }
 
     /** Wrap a collection to provide heartbeat progress updates as it's iterated through */

code/common/service/java/nu/marginalia/service/control/ServiceAdHocTaskHeartbeatImpl.java

@@ -57,16 +57,13 @@ public class ServiceAdHocTaskHeartbeatImpl implements AutoCloseable, ServiceAdHo
      */
     @Override
     public void progress(String step, int stepProgress, int stepCount) {
+        int lastProgress = this.progress;
         this.step = step;
-
-
-        // off by one since we calculate the progress based on the number of steps,
-        // and Enum.ordinal() is zero-based (so the 5th step in a 5 step task is 4, not 5; resulting in the
-        // final progress being 80% and not 100%)
-
         this.progress = (int) Math.round(100. * stepProgress / (double) stepCount);
 
-        logger.info("ServiceTask {} progress: {}%", taskBase, progress);
+        if (this.progress / 10 != lastProgress / 10) {
+            logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        }
     }
 
     public void shutDown() {

code/execution/api/java/nu/marginalia/executor/client/ExecutorExportClient.java

@@ -48,12 +48,13 @@ public class ExecutorExportClient {
         return msgId;
     }
 
-    public void exportSampleData(int node, FileStorageId fid, int size, String name) {
+    public void exportSampleData(int node, FileStorageId fid, int size, String ctFilter, String name) {
         channelPool.call(ExecutorExportApiBlockingStub::exportSampleData)
                 .forNode(node)
                 .run(RpcExportSampleData.newBuilder()
                         .setFileStorageId(fid.id())
                         .setSize(size)
+                        .setCtFilter(ctFilter)
                         .setName(name)
                         .build());
     }

code/execution/api/src/main/protobuf/executor-api.proto

@@ -100,6 +100,7 @@ message RpcExportSampleData {
   int64 fileStorageId = 1;
   int32 size = 2;
   string name = 3;
+  string ctFilter = 4;
 }
 message RpcDownloadSampleData {
   string sampleSet = 1;

code/execution/java/nu/marginalia/actor/task/ExportSampleDataActor.java

@@ -26,32 +26,32 @@ public class ExportSampleDataActor extends RecordActorPrototype {
     private final MqOutbox exportTasksOutbox;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    public record Export(FileStorageId crawlId, int size, String name) implements ActorStep {}
+    public record Export(FileStorageId crawlId, int size, String ctFilter, String name) implements ActorStep {}
-    public record Run(FileStorageId crawlId, FileStorageId destId, int size, String name, long msgId) implements ActorStep {
+    public record Run(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name, long msgId) implements ActorStep {
-        public Run(FileStorageId crawlId, FileStorageId destId, int size, String name) {
+        public Run(FileStorageId crawlId, FileStorageId destId, int size, String name, String ctFilter) {
-            this(crawlId, destId, size, name, -1);
+            this(crawlId, destId, size, name, ctFilter,-1);
         }
     }
 
     @Override
     public ActorStep transition(ActorStep self) throws Exception {
         return switch(self) {
-            case Export(FileStorageId crawlId, int size, String name) -> {
+            case Export(FileStorageId crawlId, int size, String ctFilter, String name) -> {
                 var storage = storageService.allocateStorage(FileStorageType.EXPORT,
                         "crawl-sample-export",
                         "Crawl Data Sample " + name + "/" + size + " " + LocalDateTime.now()
                 );
 
                 if (storage == null) yield new Error("Bad storage id");
-                yield new Run(crawlId, storage.id(), size, name);
+                yield new Run(crawlId, storage.id(), size, ctFilter, name);
             }
-            case Run(FileStorageId crawlId, FileStorageId destId, int size, String name, long msgId) when msgId < 0 -> {
+            case Run(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name, long msgId) when msgId < 0 -> {
                 storageService.setFileStorageState(destId, FileStorageState.NEW);
 
-                long newMsgId = exportTasksOutbox.sendAsync(ExportTaskRequest.sampleData(crawlId, destId, size, name));
+                long newMsgId = exportTasksOutbox.sendAsync(ExportTaskRequest.sampleData(crawlId, destId, ctFilter, size, name));
-                yield new Run(crawlId, destId, size, name, newMsgId);
+                yield new Run(crawlId, destId, size, ctFilter, name, newMsgId);
             }
-            case Run(_, FileStorageId destId, _, _, long msgId) -> {
+            case Run(_, FileStorageId destId, _, _, _, long msgId) -> {
                 var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
@@ -70,7 +70,7 @@ public class ExportSampleDataActor extends RecordActorPrototype {
 
     @Override
     public String describe() {
-        return "Export RSS/Atom feeds from crawl data";
+        return "Export sample crawl data";
     }
 
     @Inject

code/execution/java/nu/marginalia/execution/ExecutorExportGrpcService.java

@@ -49,6 +49,7 @@ public class ExecutorExportGrpcService
                     new ExportSampleDataActor.Export(
                             FileStorageId.of(request.getFileStorageId()),
                             request.getSize(),
+                            request.getCtFilter(),
                             request.getName()
                     )
             );

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedFetcherService.java

@@ -229,13 +229,15 @@ public class FeedFetcherService {
                     .timeout(Duration.ofSeconds(15))
                     ;
 
-            if (ifModifiedSinceDate != null) {
+            // Set the If-Modified-Since or If-None-Match headers if we have them
+            // though since there are certain idiosyncrasies in server implementations,
+            // we avoid setting both at the same time as that may turn a 304 into a 200.
+            if (ifNoneMatchTag != null) {
+                requestBuilder.header("If-None-Match", ifNoneMatchTag);
+            } else if (ifModifiedSinceDate != null) {
                 requestBuilder.header("If-Modified-Since", ifModifiedSinceDate);
             }
 
-            if (ifNoneMatchTag != null) {
-                requestBuilder.header("If-None-Match", ifNoneMatchTag);
-            }
 
             HttpRequest getRequest = requestBuilder.build();
 

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -264,17 +264,16 @@ public class CrawlerMain extends ProcessMainClass {
                 if (workLog.isJobFinished(crawlSpec.domain))
                     continue;
 
-                var task = new CrawlTask(
+                var task = new CrawlTask(crawlSpec, anchorTagsSource, outputDir, warcArchiver, domainStateDb, workLog);
-                        crawlSpec,
-                        anchorTagsSource,
-                        outputDir,
-                        warcArchiver,
-                        domainStateDb,
-                        workLog);
 
                 // Try to run immediately, to avoid unnecessarily keeping the entire work set in RAM
                 if (!trySubmitDeferredTask(task)) {
-                    // Otherwise add to the taskList for deferred execution
+
+                    // Drain the retry queue to the taskList, and try to submit any tasks that are in the retry queue
+                    retryQueue.drainTo(taskList);
+                    taskList.removeIf(this::trySubmitDeferredTask);
+
+                    // Then add this new task to the retry queue
                     taskList.add(task);
                 }
             }

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/ContentTags.java

@@ -19,11 +19,13 @@ public record ContentTags(String etag, String lastMod) {
     /** Paints the tags onto the request builder. */
     public void paint(HttpGet request) {
 
+        // Paint the ETag header if present,
+        // otherwise paint the Last-Modified header
+        // (but not both at the same time due to some servers not liking it)
+
         if (etag != null) {
             request.addHeader("If-None-Match", etag);
-        }
+        } else if (lastMod != null) {
-
-        if (lastMod != null) {
             request.addHeader("If-Modified-Since", lastMod);
         }
     }

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -51,6 +51,7 @@ import javax.net.ssl.SSLException;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.net.URISyntaxException;
+import java.net.UnknownHostException;
 import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
 import java.time.Instant;
@@ -635,14 +636,12 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
 
     @Override
     public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context) {
-        if (exception instanceof SocketTimeoutException) { // Timeouts are not recoverable
+        return switch (exception) {
-            return false;
+            case SocketTimeoutException ste -> false;
-        }
+            case SSLException ssle -> false;
-        if (exception instanceof SSLException) { // SSL exceptions are unlikely to be recoverable
+            case UnknownHostException uhe -> false;
-            return false;
+            default -> executionCount <= 3;
-        }
+        };
-
-        return executionCount <= 3;
     }
 
     @Override

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcInputBuffer.java

@@ -57,6 +57,7 @@ public abstract class WarcInputBuffer implements AutoCloseable {
             return new ErrorBuffer();
         }
 
+        Instant start = Instant.now();
         InputStream is = null;
         try {
             is = entity.getContent();
@@ -71,8 +72,25 @@ public abstract class WarcInputBuffer implements AutoCloseable {
             }
         }
         finally {
+            // We're required to consume the stream to avoid leaking connections,
+            // but we also don't want to get stuck on slow or malicious connections
+            // forever, so we set a time limit on this phase and call abort() if it's exceeded.
             try {
-                is.skip(Long.MAX_VALUE);
+                while (is != null) {
+                    // Consume some data
+                    if (is.skip(65536) == 0) {
+                        // Note that skip may return 0 if the stream is empty
+                        // or for other unspecified reasons, so we need to check
+                        // with read() as well to determine if the stream is done
+                        if (is.read() == -1)
+                            is = null;
+                    }
+                    // Check if the time limit has been exceeded
+                    else if (Duration.between(start, Instant.now()).compareTo(timeLimit) > 0) {
+                        request.abort();
+                        is = null;
+                    }
+                }
             }
             catch (IOException e) {
                 // Ignore the exception

code/processes/export-task-process/build.gradle

@@ -53,6 +53,8 @@ dependencies {
     implementation libs.commons.compress
     implementation libs.commons.codec
     implementation libs.jsoup
+    implementation libs.slop
+    implementation libs.jwarc
 
 
 

code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java

@@ -3,11 +3,15 @@ package nu.marginalia.extractor;
 import com.google.inject.Inject;
 import nu.marginalia.process.log.WorkLog;
 import nu.marginalia.process.log.WorkLogEntry;
+import nu.marginalia.slop.SlopCrawlDataRecord;
+import nu.marginalia.slop.SlopTablePacker;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorage;
 import nu.marginalia.storage.model.FileStorageId;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -27,7 +31,7 @@ public class SampleDataExporter {
     public SampleDataExporter(FileStorageService storageService) {
         this.storageService = storageService;
     }
-    public void export(FileStorageId crawlId, FileStorageId destId, int size, String name) throws SQLException, IOException {
+    public void export(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name) throws SQLException, IOException {
         FileStorage destStorage = storageService.getStorage(destId);
         Path inputDir = storageService.getStorage(crawlId).asPath();
 
@@ -54,6 +58,7 @@ public class SampleDataExporter {
 
         Path newCrawlerLogFile = Files.createTempFile(destStorage.asPath(), "crawler", ".log",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
+
         try (var bw = Files.newBufferedWriter(newCrawlerLogFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
             for (var item : entriesAll) {
                 bw.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
@@ -72,7 +77,22 @@ public class SampleDataExporter {
                 Path crawlDataPath = inputDir.resolve(item.relPath());
                 if (!Files.exists(crawlDataPath)) continue;
 
-                addFileToTar(stream, crawlDataPath, item.relPath());
+                if (StringUtils.isBlank(ctFilter)) {
+                    addFileToTar(stream, crawlDataPath, item.relPath());
+                }
+                else /* filter != null */ {
+                    boolean didFilterData = false;
+                    try {
+                        crawlDataPath = filterEntries(crawlDataPath, ctFilter);
+                        didFilterData = true;
+                        addFileToTar(stream, crawlDataPath, item.relPath());
+                    }
+                    finally {
+                        if (didFilterData) {
+                            Files.deleteIfExists(crawlDataPath);
+                        }
+                    }
+                }
             }
 
             addFileToTar(stream, newCrawlerLogFile, "crawler.log");
@@ -86,6 +106,46 @@ public class SampleDataExporter {
         Files.move(tmpTarFile, destStorage.asPath().resolve("crawl-data.tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
     }
 
+    /** Filters the entries in the crawl data file based on the content type.
+     * @param crawlDataPath The path to the crawl data file.
+     * @param contentTypeFilter The content type to filter by.
+     * @return The path to the filtered crawl data file, or null if an error occurred.
+     */
+    private Path filterEntries(Path crawlDataPath, String contentTypeFilter) throws IOException {
+        Path tempDir = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered");
+        Path tempFile = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered.slop.zip");
+
+        Files.createDirectory(tempDir);
+
+        try (var writer = new SlopCrawlDataRecord.Writer(tempDir);
+             var reader = new SlopCrawlDataRecord.FilteringReader(crawlDataPath) {
+                 @Override
+                 public boolean filter(String url, int status, String contentType) {
+                     if (contentTypeFilter.equals(contentType))
+                         return true;
+                     else if (contentType.startsWith("x-marginalia/"))
+                         // This is a metadata entry, typically domain or redirect information
+                         // let's keep those to not confuse the consumer of the data, which might
+                         // expect at least the domain summary
+                         return true;
+                     return false;
+                 }
+             }
+        ) {
+            while (reader.hasRemaining()) {
+                writer.write(reader.get());
+            }
+
+            SlopTablePacker.packToSlopZip(tempDir, tempFile);
+        }
+        finally {
+            FileUtils.deleteDirectory(tempDir.toFile());
+        }
+
+
+        return tempFile;
+    }
+
     private void addFileToTar(TarArchiveOutputStream outputStream, Path file, String fileName) throws IOException {
         var entry = outputStream.createArchiveEntry(file.toFile(), fileName);
         entry.setSize(Files.size(file));

code/processes/export-task-process/java/nu/marginalia/task/ExportTasksMain.java

@@ -92,7 +92,7 @@ public class ExportTasksMain extends ProcessMainClass {
                     termFrequencyExporter.export(request.crawlId, request.destId);
                     break;
                 case SAMPLE_DATA:
-                    sampleDataExporter.export(request.crawlId, request.destId, request.size, request.name);
+                    sampleDataExporter.export(request.crawlId, request.destId, request.size, request.ctFilter, request.name);
                     break;
                 case ADJACENCIES:
                     websiteAdjacenciesCalculator.export();

code/processes/process-mq-api/java/nu/marginalia/mqapi/tasks/ExportTaskRequest.java

@@ -16,6 +16,7 @@ public class ExportTaskRequest {
     public FileStorageId destId;
     public int size;
     public String name;
+    public String ctFilter;
 
     public ExportTaskRequest(Task task) {
         this.task = task;
@@ -42,12 +43,13 @@ public class ExportTaskRequest {
         return request;
     }
 
-    public static ExportTaskRequest sampleData(FileStorageId crawlId, FileStorageId destId, int size, String name) {
+    public static ExportTaskRequest sampleData(FileStorageId crawlId, FileStorageId destId, String ctFilter, int size, String name) {
         ExportTaskRequest request = new ExportTaskRequest(Task.SAMPLE_DATA);
         request.crawlId = crawlId;
         request.destId = destId;
         request.size = size;
         request.name = name;
+        request.ctFilter = ctFilter;
         return request;
     }
 

code/services-application/api-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-application/dating-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/explorer-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/search-service-legacy/build.gradle

@@ -5,7 +5,7 @@ plugins {
     id 'application'
     id 'jvm-test-suite'
 
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/search-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
     id 'application'
     id 'jvm-test-suite'
     id 'gg.jte.gradle' version '3.1.15'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-application/search-service/resources/jte/part/head.jte

@@ -26,4 +26,10 @@
 
     <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Marginalia">
 
-</head>
+</head>
+<noscript>
+    <h1>Users of text-based browsers</h1>
+    <p>Consider using the old interface at <a href="https://old-search.marginalia.nu/">https://old-search.marginalia.nu/</a>,
+    as it uses fewer modern CSS tricks, and should work better than the new UI.  It's functionally nearly identical, but just renders it using a different layout.</p>
+    <hr>
+</noscript>

code/services-application/search-service/resources/jte/part/warmup.jte

@@ -1,10 +1,16 @@
 This is a bit of a hack!
 
 This class exists to let tailwind we're using these classes even though they aren't visible in the code,
-as we sometimes generate classes from Java code!
+as we sometimes generate classes from Java code or javascript!
 
 <i class="text-blue-800 bg-blue-50 dark:text-blue-200 dark:bg-blue-950"></i>
 <i class="text-green-800 bg-green-50 dark:text-green-200 dark:bg-green-950"></i>
 <i class="text-purple-800 bg-purple-50 dark:text-purple-200 dark:bg-purple-950"></i>
 <i class="text-blue-950 bg-gray-100 dark:text-blue-50 dark:bg-gray-900"></i>
-<span class="hover:bg-gray-300 "></span>
+<span class="hover:bg-gray-300 "></span>
+
+<label class="suggestion group block relative">
+    <input type="radio" name="suggestion" class="peer hidden" checked>
+    <div class="px-4 py-2 cursor-pointer dark:peer-checked:bg-gray-700 dark:hover:bg-gray-700 peer-checked:bg-gray-300 hover:bg-gray-300 w-full">
+    </div>
+</label>

code/services-application/search-service/resources/jte/serp/error.jte

@@ -26,7 +26,7 @@
 
     <!-- Main content -->
     <main class="flex-1 p-4 max-w-2xl space-y-4">
-        <div class="border dark:border-gray-600 rounded bg-white text-black dark:bg-gray-800 dark:text-white text-m p-4">
+        <div class="border border-gray-300 dark:border-gray-600 rounded bg-white text-black dark:bg-gray-800 dark:text-white text-m p-4">
             <div class="flex space-x-3  place-items-baseline">
                 <i class="fa fa-circle-exclamation text-red-800"></i>
                 <div class="grow">${model.errorTitle()}</div>

code/services-application/search-service/resources/jte/serp/part/footerHowto.jte

@@ -80,10 +80,6 @@
             <tr><td>rank&gt;50</td><td>The ranking of the website is at least 50 in a span of 1 - 255</td></tr>
             <tr><td>rank&lt;50</td><td>The ranking of the website is at most 50 in a span of 1 - 255</td></tr>
 
-            <tr><td>count&gt;10</td><td> The search term must appear in at least 10 results form the domain</td></tr>
-            <tr><td>count&lt;10</td><td> The search term must appear in at most 10 results from the domain</td></tr>
-
-
             <tr><td>format:html5</td><td>Filter documents using the HTML5 standard. This is typically modern websites.</td></tr>
             <tr><td>format:xhtml</td><td>Filter documents using the XHTML standard</td></tr>
             <tr><td>format:html123</td><td>Filter documents using the HTML standards 1, 2, and 3. This is typically very old websites. </td></tr>

code/services-application/search-service/resources/jte/serp/part/searchform.jte

@@ -7,7 +7,7 @@
 
 <form class="flex-1 max-w-2xl" action="/search">
     <div class="flex">
-        @if (query.isBlank())
+        @if (query != null && query.isBlank())
         <%-- Add autofocus if the query is blank --%>
             <input type="text"
                    class="shadow-inner flex-1 dark:bg-black dark:text-gray-100 bg-gray-50 border dark:border-gray-600 border-gray-300 text-gray-900 text-sm rounded-sm block w-full p-2.5"
@@ -27,7 +27,7 @@
                    id="searchInput" />
         @endif
 
-        <div id="searchSuggestions" class="text-sm absolute top-2 mt-10 w-96 bg-white dark:bg-black border dark:border-gray-600 border-gray-200 rounded-lg shadow-lg hidden"></div>
+        <div aria-hidden="true" id="searchSuggestions" class="text-sm absolute top-3 mt-10 w-96 bg-white dark:bg-black border dark:border-gray-600 border-gray-300 rounded-lg shadow-lg hidden"></div>
 
         <button class="px-4 py-2 bg-margeblue text-white ml-2 rounded whitespace-nowrap active:text-slate-200">
             <i class="fas fa-search text-sm sm:mr-3"></i>

code/services-application/search-service/resources/static/js/typeahead.js

@@ -43,13 +43,13 @@ function displaySuggestions(suggestions) {
     }
 
     suggestionsContainer.innerHTML = suggestions.map((suggestion, index) => `
-                <div 
+        <label class="suggestion group block relative">
-                    class="suggestion px-4 py-2 cursor-pointer hover:bg-gray-300 ${index === selectedIndex ? 'bg-blue-50' : ''}"
+            <input type="radio" name="suggestion" class="peer hidden" ${index === selectedIndex ? 'checked' : ''}>
-                    data-index="${index}"
+            <div class="px-4 py-2 cursor-pointer dark:peer-checked:bg-gray-700 dark:hover:bg-gray-700 peer-checked:bg-gray-300 hover:bg-gray-300 w-full" data-index="${index}">
-                >
+                ${suggestion}
-                    ${suggestion}
+            </div>
-                </div>
+        </label>
-            `).join('');
+    `).join('');
 
     suggestionsContainer.classList.remove('hidden');
 

code/services-application/status-service/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'java'
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-core/assistant-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantModule.java

@@ -10,7 +10,8 @@ import static com.google.inject.name.Names.named;
 
 public class AssistantModule extends AbstractModule {
     public void configure() {
-        bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(WmsaHome.getHomePath().resolve("data/suggestions2.txt.gz"));
+        bind(Path.class).annotatedWith(named("suggestions-file1")).toInstance(WmsaHome.getHomePath().resolve("data/suggestions2.txt.gz"));
+        bind(Path.class).annotatedWith(named("suggestions-file2")).toInstance(WmsaHome.getHomePath().resolve("data/suggestions3.txt.gz"));
 
         bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
     }

code/services-core/assistant-service/java/nu/marginalia/assistant/suggest/PrefixSearchStructure.java

@@ -1,6 +1,7 @@
 package nu.marginalia.assistant.suggest;
 
 import gnu.trove.list.array.TIntArrayList;
+import org.jetbrains.annotations.NotNull;
 
 import java.util.*;
 
@@ -434,7 +435,7 @@ public class PrefixSearchStructure {
     /**
      * Class representing a suggested completion.
      */
-    public static class ScoredSuggestion {
+    public static class ScoredSuggestion implements Comparable<ScoredSuggestion> {
         private final String word;
         private final int score;
 
@@ -455,5 +456,10 @@ public class PrefixSearchStructure {
         public String toString() {
             return word + " (" + score + ")";
         }
+
+        @Override
+        public int compareTo(@NotNull PrefixSearchStructure.ScoredSuggestion o) {
+            return Integer.compare(this.score, o.score);
+        }
     }
 }

code/services-core/assistant-service/java/nu/marginalia/assistant/suggest/Suggestions.java

@@ -2,8 +2,6 @@ package nu.marginalia.assistant.suggest;
 
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
-import nu.marginalia.functions.math.dict.SpellChecker;
-import nu.marginalia.term_frequency_dict.TermFrequencyDict;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,35 +11,27 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.List;
-import java.util.Scanner;
-import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 
 public class Suggestions {
-    private PrefixSearchStructure searchStructure = null;
+    List<PrefixSearchStructure> searchStructures = new ArrayList<>();
-    private TermFrequencyDict termFrequencyDict = null;
+
-    private volatile boolean ready = false;
+    private volatile boolean ready = false;
-    private final SpellChecker spellChecker;
 
-    private static final Pattern suggestionPattern = Pattern.compile("^[a-zA-Z0-9]+( [a-zA-Z0-9]+)*$");
     private static final Logger logger = LoggerFactory.getLogger(Suggestions.class);
 
     private static final int MIN_SUGGEST_LENGTH = 3;
     @Inject
-    public Suggestions(@Named("suggestions-file") Path suggestionsFile,
+    public Suggestions(@Named("suggestions-file1") Path suggestionsFile1,
-                       SpellChecker spellChecker,
+                       @Named("suggestions-file2") Path suggestionsFile2
-                       TermFrequencyDict dict
                        ) {
-        this.spellChecker = spellChecker;
 
         Thread.ofPlatform().start(() -> {
-            searchStructure = loadSuggestions(suggestionsFile);
+            searchStructures.add(loadSuggestions(suggestionsFile1));
-            termFrequencyDict = dict;
+            searchStructures.add(loadSuggestions(suggestionsFile2));
             ready = true;
-            logger.info("Loaded {} suggestions", searchStructure.size());
+            logger.info("Loaded suggestions");
         });
     }
 
@@ -55,8 +45,8 @@ public class Suggestions {
 
         try (var scanner = new Scanner(new GZIPInputStream(new BufferedInputStream(Files.newInputStream(file, StandardOpenOption.READ))))) {
             while (scanner.hasNextLine()) {
-                String line = scanner.nextLine();
+                String line = scanner.nextLine().trim();
-                String[] parts = StringUtils.split(line, " ", 2);
+                String[] parts = StringUtils.split(line, " ,", 2);
                 if (parts.length != 2) {
                     logger.warn("Invalid suggestion line: {}", line);
                     continue;
@@ -64,7 +54,30 @@ public class Suggestions {
                 int cnt = Integer.parseInt(parts[0]);
                 if (cnt > 1) {
                     String word = parts[1];
-                    ret.insert(word, cnt);
+
+                    // Remove quotes and trailing periods if this is a CSV
+                    if (word.startsWith("\"") && word.endsWith("\"")) {
+                        word = word.substring(1, word.length() - 1);
+                    }
+
+                    // Remove trailing periods
+                    while (word.endsWith(".")) {
+                        word = word.substring(0, word.length() - 1);
+                    }
+
+                    // Remove junk items we may have gotten from link extraction
+                    if (word.startsWith("click here"))
+                        continue;
+                    if (word.contains("new window"))
+                        continue;
+                    if (word.contains("click to"))
+                        continue;
+                    if (word.startsWith("share "))
+                        continue;
+
+                    if (word.length() > 3) {
+                        ret.insert(word, cnt);
+                    }
                 }
             }
             return ret;
@@ -96,10 +109,22 @@ public class Suggestions {
             return List.of();
         }
 
-        var results = searchStructure.getTopCompletions(prefix, count);
+        List<PrefixSearchStructure.ScoredSuggestion> resultsAll = new ArrayList<>();
+
+        for (var searchStructure : searchStructures) {
+            resultsAll.addAll(searchStructure.getTopCompletions(prefix, count));
+        }
+        resultsAll.sort(Comparator.reverseOrder());
         List<String> ret = new ArrayList<>(count);
-        for (var result : results) {
+
-            ret.add(result.getWord());
+        Set<String> seen = new HashSet<>();
+        for (var result : resultsAll) {
+            if (seen.add(result.getWord())) {
+                ret.add(result.getWord());
+            }
+            if (ret.size() >= count) {
+                break;
+            }
         }
 
         return ret;

code/services-core/control-service/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'java'
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-core/control-service/java/nu/marginalia/control/ControlMain.java

@@ -64,6 +64,11 @@ public class ControlMain extends MainClass {
             download(suggestionsFile, new URI("https://downloads.marginalia.nu/data/suggestions2.txt.gz"));
         }
 
+        Path altSuggestionsFile = dataPath.resolve("suggestions3.txt.gz");
+        if (!Files.exists(altSuggestionsFile)) {
+            download(altSuggestionsFile, new URI("https://downloads.marginalia.nu/data/suggestions3.txt.gz"));
+        }
+
         Path asnRawData = dataPath.resolve("asn-data-raw-table");
         if (!Files.exists(asnRawData)) {
             download(asnRawData, new URI("https://thyme.apnic.net/current/data-raw-table"));

code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java

@@ -321,9 +321,10 @@ public class ControlNodeActionsService {
     private Object exportSampleData(Request req, Response rsp) {
         FileStorageId source = parseSourceFileStorageId(req.queryParams("source"));
         int size = Integer.parseInt(req.queryParams("size"));
+        String ctFilter = req.queryParams("ctFilter");
         String name = req.queryParams("name");
 
-        exportClient.exportSampleData(Integer.parseInt(req.params("id")), source, size, name);
+        exportClient.exportSampleData(Integer.parseInt(req.params("id")), source, size, ctFilter, name);
 
         return "";
     }

code/services-core/control-service/resources/templates/control/node/actions/partial-export-sample-data.hdb

@@ -35,6 +35,11 @@
         <div><input type="text" name="size" id="size" pattern="\d+" /></div>
         <small class="text-muted">How many domains to include in the sample set</small>
     </div>
+    <div class="mb-3">
+        <label for="ctFilter">Content Type Filter</label>
+        <div><input type="text" name="ctFilter" id="ctFilter" /></div>
+        <small class="text-muted">If set, includes only documents with the specified content type value</small>
+    </div>
     <div class="mb-3">
         <label for="name">Name</label>
         <div><input type="text" name="name" id="name" /></div>

code/services-core/executor-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-core/index-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/services-core/query-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {

code/tools/screenshot-capture-tool/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.4'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

deploy.txt

@@ -1,4 +1,6 @@
-## This is a token file for automatic deployment
+## This is a token file for triggering automatic deployment when no commit is made.
 
 2025-01-08:  Deploy executor.
-2025-01-07:  Deploy executor.
+2025-01-07:  Deploy executor.
+2025-04-24:  Deploy executor.
+2025-04-24:  Deploy assistant.

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists