(sample-actor) Make content type matching lenient with regard to ct parameters such as charset

code/common/model/java/nu/marginalia/model/EdgeUrl.java

@@ -121,6 +121,28 @@ public class EdgeUrl implements Serializable {
         return sb.toString();
     }
 
+
+    public String toDisplayString() {
+        StringBuilder sb = new StringBuilder(256);
+
+        sb.append(proto);
+        sb.append("://");
+        sb.append(domain);
+
+        if (port != null) {
+            sb.append(':');
+            sb.append(port);
+        }
+
+        sb.append(path);
+
+        if (param != null) {
+            sb.append('?').append(param);
+        }
+
+        return sb.toString();
+    }
+
     public String dir() {
         return path.replaceAll("/[^/]+$", "/");
     }
@@ -303,7 +325,14 @@ class EdgeUriFactory {
             }
 
             if (shouldUrlEncode) {
-                sb.append(URLEncoder.encode(queryPart, StandardCharsets.UTF_8));
+                int idx = queryPart.indexOf('=');
+                if (idx < 0) {
+                    sb.append(URLEncoder.encode(queryPart, StandardCharsets.UTF_8));
+                } else {
+                    sb.append(URLEncoder.encode(queryPart.substring(0, idx), StandardCharsets.UTF_8));
+                    sb.append('=');
+                    sb.append(URLEncoder.encode(queryPart.substring(idx + 1), StandardCharsets.UTF_8));
+                }
             } else {
                 sb.append(queryPart);
             }

code/common/model/test/nu/marginalia/model/EdgeUrlTest.java

@@ -58,16 +58,32 @@ class EdgeUrlTest {
         Assertions.assertEquals("/wiki/Sámi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").getPath());
         Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").toString());
         Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", new EdgeUrl("https://en.wikipedia.org/wiki/Sámi").toString());
+
+        Assertions.assertEquals("https://www.prijatelji-zivotinja.hr/index.en.php?id=2301k", new EdgeUrl("https://www.prijatelji-zivotinja.hr/index.en.php?id=2301k").toString());
     }
 
     @Test
     void testParms() throws URISyntaxException {
         Assertions.assertEquals("id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").toString());
+
         Assertions.assertEquals("t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").toString());
+
         Assertions.assertEquals("v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").toString());
+
         Assertions.assertEquals("id=1", new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/showthread.php?id=1",
+                new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").toString());
+
+
         Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/sh%C3%B6wthr%C3%ABad.php?id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").toString());
+
         Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/sh%C3%B6wthr%C3%ABad.php?id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").toString());
+
         Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?m=123").param);
         Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?follow=123").param);
     }

code/common/service/java/nu/marginalia/service/server/JoobyService.java

@@ -122,6 +122,11 @@ public class JoobyService {
         // single digit percentage difference since HTML already compresses very well with level = 1.
         options.setCompressionLevel(1);
 
+        // Set a cap on the number of worker threads, as Jooby's default value does not seem to consider
+        // multi-tenant servers with high thread counts, and spins up an exorbitant number of threads in that
+        // scenario
+        options.setWorkerThreads(Math.min(128, options.getWorkerThreads()));
+
 
         jooby.setServerOptions(options);
 

code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java

@@ -1,6 +1,7 @@
 package nu.marginalia.extractor;
 
 import com.google.inject.Inject;
+import nu.marginalia.process.control.ProcessHeartbeat;
 import nu.marginalia.process.log.WorkLog;
 import nu.marginalia.process.log.WorkLogEntry;
 import nu.marginalia.slop.SlopCrawlDataRecord;
@@ -23,14 +24,18 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.NoSuchElementException;
 
 public class SampleDataExporter {
     private final FileStorageService storageService;
+    private final ProcessHeartbeat processHeartbeat;
 
     @Inject
-    public SampleDataExporter(FileStorageService storageService) {
+    public SampleDataExporter(FileStorageService storageService, ProcessHeartbeat processHeartbeat) {
         this.storageService = storageService;
+        this.processHeartbeat = processHeartbeat;
     }
+
     public void export(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name) throws SQLException, IOException {
         FileStorage destStorage = storageService.getStorage(destId);
         Path inputDir = storageService.getStorage(crawlId).asPath();
@@ -59,12 +64,6 @@ public class SampleDataExporter {
         Path newCrawlerLogFile = Files.createTempFile(destStorage.asPath(), "crawler", ".log",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
 
-        try (var bw = Files.newBufferedWriter(newCrawlerLogFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
-            for (var item : entriesAll) {
-                bw.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
-            }
-        }
-
         Path newManifestJsonFile = Files.createTempFile(destStorage.asPath(), "manifest", ".json",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
         Files.writeString(newManifestJsonFile, " { \"description\": \"" + name.replace("[\"\\]", "_") + "\",\n      \"type\": \"CRAWL_DATA\" }\n");
@@ -72,24 +71,31 @@ public class SampleDataExporter {
         var tmpTarFile = Files.createTempFile(destStorage.asPath(), "data", ".tar",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
 
-        try (var stream = new TarArchiveOutputStream(Files.newOutputStream(tmpTarFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) {
+        try (var stream = new TarArchiveOutputStream(Files.newOutputStream(tmpTarFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING));
-            for (var item : entriesAll) {
+             var logWriter = Files.newBufferedWriter(newCrawlerLogFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+             var hb = processHeartbeat.createAdHocTaskHeartbeat("Generating Sample")
+        ) {
+            for (var item : hb.wrap("Scanning", entriesAll)) {
                 Path crawlDataPath = inputDir.resolve(item.relPath());
                 if (!Files.exists(crawlDataPath)) continue;
 
                 if (StringUtils.isBlank(ctFilter)) {
                     addFileToTar(stream, crawlDataPath, item.relPath());
+                    logWriter.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
                 }
                 else /* filter != null */ {
-                    boolean didFilterData = false;
+                    Path filteredData = null;
                     try {
-                        crawlDataPath = filterEntries(crawlDataPath, ctFilter);
+                        filteredData = filterEntries(crawlDataPath, ctFilter);
-                        didFilterData = true;
+                        addFileToTar(stream, filteredData, item.relPath());
-                        addFileToTar(stream, crawlDataPath, item.relPath());
+                        logWriter.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
+                    }
+                    catch (NoSuchElementException ex) {
+                        // Ignore
                     }
                     finally {
-                        if (didFilterData) {
+                        if (filteredData != null) {
-                            Files.deleteIfExists(crawlDataPath);
+                            Files.deleteIfExists(filteredData);
                         }
                     }
                 }
@@ -106,34 +112,36 @@ public class SampleDataExporter {
         Files.move(tmpTarFile, destStorage.asPath().resolve("crawl-data.tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
     }
 
-    /** Filters the entries in the crawl data file based on the content type.
+    /** Filters the entries in the crawl data file based on the content type. */
-     * @param crawlDataPath The path to the crawl data file.
+    private Path filterEntries(Path crawlDataPath, String contentTypeFilter) throws IOException, NoSuchElementException {
-     * @param contentTypeFilter The content type to filter by.
-     * @return The path to the filtered crawl data file, or null if an error occurred.
-     */
-    private Path filterEntries(Path crawlDataPath, String contentTypeFilter) throws IOException {
         Path tempDir = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered");
         Path tempFile = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered.slop.zip");
 
+        // We may have debris from a previous run, so let's clean it up
+        if (Files.isDirectory(tempDir)) {
+            FileUtils.deleteDirectory(tempDir.toFile());
+        }
         Files.createDirectory(tempDir);
 
         try (var writer = new SlopCrawlDataRecord.Writer(tempDir);
              var reader = new SlopCrawlDataRecord.FilteringReader(crawlDataPath) {
                  @Override
                  public boolean filter(String url, int status, String contentType) {
-                     if (contentTypeFilter.equals(contentType))
+                     return matchContentTypeHeaderWithMime(contentType, contentTypeFilter)
-                         return true;
+                                || contentType.startsWith("x-marginalia/"); // metadata records
-                     else if (contentType.startsWith("x-marginalia/"))
-                         // This is a metadata entry, typically domain or redirect information
-                         // let's keep those to not confuse the consumer of the data, which might
-                         // expect at least the domain summary
-                         return true;
-                     return false;
                  }
              }
         ) {
+            boolean wroteEntry = false;
             while (reader.hasRemaining()) {
-                writer.write(reader.get());
+                var entry = reader.get();
+                writer.write(entry);
+
+                wroteEntry = wroteEntry || contentTypeFilter.equals(entry.contentType());
+            }
+
+            if (!wroteEntry) {
+                throw new NoSuchElementException("No relevant entries");
             }
 
             SlopTablePacker.packToSlopZip(tempDir, tempFile);
@@ -146,6 +154,21 @@ public class SampleDataExporter {
         return tempFile;
     }
 
+    private boolean matchContentTypeHeaderWithMime(String contentType, String mime) {
+        if (null == contentType) {
+            return false;
+        }
+
+        /* The content type header may have a charset or other parameters, so we need to
+         * check if the mime type is a prefix of the content type. */
+
+        int semicolonIndex = contentType.indexOf(';');
+        if (semicolonIndex >= 0) {
+            return contentType.substring(0, semicolonIndex).equals(mime);
+        }
+        return contentType.equals(mime);
+    }
+
     private void addFileToTar(TarArchiveOutputStream outputStream, Path file, String fileName) throws IOException {
         var entry = outputStream.createArchiveEntry(file.toFile(), fileName);
         entry.setSize(Files.size(file));

code/services-application/search-service/java/nu/marginalia/search/model/UrlDetails.java

@@ -180,7 +180,7 @@ public class UrlDetails implements Comparable<UrlDetails> {
      * semantically meaningful codepoints into entity codes */
     public String displayUrl() {
         StringBuilder sb = new StringBuilder();
-        String urlStr = url.toString();
+        String urlStr = url.toDisplayString();
         for (int i = 0; i < urlStr.length(); i++) {
             char c = urlStr.charAt(i);
 

code/services-application/status-service/java/nu/marginalia/status/StatusModule.java

@@ -20,6 +20,6 @@ public class StatusModule extends AbstractModule {
         bind(String.class)
                 .annotatedWith(Names.named("searchEngineTestQuery"))
                 .toInstance(System.getProperty("status-service.public-query",
-                        "https://search.marginalia.nu/search?query=plato&ref=marginalia-automatic-metrics"));
+                        "https://marginalia-search.com/search?query=plato&ref=marginalia-automatic-metrics"));
     }
 }

deploy.txt

@@ -4,4 +4,6 @@
 2025-01-07:  Deploy executor.
 2025-04-24:  Deploy executor.
 2025-04-24:  Deploy assistant.
-2025-04-24:  Deploy qs, search and api-services.
+2025-05-04:  Deploy qs, search and api-services.
+2025-05-05:  Deploy executor partition 4.
+2025-05-05:  Deploy control.

tools/deployment/deployment.py

@@ -314,6 +314,13 @@ if __name__ == '__main__':
             deploy_tier=0,
             groups={"all", "core"}
         ),
+        'status': ServiceConfig(
+            gradle_target=':code:services-application:status-service:docker',
+            docker_name='status-service',
+            instances=None,
+            deploy_tier=4,
+            groups={"all"}
+        ),
         'query': ServiceConfig(
             gradle_target=':code:services-core:query-service:docker',
             docker_name='query-service',