(deploy) Executor partition 4.

Optimize the code by adding a simple heuristic for guessing whether we need to repair the URI before we pass it to Java's parser.

build.gradle

@@ -5,7 +5,7 @@ plugins {
 
     // This is a workaround for a bug in the Jib plugin that causes it to stall randomly
     // https://github.com/GoogleContainerTools/jib/issues/3347
-    id 'com.google.cloud.tools.jib' version '3.4.3' apply(false)
+    id 'com.google.cloud.tools.jib' version '3.4.5' apply(false)
 }
 
 group 'marginalia'
@@ -43,12 +43,11 @@ subprojects.forEach {it ->
 }
 
 ext {
-    jvmVersion=23
-    dockerImageBase='container-registry.oracle.com/graalvm/jdk:23'
+    jvmVersion = 24
+    dockerImageBase='container-registry.oracle.com/graalvm/jdk:24'
     dockerImageTag='latest'
     dockerImageRegistry='marginalia'
-    jibVersion = '3.4.4'
-
+    jibVersion = '3.4.5'
 }
 
 idea {

code/common/config/java/nu/marginalia/LanguageModels.java

@@ -24,58 +24,4 @@ public class LanguageModels {
         this.fasttextLanguageModel = fasttextLanguageModel;
         this.segments = segments;
     }
-
-    public static LanguageModelsBuilder builder() {
-        return new LanguageModelsBuilder();
-    }
-
-    public static class LanguageModelsBuilder {
-        private Path termFrequencies;
-        private Path openNLPSentenceDetectionData;
-        private Path posRules;
-        private Path posDict;
-        private Path fasttextLanguageModel;
-        private Path segments;
-
-        LanguageModelsBuilder() {
-        }
-
-        public LanguageModelsBuilder termFrequencies(Path termFrequencies) {
-            this.termFrequencies = termFrequencies;
-            return this;
-        }
-
-        public LanguageModelsBuilder openNLPSentenceDetectionData(Path openNLPSentenceDetectionData) {
-            this.openNLPSentenceDetectionData = openNLPSentenceDetectionData;
-            return this;
-        }
-
-        public LanguageModelsBuilder posRules(Path posRules) {
-            this.posRules = posRules;
-            return this;
-        }
-
-        public LanguageModelsBuilder posDict(Path posDict) {
-            this.posDict = posDict;
-            return this;
-        }
-
-        public LanguageModelsBuilder fasttextLanguageModel(Path fasttextLanguageModel) {
-            this.fasttextLanguageModel = fasttextLanguageModel;
-            return this;
-        }
-
-        public LanguageModelsBuilder segments(Path segments) {
-            this.segments = segments;
-            return this;
-        }
-
-        public LanguageModels build() {
-            return new LanguageModels(this.termFrequencies, this.openNLPSentenceDetectionData, this.posRules, this.posDict, this.fasttextLanguageModel, this.segments);
-        }
-
-        public String toString() {
-            return "LanguageModels.LanguageModelsBuilder(termFrequencies=" + this.termFrequencies + ", openNLPSentenceDetectionData=" + this.openNLPSentenceDetectionData + ", posRules=" + this.posRules + ", posDict=" + this.posDict + ", fasttextLanguageModel=" + this.fasttextLanguageModel + ", segments=" + this.segments + ")";
-        }
-    }
 }

code/common/db/java/nu/marginalia/db/DbDomainQueries.java

@@ -22,6 +22,7 @@ public class DbDomainQueries {
     private static final Logger logger = LoggerFactory.getLogger(DbDomainQueries.class);
 
     private final Cache<EdgeDomain, Integer> domainIdCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
+    private final Cache<EdgeDomain, DomainIdWithNode> domainWithNodeCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
     private final Cache<Integer, EdgeDomain> domainNameCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
     private final Cache<String, List<DomainWithNode>> siblingsCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
 
@@ -59,6 +60,34 @@ public class DbDomainQueries {
         }
     }
 
+
+    public DomainIdWithNode getDomainIdWithNode(EdgeDomain domain) throws NoSuchElementException {
+        try {
+            return domainWithNodeCache.get(domain, () -> {
+                try (var connection = dataSource.getConnection();
+                     var stmt = connection.prepareStatement("SELECT ID, NODE_AFFINITY FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) {
+
+                    stmt.setString(1, domain.toString());
+                    var rsp = stmt.executeQuery();
+                    if (rsp.next()) {
+                        return new DomainIdWithNode(rsp.getInt(1), rsp.getInt(2));
+                    }
+                }
+                catch (SQLException ex) {
+                    throw new RuntimeException(ex);
+                }
+
+                throw new NoSuchElementException();
+            });
+        }
+        catch (UncheckedExecutionException ex) {
+            throw new NoSuchElementException();
+        }
+        catch (ExecutionException ex) {
+            throw new RuntimeException(ex.getCause());
+        }
+    }
+
     public OptionalInt tryGetDomainId(EdgeDomain domain) {
 
         Integer maybeId = domainIdCache.getIfPresent(domain);
@@ -145,4 +174,6 @@ public class DbDomainQueries {
             return nodeAffinity > 0;
         }
     }
+
+    public record DomainIdWithNode (int domainId, int nodeAffinity) { }
 }

code/common/model/java/nu/marginalia/model/EdgeDomain.java

@@ -14,7 +14,7 @@ public class EdgeDomain implements Serializable {
     @Nonnull
     public final String topDomain;
 
-    public EdgeDomain(String host) {
+    public EdgeDomain(@Nonnull String host) {
         Objects.requireNonNull(host, "domain name must not be null");
 
         host = host.toLowerCase();
@@ -61,6 +61,10 @@ public class EdgeDomain implements Serializable {
         this.topDomain = topDomain;
     }
 
+    public static String getTopDomain(String host) {
+        return new EdgeDomain(host).topDomain;
+    }
+
     private boolean looksLikeGovTld(String host) {
         if (host.length() < 8)
             return false;
@@ -116,24 +120,6 @@ public class EdgeDomain implements Serializable {
         return topDomain.substring(0, cutPoint).toLowerCase();
     }
 
-    public String getLongDomainKey() {
-        StringBuilder ret = new StringBuilder();
-
-        int cutPoint = topDomain.indexOf('.');
-        if (cutPoint < 0) {
-            ret.append(topDomain);
-        } else {
-            ret.append(topDomain, 0, cutPoint);
-        }
-
-        if (!subDomain.isEmpty() && !"www".equals(subDomain)) {
-            ret.append(":");
-            ret.append(subDomain);
-        }
-
-        return ret.toString().toLowerCase();
-    }
-
     /** If possible, try to provide an alias domain,
      * i.e. a domain name that is very likely to link to this one
      * */

code/common/model/java/nu/marginalia/model/EdgeUrl.java

@@ -1,16 +1,14 @@
 package nu.marginalia.model;
 
 import nu.marginalia.util.QueryParams;
+import org.apache.commons.lang3.StringUtils;
 
 import javax.annotation.Nullable;
 import java.io.Serializable;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
+import java.net.*;
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.regex.Pattern;
 
 public class EdgeUrl implements Serializable {
     public final String proto;
@@ -33,7 +31,7 @@ public class EdgeUrl implements Serializable {
 
     private static URI parseURI(String url) throws URISyntaxException {
         try {
-            return new URI(urlencodeFixer(url));
+            return EdgeUriFactory.parseURILenient(url);
         } catch (URISyntaxException ex) {
             throw new URISyntaxException("Failed to parse URI '" + url + "'", ex.getMessage());
         }
@@ -51,58 +49,6 @@ public class EdgeUrl implements Serializable {
         }
     }
 
-    private static Pattern badCharPattern = Pattern.compile("[ \t\n\"<>\\[\\]()',|]");
-
-    /* Java's URI parser is a bit too strict in throwing exceptions when there's an error.
-
-       Here on the Internet, standards are like the picture on the box of the frozen pizza,
-       and what you get is more like what's on the inside, we try to patch things instead,
-       just give it a best-effort attempt att cleaning out broken or unnecessary constructions
-       like bad or missing URLEncoding
-     */
-    public static String urlencodeFixer(String url) throws URISyntaxException {
-        var s = new StringBuilder();
-        String goodChars = "&.?:/-;+$#";
-        String hexChars = "0123456789abcdefABCDEF";
-
-        int pathIdx = findPathIdx(url);
-        if (pathIdx < 0) { // url looks like http://marginalia.nu
-            return url + "/";
-        }
-        s.append(url, 0, pathIdx);
-
-        // We don't want the fragment, and multiple fragments breaks the Java URIParser for some reason
-        int end = url.indexOf("#");
-        if (end < 0) end = url.length();
-
-        for (int i = pathIdx; i < end; i++) {
-            int c = url.charAt(i);
-
-            if (goodChars.indexOf(c) >= 0 || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
-                s.appendCodePoint(c);
-            } else if (c == '%' && i + 2 < end) {
-                int cn = url.charAt(i + 1);
-                int cnn = url.charAt(i + 2);
-                if (hexChars.indexOf(cn) >= 0 && hexChars.indexOf(cnn) >= 0) {
-                    s.appendCodePoint(c);
-                } else {
-                    s.append("%25");
-                }
-            } else {
-                s.append(String.format("%%%02X", c));
-            }
-        }
-
-        return s.toString();
-    }
-
-    private static int findPathIdx(String url) throws URISyntaxException {
-        int colonIdx = url.indexOf(':');
-        if (colonIdx < 0 || colonIdx + 2 >= url.length()) {
-            throw new URISyntaxException(url, "Lacking protocol");
-        }
-        return url.indexOf('/', colonIdx + 2);
-    }
 
     public EdgeUrl(URI URI) {
         try {
@@ -166,11 +112,32 @@ public class EdgeUrl implements Serializable {
             sb.append(port);
         }
 
+        EdgeUriFactory.urlencodePath(sb, path);
+
+        if (param != null) {
+            EdgeUriFactory.urlencodeQuery(sb, param);
+        }
+
+        return sb.toString();
+    }
+
+
+    public String toDisplayString() {
+        StringBuilder sb = new StringBuilder(256);
+
+        sb.append(proto);
+        sb.append("://");
+        sb.append(domain);
+
+        if (port != null) {
+            sb.append(':');
+            sb.append(port);
+        }
+
         sb.append(path);
 
         if (param != null) {
-            sb.append('?');
-            sb.append(param);
+            sb.append('?').append(param);
         }
 
         return sb.toString();
@@ -247,3 +214,244 @@ public class EdgeUrl implements Serializable {
     }
 
 }
+
+class EdgeUriFactory {
+    public static URI parseURILenient(String url) throws URISyntaxException {
+
+        if (shouldOmitUrlencodeRepair(url)) {
+            try {
+                return new URI(url);
+            }
+            catch (URISyntaxException ex) {
+                // ignore and run the lenient parser
+            }
+        }
+
+        var s = new StringBuilder(url.length()+8);
+
+        int pathIdx = findPathIdx(url);
+        if (pathIdx < 0) { // url looks like http://marginalia.nu
+            return new URI(url + "/");
+        }
+        s.append(url, 0, pathIdx);
+
+        // We don't want the fragment, and multiple fragments breaks the Java URIParser for some reason
+        int end = url.indexOf("#");
+        if (end < 0) end = url.length();
+
+        int queryIdx = url.indexOf('?');
+        if (queryIdx < 0) queryIdx = end;
+
+        urlencodePath(s, url.substring(pathIdx, queryIdx));
+        if (queryIdx < end) {
+            urlencodeQuery(s, url.substring(queryIdx + 1, end));
+        }
+        return new URI(s.toString());
+    }
+
+    /** Break apart the path element of an URI into its components, and then
+     * urlencode any component that needs it, and recombine it into a single
+     * path element again.
+     */
+    public static void urlencodePath(StringBuilder sb, String path) {
+        if (path == null || path.isEmpty()) {
+            return;
+        }
+
+        String[] pathParts = StringUtils.split(path, '/');
+        if (pathParts.length == 0) {
+            sb.append('/');
+            return;
+        }
+
+        boolean shouldUrlEncode = false;
+        for (String pathPart : pathParts) {
+            if (pathPart.isEmpty()) continue;
+
+            if (needsUrlEncode(pathPart)) {
+                shouldUrlEncode = true;
+                break;
+            }
+        }
+
+        for (String pathPart : pathParts) {
+            if (pathPart.isEmpty()) continue;
+
+            if (shouldUrlEncode) {
+                sb.append('/');
+                sb.append(URLEncoder.encode(pathPart, StandardCharsets.UTF_8).replace("+", "%20"));
+            } else {
+                sb.append('/');
+                sb.append(pathPart);
+            }
+        }
+
+        if (path.endsWith("/")) {
+            sb.append('/');
+        }
+
+    }
+
+    /** Break apart the query element of a URI into its components, and then
+     * urlencode any component that needs it, and recombine it into a single
+     * query element again.
+     */
+    public static void urlencodeQuery(StringBuilder sb, String param) {
+        if (param == null || param.isEmpty()) {
+            return;
+        }
+
+        String[] queryParts = StringUtils.split(param, '&');
+
+        boolean shouldUrlEncode = false;
+        for (String queryPart : queryParts) {
+            if (queryPart.isEmpty()) continue;
+
+            if (needsUrlEncode(queryPart)) {
+                shouldUrlEncode = true;
+                break;
+            }
+        }
+
+        boolean first = true;
+        for (String queryPart : queryParts) {
+            if (queryPart.isEmpty()) continue;
+
+            if (first) {
+                sb.append('?');
+                first = false;
+            } else {
+                sb.append('&');
+            }
+
+            if (shouldUrlEncode) {
+                int idx = queryPart.indexOf('=');
+                if (idx < 0) {
+                    sb.append(URLEncoder.encode(queryPart, StandardCharsets.UTF_8));
+                } else {
+                    sb.append(URLEncoder.encode(queryPart.substring(0, idx), StandardCharsets.UTF_8));
+                    sb.append('=');
+                    sb.append(URLEncoder.encode(queryPart.substring(idx + 1), StandardCharsets.UTF_8));
+                }
+            } else {
+                sb.append(queryPart);
+            }
+        }
+    }
+
+    /** Test if the url element needs URL encoding.
+     * <p></p>
+     * Note we may have been given an already encoded path element,
+     * so we include % and + in the list of good characters
+     */
+    static boolean needsUrlEncode(String urlElement) {
+        for (int i = 0; i < urlElement.length(); i++) {
+            char c = urlElement.charAt(i);
+
+            if (isUrlSafe(c)) continue;
+            if ("+".indexOf(c) >= 0) continue;
+            if (c == '%' && i + 2 < urlElement.length()) {
+                char c1 = urlElement.charAt(i + 1);
+                char c2 = urlElement.charAt(i + 2);
+                if (isHexDigit(c1) && isHexDigit(c2)) {
+                    i += 2;
+                    continue;
+                }
+            }
+
+            return true;
+        }
+
+        return false;
+    }
+
+
+    static boolean isUrlSafe(int c) {
+        if (c >= 'a' && c <= 'z') return true;
+        if (c >= 'A' && c <= 'Z') return true;
+        if (c >= '0' && c <= '9') return true;
+        if (c == '-' || c == '_' || c == '.' || c == '~') return true;
+
+        return false;
+    }
+
+    /** Test if the URL is a valid URL that does not need to be
+     * urlencoded.
+     * <p></p>
+     * This is a very simple heuristic test that does not guarantee
+     * that the URL is valid, but it will identify cases where we
+     * are fairly certain that the URL does not need encoding,
+     * so we can skip a bunch of allocations and string operations
+     * that would otherwise be needed to fix the URL.
+     */
+    static boolean shouldOmitUrlencodeRepair(String url) {
+        int idx = 0;
+        final int len = url.length();
+
+        // Validate the scheme
+        while (idx < len - 2) {
+            char c = url.charAt(idx++);
+            if (c == ':') break;
+            if (!isAsciiAlphabetic(c)) return false;
+        }
+        if (url.charAt(idx++) != '/') return false;
+        if (url.charAt(idx++) != '/') return false;
+
+        // Validate the authority
+        while (idx < len) {
+            char c = url.charAt(idx++);
+            if (c == '/') break;
+            if (c == ':') continue;
+            if (c == '@') continue;
+            if (!isUrlSafe(c)) return false;
+        }
+
+        // Validate the path
+        if (idx >= len) return true;
+
+        while (idx < len) {
+            char c = url.charAt(idx++);
+            if (c == '?') break;
+            if (c == '/') continue;
+            if (c == '#') return true;
+            if (!isUrlSafe(c)) return false;
+        }
+
+        if (idx >= len) return true;
+
+        // Validate the query
+        while (idx < len) {
+            char c = url.charAt(idx++);
+            if (c == '&') continue;
+            if (c == '=') continue;
+            if (c == '#') return true;
+            if (!isUrlSafe(c)) return false;
+        }
+
+        return true;
+    }
+
+
+    private static boolean isAsciiAlphabetic(int c) {
+        return (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    private static boolean isHexDigit(int c) {
+        return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    /** Find the index of the path element in a URL.
+     * <p></p>
+     * The path element starts after the scheme and authority part of the URL,
+     * which is everything up to and including the first slash after the colon.
+     */
+    private static int findPathIdx(String url) throws URISyntaxException {
+        int colonIdx = url.indexOf(':');
+        if (colonIdx < 0 || colonIdx + 3 >= url.length()) {
+            throw new URISyntaxException(url, "Lacking scheme");
+        }
+        return url.indexOf('/', colonIdx + 3);
+    }
+
+
+}

code/common/model/test/nu/marginalia/model/EdgeUrlTest.java

@@ -1,6 +1,6 @@
 package nu.marginalia.model;
 
-import nu.marginalia.model.EdgeUrl;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.net.URISyntaxException;
@@ -21,25 +21,70 @@ class EdgeUrlTest {
                 new EdgeUrl("https://memex.marginalia.nu/#here")
         );
     }
+
     @Test
-    public void testParam() throws URISyntaxException {
-        System.out.println(new EdgeUrl("https://memex.marginalia.nu/index.php?id=1").toString());
-        System.out.println(new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").toString());
-    }
-    @Test
-    void urlencodeFixer() throws URISyntaxException {
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/#heredoc"));
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/%-sign"));
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/%22-sign"));
-        System.out.println(EdgeUrl.urlencodeFixer("https://www.example.com/\n \"huh\""));
+    void testUriFromString() throws URISyntaxException {
+        // We test these URLs several times as we perform URLEncode-fixing both when parsing the URL and when
+        // converting it back to a string, we want to ensure there is no changes along the way.
+
+        Assertions.assertEquals("/", EdgeUriFactory.parseURILenient("https://www.example.com/").getPath());
+        Assertions.assertEquals("https://www.example.com/", EdgeUriFactory.parseURILenient("https://www.example.com/").toString());
+        Assertions.assertEquals("https://www.example.com/", new EdgeUrl("https://www.example.com/").toString());
+
+        Assertions.assertEquals("/", EdgeUriFactory.parseURILenient("https://www.example.com/#heredoc").getPath());
+        Assertions.assertEquals("https://www.example.com/", EdgeUriFactory.parseURILenient("https://www.example.com/#heredoc").toString());
+        Assertions.assertEquals("https://www.example.com/", new EdgeUrl("https://www.example.com/#heredoc").toString());
+
+        Assertions.assertEquals("/trailingslash/", EdgeUriFactory.parseURILenient("https://www.example.com/trailingslash/").getPath());
+        Assertions.assertEquals("https://www.example.com/trailingslash/", EdgeUriFactory.parseURILenient("https://www.example.com/trailingslash/").toString());
+        Assertions.assertEquals("https://www.example.com/trailingslash/", new EdgeUrl("https://www.example.com/trailingslash/").toString());
+
+        Assertions.assertEquals("/%-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%25-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%25-sign", new EdgeUrl("https://www.example.com/%-sign").toString());
+
+        Assertions.assertEquals("/%-sign/\"-sign", EdgeUriFactory.parseURILenient("https://www.example.com//%-sign/\"-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%25-sign/%22-sign", EdgeUriFactory.parseURILenient("https://www.example.com//%-sign/\"-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%25-sign/%22-sign", new EdgeUrl("https://www.example.com//%-sign/\"-sign").toString());
+
+        Assertions.assertEquals("/\"-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%22-sign").getPath());
+        Assertions.assertEquals("https://www.example.com/%22-sign", EdgeUriFactory.parseURILenient("https://www.example.com/%22-sign").toString());
+        Assertions.assertEquals("https://www.example.com/%22-sign", new EdgeUrl("https://www.example.com/%22-sign").toString());
+
+        Assertions.assertEquals("/\n \"huh\"", EdgeUriFactory.parseURILenient("https://www.example.com/\n \"huh\"").getPath());
+        Assertions.assertEquals("https://www.example.com/%0A%20%22huh%22", EdgeUriFactory.parseURILenient("https://www.example.com/\n \"huh\"").toString());
+        Assertions.assertEquals("https://www.example.com/%0A%20%22huh%22", new EdgeUrl("https://www.example.com/\n \"huh\"").toString());
+
+        Assertions.assertEquals("/wiki/Sámi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").getPath());
+        Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", EdgeUriFactory.parseURILenient("https://en.wikipedia.org/wiki/Sámi").toString());
+        Assertions.assertEquals("https://en.wikipedia.org/wiki/S%C3%A1mi", new EdgeUrl("https://en.wikipedia.org/wiki/Sámi").toString());
+
+        Assertions.assertEquals("https://www.prijatelji-zivotinja.hr/index.en.php?id=2301k", new EdgeUrl("https://www.prijatelji-zivotinja.hr/index.en.php?id=2301k").toString());
     }
 
     @Test
     void testParms() throws URISyntaxException {
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?id=123"));
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?t=123"));
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?v=123"));
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?m=123"));
-        System.out.println(new EdgeUrl("https://search.marginalia.nu/?follow=123"));
+        Assertions.assertEquals("id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?id=123", new EdgeUrl("https://search.marginalia.nu/?id=123").toString());
+
+        Assertions.assertEquals("t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?t=123", new EdgeUrl("https://search.marginalia.nu/?t=123").toString());
+
+        Assertions.assertEquals("v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").param);
+        Assertions.assertEquals("https://search.marginalia.nu/?v=123", new EdgeUrl("https://search.marginalia.nu/?v=123").toString());
+
+        Assertions.assertEquals("id=1", new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/showthread.php?id=1",
+                new EdgeUrl("https://memex.marginalia.nu/showthread.php?id=1&count=5&tracking=123").toString());
+
+
+        Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/sh%C3%B6wthr%C3%ABad.php?id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?id=1&t=5&tracking=123").toString());
+
+        Assertions.assertEquals("id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").param);
+        Assertions.assertEquals("https://memex.marginalia.nu/sh%C3%B6wthr%C3%ABad.php?id=1&t=5", new EdgeUrl("https://memex.marginalia.nu/shöwthrëad.php?trëaking=123&id=1&t=5&").toString());
+
+        Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?m=123").param);
+        Assertions.assertNull(new EdgeUrl("https://search.marginalia.nu/?follow=123").param);
     }
 }

code/common/service/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java

@@ -59,16 +59,13 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo
      */
     @Override
     public void progress(String step, int stepProgress, int stepCount) {
+        int lastProgress = this.progress;
         this.step = step;
-
-
-        // off by one since we calculate the progress based on the number of steps,
-        // and Enum.ordinal() is zero-based (so the 5th step in a 5 step task is 4, not 5; resulting in the
-        // final progress being 80% and not 100%)
-
         this.progress = (int) Math.round(100. * stepProgress / (double) stepCount);
 
-        logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        if (this.progress / 10 != lastProgress / 10) {
+            logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        }
     }
 
     /** Wrap a collection to provide heartbeat progress updates as it's iterated through */

code/common/service/java/nu/marginalia/process/log/WorkLog.java

@@ -10,7 +10,9 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.LocalDateTime;
-import java.util.*;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 
 /** WorkLog is a journal of work done by a process,
@@ -61,6 +63,12 @@ public class WorkLog implements AutoCloseable, Closeable {
         return new WorkLoadIterable<>(logFile, mapper);
     }
 
+    public static int countEntries(Path crawlerLog) throws IOException{
+        try (var linesStream = Files.lines(crawlerLog)) {
+            return (int) linesStream.filter(WorkLogEntry::isJobId).count();
+        }
+    }
+
     // Use synchro over concurrent set to avoid competing writes
     // - correct is better than fast here, it's sketchy enough to use
     // a PrintWriter

code/common/service/java/nu/marginalia/service/control/ServiceAdHocTaskHeartbeatImpl.java

@@ -57,16 +57,13 @@ public class ServiceAdHocTaskHeartbeatImpl implements AutoCloseable, ServiceAdHo
      */
     @Override
     public void progress(String step, int stepProgress, int stepCount) {
+        int lastProgress = this.progress;
         this.step = step;
-
-
-        // off by one since we calculate the progress based on the number of steps,
-        // and Enum.ordinal() is zero-based (so the 5th step in a 5 step task is 4, not 5; resulting in the
-        // final progress being 80% and not 100%)
-
         this.progress = (int) Math.round(100. * stepProgress / (double) stepCount);
 
-        logger.info("ServiceTask {} progress: {}%", taskBase, progress);
+        if (this.progress / 10 != lastProgress / 10) {
+            logger.info("ProcessTask {} progress: {}%", taskBase, progress);
+        }
     }
 
     public void shutDown() {

code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java

@@ -6,6 +6,7 @@ import nu.marginalia.service.ServiceId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.util.Enumeration;
@@ -115,11 +116,12 @@ public class ServiceConfigurationModule extends AbstractModule {
         }
     }
 
-    public static String getLocalNetworkIP() throws Exception {
+    public static String getLocalNetworkIP() throws IOException {
         Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
 
         while (nets.hasMoreElements()) {
             NetworkInterface netif = nets.nextElement();
+            logger.info("Considering network interface {}:  Up? {},  Loopback? {}", netif.getDisplayName(), netif.isUp(), netif.isLoopback());
             if (!netif.isUp() || netif.isLoopback()) {
                 continue;
             }
@@ -127,6 +129,7 @@ public class ServiceConfigurationModule extends AbstractModule {
             Enumeration<InetAddress> inetAddresses = netif.getInetAddresses();
             while (inetAddresses.hasMoreElements()) {
                 InetAddress addr = inetAddresses.nextElement();
+                logger.info("Considering address {}: SiteLocal? {}, Loopback? {}", addr.getHostAddress(), addr.isSiteLocalAddress(), addr.isLoopbackAddress());
                 if (addr.isSiteLocalAddress() && !addr.isLoopbackAddress()) {
                     return addr.getHostAddress();
                 }

code/common/service/java/nu/marginalia/service/server/JoobyService.java

@@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
 import org.slf4j.MarkerFactory;
 
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
@@ -106,9 +107,12 @@ public class JoobyService {
                 config.externalAddress());
 
         // FIXME:  This won't work outside of docker, may need to submit a PR to jooby to allow classpaths here
-        jooby.install(new JteModule(Path.of("/app/resources/jte"), Path.of("/app/classes/jte-precompiled")));
-        jooby.assets("/*", Paths.get("/app/resources/static"));
-
+        if (Files.exists(Path.of("/app/resources/jte")) || Files.exists(Path.of("/app/classes/jte-precompiled"))) {
+            jooby.install(new JteModule(Path.of("/app/resources/jte"), Path.of("/app/classes/jte-precompiled")));
+        }
+        if (Files.exists(Path.of("/app/resources/static"))) {
+            jooby.assets("/*", Paths.get("/app/resources/static"));
+        }
         var options = new ServerOptions();
         options.setHost(config.bindAddress());
         options.setPort(restEndpoint.port());

code/common/service/java/nu/marginalia/service/server/MetricsServer.java

@@ -6,25 +6,36 @@ import nu.marginalia.service.module.ServiceConfiguration;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.net.InetSocketAddress;
 
 public class MetricsServer {
 
+    private static final Logger logger = LoggerFactory.getLogger(MetricsServer.class);
+
     @Inject
-    public MetricsServer(ServiceConfiguration configuration) throws Exception {
+    public MetricsServer(ServiceConfiguration configuration) {
         // If less than zero, we forego setting up a metrics server
         if (configuration.metricsPort() < 0)
             return;
 
-        Server server = new Server(new InetSocketAddress(configuration.bindAddress(), configuration.metricsPort()));
+        try {
+            Server server = new Server(new InetSocketAddress(configuration.bindAddress(), configuration.metricsPort()));
 
-        ServletContextHandler context = new ServletContextHandler();
-        context.setContextPath("/");
-        server.setHandler(context);
+            ServletContextHandler context = new ServletContextHandler();
+            context.setContextPath("/");
+            server.setHandler(context);
 
-        context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
+            context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
 
-        server.start();
+            logger.info("MetricsServer listening on {}:{}", configuration.bindAddress(), configuration.metricsPort());
+
+            server.start();
+        }
+        catch (Exception|NoSuchMethodError ex) {
+            logger.error("Failed to set up metrics server", ex);
+        }
     }
 }

code/common/service/java/nu/marginalia/service/server/RateLimiter.java

@@ -35,21 +35,8 @@ public class RateLimiter {
     }
 
 
-    public static RateLimiter forExpensiveRequest() {
-        return new RateLimiter(5, 10);
-    }
-
     public static RateLimiter custom(int perMinute) {
-        return new RateLimiter(perMinute, 60);
-    }
-
-    public static RateLimiter forSpamBots() {
-        return new RateLimiter(120, 3600);
-    }
-
-
-    public static RateLimiter forLogin() {
-        return new RateLimiter(3, 15);
+        return new RateLimiter(4 * perMinute, perMinute);
     }
 
     private void cleanIdleBuckets() {
@@ -62,7 +49,7 @@ public class RateLimiter {
     }
 
     private Bucket createBucket() {
-        var refill = Refill.greedy(1, Duration.ofSeconds(refillRate));
+        var refill = Refill.greedy(refillRate, Duration.ofSeconds(60));
         var bw = Bandwidth.classic(capacity, refill);
         return Bucket.builder().addLimit(bw).build();
     }

code/common/service/resources/log4j2-json.xml

@@ -5,6 +5,7 @@
             <Filters>
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
@@ -13,9 +14,20 @@
             <Filters>
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
             <SizeBasedTriggeringPolicy size="10MB" />
         </RollingFile>
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/crawler-audit-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/crawler-audit-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+                     ignoreExceptions="false">
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS}: %msg{nolookups}%n</Pattern>
+            </PatternLayout>
+            <SizeBasedTriggeringPolicy size="100MB" />
+            <Filters>
+                <MarkerFilter marker="CRAWLER" onMatch="ALLOW" onMismatch="DENY" />
+            </Filters>
+        </RollingFile>
     </Appenders>
     <Loggers>
         <Logger name="org.apache.zookeeper" level="WARN" />

code/common/service/resources/log4j2-prod.xml

@@ -5,6 +5,7 @@
             <Filters>
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
@@ -17,6 +18,17 @@
                 <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </RollingFile>
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/crawler-audit-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/crawler-audit-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+                     ignoreExceptions="false">
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS}: %msg{nolookups}%n</Pattern>
+            </PatternLayout>
+            <SizeBasedTriggeringPolicy size="100MB" />
+            <Filters>
+                <MarkerFilter marker="CRAWLER" onMatch="ALLOW" onMismatch="DENY" />
             </Filters>
         </RollingFile>
     </Appenders>

code/common/service/test/nu/marginalia/service/discovery/ZkServiceRegistryTest.java

@@ -25,7 +25,7 @@ import static org.mockito.Mockito.when;
 class ZkServiceRegistryTest {
     private static final int ZOOKEEPER_PORT = 2181;
     private static final GenericContainer<?> zookeeper =
-            new GenericContainer<>("zookeeper:3.8.0")
+            new GenericContainer<>("zookeeper:3.8")
                     .withExposedPorts(ZOOKEEPER_PORT);
 
     List<ZkServiceRegistry> registries = new ArrayList<>();

code/execution/api/java/nu/marginalia/executor/client/ExecutorExportClient.java

@@ -48,12 +48,13 @@ public class ExecutorExportClient {
         return msgId;
     }
 
-    public void exportSampleData(int node, FileStorageId fid, int size, String name) {
+    public void exportSampleData(int node, FileStorageId fid, int size, String ctFilter, String name) {
         channelPool.call(ExecutorExportApiBlockingStub::exportSampleData)
                 .forNode(node)
                 .run(RpcExportSampleData.newBuilder()
                         .setFileStorageId(fid.id())
                         .setSize(size)
+                        .setCtFilter(ctFilter)
                         .setName(name)
                         .build());
     }

code/execution/api/src/main/protobuf/executor-api.proto

@@ -100,6 +100,7 @@ message RpcExportSampleData {
   int64 fileStorageId = 1;
   int32 size = 2;
   string name = 3;
+  string ctFilter = 4;
 }
 message RpcDownloadSampleData {
   string sampleSet = 1;

code/execution/java/nu/marginalia/actor/proc/UpdateRssActor.java

@@ -14,6 +14,8 @@ import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.nodecfg.model.NodeProfile;
 import nu.marginalia.service.module.ServiceConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.time.Duration;
 import java.time.LocalDateTime;
@@ -29,6 +31,7 @@ public class UpdateRssActor extends RecordActorPrototype {
 
     private final NodeConfigurationService nodeConfigurationService;
     private final MqPersistence persistence;
+    private static final Logger logger = LoggerFactory.getLogger(UpdateRssActor.class);
 
     @Inject
     public UpdateRssActor(Gson gson,
@@ -101,8 +104,8 @@ public class UpdateRssActor extends RecordActorPrototype {
             case UpdateRefresh(int count, long msgId) -> {
                 MqMessage msg = persistence.waitForMessageTerminalState(msgId, Duration.ofSeconds(10), Duration.ofHours(12));
                 if (msg == null) {
-                    // Retry the update
-                    yield new Error("Failed to update feeds: message not found");
+                    logger.warn("UpdateRefresh is taking a very long time");
+                    yield new UpdateRefresh(count, msgId);
                 } else if (msg.state() != MqMessageState.OK) {
                     // Retry the update
                     yield new Error("Failed to update feeds: " + msg.state());
@@ -119,8 +122,8 @@ public class UpdateRssActor extends RecordActorPrototype {
             case UpdateClean(long msgId) -> {
                 MqMessage msg = persistence.waitForMessageTerminalState(msgId, Duration.ofSeconds(10), Duration.ofHours(12));
                 if (msg == null) {
-                    // Retry the update
-                    yield new Error("Failed to update feeds: message not found");
+                    logger.warn("UpdateClean is taking a very long time");
+                    yield new UpdateClean(msgId);
                 } else if (msg.state() != MqMessageState.OK) {
                     // Retry the update
                     yield new Error("Failed to update feeds: " + msg.state());

code/execution/java/nu/marginalia/actor/task/DownloadSampleActor.java

@@ -8,6 +8,7 @@ import nu.marginalia.actor.state.ActorResumeBehavior;
 import nu.marginalia.actor.state.ActorStep;
 import nu.marginalia.actor.state.Resume;
 import nu.marginalia.service.control.ServiceEventLog;
+import nu.marginalia.service.control.ServiceHeartbeat;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorage;
 import nu.marginalia.storage.model.FileStorageId;
@@ -19,6 +20,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
@@ -32,6 +34,7 @@ public class DownloadSampleActor extends RecordActorPrototype {
 
     private final FileStorageService storageService;
     private final ServiceEventLog eventLog;
+    private final ServiceHeartbeat heartbeat;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resume(behavior = ActorResumeBehavior.ERROR)
@@ -66,15 +69,39 @@ public class DownloadSampleActor extends RecordActorPrototype {
 
                 Files.deleteIfExists(Path.of(tarFileName));
 
-                try (var is = new BufferedInputStream(new URI(downloadURI).toURL().openStream());
-                     var os = new BufferedOutputStream(Files.newOutputStream(Path.of(tarFileName), StandardOpenOption.CREATE))) {
-                    is.transferTo(os);
+                HttpURLConnection urlConnection = (HttpURLConnection) new URI(downloadURI).toURL().openConnection();
+
+                try (var hb = heartbeat.createServiceAdHocTaskHeartbeat("Downloading sample")) {
+                    long size = urlConnection.getContentLengthLong();
+                    byte[] buffer = new byte[8192];
+
+                    try (var is = new BufferedInputStream(urlConnection.getInputStream());
+                         var os = new BufferedOutputStream(Files.newOutputStream(Path.of(tarFileName), StandardOpenOption.CREATE))) {
+                        long copiedSize = 0;
+
+                        while (copiedSize < size) {
+                            int read = is.read(buffer);
+
+                            if (read < 0) // We've been promised a file of length 'size'
+                                throw new IOException("Unexpected end of stream");
+
+                            os.write(buffer, 0, read);
+                            copiedSize += read;
+
+                            // Update progress bar
+                            hb.progress(String.format("%d MB", copiedSize / 1024 / 1024), (int) (copiedSize / 1024), (int) (size / 1024));
+                        }
+                    }
+
                 }
                 catch (Exception ex) {
                     eventLog.logEvent(DownloadSampleActor.class, "Error downloading sample");
                     logger.error("Error downloading sample", ex);
                     yield new Error();
                 }
+                finally {
+                    urlConnection.disconnect();
+                }
 
                 eventLog.logEvent(DownloadSampleActor.class, "Download complete");
                 yield new Extract(fileStorageId, tarFileName);
@@ -170,11 +197,12 @@ public class DownloadSampleActor extends RecordActorPrototype {
     @Inject
     public DownloadSampleActor(Gson gson,
                                FileStorageService storageService,
-                               ServiceEventLog eventLog)
+                               ServiceEventLog eventLog, ServiceHeartbeat heartbeat)
     {
         super(gson);
         this.storageService = storageService;
         this.eventLog = eventLog;
+        this.heartbeat = heartbeat;
     }
 
 }

code/execution/java/nu/marginalia/actor/task/ExportSampleDataActor.java

@@ -26,32 +26,32 @@ public class ExportSampleDataActor extends RecordActorPrototype {
     private final MqOutbox exportTasksOutbox;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    public record Export(FileStorageId crawlId, int size, String name) implements ActorStep {}
-    public record Run(FileStorageId crawlId, FileStorageId destId, int size, String name, long msgId) implements ActorStep {
-        public Run(FileStorageId crawlId, FileStorageId destId, int size, String name) {
-            this(crawlId, destId, size, name, -1);
+    public record Export(FileStorageId crawlId, int size, String ctFilter, String name) implements ActorStep {}
+    public record Run(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name, long msgId) implements ActorStep {
+        public Run(FileStorageId crawlId, FileStorageId destId, int size, String name, String ctFilter) {
+            this(crawlId, destId, size, name, ctFilter,-1);
         }
     }
 
     @Override
     public ActorStep transition(ActorStep self) throws Exception {
         return switch(self) {
-            case Export(FileStorageId crawlId, int size, String name) -> {
+            case Export(FileStorageId crawlId, int size, String ctFilter, String name) -> {
                 var storage = storageService.allocateStorage(FileStorageType.EXPORT,
                         "crawl-sample-export",
                         "Crawl Data Sample " + name + "/" + size + " " + LocalDateTime.now()
                 );
 
                 if (storage == null) yield new Error("Bad storage id");
-                yield new Run(crawlId, storage.id(), size, name);
+                yield new Run(crawlId, storage.id(), size, ctFilter, name);
             }
-            case Run(FileStorageId crawlId, FileStorageId destId, int size, String name, long msgId) when msgId < 0 -> {
+            case Run(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name, long msgId) when msgId < 0 -> {
                 storageService.setFileStorageState(destId, FileStorageState.NEW);
 
-                long newMsgId = exportTasksOutbox.sendAsync(ExportTaskRequest.sampleData(crawlId, destId, size, name));
-                yield new Run(crawlId, destId, size, name, newMsgId);
+                long newMsgId = exportTasksOutbox.sendAsync(ExportTaskRequest.sampleData(crawlId, destId, ctFilter, size, name));
+                yield new Run(crawlId, destId, size, ctFilter, name, newMsgId);
             }
-            case Run(_, FileStorageId destId, _, _, long msgId) -> {
+            case Run(_, FileStorageId destId, _, _, _, long msgId) -> {
                 var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
@@ -70,7 +70,7 @@ public class ExportSampleDataActor extends RecordActorPrototype {
 
     @Override
     public String describe() {
-        return "Export RSS/Atom feeds from crawl data";
+        return "Export sample crawl data";
     }
 
     @Inject

code/execution/java/nu/marginalia/actor/task/MigrateCrawlDataActor.java

@@ -8,6 +8,7 @@ import nu.marginalia.actor.state.ActorStep;
 import nu.marginalia.io.CrawlerOutputFile;
 import nu.marginalia.process.log.WorkLog;
 import nu.marginalia.process.log.WorkLogEntry;
+import nu.marginalia.service.control.ServiceHeartbeat;
 import nu.marginalia.slop.SlopCrawlDataRecord;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorage;
@@ -18,6 +19,7 @@ import org.slf4j.LoggerFactory;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
@@ -26,14 +28,15 @@ import java.util.function.Function;
 public class MigrateCrawlDataActor extends RecordActorPrototype {
 
     private final FileStorageService fileStorageService;
-
+    private final ServiceHeartbeat serviceHeartbeat;
     private static final Logger logger = LoggerFactory.getLogger(MigrateCrawlDataActor.class);
 
     @Inject
-    public MigrateCrawlDataActor(Gson gson, FileStorageService fileStorageService) {
+    public MigrateCrawlDataActor(Gson gson, FileStorageService fileStorageService, ServiceHeartbeat serviceHeartbeat) {
         super(gson);
 
         this.fileStorageService = fileStorageService;
+        this.serviceHeartbeat = serviceHeartbeat;
     }
 
     public record Run(long fileStorageId) implements ActorStep {}
@@ -49,33 +52,50 @@ public class MigrateCrawlDataActor extends RecordActorPrototype {
                 Path crawlerLog = root.resolve("crawler.log");
                 Path newCrawlerLog = Files.createTempFile(root, "crawler", ".migrate.log");
 
-                try (WorkLog workLog = new WorkLog(newCrawlerLog)) {
+                int totalEntries = WorkLog.countEntries(crawlerLog);
+
+                try (WorkLog workLog = new WorkLog(newCrawlerLog);
+                     var heartbeat = serviceHeartbeat.createServiceAdHocTaskHeartbeat("Migrating")
+                ) {
+                    int entryIdx = 0;
+
                     for (Map.Entry<WorkLogEntry, Path> item : WorkLog.iterableMap(crawlerLog, new CrawlDataLocator(root))) {
 
-                        var entry = item.getKey();
-                        var path = item.getValue();
+                        final WorkLogEntry entry = item.getKey();
+                        final Path inputPath = item.getValue();
 
-                        logger.info("Converting {}", entry.id());
+                        Path outputPath = inputPath;
+                        heartbeat.progress("Migrating" + inputPath.getFileName(), entryIdx++, totalEntries);
 
-
-                        if (path.toFile().getName().endsWith(".parquet")) {
+                        if (inputPath.toString().endsWith(".parquet")) {
                             String domain = entry.id();
                             String id = Integer.toHexString(domain.hashCode());
 
-                            Path outputFile = CrawlerOutputFile.createSlopPath(root, id, domain);
+                            outputPath = CrawlerOutputFile.createSlopPath(root, id, domain);
 
-                            SlopCrawlDataRecord.convertFromParquet(path, outputFile);
+                            if (Files.exists(inputPath)) {
+                                try {
+                                    SlopCrawlDataRecord.convertFromParquet(inputPath, outputPath);
+                                    Files.deleteIfExists(inputPath);
+                                } catch (Exception ex) {
+                                    outputPath = inputPath; // don't update the work log on error
+                                    logger.error("Failed to convert " + inputPath, ex);
+                                }
+                            }
+                            else if (!Files.exists(inputPath) && !Files.exists(outputPath)) {
+                                // if the input file is missing, and the output file is missing, we just write the log
+                                // record identical to the old one
+                                outputPath = inputPath;
+                            }
+                        }
 
-                            workLog.setJobToFinished(entry.id(), outputFile.toString(), entry.cnt());
-                        }
-                        else {
-                            workLog.setJobToFinished(entry.id(), path.toString(), entry.cnt());
-                        }
+                        // Write a log entry for the (possibly) converted file
+                        workLog.setJobToFinished(entry.id(), outputPath.toString(), entry.cnt());
                     }
                 }
 
                 Path oldCrawlerLog = Files.createTempFile(root, "crawler-", ".migrate.old.log");
-                Files.move(crawlerLog, oldCrawlerLog);
+                Files.move(crawlerLog, oldCrawlerLog, StandardCopyOption.REPLACE_EXISTING);
                 Files.move(newCrawlerLog, crawlerLog);
 
                 yield new End();

code/execution/java/nu/marginalia/execution/ExecutorExportGrpcService.java

@@ -49,6 +49,7 @@ public class ExecutorExportGrpcService
                     new ExportSampleDataActor.Export(
                             FileStorageId.of(request.getFileStorageId()),
                             request.getSize(),
+                            request.getCtFilter(),
                             request.getName()
                     )
             );

code/functions/favicon/api/build.gradle

@@ -0,0 +1,47 @@
+plugins {
+    id 'java'
+
+    id "com.google.protobuf" version "0.9.4"
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
+    }
+}
+
+jar.archiveBaseName = 'favicon-api'
+
+apply from: "$rootProject.projectDir/protobuf.gradle"
+apply from: "$rootProject.projectDir/srcsets.gradle"
+
+dependencies {
+    implementation project(':code:common:model')
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.notnull
+    implementation libs.guava
+    implementation dependencies.create(libs.guice.get()) {
+        exclude group: 'com.google.guava'
+    }
+    implementation libs.gson
+    implementation libs.bundles.protobuf
+    implementation libs.guava
+    libs.bundles.grpc.get().each {
+        implementation dependencies.create(it) {
+            exclude group: 'com.google.guava'
+        }
+    }
+
+
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+}

code/functions/favicon/api/java/nu/marginalia/api/favicon/FaviconClient.java

@@ -0,0 +1,39 @@
+package nu.marginalia.api.favicon;
+
+import com.google.inject.Inject;
+import nu.marginalia.service.client.GrpcChannelPoolFactory;
+import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Optional;
+
+public class FaviconClient {
+    private static final Logger logger = LoggerFactory.getLogger(FaviconClient.class);
+
+    private final GrpcMultiNodeChannelPool<FaviconAPIGrpc.FaviconAPIBlockingStub> channelPool;
+
+    @Inject
+    public FaviconClient(GrpcChannelPoolFactory factory) {
+        this.channelPool = factory.createMulti(
+                ServiceKey.forGrpcApi(FaviconAPIGrpc.class, ServicePartition.multi()),
+                FaviconAPIGrpc::newBlockingStub);
+    }
+
+    public record FaviconData(byte[] bytes, String contentType) {}
+
+
+    public Optional<FaviconData> getFavicon(String domain, int node) {
+        RpcFaviconResponse rsp = channelPool.call(FaviconAPIGrpc.FaviconAPIBlockingStub::getFavicon)
+                .forNode(node)
+                .run(RpcFaviconRequest.newBuilder().setDomain(domain).build());
+
+        if (rsp.getData().isEmpty())
+            return Optional.empty();
+
+        return Optional.of(new FaviconData(rsp.getData().toByteArray(), rsp.getContentType()));
+    }
+
+}

code/functions/favicon/api/src/main/protobuf/favicon.proto

@@ -0,0 +1,20 @@
+syntax="proto3";
+package marginalia.api.favicon;
+
+option java_package="nu.marginalia.api.favicon";
+option java_multiple_files=true;
+
+service FaviconAPI {
+  /** Fetches information about a domain. */
+  rpc getFavicon(RpcFaviconRequest) returns (RpcFaviconResponse) {}
+}
+
+message RpcFaviconRequest {
+  string domain = 1;
+}
+
+message RpcFaviconResponse {
+  string domain = 1;
+  bytes data = 2;
+  string contentType = 3;
+}

code/functions/favicon/build.gradle

@@ -0,0 +1,49 @@
+plugins {
+    id 'java'
+
+    id 'application'
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
+    }
+}
+
+apply from: "$rootProject.projectDir/srcsets.gradle"
+
+dependencies {
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+    implementation project(':code:common:model')
+    implementation project(':code:common:db')
+    implementation project(':code:functions:favicon:api')
+    implementation project(':code:processes:crawling-process')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.guava
+    libs.bundles.grpc.get().each {
+        implementation dependencies.create(it) {
+            exclude group: 'com.google.guava'
+        }
+    }
+
+
+    implementation libs.notnull
+    implementation libs.guava
+    implementation dependencies.create(libs.guice.get()) {
+        exclude group: 'com.google.guava'
+    }
+    implementation dependencies.create(libs.spark.get()) {
+        exclude group: 'org.eclipse.jetty'
+    }
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+
+}

code/functions/favicon/java/nu/marginalia/functions/favicon/FaviconGrpcService.java

@@ -0,0 +1,48 @@
+package nu.marginalia.functions.favicon;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.protobuf.ByteString;
+import io.grpc.stub.StreamObserver;
+import nu.marginalia.api.favicon.FaviconAPIGrpc;
+import nu.marginalia.api.favicon.RpcFaviconRequest;
+import nu.marginalia.api.favicon.RpcFaviconResponse;
+import nu.marginalia.crawl.DomainStateDb;
+import nu.marginalia.service.server.DiscoverableService;
+
+import java.util.Optional;
+
+@Singleton
+public class FaviconGrpcService extends FaviconAPIGrpc.FaviconAPIImplBase implements DiscoverableService {
+    private final DomainStateDb domainStateDb;
+
+    @Inject
+    public FaviconGrpcService(DomainStateDb domainStateDb) {
+        this.domainStateDb = domainStateDb;
+    }
+
+    public boolean shouldRegisterService() {
+        return domainStateDb.isAvailable();
+    }
+
+    @Override
+    public void getFavicon(RpcFaviconRequest request, StreamObserver<RpcFaviconResponse> responseObserver) {
+        Optional<DomainStateDb.FaviconRecord> icon = domainStateDb.getIcon(request.getDomain());
+
+        RpcFaviconResponse response;
+        if (icon.isEmpty()) {
+            response = RpcFaviconResponse.newBuilder().build();
+        }
+        else {
+            var iconRecord = icon.get();
+            response = RpcFaviconResponse.newBuilder()
+                            .setContentType(iconRecord.contentType())
+                            .setDomain(request.getDomain())
+                            .setData(ByteString.copyFrom(iconRecord.imageData()))
+                            .build();
+        }
+
+        responseObserver.onNext(response);
+        responseObserver.onCompleted();
+    }
+}

code/functions/live-capture/build.gradle

@@ -34,6 +34,7 @@ dependencies {
     implementation libs.bundles.slf4j
     implementation libs.commons.lang3
     implementation libs.commons.io
+    implementation libs.wiremock
 
     implementation libs.prometheus
     implementation libs.guava

code/functions/live-capture/java/nu/marginalia/livecapture/BrowserlessClient.java

@@ -1,6 +1,7 @@
 package nu.marginalia.livecapture;
 
 import com.google.gson.Gson;
+import nu.marginalia.WmsaHome;
 import nu.marginalia.model.gson.GsonFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -12,6 +13,7 @@ import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.time.Duration;
 import java.util.Map;
+import java.util.Optional;
 
 /** Client for local browserless.io API */
 public class BrowserlessClient implements AutoCloseable {
@@ -27,13 +29,16 @@ public class BrowserlessClient implements AutoCloseable {
     private final URI browserlessURI;
     private final Gson gson = GsonFactory.get();
 
+    private final String userAgent = WmsaHome.getUserAgent().uaString();
+
     public BrowserlessClient(URI browserlessURI) {
         this.browserlessURI = browserlessURI;
     }
 
-    public String content(String url, GotoOptions gotoOptions) throws IOException, InterruptedException {
+    public Optional<String> content(String url, GotoOptions gotoOptions) throws IOException, InterruptedException {
         Map<String, Object> requestData = Map.of(
                 "url", url,
+                "userAgent", userAgent,
                 "gotoOptions", gotoOptions
         );
 
@@ -49,10 +54,10 @@ public class BrowserlessClient implements AutoCloseable {
 
         if (rsp.statusCode() >= 300) {
             logger.info("Failed to fetch content for {}, status {}", url, rsp.statusCode());
-            return null;
+            return Optional.empty();
         }
 
-        return rsp.body();
+        return Optional.of(rsp.body());
     }
 
     public byte[] screenshot(String url, GotoOptions gotoOptions, ScreenshotOptions screenshotOptions)
@@ -60,6 +65,7 @@ public class BrowserlessClient implements AutoCloseable {
 
         Map<String, Object> requestData = Map.of(
                 "url", url,
+                "userAgent", userAgent,
                 "options", screenshotOptions,
                 "gotoOptions", gotoOptions
         );
@@ -84,7 +90,7 @@ public class BrowserlessClient implements AutoCloseable {
     }
 
     @Override
-    public void close() throws Exception {
+    public void close() {
         httpClient.shutdownNow();
     }
 

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedFetcherService.java

@@ -33,6 +33,7 @@ import java.sql.SQLException;
 import java.time.*;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -71,7 +72,7 @@ public class FeedFetcherService {
     public enum UpdateMode {
         CLEAN,
         REFRESH
-    };
+    }
 
     public void updateFeeds(UpdateMode updateMode) throws IOException {
         if (updating) // Prevent concurrent updates
@@ -87,6 +88,7 @@ public class FeedFetcherService {
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .version(HttpClient.Version.HTTP_2)
                 .build();
+             ExecutorService fetchExecutor = Executors.newCachedThreadPool();
              FeedJournal feedJournal = FeedJournal.create();
              var heartbeat = serviceHeartbeat.createServiceAdHocTaskHeartbeat("Update Rss Feeds")
         ) {
@@ -131,7 +133,7 @@ public class FeedFetcherService {
 
                         FetchResult feedData;
                         try (DomainLocks.DomainLock domainLock = domainLocks.lockDomain(new EdgeDomain(feed.domain()))) {
-                            feedData = fetchFeedData(feed, client, ifModifiedSinceDate, ifNoneMatchTag);
+                            feedData = fetchFeedData(feed, client, fetchExecutor, ifModifiedSinceDate, ifNoneMatchTag);
                         } catch (Exception ex) {
                             feedData = new FetchResult.TransientError();
                         }
@@ -211,6 +213,7 @@ public class FeedFetcherService {
 
     private FetchResult fetchFeedData(FeedDefinition feed,
                                       HttpClient client,
+                                      ExecutorService executorService,
                                       @Nullable String ifModifiedSinceDate,
                                       @Nullable String ifNoneMatchTag)
     {
@@ -226,18 +229,27 @@ public class FeedFetcherService {
                     .timeout(Duration.ofSeconds(15))
                     ;
 
-            if (ifModifiedSinceDate != null) {
+            // Set the If-Modified-Since or If-None-Match headers if we have them
+            // though since there are certain idiosyncrasies in server implementations,
+            // we avoid setting both at the same time as that may turn a 304 into a 200.
+            if (ifNoneMatchTag != null) {
+                requestBuilder.header("If-None-Match", ifNoneMatchTag);
+            } else if (ifModifiedSinceDate != null) {
                 requestBuilder.header("If-Modified-Since", ifModifiedSinceDate);
             }
 
-            if (ifNoneMatchTag != null) {
-                requestBuilder.header("If-None-Match", ifNoneMatchTag);
-            }
 
             HttpRequest getRequest = requestBuilder.build();
 
             for (int i = 0; i < 3; i++) {
-                HttpResponse<byte[]> rs = client.send(getRequest, HttpResponse.BodyHandlers.ofByteArray());
+
+                /* Note we need to use an executor to time-limit the send() method in HttpClient, as
+                 * its support for timeouts only applies to the time until response starts to be received,
+                 * and does not catch the case when the server starts to send data but then hangs.
+                 */
+                HttpResponse<byte[]> rs = executorService.submit(
+                        () -> client.send(getRequest, HttpResponse.BodyHandlers.ofByteArray()))
+                                .get(15, TimeUnit.SECONDS);
 
                 if (rs.statusCode() == 429) { // Too Many Requests
                     int retryAfter = Integer.parseInt(rs.headers().firstValue("Retry-After").orElse("2"));

code/functions/live-capture/test/nu/marginalia/livecapture/BrowserlessClientTest.java

@@ -1,5 +1,9 @@
 package nu.marginalia.livecapture;
 
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import nu.marginalia.WmsaHome;
+import nu.marginalia.service.module.ServiceConfigurationModule;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -8,34 +12,86 @@ import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.junit.jupiter.Testcontainers;
 import org.testcontainers.utility.DockerImageName;
 
+import java.io.IOException;
 import java.net.URI;
 import java.util.Map;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.*;
+
+
 @Testcontainers
 @Tag("slow")
 public class BrowserlessClientTest {
     static GenericContainer<?> container = new GenericContainer<>(DockerImageName.parse("browserless/chrome"))
             .withEnv(Map.of("TOKEN", "BROWSERLESS_TOKEN"))
+            .withNetworkMode("bridge")
             .withExposedPorts(3000);
 
+    static WireMockServer wireMockServer =
+            new WireMockServer(WireMockConfiguration.wireMockConfig()
+                    .port(18089));
+
+    static String localIp;
+
+    static URI browserlessURI;
+
     @BeforeAll
-    public static void setup() {
+    public static void setup() throws IOException {
         container.start();
+
+        browserlessURI = URI.create(String.format("http://%s:%d/",
+                container.getHost(),
+                container.getMappedPort(3000))
+        );
+
+        wireMockServer.start();
+        wireMockServer.stubFor(get("/").willReturn(aResponse().withStatus(200).withBody("Ok")));
+
+        localIp = ServiceConfigurationModule.getLocalNetworkIP();
+
+    }
+
+    @Tag("flaky")
+    @Test
+    public void testInspectContentUA__Flaky() throws Exception {
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            client.content("http://" + localIp + ":18089/",
+                    BrowserlessClient.GotoOptions.defaultValues()
+            );
+        }
+
+        wireMockServer.verify(getRequestedFor(urlEqualTo("/")).withHeader("User-Agent", equalTo(WmsaHome.getUserAgent().uaString())));
+    }
+
+    @Tag("flaky")
+    @Test
+    public void testInspectScreenshotUA__Flaky() throws Exception {
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            client.screenshot("http://" + localIp + ":18089/",
+                    BrowserlessClient.GotoOptions.defaultValues(),
+                    BrowserlessClient.ScreenshotOptions.defaultValues()
+            );
+        }
+
+        wireMockServer.verify(getRequestedFor(urlEqualTo("/")).withHeader("User-Agent", equalTo(WmsaHome.getUserAgent().uaString())));
     }
 
     @Test
     public void testContent() throws Exception {
-        try (var client = new BrowserlessClient(URI.create("http://" + container.getHost() + ":" + container.getMappedPort(3000)))) {
-            var content = client.content("https://www.marginalia.nu/", BrowserlessClient.GotoOptions.defaultValues());
-            Assertions.assertNotNull(content, "Content should not be null");
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            var content = client.content("https://www.marginalia.nu/", BrowserlessClient.GotoOptions.defaultValues()).orElseThrow();
+
             Assertions.assertFalse(content.isBlank(), "Content should not be empty");
         }
     }
 
     @Test
     public void testScreenshot() throws Exception {
-        try (var client = new BrowserlessClient(URI.create("http://" + container.getHost() + ":" + container.getMappedPort(3000)))) {
-            var screenshot = client.screenshot("https://www.marginalia.nu/", BrowserlessClient.GotoOptions.defaultValues(), BrowserlessClient.ScreenshotOptions.defaultValues());
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            var screenshot = client.screenshot("https://www.marginalia.nu/",
+                    BrowserlessClient.GotoOptions.defaultValues(),
+                    BrowserlessClient.ScreenshotOptions.defaultValues());
+
             Assertions.assertNotNull(screenshot, "Screenshot should not be null");
         }
     }

code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryExpansion.java

@@ -134,6 +134,10 @@ public class QueryExpansion {
                 if (scoreCombo > scoreA + scoreB || scoreCombo > 1000) {
                     graph.addVariantForSpan(prev, qw, joinedWord);
                 }
+                else if (StringUtils.isAlpha(prev.word()) && StringUtils.isNumeric(qw.word())) { // join e.g. trs 80 to trs80 and trs-80
+                    graph.addVariantForSpan(prev, qw, prev.word() + qw.word());
+                    graph.addVariantForSpan(prev, qw, prev.word() + "-" + qw.word());
+                }
             }
 
             prev = qw;

code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java

@@ -213,6 +213,18 @@ public class QueryFactoryTest {
         System.out.println(subquery);
     }
 
+
+    @Test
+    public void testContractionWordNum() {
+        var subquery = parseAndGetSpecs("glove 80");
+
+        Assertions.assertTrue(subquery.query.compiledQuery.contains(" glove "));
+        Assertions.assertTrue(subquery.query.compiledQuery.contains(" 80 "));
+        Assertions.assertTrue(subquery.query.compiledQuery.contains(" glove-80 "));
+        Assertions.assertTrue(subquery.query.compiledQuery.contains(" glove80 "));
+    }
+
+
     @Test
     public void testCplusPlus() {
         var subquery = parseAndGetSpecs("std::vector::push_back vector");

code/libraries/blocking-thread-pool/java/nu/marginalia/util/SimpleBlockingThreadPool.java

@@ -23,16 +23,33 @@ public class SimpleBlockingThreadPool {
     private final Logger logger = LoggerFactory.getLogger(SimpleBlockingThreadPool.class);
 
     public SimpleBlockingThreadPool(String name, int poolSize, int queueSize) {
+        this(name, poolSize, queueSize, ThreadType.PLATFORM);
+    }
+
+    public SimpleBlockingThreadPool(String name, int poolSize, int queueSize, ThreadType threadType) {
         tasks = new ArrayBlockingQueue<>(queueSize);
 
         for (int i = 0; i < poolSize; i++) {
-            Thread worker = new Thread(this::worker, name  + "[" + i + "]");
-            worker.setDaemon(true);
-            worker.start();
+
+            Thread.Builder threadBuilder = switch (threadType) {
+                case VIRTUAL -> Thread.ofVirtual();
+                case PLATFORM -> Thread.ofPlatform().daemon(true);
+            };
+
+            Thread worker = threadBuilder
+                    .name(name  + "[" + i + "]")
+                    .start(this::worker);
+
             workers.add(worker);
         }
 
     }
+
+    public enum ThreadType {
+        VIRTUAL,
+        PLATFORM
+    }
+
     public void submit(Task task) throws InterruptedException {
         tasks.put(task);
     }

code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractor.java

@@ -155,8 +155,15 @@ public class SentenceExtractor {
     public List<DocumentSentence> extractSentencesFromString(String text, EnumSet<HtmlTag> htmlTags) {
         String[] sentences;
 
-        // Normalize spaces
+        // Safety net against malformed data DOS attacks,
+        // found 5+ MB <p>-tags in the wild that just break
+        // the sentence extractor causing it to stall forever.
+        if (text.length() > 50_000) {
+            // 50k chars can hold a small novel, let alone single html tags
+            text = text.substring(0, 50_000);
+        }
 
+        // Normalize spaces
         text = normalizeSpaces(text);
 
         // Split into sentences

code/libraries/message-queue/java/nu/marginalia/actor/prototype/RecordActorPrototype.java

@@ -5,9 +5,7 @@ import nu.marginalia.actor.state.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 
 public abstract class RecordActorPrototype implements ActorPrototype {
 
@@ -118,7 +116,7 @@ public abstract class RecordActorPrototype implements ActorPrototype {
         }
 
         private String functionName(Class<? extends ActorStep> functionClass) {
-            return functionClass.getSimpleName().toUpperCase();
+            return ActorStep.functionName(functionClass);
         }
 
         private ActorStep constructState(String message) throws ReflectiveOperationException {
@@ -145,4 +143,43 @@ public abstract class RecordActorPrototype implements ActorPrototype {
         }
     }
 
+    /** Get a list of JSON prototypes for each actor step declared by this actor */
+    @SuppressWarnings("unchecked")
+    public Map<String, String> getMessagePrototypes() {
+        Map<String, String> messagePrototypes = new HashMap<>();
+
+        for (var clazz : getClass().getDeclaredClasses()) {
+            if (!clazz.isRecord() || !ActorStep.class.isAssignableFrom(clazz))
+                continue;
+
+            StringJoiner sj = new StringJoiner(",\n\t", "{\n\t", "\n}");
+
+            renderToJsonPrototype(sj, (Class<? extends Record>) clazz);
+
+            messagePrototypes.put(ActorStep.functionName((Class<? extends ActorStep>) clazz), sj.toString());
+        }
+
+        return messagePrototypes;
+    }
+
+    @SuppressWarnings("unchecked")
+    private void renderToJsonPrototype(StringJoiner sj, Class<? extends Record> recordType) {
+        for (var field : recordType.getDeclaredFields()) {
+            String typeName = field.getType().getSimpleName();
+
+            if ("List".equals(typeName)) {
+                sj.add(String.format("\"%s\": [ ]", field.getName()));
+            }
+            else if (field.getType().isRecord()) {
+                var innerSj = new StringJoiner(",", "{", "}");
+                renderToJsonPrototype(innerSj, (Class<? extends Record>) field.getType());
+                sj.add(String.format("\"%s\": %s", field.getName(), sj));
+            }
+            else {
+                sj.add(String.format("\"%s\": \"%s\"", field.getName(), typeName));
+            }
+        }
+
+    }
+
 }

code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStep.java

@@ -1,3 +1,7 @@
 package nu.marginalia.actor.state;
 
-public interface ActorStep {}
+public interface ActorStep {
+    static String functionName(Class<? extends ActorStep> type) {
+        return type.getSimpleName().toUpperCase();
+    }
+}

code/processes/converting-process/build.gradle

@@ -87,6 +87,8 @@ dependencies {
     implementation libs.commons.compress
     implementation libs.sqlite
 
+    implementation libs.bundles.httpcomponents
+
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit
     testImplementation libs.mockito

code/processes/converting-process/java/nu/marginalia/converting/ConverterMain.java

@@ -12,6 +12,7 @@ import nu.marginalia.converting.sideload.SideloadSourceFactory;
 import nu.marginalia.converting.writer.ConverterBatchWritableIf;
 import nu.marginalia.converting.writer.ConverterBatchWriter;
 import nu.marginalia.converting.writer.ConverterWriter;
+import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.mq.MessageQueueFactory;
 import nu.marginalia.mqapi.converting.ConvertRequest;
 import nu.marginalia.process.ProcessConfiguration;
@@ -34,6 +35,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
@@ -49,6 +51,7 @@ public class ConverterMain extends ProcessMainClass {
     private final ProcessHeartbeat heartbeat;
     private final FileStorageService fileStorageService;
     private final SideloadSourceFactory sideloadSourceFactory;
+    private static final int SIDELOAD_THRESHOLD = Integer.getInteger("converter.sideloadThreshold", 10_000);
 
     public static void main(String... args) throws Exception {
 
@@ -199,12 +202,26 @@ public class ConverterMain extends ProcessMainClass {
             processedDomains.set(batchingWorkLog.size());
             heartbeat.setProgress(processedDomains.get() / (double) totalDomains);
 
-            for (var domain : WorkLog.iterableMap(crawlDir.getLogFile(),
+            logger.info("Processing small items");
+
+            // We separate the large and small domains to reduce the number of critical sections,
+            // as the large domains have a separate processing track that doesn't store everything
+            // in memory
+
+            final List<Path> bigTasks = new ArrayList<>();
+
+            // First process the small items
+            for (var dataPath : WorkLog.iterableMap(crawlDir.getLogFile(),
                     new CrawlDataLocator(crawlDir.getDir(), batchingWorkLog)))
             {
+                if (SerializableCrawlDataStream.getSizeHint(dataPath) >= SIDELOAD_THRESHOLD) {
+                    bigTasks.add(dataPath);
+                    continue;
+                }
+
                 pool.submit(() -> {
-                    try {
-                        ConverterBatchWritableIf writable = processor.createWritable(domain);
+                    try (var dataStream = SerializableCrawlDataStream.openDataStream(dataPath)) {
+                        ConverterBatchWritableIf writable = processor.fullProcessing(dataStream) ;
                         converterWriter.accept(writable);
                     }
                     catch (Exception ex) {
@@ -223,6 +240,35 @@ public class ConverterMain extends ProcessMainClass {
             do {
                 System.out.println("Waiting for pool to terminate... " + pool.getActiveCount() + " remaining");
             } while (!pool.awaitTermination(60, TimeUnit.SECONDS));
+
+            logger.info("Processing large items");
+
+            try (var hb = heartbeat.createAdHocTaskHeartbeat("Large Domains")) {
+                int bigTaskIdx = 0;
+                // Next the big items domain-by-domain
+                for (var dataPath : bigTasks) {
+                    hb.progress(dataPath.toFile().getName(), bigTaskIdx++, bigTasks.size());
+
+                    try {
+                        // SerializableCrawlDataStream is autocloseable, we can't try-with-resources because then it will be
+                        // closed before it's consumed by the converterWriter.  Instead, the converterWriter guarantees it
+                        // will close it after it's consumed.
+
+                        var stream = SerializableCrawlDataStream.openDataStream(dataPath);
+                        ConverterBatchWritableIf writable = processor.simpleProcessing(stream, SerializableCrawlDataStream.getSizeHint(dataPath));
+
+                        converterWriter.accept(writable);
+                    }
+                    catch (Exception ex) {
+                        logger.info("Error in processing", ex);
+                    }
+                    finally {
+                        heartbeat.setProgress(processedDomains.incrementAndGet() / (double) totalDomains);
+                    }
+                }
+            }
+
+            logger.info("Processing complete");
         }
     }
 

code/processes/converting-process/java/nu/marginalia/converting/processor/DomainProcessor.java

@@ -14,7 +14,6 @@ import nu.marginalia.converting.writer.ConverterBatchWritableIf;
 import nu.marginalia.converting.writer.ConverterBatchWriter;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.geoip.sources.AsnTable;
-import nu.marginalia.io.CrawledDomainReader;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.crawl.DomainIndexingState;
@@ -28,13 +27,11 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.sql.SQLException;
 import java.util.*;
 import java.util.regex.Pattern;
 
 public class DomainProcessor {
-    private static final int SIDELOAD_THRESHOLD = Integer.getInteger("converter.sideloadThreshold", 10_000);
     private final DocumentProcessor documentProcessor;
     private final SiteWords siteWords;
     private final AnchorTagsSource anchorTagsSource;
@@ -56,21 +53,6 @@ public class DomainProcessor {
         geoIpDictionary.waitReady();
     }
 
-    public ConverterBatchWritableIf createWritable(Path path) throws IOException {
-
-        var dataStream = CrawledDomainReader.createDataStream(path);
-
-        final int sizeHint = dataStream.sizeHint();
-
-        if (sizeHint > SIDELOAD_THRESHOLD) {
-            // If the file is too big, we run a processing mode that doesn't
-            // require loading the entire dataset into RAM
-            return simpleProcessing(dataStream, sizeHint);
-        }
-
-        return fullProcessing(dataStream);
-    }
-
     public SimpleProcessing simpleProcessing(SerializableCrawlDataStream dataStream, int sizeHint, Collection<String> extraKeywords) {
         try {
             return new SimpleProcessing(dataStream, sizeHint, extraKeywords);
@@ -159,6 +141,7 @@ public class DomainProcessor {
         private final Set<String> processedUrls = new HashSet<>();
         private final DomainLinks externalDomainLinks;
         private final LshDocumentDeduplicator deduplicator = new LshDocumentDeduplicator();
+
         private static final ProcessingIterator.Factory iteratorFactory = ProcessingIterator.factory(8,
                 Integer.getInteger("java.util.concurrent.ForkJoinPool.common.parallelism", Runtime.getRuntime().availableProcessors())
         );
@@ -194,6 +177,7 @@ public class DomainProcessor {
         @Override
         public Iterator<ProcessedDocument> getDocumentsStream() {
             return iteratorFactory.create((taskConsumer) -> {
+
                 while (dataStream.hasNext())
                 {
                     if (!(dataStream.next() instanceof CrawledDocument doc))

code/processes/converting-process/java/nu/marginalia/converting/processor/classifier/adblock/AdblockSimulator.java

@@ -116,7 +116,7 @@ public class AdblockSimulator {
 
 
     // Refrain from cleaning up this code, it's very hot code and needs to be fast.
-    // This version is about 100x faster than the a "clean" first stab implementation.
+    // This version is about 100x faster than a "clean" first stab implementation.
 
     class RuleVisitor implements NodeFilter {
         public boolean sawAds;

code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentGeneratorExtractor.java

@@ -23,7 +23,7 @@ public class DocumentGeneratorExtractor {
 
         var tags = doc.select("meta[name=generator]");
 
-        if (tags.size() == 0) {
+        if (tags.isEmpty()) {
             // Some sites have a comment in the head instead of a meta tag
             return fingerprintServerTech(doc, responseHeaders);
         }

code/processes/converting-process/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java

@@ -127,7 +127,7 @@ public class EncyclopediaMarginaliaNuSideloader implements SideloadSource, AutoC
         }
         fullHtml.append("</div></body></html>");
 
-        var doc = sideloaderProcessing
+        return sideloaderProcessing
                 .processDocument(fullUrl,
                         fullHtml.toString(),
                         List.of("encyclopedia", "wiki"),
@@ -137,8 +137,6 @@ public class EncyclopediaMarginaliaNuSideloader implements SideloadSource, AutoC
                         anchorTextKeywords.getAnchorTextKeywords(domainLinks, new EdgeUrl(fullUrl)),
                         LocalDate.now().getYear(),
                         10_000_000);
-
-        return doc;
     }
 
     private String normalizeUtf8(String url) {

code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterWriter.java

@@ -39,6 +39,9 @@ public class ConverterWriter implements AutoCloseable {
         workerThread.start();
     }
 
+    /** Queue and eventually write the domain into the converter journal
+     *  The domain object will be closed after it's processed.
+     * */
     public void accept(@Nullable ConverterBatchWritableIf domain) {
         if (null == domain)
             return;
@@ -72,15 +75,15 @@ public class ConverterWriter implements AutoCloseable {
 
                 if (workLog.isItemCommitted(id) || workLog.isItemInCurrentBatch(id)) {
                     logger.warn("Skipping already logged item {}", id);
+                }
+                else {
+                    currentWriter.write(data);
+                    workLog.logItem(id);
                     data.close();
-                    continue;
                 }
 
-                currentWriter.write(data);
-
-                workLog.logItem(id);
-
                 switcher.tick();
+                data.close();
             }
         }
         catch (Exception ex) {

code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDocumentRecord.java

@@ -11,7 +11,6 @@ import nu.marginalia.slop.column.primitive.IntColumn;
 import nu.marginalia.slop.column.primitive.LongColumn;
 import nu.marginalia.slop.column.string.EnumColumn;
 import nu.marginalia.slop.column.string.StringColumn;
-import nu.marginalia.slop.column.string.TxtStringColumn;
 import nu.marginalia.slop.desc.StorageType;
 import org.jetbrains.annotations.Nullable;
 
@@ -182,8 +181,8 @@ public record SlopDocumentRecord(
     }
 
     // Basic information
-    private static final TxtStringColumn domainsColumn = new TxtStringColumn("domain", StandardCharsets.UTF_8, StorageType.GZIP);
-    private static final TxtStringColumn urlsColumn = new TxtStringColumn("url", StandardCharsets.UTF_8, StorageType.GZIP);
+    private static final StringColumn domainsColumn = new StringColumn("domain", StandardCharsets.UTF_8, StorageType.GZIP);
+    private static final StringColumn urlsColumn = new StringColumn("url", StandardCharsets.UTF_8, StorageType.GZIP);
     private static final VarintColumn ordinalsColumn = new VarintColumn("ordinal", StorageType.PLAIN);
     private static final EnumColumn statesColumn = new EnumColumn("state", StandardCharsets.US_ASCII, StorageType.PLAIN);
     private static final StringColumn stateReasonsColumn = new StringColumn("stateReason", StandardCharsets.US_ASCII, StorageType.GZIP);
@@ -211,7 +210,7 @@ public record SlopDocumentRecord(
     private static final VarintCodedSequenceArrayColumn spansColumn = new VarintCodedSequenceArrayColumn("spans", StorageType.ZSTD);
 
     public static class KeywordsProjectionReader extends SlopTable {
-        private final TxtStringColumn.Reader domainsReader;
+        private final StringColumn.Reader domainsReader;
         private final VarintColumn.Reader ordinalsReader;
         private final IntColumn.Reader htmlFeaturesReader;
         private final LongColumn.Reader domainMetadataReader;
@@ -275,8 +274,8 @@ public record SlopDocumentRecord(
     }
 
     public static class MetadataReader extends SlopTable {
-        private final TxtStringColumn.Reader domainsReader;
-        private final TxtStringColumn.Reader urlsReader;
+        private final StringColumn.Reader domainsReader;
+        private final StringColumn.Reader urlsReader;
         private final VarintColumn.Reader ordinalsReader;
         private final StringColumn.Reader titlesReader;
         private final StringColumn.Reader descriptionsReader;
@@ -332,8 +331,8 @@ public record SlopDocumentRecord(
     }
 
     public static class Writer extends SlopTable {
-        private final TxtStringColumn.Writer domainsWriter;
-        private final TxtStringColumn.Writer urlsWriter;
+        private final StringColumn.Writer domainsWriter;
+        private final StringColumn.Writer urlsWriter;
         private final VarintColumn.Writer ordinalsWriter;
         private final EnumColumn.Writer statesWriter;
         private final StringColumn.Writer stateReasonsWriter;

code/processes/converting-process/test/nu/marginalia/converting/CrawlingThenConvertingIntegrationTest.java

@@ -8,7 +8,6 @@ import nu.marginalia.converting.model.ProcessedDomain;
 import nu.marginalia.converting.processor.DomainProcessor;
 import nu.marginalia.crawl.CrawlerMain;
 import nu.marginalia.crawl.DomainStateDb;
-import nu.marginalia.crawl.fetcher.Cookies;
 import nu.marginalia.crawl.fetcher.HttpFetcher;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
@@ -247,7 +246,7 @@ public class CrawlingThenConvertingIntegrationTest {
     private CrawledDomain crawl(CrawlerMain.CrawlSpecRecord specs, Predicate<EdgeDomain> domainBlacklist) throws Exception {
         List<SerializableCrawlData> data = new ArrayList<>();
 
-        try (var recorder = new WarcRecorder(fileName, new Cookies());
+        try (var recorder = new WarcRecorder(fileName);
              var db = new DomainStateDb(dbTempFile))
         {
             new CrawlerRetreiver(httpFetcher, new DomainProber(domainBlacklist), specs, db, recorder).crawlDomain();

code/processes/crawling-process/build.gradle

@@ -60,10 +60,14 @@ dependencies {
     implementation libs.fastutil
 
     implementation libs.bundles.mariadb
+    implementation libs.bundles.httpcomponents
 
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit
     testImplementation libs.mockito
+    testImplementation libs.wiremock
+
+
 
     testImplementation project(':code:processes:test-data')
 }

code/processes/crawling-process/ft-content-type/java/nu/marginalia/contenttype/DocumentBodyToString.java

@@ -26,7 +26,7 @@ public class DocumentBodyToString {
         return new String(data, charset);
     }
 
-    public static Document getParsedData(ContentType type, byte[] data, String url) throws IOException {
+    public static Document getParsedData(ContentType type, byte[] data, int maxLength, String url) throws IOException {
         final Charset charset;
 
         if (type.charset() == null || type.charset().isBlank()) {
@@ -35,7 +35,7 @@ public class DocumentBodyToString {
             charset = charsetMap.computeIfAbsent(type, DocumentBodyToString::computeCharset);
         }
 
-        ByteArrayInputStream bais = new ByteArrayInputStream(data);
+        ByteArrayInputStream bais = new ByteArrayInputStream(data, 0, Math.min(data.length, maxLength));
 
         return Jsoup.parse(bais, charset.name(), url);
     }

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -19,7 +19,6 @@ import nu.marginalia.crawl.retreival.DomainProber;
 import nu.marginalia.crawl.warc.WarcArchiverFactory;
 import nu.marginalia.crawl.warc.WarcArchiverIf;
 import nu.marginalia.db.DomainBlacklist;
-import nu.marginalia.io.CrawledDomainReader;
 import nu.marginalia.io.CrawlerOutputFile;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.mq.MessageQueueFactory;
@@ -28,13 +27,11 @@ import nu.marginalia.process.ProcessConfigurationModule;
 import nu.marginalia.process.ProcessMainClass;
 import nu.marginalia.process.control.ProcessHeartbeatImpl;
 import nu.marginalia.process.log.WorkLog;
-import nu.marginalia.process.log.WorkLogEntry;
 import nu.marginalia.service.module.DatabaseModule;
 import nu.marginalia.slop.SlopCrawlDataRecord;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.util.SimpleBlockingThreadPool;
-import org.apache.logging.log4j.util.Strings;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,9 +43,9 @@ import java.nio.file.StandardCopyOption;
 import java.security.Security;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
 
 import static nu.marginalia.mqapi.ProcessInboxNames.CRAWLER_INBOX;
 
@@ -70,6 +67,8 @@ public class CrawlerMain extends ProcessMainClass {
 
     private final Map<String, CrawlTask> pendingCrawlTasks = new ConcurrentHashMap<>();
 
+    private final LinkedBlockingQueue<CrawlTask> retryQueue = new LinkedBlockingQueue<>();
+
     private final AtomicInteger tasksDone = new AtomicInteger(0);
     private final HttpFetcherImpl fetcher;
 
@@ -107,9 +106,18 @@ public class CrawlerMain extends ProcessMainClass {
         this.blacklist = blacklist;
         this.node = processConfiguration.node();
 
+        SimpleBlockingThreadPool.ThreadType threadType;
+        if (Boolean.getBoolean("crawler.useVirtualThreads")) {
+            threadType = SimpleBlockingThreadPool.ThreadType.VIRTUAL;
+        }
+        else {
+            threadType = SimpleBlockingThreadPool.ThreadType.PLATFORM;
+        }
+
         pool = new SimpleBlockingThreadPool("CrawlerPool",
                 Integer.getInteger("crawler.poolSize", 256),
-                1);
+                1,
+                threadType);
 
 
         // Wait for the blacklist to be loaded before starting the crawl
@@ -182,8 +190,6 @@ public class CrawlerMain extends ProcessMainClass {
         // Assign any domains with node_affinity=0 to this node, and then fetch all domains assigned to this node
         // to be crawled.
 
-        performMigration(outputDir);
-
         try (var conn = dataSource.getConnection()) {
             try (var assignFreeDomains = conn.prepareStatement(
                     """
@@ -227,10 +233,7 @@ public class CrawlerMain extends ProcessMainClass {
 
         logger.info("Loaded {} domains", crawlSpecRecords.size());
 
-        // Shuffle the domains to ensure we get a good mix of domains in each crawl,
-        // so that e.g. the big domains don't get all crawled at once, or we end up
-        // crawling the same server in parallel from different subdomains...
-        Collections.shuffle(crawlSpecRecords);
+        crawlSpecRecords.sort(crawlSpecArrangement(crawlSpecRecords));
 
         // First a validation run to ensure the file is all good to parse
         if (crawlSpecRecords.isEmpty()) {
@@ -251,21 +254,53 @@ public class CrawlerMain extends ProcessMainClass {
             // (this happens when the process is restarted after a crash or a shutdown)
             tasksDone.set(workLog.countFinishedJobs());
 
-            // Create crawl tasks and submit them to the pool for execution
+            // List of deferred tasks used to ensure beneficial scheduling of domains with regard to DomainLocks,
+            // merely shuffling the domains tends to lead to a lot of threads being blocked waiting for a semphore,
+            // this will more aggressively attempt to schedule the jobs to avoid blocking
+            List<CrawlTask> taskList = new ArrayList<>();
+
+            // Create crawl tasks
             for (CrawlSpecRecord crawlSpec : crawlSpecRecords) {
-                if (workLog.isJobFinished(crawlSpec.domain()))
+                if (workLog.isJobFinished(crawlSpec.domain))
                     continue;
 
-                var task = new CrawlTask(
-                        crawlSpec,
-                        anchorTagsSource,
-                        outputDir,
-                        warcArchiver,
-                        domainStateDb,
-                        workLog);
+                var task = new CrawlTask(crawlSpec, anchorTagsSource, outputDir, warcArchiver, domainStateDb, workLog);
 
-                if (pendingCrawlTasks.putIfAbsent(crawlSpec.domain(), task) == null) {
-                    pool.submitQuietly(task);
+                // Try to run immediately, to avoid unnecessarily keeping the entire work set in RAM
+                if (!trySubmitDeferredTask(task)) {
+
+                    // Drain the retry queue to the taskList, and try to submit any tasks that are in the retry queue
+                    retryQueue.drainTo(taskList);
+                    taskList.removeIf(this::trySubmitDeferredTask);
+
+                    // Then add this new task to the retry queue
+                    taskList.add(task);
+                }
+            }
+
+             // Schedule viable tasks for execution until list is empty
+            for (int emptyRuns = 0;emptyRuns < 300;) {
+                boolean hasTasks = !taskList.isEmpty();
+
+                // The order of these checks  very important to avoid a race condition
+                // where we miss a task that is put into the retry queue
+                boolean hasRunningTasks = pool.getActiveCount() > 0;
+                boolean hasRetryTasks = !retryQueue.isEmpty();
+
+                if (hasTasks || hasRetryTasks || hasRunningTasks) {
+                    retryQueue.drainTo(taskList);
+
+                    // Try to submit any tasks that are in the retry queue (this will block if the pool is full)
+                    taskList.removeIf(this::trySubmitDeferredTask);
+
+                    // Add a small pause here to avoid busy looping toward the end of the execution cycle when
+                    // we might have no new viable tasks to run for hours on end
+                    TimeUnit.MILLISECONDS.sleep(5);
+                } else {
+                    // We have no tasks to run, and no tasks in the retry queue
+                    // but we wait a bit to see if any new tasks come in via the retry queue
+                    emptyRuns++;
+                    TimeUnit.SECONDS.sleep(1);
                 }
             }
 
@@ -293,6 +328,52 @@ public class CrawlerMain extends ProcessMainClass {
         }
     }
 
+    /** Create a comparator that sorts the crawl specs in a way that is beneficial for the crawl,
+     * we want to enqueue domains that have common top domains first, but otherwise have a random
+     * order.
+     * <p></p>
+     * Note, we can't use hash codes for randomization as it is not desirable to have the same order
+     * every time the process is restarted (and CrawlSpecRecord is a record, which defines equals and
+     * hashcode based on the fields).
+     * */
+    private Comparator<CrawlSpecRecord> crawlSpecArrangement(List<CrawlSpecRecord> records) {
+        Random r = new Random();
+        Map<String, Integer> topDomainCounts = new HashMap<>(4 + (int) Math.sqrt(records.size()));
+        Map<String, Integer> randomOrder = new HashMap<>(records.size());
+
+        for (var spec : records) {
+            topDomainCounts.merge(EdgeDomain.getTopDomain(spec.domain), 1, Integer::sum);
+            randomOrder.put(spec.domain, r.nextInt());
+        }
+
+        return Comparator.comparing((CrawlSpecRecord spec) -> topDomainCounts.getOrDefault(EdgeDomain.getTopDomain(spec.domain), 0) >= 8)
+                .reversed()
+                .thenComparing(spec -> randomOrder.get(spec.domain))
+                .thenComparing(Record::hashCode); // non-deterministic tie-breaker to
+    }
+
+    /** Submit a task for execution if it can be run, returns true if it was submitted
+     * or if it can be discarded */
+    private boolean trySubmitDeferredTask(CrawlTask task) {
+        if (!task.canRun()) {
+            return false;
+        }
+
+        if (pendingCrawlTasks.putIfAbsent(task.domain, task) != null) {
+            return true; // task has already run, duplicate in crawl specs
+        }
+
+        try {
+            // This blocks the caller when the pool is full
+            pool.submitQuietly(task);
+            return true;
+        }
+        catch (RuntimeException ex) {
+            logger.error("Failed to submit task " + task.domain, ex);
+            return false;
+        }
+    }
+
     public void runForSingleDomain(String targetDomainName, FileStorageId fileStorageId) throws Exception {
         runForSingleDomain(targetDomainName, fileStorageService.getStorage(fileStorageId).asPath());
     }
@@ -349,79 +430,117 @@ public class CrawlerMain extends ProcessMainClass {
             this.id = Integer.toHexString(domain.hashCode());
         }
 
+        /** Best effort indicator whether we could start this now without getting stuck in
+         * DomainLocks purgatory */
+        public boolean canRun() {
+            return domainLocks.isLockableHint(new EdgeDomain(domain));
+        }
+
         @Override
         public void run() throws Exception {
 
-            Path newWarcFile = CrawlerOutputFile.createWarcPath(outputDir, id, domain, CrawlerOutputFile.WarcFileVersion.LIVE);
-            Path tempFile = CrawlerOutputFile.createWarcPath(outputDir, id, domain, CrawlerOutputFile.WarcFileVersion.TEMP);
-            Path slopFile = CrawlerOutputFile.createSlopPath(outputDir, id, domain);
-
-            // Move the WARC file to a temp file if it exists, so we can resume the crawl using the old data
-            // while writing to the same file name as before
-            if (Files.exists(newWarcFile)) {
-                Files.move(newWarcFile, tempFile, StandardCopyOption.REPLACE_EXISTING);
-            }
-            else {
-                Files.deleteIfExists(tempFile);
+            if (workLog.isJobFinished(domain)) { // No-Op
+                logger.info("Omitting task {}, as it is already run", domain);
+                return;
             }
 
-            try (var warcRecorder = new WarcRecorder(newWarcFile, fetcher); // write to a temp file for now
-                 var retriever = new CrawlerRetreiver(fetcher, domainProber, specification, domainStateDb, warcRecorder);
-                 CrawlDataReference reference = getReference()
-            )
-            {
-                // Resume the crawl if it was aborted
-                if (Files.exists(tempFile)) {
-                    retriever.syncAbortedRun(tempFile);
-                    Files.delete(tempFile);
+            Optional<DomainLocks.DomainLock> lock = domainLocks.tryLockDomain(new EdgeDomain(domain));
+            // We don't have a lock, so we can't run this task
+            // we return to avoid blocking the pool for too long
+            if (lock.isEmpty()) {
+                if (retryQueue.remainingCapacity() > 0) {
+                    // Sleep a moment to avoid busy looping via the retry queue
+                    // in the case when few tasks remain and almost all are ineligible for
+                    // immediate restart
+                    Thread.sleep(5);
                 }
 
-                DomainLinks domainLinks = anchorTagsSource.getAnchorTags(domain);
+                retryQueue.put(this);
+                return;
+            }
+            DomainLocks.DomainLock domainLock = lock.get();
 
-                int size;
-                try (var lock = domainLocks.lockDomain(new EdgeDomain(domain))) {
-                    size = retriever.crawlDomain(domainLinks, reference);
+            try (domainLock) {
+                Thread.currentThread().setName("crawling:" + domain);
+
+                Path newWarcFile = CrawlerOutputFile.createWarcPath(outputDir, id, domain, CrawlerOutputFile.WarcFileVersion.LIVE);
+                Path tempFile = CrawlerOutputFile.createWarcPath(outputDir, id, domain, CrawlerOutputFile.WarcFileVersion.TEMP);
+                Path slopFile = CrawlerOutputFile.createSlopPath(outputDir, id, domain);
+
+                // Move the WARC file to a temp file if it exists, so we can resume the crawl using the old data
+                // while writing to the same file name as before
+                if (Files.exists(newWarcFile)) {
+                    Files.move(newWarcFile, tempFile, StandardCopyOption.REPLACE_EXISTING);
+                }
+                else {
+                    Files.deleteIfExists(tempFile);
                 }
 
-                // Delete the reference crawl data if it's not the same as the new one
-                // (mostly a case when migrating from legacy->warc)
-                reference.delete();
+                try (var warcRecorder = new WarcRecorder(newWarcFile); // write to a temp file for now
+                     var retriever = new CrawlerRetreiver(fetcher, domainProber, specification, domainStateDb, warcRecorder);
+                     CrawlDataReference reference = getReference())
+                {
+                    // Resume the crawl if it was aborted
+                    if (Files.exists(tempFile)) {
+                        retriever.syncAbortedRun(tempFile);
+                        Files.delete(tempFile);
+                    }
 
-                // Convert the WARC file to Parquet
-                SlopCrawlDataRecord
-                        .convertWarc(domain, userAgent, newWarcFile, slopFile);
+                    DomainLinks domainLinks = anchorTagsSource.getAnchorTags(domain);
 
-                // Optionally archive the WARC file if full retention is enabled,
-                // otherwise delete it:
-                warcArchiver.consumeWarc(newWarcFile, domain);
+                    int size = retriever.crawlDomain(domainLinks, reference);
 
-                // Mark the domain as finished in the work log
-                workLog.setJobToFinished(domain, slopFile.toString(), size);
+                    // Delete the reference crawl data if it's not the same as the new one
+                    // (mostly a case when migrating from legacy->warc)
+                    reference.delete();
 
-                // Update the progress bar
-                heartbeat.setProgress(tasksDone.incrementAndGet() / (double) totalTasks);
+                    // Convert the WARC file to Slop
+                    SlopCrawlDataRecord
+                            .convertWarc(domain, userAgent, newWarcFile, slopFile);
 
-                logger.info("Fetched {}", domain);
-            } catch (Exception e) {
-                logger.error("Error fetching domain " + domain, e);
-            }
-            finally {
-                // We don't need to double-count these; it's also kept int he workLog
-                pendingCrawlTasks.remove(domain);
-                Thread.currentThread().setName("[idle]");
+                    // Optionally archive the WARC file if full retention is enabled,
+                    // otherwise delete it:
+                    warcArchiver.consumeWarc(newWarcFile, domain);
 
-                Files.deleteIfExists(newWarcFile);
-                Files.deleteIfExists(tempFile);
+                    // Mark the domain as finished in the work log
+                    workLog.setJobToFinished(domain, slopFile.toString(), size);
+
+                    // Update the progress bar
+                    heartbeat.setProgress(tasksDone.incrementAndGet() / (double) totalTasks);
+
+                    logger.info("Fetched {}", domain);
+                } catch (Exception e) {
+                    logger.error("Error fetching domain " + domain, e);
+                }
+                finally {
+                    // We don't need to double-count these; it's also kept in the workLog
+                    pendingCrawlTasks.remove(domain);
+                    Thread.currentThread().setName("[idle]");
+
+                    Files.deleteIfExists(newWarcFile);
+                    Files.deleteIfExists(tempFile);
+                }
             }
         }
 
         private CrawlDataReference getReference() {
             try {
-                return new CrawlDataReference(CrawledDomainReader.createDataStream(outputDir, domain, id));
-            } catch (IOException e) {
+                Path slopPath = CrawlerOutputFile.getSlopPath(outputDir, id, domain);
+                if (Files.exists(slopPath)) {
+                    return new CrawlDataReference(slopPath);
+                }
+
+                Path parquetPath = CrawlerOutputFile.getParquetPath(outputDir, id, domain);
+                if (Files.exists(parquetPath)) {
+                    slopPath = migrateParquetData(parquetPath, domain, outputDir);
+                    return new CrawlDataReference(slopPath);
+                }
+
+            } catch (Exception e) {
                 logger.debug("Failed to read previous crawl data for {}", specification.domain());
-                return new CrawlDataReference();
             }
+
+            return new CrawlDataReference();
         }
 
     }
@@ -482,92 +601,19 @@ public class CrawlerMain extends ProcessMainClass {
         }
     }
 
-    // Data migration logic
-
-    private void performMigration(Path root) throws IOException {
-        Path crawlerLog = root.resolve("crawler.log");
-        Path newCrawlerLog = Files.createTempFile(root, "crawler", ".migrate.log");
-
-
-        int finishedTasks = 0;
-        int totalTasks;
-        try (var oldLog = new WorkLog(crawlerLog)) {
-            totalTasks = oldLog.countFinishedJobs();
+    // Migrate from parquet to slop if necessary
+    //
+    // This must be synchronized as chewing through parquet files in parallel leads to enormous memory overhead
+    private synchronized Path migrateParquetData(Path inputPath, String domain, Path crawlDataRoot) throws IOException {
+        if (!inputPath.toString().endsWith(".parquet")) {
+            return inputPath;
         }
 
-        try (WorkLog workLog = new WorkLog(newCrawlerLog);
-            var migrationHeartbeat = heartbeat.createAdHocTaskHeartbeat("MIGRATING")) {
+        Path outputFile = CrawlerOutputFile.createSlopPath(crawlDataRoot, Integer.toHexString(domain.hashCode()), domain);
 
+        SlopCrawlDataRecord.convertFromParquet(inputPath, outputFile);
 
-
-            for (Map.Entry<WorkLogEntry, Path> item : WorkLog.iterableMap(crawlerLog, new CrawlDataLocator(root))) {
-
-                var entry = item.getKey();
-                var path = item.getValue();
-
-                if (path.toFile().getName().endsWith(".parquet")) {
-                    logger.info("Converting {}", entry.id());
-
-                    String domain = entry.id();
-                    String id = Integer.toHexString(domain.hashCode());
-
-                    Path outputFile = CrawlerOutputFile.createSlopPath(root, id, domain);
-
-                    SlopCrawlDataRecord.convertFromParquet(path, outputFile);
-
-                    workLog.setJobToFinished(entry.id(), outputFile.toString(), entry.cnt());
-                }
-                else {
-                    workLog.setJobToFinished(entry.id(), path.toString(), entry.cnt());
-                }
-
-                migrationHeartbeat.progress("Parquet To Slop", ++finishedTasks, totalTasks);
-            }
-        }
-
-        Path oldCrawlerLog = Files.createTempFile(root, "crawler-", ".migrate.old.log");
-        Files.move(crawlerLog, oldCrawlerLog, StandardCopyOption.REPLACE_EXISTING);
-        Files.move(newCrawlerLog, crawlerLog);
-    }
-
-
-    private static class CrawlDataLocator implements Function<WorkLogEntry, Optional<Map.Entry<WorkLogEntry, Path>>> {
-
-        private final Path crawlRootDir;
-
-        CrawlDataLocator(Path crawlRootDir) {
-            this.crawlRootDir = crawlRootDir;
-        }
-
-        @Override
-        public Optional<Map.Entry<WorkLogEntry, Path>> apply(WorkLogEntry entry) {
-            var path = getCrawledFilePath(crawlRootDir, entry.path());
-
-            if (!Files.exists(path)) {
-                return Optional.empty();
-            }
-
-            try {
-                return Optional.of(Map.entry(entry, path));
-            }
-            catch (Exception ex) {
-                return Optional.empty();
-            }
-        }
-
-        private Path getCrawledFilePath(Path crawlDir, String fileName) {
-            int sp = fileName.lastIndexOf('/');
-
-            // Normalize the filename
-            if (sp >= 0 && sp + 1< fileName.length())
-                fileName = fileName.substring(sp + 1);
-            if (fileName.length() < 4)
-                fileName = Strings.repeat("0", 4 - fileName.length()) + fileName;
-
-            String sp1 = fileName.substring(0, 2);
-            String sp2 = fileName.substring(2, 4);
-            return crawlDir.resolve(sp1).resolve(sp2).resolve(fileName);
-        }
+        return outputFile;
     }
 
 }

code/processes/crawling-process/java/nu/marginalia/crawl/DomainStateDb.java

@@ -1,5 +1,8 @@
 package nu.marginalia.crawl;
 
+import com.google.inject.Inject;
+import nu.marginalia.storage.FileStorageService;
+import nu.marginalia.storage.model.FileStorageType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -8,7 +11,9 @@ import java.nio.file.Path;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.time.Duration;
 import java.time.Instant;
+import java.util.Objects;
 import java.util.Optional;
 
 /** Supplemental sqlite database for storing the summary of a crawl.
@@ -20,6 +25,17 @@ public class DomainStateDb implements AutoCloseable {
 
     private final Connection connection;
 
+
+    public record CrawlMeta(
+            String domainName,
+            Instant lastFullCrawl,
+            Duration recrawlTime,
+            Duration crawlTime,
+            int recrawlErrors,
+            int crawlChanges,
+            int totalCrawlSize
+    ) {}
+
     public record SummaryRecord(
             String domainName,
             Instant lastUpdated,
@@ -60,7 +76,31 @@ public class DomainStateDb implements AutoCloseable {
 
     }
 
-    public DomainStateDb(Path filename) throws SQLException {
+    public record FaviconRecord(String contentType, byte[] imageData) {}
+
+    @Inject
+    public DomainStateDb(FileStorageService fileStorageService) throws SQLException {
+        this(findFilename(fileStorageService));
+    }
+
+    private static Path findFilename(FileStorageService fileStorageService) throws SQLException {
+        var fsId = fileStorageService.getOnlyActiveFileStorage(FileStorageType.CRAWL_DATA);
+
+        if (fsId.isPresent()) {
+            var fs = fileStorageService.getStorage(fsId.get());
+            return fs.asPath().resolve("domainstate.db");
+        }
+        else {
+            return null;
+        }
+    }
+
+    public DomainStateDb(@Nullable Path filename) throws SQLException {
+        if (null == filename) {
+            connection = null;
+            return;
+        }
+
         String sqliteDbString = "jdbc:sqlite:" + filename.toString();
         connection = DriverManager.getConnection(sqliteDbString);
 
@@ -74,18 +114,102 @@ public class DomainStateDb implements AutoCloseable {
                         feedUrl TEXT
                     )
                     """);
-
+            stmt.executeUpdate("""
+                    CREATE TABLE IF NOT EXISTS crawl_meta (
+                        domain TEXT PRIMARY KEY,
+                        lastFullCrawlEpochMs LONG NOT NULL,
+                        recrawlTimeMs LONG NOT NULL,
+                        recrawlErrors INTEGER NOT NULL,
+                        crawlTimeMs LONG NOT NULL,
+                        crawlChanges INTEGER NOT NULL,
+                        totalCrawlSize INTEGER NOT NULL
+                    )
+                    """);
+            stmt.executeUpdate("""
+                    CREATE TABLE IF NOT EXISTS favicon (
+                        domain TEXT PRIMARY KEY,
+                        contentType TEXT NOT NULL,
+                        icon BLOB NOT NULL
+                    )
+                    """);
             stmt.execute("PRAGMA journal_mode=WAL");
         }
     }
 
     @Override
     public void close() throws SQLException {
-        connection.close();
+        if (connection != null) {
+            connection.close();
+        }
     }
 
+    public boolean isAvailable() {
+        return connection != null;
+    }
+
+    public void saveIcon(String domain, FaviconRecord faviconRecord) {
+        if (connection == null) throw new IllegalStateException("No connection to domainstate db");
+
+        try (var stmt = connection.prepareStatement("""
+                INSERT OR REPLACE INTO favicon (domain, contentType, icon)
+                       VALUES(?, ?, ?)
+            """)) {
+            stmt.setString(1, domain);
+            stmt.setString(2, Objects.requireNonNullElse(faviconRecord.contentType, "application/octet-stream"));
+            stmt.setBytes(3, faviconRecord.imageData);
+            stmt.executeUpdate();
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to insert favicon", ex);
+        }
+    }
+
+    public Optional<FaviconRecord> getIcon(String domain) {
+        if (connection == null)
+            return Optional.empty();
+
+        try (var stmt = connection.prepareStatement("SELECT contentType, icon FROM favicon WHERE DOMAIN = ?")) {
+            stmt.setString(1, domain);
+            var rs = stmt.executeQuery();
+
+            if (rs.next()) {
+                return Optional.of(
+                    new FaviconRecord(
+                        rs.getString("contentType"),
+                        rs.getBytes("icon")
+                    )
+                );
+            }
+        } catch (SQLException e) {
+            logger.error("Failed to retrieve favicon", e);
+        }
+
+        return Optional.empty();
+    }
+
+    public void save(CrawlMeta crawlMeta) {
+        if (connection == null) throw new IllegalStateException("No connection to domainstate db");
+
+        try (var stmt = connection.prepareStatement("""
+                INSERT OR REPLACE INTO crawl_meta (domain, lastFullCrawlEpochMs, recrawlTimeMs, recrawlErrors, crawlTimeMs, crawlChanges, totalCrawlSize)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                """)) {
+            stmt.setString(1, crawlMeta.domainName());
+            stmt.setLong(2, crawlMeta.lastFullCrawl.toEpochMilli());
+            stmt.setLong(3, crawlMeta.recrawlTime.toMillis());
+            stmt.setInt(4, crawlMeta.recrawlErrors);
+            stmt.setLong(5, crawlMeta.crawlTime.toMillis());
+            stmt.setInt(6, crawlMeta.crawlChanges);
+            stmt.setInt(7, crawlMeta.totalCrawlSize);
+            stmt.executeUpdate();
+        } catch (SQLException e) {
+            logger.error("Failed to insert crawl meta record", e);
+        }
+    }
 
     public void save(SummaryRecord record) {
+        if (connection == null) throw new IllegalStateException("No connection to domainstate db");
+
         try (var stmt = connection.prepareStatement("""
                 INSERT OR REPLACE INTO summary (domain, lastUpdatedEpochMs, state, stateDesc, feedUrl)
                 VALUES (?, ?, ?, ?, ?)
@@ -101,7 +225,38 @@ public class DomainStateDb implements AutoCloseable {
         }
     }
 
-    public Optional<SummaryRecord> get(String domainName) {
+    public Optional<CrawlMeta> getMeta(String domainName) {
+        if (connection == null)
+            return Optional.empty();
+
+        try (var stmt = connection.prepareStatement("""
+                SELECT domain, lastFullCrawlEpochMs, recrawlTimeMs, recrawlErrors, crawlTimeMs, crawlChanges, totalCrawlSize
+                FROM crawl_meta
+                WHERE domain = ?
+                """)) {
+            stmt.setString(1, domainName);
+            var rs = stmt.executeQuery();
+            if (rs.next()) {
+                return Optional.of(new CrawlMeta(
+                        rs.getString("domain"),
+                        Instant.ofEpochMilli(rs.getLong("lastFullCrawlEpochMs")),
+                        Duration.ofMillis(rs.getLong("recrawlTimeMs")),
+                        Duration.ofMillis(rs.getLong("crawlTimeMs")),
+                        rs.getInt("recrawlErrors"),
+                        rs.getInt("crawlChanges"),
+                        rs.getInt("totalCrawlSize")
+                ));
+            }
+        } catch (SQLException ex) {
+            logger.error("Failed to get crawl meta record", ex);
+        }
+        return Optional.empty();
+    }
+
+    public Optional<SummaryRecord> getSummary(String domainName) {
+        if (connection == null)
+            return Optional.empty();
+
         try (var stmt = connection.prepareStatement("""
                 SELECT domain, lastUpdatedEpochMs, state, stateDesc, feedUrl
                 FROM summary

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/ContentTags.java

@@ -1,6 +1,6 @@
 package nu.marginalia.crawl.fetcher;
 
-import java.net.http.HttpRequest;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
 
 /** Encapsulates request modifiers; the ETag and Last-Modified tags for a resource */
 public record ContentTags(String etag, String lastMod) {
@@ -17,14 +17,16 @@ public record ContentTags(String etag, String lastMod) {
     }
 
     /** Paints the tags onto the request builder. */
-    public void paint(HttpRequest.Builder getBuilder) {
+    public void paint(HttpGet request) {
+
+        // Paint the ETag header if present,
+        // otherwise paint the Last-Modified header
+        // (but not both at the same time due to some servers not liking it)
 
         if (etag != null) {
-            getBuilder.header("If-None-Match", etag);
-        }
-
-        if (lastMod != null) {
-            getBuilder.header("If-Modified-Since", lastMod);
+            request.addHeader("If-None-Match", etag);
+        } else if (lastMod != null) {
+            request.addHeader("If-Modified-Since", lastMod);
         }
     }
 }

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/Cookies.java

@@ -1,34 +0,0 @@
-package nu.marginalia.crawl.fetcher;
-
-import java.io.IOException;
-import java.net.CookieHandler;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Cookies extends CookieHandler {
-    final ThreadLocal<ConcurrentHashMap<String, List<String>>> cookieJar = ThreadLocal.withInitial(ConcurrentHashMap::new);
-
-    public void clear() {
-        cookieJar.get().clear();
-    }
-
-    public boolean hasCookies() {
-        return !cookieJar.get().isEmpty();
-    }
-
-    public List<String> getCookies() {
-        return cookieJar.get().values().stream().flatMap(List::stream).toList();
-    }
-
-    @Override
-    public Map<String, List<String>> get(URI uri, Map<String, List<String>> requestHeaders) throws IOException {
-        return cookieJar.get();
-    }
-
-    @Override
-    public void put(URI uri, Map<String, List<String>> responseHeaders) throws IOException {
-        cookieJar.get().putAll(responseHeaders);
-    }
-}

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/DomainCookies.java

@@ -0,0 +1,56 @@
+package nu.marginalia.crawl.fetcher;
+
+import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.HttpResponse;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.StringJoiner;
+
+public class DomainCookies {
+    private final Map<String, String> cookies = new HashMap<>();
+
+    public boolean hasCookies() {
+        return !cookies.isEmpty();
+    }
+
+    public void updateCookieStore(HttpResponse response) {
+        for (var header : response.getHeaders()) {
+            if (header.getName().equalsIgnoreCase("Set-Cookie")) {
+                parseCookieHeader(header.getValue());
+            }
+        }
+    }
+
+    private void parseCookieHeader(String value) {
+        // Parse the Set-Cookie header value and extract the cookies
+
+        String[] parts = value.split(";");
+        String cookie = parts[0].trim();
+
+        if (cookie.contains("=")) {
+            String[] cookieParts = cookie.split("=");
+            String name = cookieParts[0].trim();
+            String val = cookieParts[1].trim();
+            cookies.put(name, val);
+        }
+    }
+
+    public void paintRequest(HttpUriRequestBase request) {
+        request.addHeader("Cookie", createCookieHeader());
+    }
+
+    public void paintRequest(ClassicHttpRequest request) {
+        request.addHeader("Cookie", createCookieHeader());
+    }
+
+    private String createCookieHeader() {
+        StringJoiner sj = new StringJoiner("; ");
+        for (var cookie : cookies.entrySet()) {
+            sj.add(cookie.getKey() + "=" + cookie.getValue());
+        }
+        return sj.toString();
+    }
+
+}

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcher.java

@@ -8,6 +8,7 @@ import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.body.HttpFetchResult;
 import nu.marginalia.model.crawldata.CrawlerDomainStatus;
+import org.apache.hc.client5.http.cookie.CookieStore;
 
 import java.util.List;
 
@@ -15,20 +16,17 @@ import java.util.List;
 public interface HttpFetcher extends AutoCloseable {
     void setAllowAllContentTypes(boolean allowAllContentTypes);
 
-    Cookies getCookies();
+    CookieStore getCookies();
     void clearCookies();
 
     DomainProbeResult probeDomain(EdgeUrl url);
 
-    ContentTypeProbeResult probeContentType(
-                                EdgeUrl url,
-                                WarcRecorder recorder,
-                                ContentTags tags) throws HttpFetcherImpl.RateLimitException;
-
     HttpFetchResult fetchContent(EdgeUrl url,
                                  WarcRecorder recorder,
+                                 DomainCookies cookies,
+                                 CrawlDelayTimer timer,
                                  ContentTags tags,
-                                 ProbeType probeType) throws Exception;
+                                 ProbeType probeType);
 
     List<EdgeUrl> fetchSitemapUrls(String rootSitemapUrl, CrawlDelayTimer delayTimer);
 
@@ -46,6 +44,7 @@ public interface HttpFetcher extends AutoCloseable {
 
         /** This domain redirects to another domain */
         record Redirect(EdgeDomain domain) implements DomainProbeResult {}
+        record RedirectSameDomain_Internal(EdgeUrl domain) implements DomainProbeResult {}
 
         /** If the retrieval of the probed url was successful, return the url as it was fetched
          * (which may be different from the url we probed, if we attempted another URL schema).
@@ -56,7 +55,10 @@ public interface HttpFetcher extends AutoCloseable {
     }
 
     sealed interface ContentTypeProbeResult {
+        record NoOp() implements ContentTypeProbeResult {}
         record Ok(EdgeUrl resolvedUrl) implements ContentTypeProbeResult { }
+        record HttpError(int statusCode, String message) implements ContentTypeProbeResult { }
+        record Redirect(EdgeUrl location) implements ContentTypeProbeResult { }
         record BadContentType(String contentType, int statusCode) implements ContentTypeProbeResult { }
         record Timeout(java.lang.Exception ex) implements ContentTypeProbeResult { }
         record Exception(java.lang.Exception ex) implements ContentTypeProbeResult { }

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -5,68 +5,169 @@ import com.google.inject.Singleton;
 import crawlercommons.robots.SimpleRobotRules;
 import crawlercommons.robots.SimpleRobotRulesParser;
 import nu.marginalia.UserAgent;
-import nu.marginalia.crawl.fetcher.socket.NoSecuritySSL;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
 import nu.marginalia.crawl.retreival.CrawlDelayTimer;
+import nu.marginalia.link_parser.LinkParser;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.body.ContentTypeLogic;
 import nu.marginalia.model.body.DocumentBodyExtractor;
 import nu.marginalia.model.body.HttpFetchResult;
 import nu.marginalia.model.crawldata.CrawlerDomainStatus;
+import org.apache.hc.client5.http.ConnectionKeepAliveStrategy;
+import org.apache.hc.client5.http.HttpRequestRetryStrategy;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.cookie.BasicCookieStore;
+import org.apache.hc.client5.http.cookie.CookieStore;
+import org.apache.hc.client5.http.cookie.StandardCookieSpec;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy;
+import org.apache.hc.core5.http.*;
+import org.apache.hc.core5.http.io.HttpClientResponseHandler;
+import org.apache.hc.core5.http.io.SocketConfig;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
+import org.apache.hc.core5.http.message.MessageSupport;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.pool.PoolStats;
+import org.apache.hc.core5.util.TimeValue;
+import org.apache.hc.core5.util.Timeout;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.parser.Parser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
 import java.io.IOException;
-import java.io.InputStream;
+import java.net.SocketTimeoutException;
 import java.net.URISyntaxException;
-import java.net.URLDecoder;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
-import java.net.http.HttpTimeoutException;
-import java.nio.charset.StandardCharsets;
+import java.net.UnknownHostException;
+import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.zip.GZIPInputStream;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 
 @Singleton
-public class HttpFetcherImpl implements HttpFetcher {
+public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
     private final String userAgentString;
     private final String userAgentIdentifier;
-    private final Cookies cookies = new Cookies();
+
+    private final CookieStore cookies = new BasicCookieStore();
 
     private static final SimpleRobotRulesParser robotsParser = new SimpleRobotRulesParser();
     private static final ContentTypeLogic contentTypeLogic = new ContentTypeLogic();
+    private final Marker crawlerAuditMarker = MarkerFactory.getMarker("CRAWLER");
 
-    private final Duration requestTimeout = Duration.ofSeconds(10);
-
+    private final LinkParser linkParser = new LinkParser();
     @Override
     public void setAllowAllContentTypes(boolean allowAllContentTypes) {
         contentTypeLogic.setAllowAllContentTypes(allowAllContentTypes);
     }
 
-    private final HttpClient client;
+    private final CloseableHttpClient client;
+    private PoolingHttpClientConnectionManager connectionManager;
 
-    private HttpClient createClient() {
-        return HttpClient.newBuilder()
-                .sslContext(NoSecuritySSL.buildSslContext())
-                .cookieHandler(cookies)
-                .followRedirects(HttpClient.Redirect.NORMAL)
-                .connectTimeout(Duration.ofSeconds(8))
-                .executor(Executors.newCachedThreadPool())
+    public PoolStats getPoolStats() {
+        return connectionManager.getTotalStats();
+    }
+
+    private CloseableHttpClient createClient() throws NoSuchAlgorithmException {
+        final ConnectionConfig connectionConfig = ConnectionConfig.custom()
+                .setSocketTimeout(10, TimeUnit.SECONDS)
+                .setConnectTimeout(30, TimeUnit.SECONDS)
+                .setValidateAfterInactivity(TimeValue.ofSeconds(5))
+                .build();
+
+        connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
+                .setMaxConnPerRoute(2)
+                .setMaxConnTotal(5000)
+                .setDefaultConnectionConfig(connectionConfig)
+                .setTlsSocketStrategy(new DefaultClientTlsStrategy(SSLContext.getDefault()))
+                .build();
+
+        connectionManager.setDefaultSocketConfig(SocketConfig.custom()
+                .setSoLinger(TimeValue.ofSeconds(-1))
+                .setSoTimeout(Timeout.ofSeconds(10))
+                .build()
+        );
+
+        Thread.ofPlatform().daemon(true).start(() -> {
+            try {
+                for (;;) {
+                    TimeUnit.SECONDS.sleep(15);
+                    logger.info("Connection pool stats: {}", connectionManager.getTotalStats());
+                }
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        });
+
+        final RequestConfig defaultRequestConfig = RequestConfig.custom()
+                .setCookieSpec(StandardCookieSpec.RELAXED)
+                .setResponseTimeout(10, TimeUnit.SECONDS)
+                .setConnectionRequestTimeout(5, TimeUnit.MINUTES)
+                .build();
+
+        return HttpClients.custom()
+                .setDefaultCookieStore(cookies)
+                .setConnectionManager(connectionManager)
+                .setRetryStrategy(this)
+                .setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
+                    // Default keep-alive duration is 3 minutes, but this is too long for us,
+                    // as we are either going to re-use it fairly quickly or close it for a long time.
+                    //
+                    // So we set it to 30 seconds or clamp the server-provided value to a minimum of 10 seconds.
+                    private static final TimeValue defaultValue = TimeValue.ofSeconds(30);
+
+                    @Override
+                    public TimeValue getKeepAliveDuration(HttpResponse response, HttpContext context) {
+                        final Iterator<HeaderElement> it = MessageSupport.iterate(response, HeaderElements.KEEP_ALIVE);
+
+                        while (it.hasNext()) {
+                            final HeaderElement he = it.next();
+                            final String param = he.getName();
+                            final String value = he.getValue();
+
+                            if (value == null)
+                                continue;
+                            if (!"timeout".equalsIgnoreCase(param))
+                                continue;
+
+                            try {
+                                long timeout = Long.parseLong(value);
+                                timeout = Math.clamp(timeout, 30, defaultValue.toSeconds());
+                                return TimeValue.ofSeconds(timeout);
+                            } catch (final NumberFormatException ignore) {
+                                break;
+                            }
+                        }
+                        return defaultValue;
+                    }
+                })
+                .disableRedirectHandling()
+                .setDefaultRequestConfig(defaultRequestConfig)
                 .build();
     }
 
     @Override
-    public Cookies getCookies() {
+    public CookieStore getCookies() {
         return cookies;
     }
 
@@ -78,19 +179,27 @@ public class HttpFetcherImpl implements HttpFetcher {
     @Inject
     public HttpFetcherImpl(UserAgent userAgent)
     {
-        this.client = createClient();
+        try {
+            this.client = createClient();
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
         this.userAgentString = userAgent.uaString();
         this.userAgentIdentifier = userAgent.uaIdentifier();
     }
 
     public HttpFetcherImpl(String userAgent) {
-        this.client = createClient();
+        try {
+            this.client = createClient();
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
         this.userAgentString = userAgent;
         this.userAgentIdentifier = userAgent;
     }
 
     // Not necessary in prod, but useful in test
-    public void close() {
+    public void close() throws IOException {
         client.close();
     }
 
@@ -103,30 +212,94 @@ public class HttpFetcherImpl implements HttpFetcher {
      */
     @Override
     public DomainProbeResult probeDomain(EdgeUrl url) {
-        HttpRequest head;
-        try {
-            head = HttpRequest.newBuilder()
-                    .HEAD()
-                    .uri(url.asURI())
-                    .header("User-agent", userAgentString)
-                    .timeout(requestTimeout)
-                    .build();
-        } catch (URISyntaxException e) {
-            return new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Invalid URL");
-        }
+        List<EdgeUrl> urls = new ArrayList<>();
+        urls.add(url);
 
-        try {
-            var rsp = client.send(head, HttpResponse.BodyHandlers.discarding());
-            EdgeUrl rspUri = new EdgeUrl(rsp.uri());
+        int redirects = 0;
+        AtomicBoolean tryGet = new AtomicBoolean(false);
 
-            if (!Objects.equals(rspUri.domain, url.domain)) {
-                return new DomainProbeResult.Redirect(rspUri.domain);
+        while (!urls.isEmpty() && ++redirects < 5) {
+            ClassicHttpRequest request;
+
+            EdgeUrl topUrl = urls.removeFirst();
+            try {
+                if (tryGet.get()) {
+                    request = ClassicRequestBuilder.get(topUrl.asURI())
+                                .addHeader("User-Agent", userAgentString)
+                                .addHeader("Accept-Encoding", "gzip")
+                                .addHeader("Range", "bytes=0-255")
+                                .build();
+                } else {
+                    request = ClassicRequestBuilder.head(topUrl.asURI())
+                                .addHeader("User-Agent", userAgentString)
+                                .addHeader("Accept-Encoding", "gzip")
+                                .build();
+                }
+            } catch (URISyntaxException e) {
+                return new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Invalid URL");
             }
-            return new DomainProbeResult.Ok(rspUri);
-        }
-        catch (Exception ex) {
-            return new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, ex.getMessage());
+
+            try {
+                var result = SendLock.wrapSend(client, request, response -> {
+                    EntityUtils.consume(response.getEntity());
+
+                    return switch (response.getCode()) {
+                        case 200 -> new DomainProbeResult.Ok(url);
+                        case 405 -> {
+                            if (!tryGet.get()) {
+                                tryGet.set(true);
+                                yield new DomainProbeResult.RedirectSameDomain_Internal(url);
+                            }
+                            else {
+                                yield new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "HTTP status 405, tried HEAD and GET?!");
+                            }
+                        }
+                        case 301, 302, 307 -> {
+                            var location = response.getFirstHeader("Location");
+
+                            if (location != null) {
+                                Optional<EdgeUrl> newUrl = linkParser.parseLink(topUrl, location.getValue());
+                                if (newUrl.isEmpty()) {
+                                    yield new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Invalid location header on redirect");
+                                }
+                                EdgeUrl newEdgeUrl = newUrl.get();
+                                if (newEdgeUrl.domain.equals(topUrl.domain)) {
+                                    yield new DomainProbeResult.RedirectSameDomain_Internal(newEdgeUrl);
+                                }
+                                else {
+                                    yield new DomainProbeResult.Redirect(newEdgeUrl.domain);
+                                }
+                            }
+
+                            yield new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "No location header on redirect");
+
+                        }
+                        default ->
+                                new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "HTTP status " + response.getCode());
+                    };
+                });
+
+                if (result instanceof DomainProbeResult.RedirectSameDomain_Internal(EdgeUrl redirUrl)) {
+                    urls.add(redirUrl);
+                }
+                else {
+                    return result;
+                }
+
+                // We don't have robots.txt yet, so we'll assume a request delay of 1 second
+                TimeUnit.SECONDS.sleep(1);
+            }
+            catch (SocketTimeoutException ex) {
+                return new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Timeout during domain probe");
+            }
+            catch (Exception ex) {
+                return new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Error during domain probe");
+            }
+
         }
+
+        return new DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Failed to resolve domain root");
+
     }
 
     /** Perform a HEAD request to fetch the content type of a URL.
@@ -137,70 +310,73 @@ public class HttpFetcherImpl implements HttpFetcher {
      * recorded in the WARC file on failure.
      */
     public ContentTypeProbeResult probeContentType(EdgeUrl url,
-                                                   WarcRecorder warcRecorder,
-                                                   ContentTags tags) throws RateLimitException {
-        if (tags.isEmpty() && contentTypeLogic.isUrlLikeBinary(url)) {
-
-            try {
-                var headBuilder = HttpRequest.newBuilder()
-                    .HEAD()
-                    .uri(url.asURI())
-                    .header("User-agent", userAgentString)
-                    .header("Accept-Encoding", "gzip")
-                    .timeout(requestTimeout)
-                    ;
-
-                var rsp = client.send(headBuilder.build(), HttpResponse.BodyHandlers.discarding());
-                var headers = rsp.headers();
-
-                var contentTypeHeader = headers.firstValue("Content-Type").orElse(null);
-
-                if (contentTypeHeader != null && !contentTypeLogic.isAllowableContentType(contentTypeHeader)) {
-                    warcRecorder.flagAsFailedContentTypeProbe(url, contentTypeHeader, rsp.statusCode());
-
-                    return new ContentTypeProbeResult.BadContentType(contentTypeHeader, rsp.statusCode());
-                }
-
-                // Update the URL to the final URL of the HEAD request, otherwise we might end up doing
-
-                // HEAD 301 url1 -> url2
-                // HEAD 200 url2
-                // GET 301 url1 -> url2
-                // GET 200 url2
-
-                // which is not what we want. Overall we want to do as few requests as possible to not raise
-                // too many eyebrows when looking at the logs on the target server.  Overall it's probably desirable
-                // that it looks like the traffic makes sense, as opposed to looking like a broken bot.
-
-                var redirectUrl = new EdgeUrl(rsp.uri());
-                EdgeUrl ret;
-
-                if (Objects.equals(redirectUrl.domain, url.domain)) ret = redirectUrl;
-                else ret = url;
-
-                // Intercept rate limiting
-                if (rsp.statusCode() == 429) {
-                    throw new HttpFetcherImpl.RateLimitException(headers.firstValue("Retry-After").orElse("1"));
-                }
-
-                return new ContentTypeProbeResult.Ok(ret);
-            }
-            catch (HttpTimeoutException ex) {
-                warcRecorder.flagAsTimeout(url);
-                return new ContentTypeProbeResult.Timeout(ex);
-            }
-            catch (RateLimitException ex) {
-                throw ex;
-            }
-            catch (Exception ex) {
-                logger.error("Error during fetching {}[{}]", ex.getClass().getSimpleName(), ex.getMessage());
-
-                warcRecorder.flagAsError(url, ex);
-
-                return new ContentTypeProbeResult.Exception(ex);
-            }
+                                                   DomainCookies cookies,
+                                                   CrawlDelayTimer timer,
+                                                   ContentTags tags) {
+        if (!tags.isEmpty() || !contentTypeLogic.isUrlLikeBinary(url)) {
+            return new ContentTypeProbeResult.NoOp();
+        }
+
+        try {
+            ClassicHttpRequest head = ClassicRequestBuilder.head(url.asURI())
+                    .addHeader("User-Agent", userAgentString)
+                    .addHeader("Accept-Encoding", "gzip")
+                    .build();
+
+            cookies.paintRequest(head);
+
+            return SendLock.wrapSend(client, head, (rsp) -> {
+                cookies.updateCookieStore(rsp);
+                EntityUtils.consume(rsp.getEntity());
+                int statusCode = rsp.getCode();
+
+                // Handle redirects
+                if (statusCode == 301 || statusCode == 302 || statusCode == 307) {
+                    var location = rsp.getFirstHeader("Location");
+                    if (location != null) {
+                        Optional<EdgeUrl> newUrl = linkParser.parseLink(url, location.getValue());
+                        if (newUrl.isEmpty())
+                            return new ContentTypeProbeResult.HttpError(statusCode, "Invalid location header on redirect");
+                        return new ContentTypeProbeResult.Redirect(newUrl.get());
+                    }
+                }
+
+                if (statusCode == 405) {
+                    // If we get a 405, we can't probe the content type with HEAD, so we'll just say it's ok
+                    return new ContentTypeProbeResult.Ok(url);
+                }
+
+                // Handle errors
+                if (statusCode < 200 || statusCode > 300) {
+                    return new ContentTypeProbeResult.HttpError(statusCode, "Bad status code");
+                }
+
+                // Handle missing content type
+                var ctHeader = rsp.getFirstHeader("Content-Type");
+                if (ctHeader == null) {
+                    return new ContentTypeProbeResult.HttpError(statusCode, "Missing Content-Type header");
+                }
+                var contentType = ctHeader.getValue();
+
+                // Check if the content type is allowed
+                if (contentTypeLogic.isAllowableContentType(contentType)) {
+                    return new ContentTypeProbeResult.Ok(url);
+                } else {
+                    return new ContentTypeProbeResult.BadContentType(contentType, statusCode);
+                }
+            });
+        }
+        catch (SocketTimeoutException ex) {
+
+            return new ContentTypeProbeResult.Timeout(ex);
+        }
+        catch (Exception ex) {
+            logger.error("Error during fetching {}[{}]", ex.getClass().getSimpleName(), ex.getMessage());
+            return new ContentTypeProbeResult.Exception(ex);
+        }
+        finally {
+            timer.waitFetchDelay();
         }
-        return new ContentTypeProbeResult.Ok(url);
     }
 
     /** Fetch the content of a URL, and record it in a WARC file,
@@ -210,37 +386,87 @@ public class HttpFetcherImpl implements HttpFetcher {
     @Override
     public HttpFetchResult fetchContent(EdgeUrl url,
                                            WarcRecorder warcRecorder,
+                                           DomainCookies cookies,
+                                           CrawlDelayTimer timer,
                                            ContentTags contentTags,
                                            ProbeType probeType)
-        throws Exception
     {
-        var getBuilder = HttpRequest.newBuilder()
-                .GET()
-                .uri(url.asURI())
-                .header("User-agent", userAgentString)
-                .header("Accept-Encoding", "gzip")
-                .header("Accept-Language", "en,*;q=0.5")
-                .header("Accept", "text/html, application/xhtml+xml, text/*;q=0.8")
-                .timeout(requestTimeout)
-                ;
+        try {
+            if (probeType == HttpFetcher.ProbeType.FULL) {
+                try {
+                    var probeResult = probeContentType(url, cookies, timer, contentTags);
 
-        contentTags.paint(getBuilder);
+                    switch (probeResult) {
+                        case HttpFetcher.ContentTypeProbeResult.NoOp():
+                            break; //
+                        case HttpFetcher.ContentTypeProbeResult.Ok(EdgeUrl resolvedUrl):
+                            logger.info(crawlerAuditMarker, "Probe result OK for {}", url);
+                            url = resolvedUrl; // If we were redirected while probing, use the final URL for fetching
+                            break;
+                        case ContentTypeProbeResult.BadContentType badContentType:
+                            warcRecorder.flagAsFailedContentTypeProbe(url, badContentType.contentType(), badContentType.statusCode());
+                            logger.info(crawlerAuditMarker, "Probe result Bad ContenType ({}) for {}", badContentType.contentType(), url);
+                            return new HttpFetchResult.ResultNone();
+                        case ContentTypeProbeResult.BadContentType.Timeout(Exception ex):
+                            logger.info(crawlerAuditMarker, "Probe result Timeout for {}", url);
+                            warcRecorder.flagAsTimeout(url);
+                            return new HttpFetchResult.ResultException(ex);
+                        case ContentTypeProbeResult.Exception(Exception ex):
+                            logger.info(crawlerAuditMarker, "Probe result Exception({}) for {}", ex.getClass().getSimpleName(), url);
+                            warcRecorder.flagAsError(url, ex);
+                            return new HttpFetchResult.ResultException(ex);
+                        case ContentTypeProbeResult.HttpError httpError:
+                            logger.info(crawlerAuditMarker, "Probe result HTTP Error ({}) for {}", httpError.statusCode(), url);
+                            return new HttpFetchResult.ResultException(new HttpException("HTTP status code " + httpError.statusCode() + ": " + httpError.message()));
+                        case ContentTypeProbeResult.Redirect redirect:
+                            logger.info(crawlerAuditMarker, "Probe result redirect for {} -> {}", url, redirect.location());
+                            return new HttpFetchResult.ResultRedirect(redirect.location());
+                    }
+                } catch (Exception ex) {
+                    logger.warn("Failed to fetch {}", url, ex);
+                    return new HttpFetchResult.ResultException(ex);
+                }
 
-        HttpFetchResult result = warcRecorder.fetch(client, getBuilder.build());
-
-        if (result instanceof HttpFetchResult.ResultOk ok) {
-            if (ok.statusCode() == 429) {
-                throw new RateLimitException(Objects.requireNonNullElse(ok.header("Retry-After"), "1"));
             }
-            if (ok.statusCode() == 304) {
-                return new HttpFetchResult.Result304Raw();
-            }
-            if (ok.statusCode() == 200) {
-                return ok;
+
+            HttpGet request = new HttpGet(url.asURI());
+            request.addHeader("User-Agent", userAgentString);
+            request.addHeader("Accept-Encoding", "gzip");
+            request.addHeader("Accept-Language", "en,*;q=0.5");
+            request.addHeader("Accept", "text/html, application/xhtml+xml, text/*;q=0.8");
+
+            contentTags.paint(request);
+
+            try (var sl = new SendLock()) {
+                Instant start = Instant.now();
+                HttpFetchResult result = warcRecorder.fetch(client, cookies, request);
+
+                Duration fetchDuration = Duration.between(start, Instant.now());
+
+                if (result instanceof HttpFetchResult.ResultOk ok) {
+                    if (ok.statusCode() == 304) {
+                        result = new HttpFetchResult.Result304Raw();
+                    }
+                }
+
+                switch (result) {
+                    case HttpFetchResult.ResultOk ok -> logger.info(crawlerAuditMarker, "Fetch result OK {} for {} ({} ms)", ok.statusCode(), url, fetchDuration.toMillis());
+                    case HttpFetchResult.ResultRedirect redirect -> logger.info(crawlerAuditMarker, "Fetch result redirect: {}  for {}", redirect.url(), url);
+                    case HttpFetchResult.ResultNone none -> logger.info(crawlerAuditMarker, "Fetch result none for {}", url);
+                    case HttpFetchResult.ResultException ex -> logger.error(crawlerAuditMarker, "Fetch result exception for {}", url, ex.ex());
+                    case HttpFetchResult.Result304Raw raw -> logger.info(crawlerAuditMarker, "Fetch result: 304 Raw for {}", url);
+                    case HttpFetchResult.Result304ReplacedWithReference ref -> logger.info(crawlerAuditMarker, "Fetch result: 304 With reference for {}", url);
+                }
+
+                return result;
             }
         }
+        catch (Exception ex) {
+            logger.error(crawlerAuditMarker, "Fetch result exception for {}", url, ex);
+
+            return new HttpFetchResult.ResultException(ex);
+        }
 
-        return result;
     }
 
     @Override
@@ -248,6 +474,7 @@ public class HttpFetcherImpl implements HttpFetcher {
         return new SitemapRetriever();
     }
 
+    /** Recursively fetch sitemaps */
     @Override
     public List<EdgeUrl> fetchSitemapUrls(String root, CrawlDelayTimer delayTimer) {
         try {
@@ -267,7 +494,7 @@ public class HttpFetcherImpl implements HttpFetcher {
             while (!sitemapQueue.isEmpty() && ret.size() < 20_000 && ++fetchedSitemaps < 10) {
                 var head = sitemapQueue.removeFirst();
 
-                switch (fetchSitemap(head)) {
+                switch (fetchSingleSitemap(head)) {
                     case SitemapResult.SitemapUrls(List<String> urls) -> {
 
                         for (var url : urls) {
@@ -303,62 +530,66 @@ public class HttpFetcherImpl implements HttpFetcher {
     }
 
 
-    private SitemapResult fetchSitemap(EdgeUrl sitemapUrl) throws URISyntaxException, IOException, InterruptedException {
-        HttpRequest getRequest = HttpRequest.newBuilder()
-                .GET()
-                .uri(sitemapUrl.asURI())
-                .header("Accept-Encoding", "gzip")
-                .header("Accept", "text/*, */*;q=0.9")
-                .header("User-agent", userAgentString)
-                .timeout(requestTimeout)
-                .build();
+    private SitemapResult fetchSingleSitemap(EdgeUrl sitemapUrl) throws URISyntaxException {
+        HttpGet getRequest = new HttpGet(sitemapUrl.asURI());
 
-        var response = client.send(getRequest, HttpResponse.BodyHandlers.ofInputStream());
-        if (response.statusCode() != 200) {
-            return new SitemapResult.SitemapError();
+        getRequest.addHeader("User-Agent", userAgentString);
+        getRequest.addHeader("Accept-Encoding", "gzip");
+        getRequest.addHeader("Accept", "text/*, */*;q=0.9");
+        getRequest.addHeader("User-Agent", userAgentString);
+
+        try (var sl = new SendLock()) {
+            return client.execute(getRequest, response -> {
+                try {
+                    if (response.getCode() != 200) {
+                        return new SitemapResult.SitemapError();
+                    }
+
+                    Document parsedSitemap = Jsoup.parse(
+                            EntityUtils.toString(response.getEntity()),
+                            sitemapUrl.toString(),
+                            Parser.xmlParser()
+                    );
+
+                    if (parsedSitemap.childrenSize() == 0) {
+                        return new SitemapResult.SitemapError();
+                    }
+
+                    String rootTagName = parsedSitemap.child(0).tagName();
+
+                    return switch (rootTagName.toLowerCase()) {
+                        case "sitemapindex" -> {
+                            List<String> references = new ArrayList<>();
+                            for (var locTag : parsedSitemap.getElementsByTag("loc")) {
+                                references.add(locTag.text().trim());
+                            }
+                            yield new SitemapResult.SitemapReferences(Collections.unmodifiableList(references));
+                        }
+                        case "urlset" -> {
+                            List<String> urls = new ArrayList<>();
+                            for (var locTag : parsedSitemap.select("url > loc")) {
+                                urls.add(locTag.text().trim());
+                            }
+                            yield new SitemapResult.SitemapUrls(Collections.unmodifiableList(urls));
+                        }
+                        case "rss", "atom" -> {
+                            List<String> urls = new ArrayList<>();
+                            for (var locTag : parsedSitemap.select("link, url")) {
+                                urls.add(locTag.text().trim());
+                            }
+                            yield new SitemapResult.SitemapUrls(Collections.unmodifiableList(urls));
+                        }
+                        default -> new SitemapResult.SitemapError();
+                    };
+                }
+                finally {
+                    EntityUtils.consume(response.getEntity());
+                }
+            });
         }
-
-        try (InputStream inputStream = response.body()) {
-
-            InputStream parserStream;
-            if (sitemapUrl.path.endsWith(".gz")) {
-                parserStream = new GZIPInputStream(inputStream);
-            }
-            else {
-                parserStream = inputStream;
-            }
-
-            Document parsedSitemap = Jsoup.parse(parserStream, "UTF-8", sitemapUrl.toString(), Parser.xmlParser());
-            if (parsedSitemap.childrenSize() == 0) {
-                return new SitemapResult.SitemapError();
-            }
-
-            String rootTagName = parsedSitemap.child(0).tagName();
-
-            return switch (rootTagName.toLowerCase()) {
-                case "sitemapindex" -> {
-                    List<String> references = new ArrayList<>();
-                    for (var locTag : parsedSitemap.getElementsByTag("loc")) {
-                        references.add(locTag.text().trim());
-                    }
-                    yield new SitemapResult.SitemapReferences(Collections.unmodifiableList(references));
-                }
-                case "urlset" -> {
-                    List<String> urls = new ArrayList<>();
-                    for (var locTag : parsedSitemap.select("url > loc")) {
-                        urls.add(locTag.text().trim());
-                    }
-                    yield new SitemapResult.SitemapUrls(Collections.unmodifiableList(urls));
-                }
-                case "rss", "atom" -> {
-                    List<String> urls = new ArrayList<>();
-                    for (var locTag : parsedSitemap.select("link, url")) {
-                        urls.add(locTag.text().trim());
-                    }
-                    yield new SitemapResult.SitemapUrls(Collections.unmodifiableList(urls));
-                }
-                default -> new SitemapResult.SitemapError();
-            };
+        catch (Exception ex) {
+            logger.warn("Error while fetching sitemap {}: {} ({})", sitemapUrl, ex.getClass().getSimpleName(), ex.getMessage());
+            return new SitemapResult.SitemapError();
         }
     }
 
@@ -382,16 +613,14 @@ public class HttpFetcherImpl implements HttpFetcher {
     }
 
     private Optional<SimpleRobotRules> fetchAndParseRobotsTxt(EdgeUrl url, WarcRecorder recorder) {
-        try {
-            var getRequest = HttpRequest.newBuilder()
-                    .GET()
-                    .uri(url.asURI())
-                    .header("Accept-Encoding", "gzip")
-                    .header("Accept", "text/*, */*;q=0.9")
-                    .header("User-agent", userAgentString)
-                    .timeout(requestTimeout);
+        try (var sl = new SendLock()) {
 
-            HttpFetchResult result = recorder.fetch(client, getRequest.build());
+            HttpGet request = new HttpGet(url.asURI());
+            request.addHeader("User-Agent", userAgentString);
+            request.addHeader("Accept-Encoding", "gzip");
+            request.addHeader("Accept", "text/*, */*;q=0.9");
+
+            HttpFetchResult result = recorder.fetch(client, new DomainCookies(), request);
 
             return DocumentBodyExtractor.asBytes(result).mapOpt((contentType, body) ->
                 robotsParser.parseContent(url.toString(),
@@ -405,6 +634,57 @@ public class HttpFetcherImpl implements HttpFetcher {
         }
     }
 
+    @Override
+    public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context) {
+        return switch (exception) {
+            case SocketTimeoutException ste -> false;
+            case SSLException ssle -> false;
+            case UnknownHostException uhe -> false;
+            default -> executionCount <= 3;
+        };
+    }
+
+    @Override
+    public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
+        return switch (response.getCode()) {
+            case 500, 503 -> executionCount <= 2;
+            case 429 -> executionCount <= 3;
+            default -> false;
+        };
+    }
+
+    @Override
+    public TimeValue getRetryInterval(HttpRequest request, IOException exception, int executionCount, HttpContext context) {
+        return TimeValue.ofSeconds(1);
+    }
+
+    @Override
+    public TimeValue getRetryInterval(HttpResponse response, int executionCount, HttpContext context) {
+
+        int statusCode = response.getCode();
+
+        // Give 503 a bit more time
+        if (statusCode == 503) return TimeValue.ofSeconds(5);
+
+        if (statusCode == 429) {
+            // get the Retry-After header
+            String retryAfter = response.getFirstHeader("Retry-After").getValue();
+            if (retryAfter == null) {
+                return TimeValue.ofSeconds(2);
+            }
+
+            try {
+                int retryAfterTime = Integer.parseInt(retryAfter);
+                retryAfterTime = Math.clamp(retryAfterTime, 1, 5);
+
+                return TimeValue.ofSeconds(retryAfterTime);
+            } catch (NumberFormatException e) {
+                logger.warn("Invalid Retry-After header: {}", retryAfter);
+            }
+        }
+
+        return TimeValue.ofSeconds(2);
+    }
 
     public static class RateLimitException extends Exception {
         private final String retryAfter;
@@ -425,5 +705,31 @@ public class HttpFetcherImpl implements HttpFetcher {
             }
         }
     }
+
+}
+
+class SendLock implements AutoCloseable {
+
+    private static final Semaphore maxConcurrentRequests = new Semaphore(Integer.getInteger("crawler.maxConcurrentRequests", 512));
+    boolean closed = false;
+
+    public SendLock() {
+        maxConcurrentRequests.acquireUninterruptibly();
+    }
+
+    public static <T> T wrapSend(HttpClient client, final ClassicHttpRequest request,
+                                               final HttpClientResponseHandler<? extends T> responseHandler) throws IOException {
+        try (var lock = new SendLock()) {
+            return client.execute(request, responseHandler);
+        }
+    }
+
+    @Override
+    public void close() {
+        if (!closed) {
+            maxConcurrentRequests.release();
+            closed = true;
+        }
+    }
 }
 

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcInputBuffer.java

@@ -1,15 +1,20 @@
 package nu.marginalia.crawl.fetcher.warc;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.BOMInputStream;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.Header;
 import org.netpreserve.jwarc.WarcTruncationReason;
 
 import java.io.*;
-import java.net.http.HttpHeaders;
-import java.net.http.HttpResponse;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Map;
-import java.util.zip.GZIPInputStream;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
+
+import static nu.marginalia.crawl.fetcher.warc.ErrorBuffer.suppressContentEncoding;
 
 /** Input buffer for temporary storage of a HTTP response
  *  This may be in-memory or on-disk, at the discretion of
@@ -17,9 +22,9 @@ import java.util.zip.GZIPInputStream;
  * */
 public abstract class WarcInputBuffer implements AutoCloseable {
     protected WarcTruncationReason truncationReason = WarcTruncationReason.NOT_TRUNCATED;
-    protected HttpHeaders headers;
+    protected Header[] headers;
 
-    WarcInputBuffer(HttpHeaders headers) {
+    WarcInputBuffer(Header[] headers) {
         this.headers = headers;
     }
 
@@ -31,7 +36,7 @@ public abstract class WarcInputBuffer implements AutoCloseable {
 
     public final WarcTruncationReason truncationReason() { return truncationReason; }
 
-    public final HttpHeaders headers() { return headers; }
+    public final Header[] headers() { return headers; }
 
     /** Create a buffer for a response.
      *  If the response is small and not compressed, it will be stored in memory.
@@ -39,34 +44,70 @@ public abstract class WarcInputBuffer implements AutoCloseable {
      *  and suppressed from the headers.
      *  If an error occurs, a buffer will be created with no content and an error status.
      */
-    static WarcInputBuffer forResponse(HttpResponse<InputStream> rsp) {
-        if (rsp == null)
+    static WarcInputBuffer forResponse(ClassicHttpResponse response,
+                                       HttpGet request,
+                                       Duration timeLimit) throws IOException {
+        if (response == null)
             return new ErrorBuffer();
 
-        var headers = rsp.headers();
 
-        try (var is = rsp.body()) {
-            int contentLength = (int) headers.firstValueAsLong("Content-Length").orElse(-1L);
-            String contentEncoding = headers.firstValue("Content-Encoding").orElse(null);
+        var entity = response.getEntity();
 
-            if (contentEncoding == null && contentLength > 0 && contentLength < 8192) {
+        if (null == entity) {
+            return new ErrorBuffer();
+        }
+
+        Instant start = Instant.now();
+        InputStream is = null;
+        try {
+            is = entity.getContent();
+            long length = entity.getContentLength();
+
+            if (length > 0 && length < 8192) {
                 // If the content is small and not compressed, we can just read it into memory
-                return new MemoryBuffer(headers, is, contentLength);
-            }
-            else {
+                return new MemoryBuffer(response.getHeaders(), request, timeLimit, is, (int) length);
+            } else {
                 // Otherwise, we unpack it into a file and read it from there
-                return new FileBuffer(headers, is);
+                return new FileBuffer(response.getHeaders(), request, timeLimit, is);
             }
         }
-        catch (Exception ex) {
-            return new ErrorBuffer();
+        finally {
+            // We're required to consume the stream to avoid leaking connections,
+            // but we also don't want to get stuck on slow or malicious connections
+            // forever, so we set a time limit on this phase and call abort() if it's exceeded.
+            try {
+                while (is != null) {
+                    // Consume some data
+                    if (is.skip(65536) == 0) {
+                        // Note that skip may return 0 if the stream is empty
+                        // or for other unspecified reasons, so we need to check
+                        // with read() as well to determine if the stream is done
+                        if (is.read() == -1)
+                            is = null;
+                    }
+                    // Check if the time limit has been exceeded
+                    else if (Duration.between(start, Instant.now()).compareTo(timeLimit) > 0) {
+                        request.abort();
+                        is = null;
+                    }
+                }
+            }
+            catch (IOException e) {
+                // Ignore the exception
+            }
+            finally {
+                // Close the input stream
+                IOUtils.closeQuietly(is);
+            }
         }
 
+
     }
 
     /** Copy an input stream to an output stream, with a maximum size and time limit */
-    protected void copy(InputStream is, OutputStream os) {
-        long startTime = System.currentTimeMillis();
+    protected void copy(InputStream is, HttpGet request, OutputStream os, Duration timeLimit) {
+        Instant start = Instant.now();
+        Instant timeout = start.plus(timeLimit);
         long size = 0;
 
         byte[] buffer = new byte[8192];
@@ -76,24 +117,105 @@ public abstract class WarcInputBuffer implements AutoCloseable {
 
         while (true) {
             try {
+                Duration remaining = Duration.between(Instant.now(), timeout);
+                if (remaining.isNegative()) {
+                    truncationReason = WarcTruncationReason.TIME;
+                    // Abort the request if the time limit is exceeded
+                    // so we don't keep the connection open forever or are forced to consume
+                    // the stream to the end
+
+                    request.abort();
+                    break;
+                }
+
                 int n = is.read(buffer);
+
                 if (n < 0) break;
                 size += n;
-                os.write(buffer, 0, n);
 
-                if (size > WarcRecorder.MAX_SIZE) {
+                // Even if we've exceeded the max length,
+                // we keep consuming the stream up until the end or a timeout,
+                // as closing the stream means resetting the connection, and
+                // that's generally not desirable.
+
+                if (size < WarcRecorder.MAX_SIZE) {
+                    os.write(buffer, 0, n);
+                }
+                else if (truncationReason != WarcTruncationReason.LENGTH) {
                     truncationReason = WarcTruncationReason.LENGTH;
                     break;
                 }
 
-                if (System.currentTimeMillis() - startTime > WarcRecorder.MAX_TIME) {
-                    truncationReason = WarcTruncationReason.TIME;
-                    break;
-                }
             } catch (IOException e) {
-                throw new RuntimeException(e);
+                truncationReason = WarcTruncationReason.UNSPECIFIED;
             }
         }
+
+    }
+
+    /** Takes a Content-Range header and checks if it is complete.
+     *  A complete range is one that covers the entire resource.
+     *  For example, "bytes 0-1023/2048" or "bytes 0-1023/*" are complete ranges.
+     *  "bytes 0-1023/2048" is not a complete range.
+     */
+    public boolean isRangeComplete(Header[] headers) {
+        // Find the Content-Range header
+        String contentRangeHeader = null;
+        for (var header : headers) {
+            if ("Content-Range".equalsIgnoreCase(header.getName())) {
+                contentRangeHeader = header.getValue();
+                break;
+            }
+        }
+
+        // Return true if header is null or empty
+        if (contentRangeHeader == null || contentRangeHeader.isEmpty()) {
+            return true;
+        }
+
+        try {
+            // Content-Range format: "bytes range-start-range-end/size"
+            // e.g., "bytes 0-1023/2048" or "bytes 0-1023/*"
+
+            // Get the part after "bytes "
+            String[] parts = contentRangeHeader.split(" ", 2);
+            if (parts.length < 2) {
+                return false;
+            }
+
+            // Get the range and size parts (e.g., "0-1023/2048")
+            String rangeAndSize = parts[1];
+            String[] rangeAndSizeParts = rangeAndSize.split("/", 2);
+            if (rangeAndSizeParts.length < 2) {
+                return false;
+            }
+
+            // Get the range (e.g., "0-1023")
+            String range = rangeAndSizeParts[0];
+            String[] rangeParts = range.split("-", 2);
+            if (rangeParts.length < 2) {
+                return false;
+            }
+
+            // Get the size (e.g., "2048" or "*")
+            String size = rangeAndSizeParts[1];
+
+            // If size is "*", we don't know the total size, so return false
+            if ("*".equals(size)) {
+                return false;
+            }
+
+            // Parse as long to handle large files
+            long rangeStart = Long.parseLong(rangeParts[0]);
+            long rangeEnd = Long.parseLong(rangeParts[1]);
+            long totalSize = Long.parseLong(size);
+
+            // Check if the range covers the entire resource
+            return rangeStart == 0 && rangeEnd == totalSize - 1;
+
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+            return false;
+        }
     }
 
 }
@@ -101,7 +223,7 @@ public abstract class WarcInputBuffer implements AutoCloseable {
 /** Pseudo-buffer for when we have an error */
 class ErrorBuffer extends WarcInputBuffer {
     public ErrorBuffer() {
-        super(HttpHeaders.of(Map.of(), (k,v)->false));
+        super(new Header[0]);
 
         truncationReason = WarcTruncationReason.UNSPECIFIED;
     }
@@ -118,17 +240,29 @@ class ErrorBuffer extends WarcInputBuffer {
 
     @Override
     public void close() throws Exception {}
+
+
+    static Header[] suppressContentEncoding(Header[] headers) {
+        return Arrays.stream(headers).filter(header -> !"Content-Encoding".equalsIgnoreCase(header.getName())).toArray(Header[]::new);
+    }
+
 }
 
 /** Buffer for when we have the response in memory */
 class MemoryBuffer extends WarcInputBuffer {
     byte[] data;
-    public MemoryBuffer(HttpHeaders headers, InputStream responseStream, int size) {
-        super(headers);
+    public MemoryBuffer(Header[] headers, HttpGet request, Duration timeLimit, InputStream responseStream, int size) {
+        super(suppressContentEncoding(headers));
+
+        if (!isRangeComplete(headers)) {
+            truncationReason = WarcTruncationReason.LENGTH;
+        } else {
+            truncationReason = WarcTruncationReason.NOT_TRUNCATED;
+        }
 
         var outputStream = new ByteArrayOutputStream(size);
 
-        copy(responseStream, outputStream);
+        copy(responseStream, request, outputStream, timeLimit);
 
         data = outputStream.toByteArray();
     }
@@ -152,40 +286,25 @@ class MemoryBuffer extends WarcInputBuffer {
 class FileBuffer extends WarcInputBuffer {
     private final Path tempFile;
 
-    public FileBuffer(HttpHeaders headers, InputStream responseStream) throws IOException {
+    public FileBuffer(Header[] headers, HttpGet request, Duration timeLimit, InputStream responseStream) throws IOException {
         super(suppressContentEncoding(headers));
 
+        if (!isRangeComplete(headers)) {
+            truncationReason = WarcTruncationReason.LENGTH;
+        } else {
+            truncationReason = WarcTruncationReason.NOT_TRUNCATED;
+        }
+
         this.tempFile = Files.createTempFile("rsp", ".html");
 
-
-        if ("gzip".equalsIgnoreCase(headers.firstValue("Content-Encoding").orElse(""))) {
-            try (var out = Files.newOutputStream(tempFile)) {
-                copy(new GZIPInputStream(responseStream), out);
-            }
-            catch (Exception ex) {
-                truncationReason = WarcTruncationReason.UNSPECIFIED;
-            }
+        try (var out = Files.newOutputStream(tempFile)) {
+            copy(responseStream, request, out, timeLimit);
         }
-        else {
-            try (var out = Files.newOutputStream(tempFile)) {
-                copy(responseStream, out);
-            }
-            catch (Exception ex) {
-                truncationReason = WarcTruncationReason.UNSPECIFIED;
-            }
+        catch (Exception ex) {
+            truncationReason = WarcTruncationReason.UNSPECIFIED;
         }
     }
 
-    private static HttpHeaders suppressContentEncoding(HttpHeaders headers) {
-        return HttpHeaders.of(headers.map(), (k, v) -> {
-            if ("Content-Encoding".equalsIgnoreCase(k)) {
-                return false;
-            }
-            return !"Transfer-Encoding".equalsIgnoreCase(k);
-        });
-    }
-
-
     public InputStream read() throws IOException {
         return Files.newInputStream(tempFile);
     }

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcProtocolReconstructor.java

@@ -1,6 +1,8 @@
 package nu.marginalia.crawl.fetcher.warc;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.Header;
 
 import java.net.URI;
 import java.net.URLEncoder;
@@ -17,7 +19,7 @@ import java.util.stream.Collectors;
 public class WarcProtocolReconstructor {
 
     static String getHttpRequestString(String method,
-                                       Map<String, List<String>> mainHeaders,
+                                       Header[] mainHeaders,
                                        Map<String, List<String>> extraHeaders,
                                        URI uri) {
         StringBuilder requestStringBuilder = new StringBuilder();
@@ -34,12 +36,13 @@ public class WarcProtocolReconstructor {
 
         Set<String> addedHeaders = new HashSet<>();
 
-        mainHeaders.forEach((k, values) -> {
-            for (var value : values) {
-                addedHeaders.add(k);
-                requestStringBuilder.append(capitalizeHeader(k)).append(": ").append(value).append("\r\n");
-            }
-        });
+        for (var header : mainHeaders) {
+            String k = header.getName();
+            String v = header.getValue();
+
+            addedHeaders.add(k);
+            requestStringBuilder.append(capitalizeHeader(k)).append(": ").append(v).append("\r\n");
+        }
 
         extraHeaders.forEach((k, values) -> {
             if (!addedHeaders.contains(k)) {
@@ -87,6 +90,12 @@ public class WarcProtocolReconstructor {
         return "HTTP/" + version + " " + statusCode + " " + statusMessage + "\r\n" + headerString + "\r\n\r\n";
     }
 
+    static String getResponseHeader(ClassicHttpResponse response, long size) {
+        String headerString = getHeadersAsString(response.getHeaders(), size);
+
+        return response.getVersion().format() + " " + response.getCode() + " " + response.getReasonPhrase() + "\r\n" + headerString + "\r\n\r\n";
+    }
+
     private static final Map<Integer, String> STATUS_CODE_MAP = Map.ofEntries(
             Map.entry(200, "OK"),
             Map.entry(201, "Created"),
@@ -149,6 +158,37 @@ public class WarcProtocolReconstructor {
         return joiner.toString();
     }
 
+
+
+    static private String getHeadersAsString(Header[] headers, long responseSize) {
+        StringJoiner joiner = new StringJoiner("\r\n");
+
+        for (var header : headers) {
+            String headerCapitalized = capitalizeHeader(header.getName());
+
+            // Omit pseudoheaders injected by the crawler itself
+            if (headerCapitalized.startsWith("X-Marginalia"))
+                continue;
+
+            // Omit Transfer-Encoding and Content-Encoding headers
+            if (headerCapitalized.equals("Transfer-Encoding"))
+                continue;
+            if (headerCapitalized.equals("Content-Encoding"))
+                continue;
+
+            // Since we're transparently decoding gzip, we need to update the Content-Length header
+            // to reflect the actual size of the response body. We'll do this at the end.
+            if (headerCapitalized.equals("Content-Length"))
+                continue;
+
+            joiner.add(headerCapitalized + ": " + header.getValue());
+        }
+
+        joiner.add("Content-Length: " + responseSize);
+
+        return joiner.toString();
+    }
+
     static private String getHeadersAsString(HttpHeaders headers, long responseSize) {
         StringJoiner joiner = new StringJoiner("\r\n");
 

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcRecorder.java

@@ -1,11 +1,16 @@
 package nu.marginalia.crawl.fetcher.warc;
 
 import nu.marginalia.crawl.fetcher.ContentTags;
-import nu.marginalia.crawl.fetcher.Cookies;
+import nu.marginalia.crawl.fetcher.DomainCookies;
+import nu.marginalia.crawl.fetcher.HttpFetcher;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
+import nu.marginalia.link_parser.LinkParser;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.body.HttpFetchResult;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.core5.http.NameValuePair;
 import org.jetbrains.annotations.Nullable;
 import org.netpreserve.jwarc.*;
 import org.slf4j.Logger;
@@ -14,10 +19,9 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.InetAddress;
+import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.http.HttpClient;
-import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -37,7 +41,7 @@ public class WarcRecorder implements AutoCloseable {
     static final int MAX_TIME = 30_000;
 
     /** Maximum (decompressed) size we'll save */
-    static final int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 10 * 1024 * 1024);
+    static final int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 32 * 1024 * 1024);
 
     private final WarcWriter writer;
     private final Path warcFile;
@@ -48,22 +52,15 @@ public class WarcRecorder implements AutoCloseable {
     // Affix a version string in case we need to change the format in the future
     // in some way
     private final String warcRecorderVersion = "1.0";
-    private final Cookies cookies;
+    private final LinkParser linkParser = new LinkParser();
     /**
      * Create a new WarcRecorder that will write to the given file
      *
      * @param warcFile The file to write to
      */
-    public WarcRecorder(Path warcFile, HttpFetcherImpl fetcher) throws IOException {
+    public WarcRecorder(Path warcFile) throws IOException {
         this.warcFile = warcFile;
         this.writer = new WarcWriter(warcFile);
-        this.cookies = fetcher.getCookies();
-    }
-
-    public WarcRecorder(Path warcFile, Cookies cookies) throws IOException {
-        this.warcFile = warcFile;
-        this.writer = new WarcWriter(warcFile);
-        this.cookies = cookies;
     }
 
     /**
@@ -73,16 +70,25 @@ public class WarcRecorder implements AutoCloseable {
     public WarcRecorder() throws IOException {
         this.warcFile = Files.createTempFile("warc", ".warc.gz");
         this.writer = new WarcWriter(this.warcFile);
-        this.cookies = new Cookies();
 
         temporaryFile = true;
     }
 
     public HttpFetchResult fetch(HttpClient client,
-                                 java.net.http.HttpRequest request)
+                                 DomainCookies cookies,
+                                 HttpGet request)
             throws NoSuchAlgorithmException, IOException, URISyntaxException, InterruptedException
     {
-        URI requestUri = request.uri();
+        return fetch(client, cookies, request, Duration.ofMillis(MAX_TIME));
+    }
+
+    public HttpFetchResult fetch(HttpClient client,
+                                 DomainCookies cookies,
+                                 HttpGet request,
+                                 Duration timeout)
+            throws NoSuchAlgorithmException, IOException, URISyntaxException, InterruptedException
+    {
+        URI requestUri = request.getUri();
 
         WarcDigestBuilder responseDigestBuilder = new WarcDigestBuilder();
         WarcDigestBuilder payloadDigestBuilder = new WarcDigestBuilder();
@@ -90,121 +96,151 @@ public class WarcRecorder implements AutoCloseable {
         Instant date = Instant.now();
 
         // Not entirely sure why we need to do this, but keeping it due to Chesterton's Fence
-        Map<String, List<String>> extraHeaders = new HashMap<>(request.headers().map());
+        Map<String, List<String>> extraHeaders = new HashMap<>(request.getHeaders().length);
 
-        HttpResponse<InputStream> response;
+        // Inject a range header to attempt to limit the size of the response
+        // to the maximum size we want to store, if the server supports it.
+        request.addHeader("Range", "bytes=0-"+MAX_SIZE);
+        cookies.paintRequest(request);
         try {
-            response = client.send(request, java.net.http.HttpResponse.BodyHandlers.ofInputStream());
-        }
-        catch (IOException ex) {
-            logger.warn("Failed to fetch URL {}:  {}", requestUri, ex.getMessage());
+            return client.execute(request,response -> {
+
+                try (WarcInputBuffer inputBuffer = WarcInputBuffer.forResponse(response, request, timeout);
+                     InputStream inputStream = inputBuffer.read()) {
+
+                    cookies.updateCookieStore(response);
+
+                    // Build and write the request
+
+                    WarcDigestBuilder requestDigestBuilder = new WarcDigestBuilder();
+
+                    byte[] httpRequestString = WarcProtocolReconstructor
+                            .getHttpRequestString(
+                                    request.getMethod(),
+                                    request.getHeaders(),
+                                    extraHeaders,
+                                    requestUri)
+                            .getBytes();
+
+                    requestDigestBuilder.update(httpRequestString);
+
+                    WarcRequest warcRequest = new WarcRequest.Builder(requestUri)
+                            .blockDigest(requestDigestBuilder.build())
+                            .date(date)
+                            .body(MediaType.HTTP_REQUEST, httpRequestString)
+                            .build();
+
+                    warcRequest.http(); // force HTTP header to be parsed before body is consumed so that caller can use it
+                    writer.write(warcRequest);
+
+
+                    if (cookies.hasCookies()) {
+                        response.addHeader("X-Has-Cookies", 1);
+                    }
+
+                    byte[] responseHeaders = WarcProtocolReconstructor.getResponseHeader(response, inputBuffer.size()).getBytes(StandardCharsets.UTF_8);
+
+                    ResponseDataBuffer responseDataBuffer = new ResponseDataBuffer(inputBuffer.size() + responseHeaders.length);
+
+                    responseDataBuffer.put(responseHeaders);
+                    responseDataBuffer.updateDigest(responseDigestBuilder, 0, responseHeaders.length);
+
+                    int dataStart = responseDataBuffer.pos();
+
+                    for (;;) {
+                        int remainingLength = responseDataBuffer.remaining();
+                        if (remainingLength == 0)
+                            break;
+
+                        int startPos = responseDataBuffer.pos();
+
+                        int n = responseDataBuffer.readFrom(inputStream, remainingLength);
+                        if (n < 0)
+                            break;
+
+                        responseDataBuffer.updateDigest(responseDigestBuilder, startPos, n);
+                        responseDataBuffer.updateDigest(payloadDigestBuilder, startPos, n);
+                    }
+
+                    // with some http client libraries, that resolve redirects transparently, this might be different
+                    // from the request URI, but currently we don't have transparent redirect resolution so it's always
+                    // the same (though let's keep the variables separate in case this changes)
+                    final URI responseUri = requestUri;
+
+                    WarcResponse.Builder responseBuilder = new WarcResponse.Builder(responseUri)
+                            .blockDigest(responseDigestBuilder.build())
+                            .date(date)
+                            .concurrentTo(warcRequest.id())
+                            .body(MediaType.HTTP_RESPONSE, responseDataBuffer.copyBytes());
+
+                    InetAddress inetAddress = InetAddress.getByName(responseUri.getHost());
+                    responseBuilder.ipAddress(inetAddress);
+                    responseBuilder.payloadDigest(payloadDigestBuilder.build());
+                    responseBuilder.truncated(inputBuffer.truncationReason());
+
+                    // Build and write the response
+
+                    var warcResponse = responseBuilder.build();
+                    warcResponse.http(); // force HTTP header to be parsed before body is consumed so that caller can use it
+                    writer.write(warcResponse);
+
+                    if (Duration.between(date, Instant.now()).compareTo(Duration.ofSeconds(9)) > 0
+                            && inputBuffer.size() < 2048
+                            && !requestUri.getPath().endsWith("robots.txt")) // don't bail on robots.txt
+                    {
+                        // Fast detection and mitigation of crawler traps that respond with slow
+                        // small responses, with a high branching factor
+
+                        // Note we bail *after* writing the warc records, this will effectively only
+                        // prevent link extraction from the document.
+
+                        logger.warn("URL {} took too long to fetch ({}s) and was too small for the effort ({}b)",
+                                requestUri,
+                                Duration.between(date, Instant.now()).getSeconds(),
+                                inputBuffer.size()
+                        );
+
+                        return new HttpFetchResult.ResultException(new IOException("Likely crawler trap"));
+                    }
+
+                    if (response.getCode() == 301 || response.getCode() == 302 || response.getCode() == 307) {
+                        // If the server responds with a redirect, we need to
+                        // update the request URI to the new location
+                        EdgeUrl redirectLocation = Optional.ofNullable(response.getFirstHeader("Location"))
+                                                           .map(NameValuePair::getValue)
+                                .flatMap(location -> linkParser.parseLink(new EdgeUrl(requestUri), location))
+                                .orElse(null);
+                        if (redirectLocation != null) {
+                            // If the redirect location is a valid URL, we need to update the request URI
+                            return new HttpFetchResult.ResultRedirect(redirectLocation);
+                        } else {
+                            // If the redirect location is not a valid URL, we need to throw an exception
+                            return new HttpFetchResult.ResultException(new IOException("Invalid redirect location: " + response.getFirstHeader("Location")));
+                        }
+                    }
+
+
+                    return new HttpFetchResult.ResultOk(responseUri,
+                            response.getCode(),
+                            inputBuffer.headers(),
+                            inetAddress.getHostAddress(),
+                            responseDataBuffer.data,
+                            dataStart,
+                            responseDataBuffer.length() - dataStart);
+                } catch (Exception ex) {
+                    flagAsError(new EdgeUrl(requestUri), ex); // write a WARC record to indicate the error
+                    logger.warn("Failed to fetch URL {}:  {}", requestUri, ex.getMessage());
+                    return new HttpFetchResult.ResultException(ex);
+                }
+            });
+        // the client.execute() method will throw an exception if the request times out
+        // or on other IO exceptions, so we need to catch those here as well as having
+        // exception handling in the response handler
+        } catch (SocketTimeoutException ex) {
+            flagAsTimeout(new EdgeUrl(requestUri)); // write a WARC record to indicate the timeout
             return new HttpFetchResult.ResultException(ex);
-        }
-
-
-        try (WarcInputBuffer inputBuffer = WarcInputBuffer.forResponse(response);
-             InputStream inputStream = inputBuffer.read())
-        {
-            if (cookies.hasCookies()) {
-                extraHeaders.put("X-Has-Cookies", List.of("1"));
-            }
-
-            byte[] responseHeaders = WarcProtocolReconstructor.getResponseHeader(response, inputBuffer.size()).getBytes(StandardCharsets.UTF_8);
-
-            ResponseDataBuffer responseDataBuffer = new ResponseDataBuffer(inputBuffer.size() + responseHeaders.length);
-
-            responseDataBuffer.put(responseHeaders);
-            responseDataBuffer.updateDigest(responseDigestBuilder, 0, responseHeaders.length);
-
-            int dataStart = responseDataBuffer.pos();
-
-            for (;;) {
-                int remainingLength = responseDataBuffer.remaining();
-                if (remainingLength == 0)
-                    break;
-
-                int startPos = responseDataBuffer.pos();
-
-                int n = responseDataBuffer.readFrom(inputStream, remainingLength);
-                if (n < 0)
-                    break;
-
-                responseDataBuffer.updateDigest(responseDigestBuilder, startPos, n);
-                responseDataBuffer.updateDigest(payloadDigestBuilder, startPos, n);
-            }
-
-            // It looks like this might be the same as requestUri, but it's not;
-            // it's the URI after resolving redirects.
-            final URI responseUri = response.uri();
-
-            WarcResponse.Builder responseBuilder = new WarcResponse.Builder(responseUri)
-                    .blockDigest(responseDigestBuilder.build())
-                    .date(date)
-                    .body(MediaType.HTTP_RESPONSE, responseDataBuffer.copyBytes());
-
-            InetAddress inetAddress = InetAddress.getByName(responseUri.getHost());
-            responseBuilder.ipAddress(inetAddress);
-            responseBuilder.payloadDigest(payloadDigestBuilder.build());
-            responseBuilder.truncated(inputBuffer.truncationReason());
-
-            // Build and write the response
-
-            var warcResponse = responseBuilder.build();
-            warcResponse.http(); // force HTTP header to be parsed before body is consumed so that caller can use it
-            writer.write(warcResponse);
-
-            // Build and write the request
-
-            WarcDigestBuilder requestDigestBuilder = new WarcDigestBuilder();
-
-            byte[] httpRequestString = WarcProtocolReconstructor
-                    .getHttpRequestString(
-                            response.request().method(),
-                            response.request().headers().map(),
-                            extraHeaders,
-                            requestUri)
-                    .getBytes();
-
-            requestDigestBuilder.update(httpRequestString);
-
-            WarcRequest warcRequest = new WarcRequest.Builder(requestUri)
-                    .blockDigest(requestDigestBuilder.build())
-                    .date(date)
-                    .body(MediaType.HTTP_REQUEST, httpRequestString)
-                    .concurrentTo(warcResponse.id())
-                    .build();
-
-            warcRequest.http(); // force HTTP header to be parsed before body is consumed so that caller can use it
-            writer.write(warcRequest);
-
-            if (Duration.between(date, Instant.now()).compareTo(Duration.ofSeconds(9)) > 0
-                    && inputBuffer.size() < 2048
-                    && !request.uri().getPath().endsWith("robots.txt")) // don't bail on robots.txt
-            {
-                // Fast detection and mitigation of crawler traps that respond with slow
-                // small responses, with a high branching factor
-
-                // Note we bail *after* writing the warc records, this will effectively only
-                // prevent link extraction from the document.
-
-                logger.warn("URL {} took too long to fetch ({}s) and was too small for the effort ({}b)",
-                        requestUri,
-                        Duration.between(date, Instant.now()).getSeconds(),
-                        inputBuffer.size()
-                );
-
-                return new HttpFetchResult.ResultException(new IOException("Likely crawler trap"));
-            }
-
-            return new HttpFetchResult.ResultOk(responseUri,
-                    response.statusCode(),
-                    inputBuffer.headers(),
-                    inetAddress.getHostAddress(),
-                    responseDataBuffer.data,
-                    dataStart,
-                    responseDataBuffer.length() - dataStart);
-        }
-        catch (Exception ex) {
+        } catch (IOException ex) {
+            flagAsError(new EdgeUrl(requestUri), ex); // write a WARC record to indicate the error
             logger.warn("Failed to fetch URL {}:  {}", requestUri, ex.getMessage());
             return new HttpFetchResult.ResultException(ex);
         }
@@ -214,7 +250,7 @@ public class WarcRecorder implements AutoCloseable {
         writer.write(item);
     }
 
-    private void saveOldResponse(EdgeUrl url, String contentType, int statusCode, byte[] documentBody, @Nullable String headers, ContentTags contentTags) {
+    private void saveOldResponse(EdgeUrl url, DomainCookies domainCookies, String contentType, int statusCode, byte[] documentBody, @Nullable String headers, ContentTags contentTags) {
         try {
             WarcDigestBuilder responseDigestBuilder = new WarcDigestBuilder();
             WarcDigestBuilder payloadDigestBuilder = new WarcDigestBuilder();
@@ -275,7 +311,7 @@ public class WarcRecorder implements AutoCloseable {
                     .date(Instant.now())
                     .body(MediaType.HTTP_RESPONSE, responseDataBuffer.copyBytes());
 
-            if (cookies.hasCookies()) {
+            if (domainCookies.hasCookies() || (headers != null && headers.contains("Set-Cookie:"))) {
                 builder.addHeader("X-Has-Cookies", "1");
             }
 
@@ -295,8 +331,8 @@ public class WarcRecorder implements AutoCloseable {
      * an E-Tag or Last-Modified header, and the server responds with a 304 Not Modified.  In this
      * scenario we want to record the data as it was in the previous crawl, but not re-fetch it.
      */
-    public void writeReferenceCopy(EdgeUrl url, String contentType, int statusCode, byte[] documentBody, @Nullable String headers, ContentTags ctags) {
-        saveOldResponse(url, contentType, statusCode, documentBody, headers, ctags);
+    public void writeReferenceCopy(EdgeUrl url, DomainCookies cookies, String contentType, int statusCode, byte[] documentBody, @Nullable String headers, ContentTags ctags) {
+        saveOldResponse(url, cookies, contentType, statusCode, documentBody, headers, ctags);
     }
 
     public void writeWarcinfoHeader(String ip, EdgeDomain domain, HttpFetcherImpl.DomainProbeResult result) throws IOException {
@@ -316,6 +352,9 @@ public class WarcRecorder implements AutoCloseable {
             case HttpFetcherImpl.DomainProbeResult.Ok ok:
                 fields.put("X-WARC-Probe-Status", List.of("OK"));
                 break;
+            case HttpFetcher.DomainProbeResult.RedirectSameDomain_Internal redirectSameDomain:
+                fields.put("X-WARC-Probe-Status", List.of("REDIR-INTERNAL"));
+                break;
         }
 
         var warcinfo = new Warcinfo.Builder()

code/processes/crawling-process/java/nu/marginalia/crawl/logic/DomainLocks.java

@@ -3,6 +3,7 @@ package nu.marginalia.crawl.logic;
 import nu.marginalia.model.EdgeDomain;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
 
@@ -19,8 +20,22 @@ public class DomainLocks {
      * and may be held by another thread.  The caller is responsible for locking and  releasing the lock.
      */
     public DomainLock lockDomain(EdgeDomain domain) throws InterruptedException {
-        return new DomainLock(domain.toString(),
-                locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits));
+        var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits);
+
+        sem.acquire();
+
+        return new DomainLock(sem);
+    }
+
+    public Optional<DomainLock> tryLockDomain(EdgeDomain domain) {
+        var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits);
+        if (sem.tryAcquire(1)) {
+            return Optional.of(new DomainLock(sem));
+        }
+        else {
+            // We don't have a lock, so we return an empty optional
+            return Optional.empty();
+        }
     }
 
     private Semaphore defaultPermits(String topDomain) {
@@ -28,39 +43,45 @@ public class DomainLocks {
             return new Semaphore(16);
         if (topDomain.equals("blogspot.com"))
             return new Semaphore(8);
-
+        if (topDomain.equals("tumblr.com"))
+            return new Semaphore(8);
         if (topDomain.equals("neocities.org"))
-            return new Semaphore(4);
+            return new Semaphore(8);
         if (topDomain.equals("github.io"))
-            return new Semaphore(4);
+            return new Semaphore(8);
 
+        // Substack really dislikes broad-scale crawlers, so we need to be careful
+        // to not get blocked.
         if (topDomain.equals("substack.com")) {
             return new Semaphore(1);
         }
-        if (topDomain.endsWith(".edu")) {
-            return new Semaphore(1);
-        }
 
         return new Semaphore(2);
     }
 
+    /** Returns true if the domain is lockable, i.e. if it is not already locked by another thread.
+     * (this is just a hint, and does not guarantee that the domain is actually lockable any time
+     * after this method returns true)
+     */
+    public boolean isLockableHint(EdgeDomain domain) {
+        Semaphore sem = locks.get(domain.topDomain.toLowerCase());
+        if (null == sem)
+            return true;
+        else
+            return sem.availablePermits() > 0;
+    }
+
     public static class DomainLock implements AutoCloseable {
-        private final String domainName;
         private final Semaphore semaphore;
 
-        DomainLock(String domainName, Semaphore semaphore) throws InterruptedException {
-            this.domainName = domainName;
+        DomainLock(Semaphore semaphore) {
             this.semaphore = semaphore;
-
-            Thread.currentThread().setName("crawling:" + domainName + " [await domain lock]");
-            semaphore.acquire();
-            Thread.currentThread().setName("crawling:" + domainName);
         }
 
         @Override
         public void close() throws Exception {
             semaphore.release();
-            Thread.currentThread().setName("crawling:" + domainName + " [wrapping up]");
+            Thread.currentThread().setName("[idle]");
         }
     }
 }

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDataReference.java

@@ -4,6 +4,7 @@ import nu.marginalia.ContentTypes;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.lsh.EasyLSH;
 import nu.marginalia.model.crawldata.CrawledDocument;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -11,51 +12,73 @@ import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Optional;
 
 /** A reference to a domain that has been crawled before. */
-public class CrawlDataReference implements AutoCloseable {
+public class CrawlDataReference implements AutoCloseable, Iterable<CrawledDocument> {
+
+    private boolean closed = false;
+
+    @Nullable
+    private final Path path;
+
+    @Nullable
+    private SerializableCrawlDataStream data = null;
 
-    private final SerializableCrawlDataStream data;
     private static final Logger logger = LoggerFactory.getLogger(CrawlDataReference.class);
 
-    public CrawlDataReference(SerializableCrawlDataStream data) {
-        this.data = data;
+    public CrawlDataReference(@Nullable Path path) {
+        this.path = path;
     }
 
     public CrawlDataReference() {
-        this(SerializableCrawlDataStream.empty());
+        this(null);
     }
 
     /** Delete the associated data from disk, if it exists */
     public void delete() throws IOException {
-        Path filePath = data.path();
-
-        if (filePath != null) {
-            Files.deleteIfExists(filePath);
+        if (path != null) {
+            Files.deleteIfExists(path);
         }
     }
 
-    /** Get the next document from the crawl data,
-     * returning null when there are no more documents
-     * available
-     */
-    @Nullable
-    public CrawledDocument nextDocument() {
-        try {
-            while (data.hasNext()) {
-                if (data.next() instanceof CrawledDocument doc) {
-                    if (!ContentTypes.isAccepted(doc.contentType))
-                        continue;
+    public @NotNull Iterator<CrawledDocument> iterator() {
 
-                    return doc;
+        requireStream();
+        // Guaranteed by requireStream, but helps java
+        Objects.requireNonNull(data);
+
+        return data.map(next -> {
+            if (next instanceof CrawledDocument doc && ContentTypes.isAccepted(doc.contentType)) {
+                return Optional.of(doc);
+            }
+            else {
+                return Optional.empty();
+            }
+        });
+    }
+
+    /** After calling this method, data is guaranteed to be non-null */
+    private void requireStream() {
+        if (closed) {
+            throw new IllegalStateException("Use after close()");
+        }
+
+        if (data == null) {
+            try {
+                if (path != null) {
+                    data = SerializableCrawlDataStream.openDataStream(path);
+                    return;
                 }
             }
-        }
-        catch (IOException ex) {
-            logger.error("Failed to read next document", ex);
-        }
+            catch (Exception ex) {
+                logger.error("Failed to open stream", ex);
+            }
 
-        return null;
+            data = SerializableCrawlDataStream.empty();
+        }
     }
 
     public static boolean isContentBodySame(byte[] one, byte[] other) {
@@ -98,7 +121,12 @@ public class CrawlDataReference implements AutoCloseable {
     }
 
     @Override
-    public void close() throws Exception {
-        data.close();
+    public void close() throws IOException {
+        if (!closed) {
+            if (data != null) {
+                data.close();
+            }
+            closed = true;
+        }
     }
 }

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java

@@ -3,6 +3,7 @@ package nu.marginalia.crawl.retreival;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 
 import java.time.Duration;
+import java.util.concurrent.ThreadLocalRandom;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
@@ -50,15 +51,20 @@ public class CrawlDelayTimer {
         waitFetchDelay(0);
     }
 
+    public void waitFetchDelay(Duration spentTime) {
+        waitFetchDelay(spentTime.toMillis());
+    }
+
     public void waitFetchDelay(long spentTime) {
         long sleepTime = delayTime;
 
+        long jitter = ThreadLocalRandom.current().nextLong(0, 150);
         try {
             if (sleepTime >= 1) {
                 if (spentTime > sleepTime)
                     return;
 
-                Thread.sleep(min(sleepTime - spentTime, 5000));
+                Thread.sleep(min(sleepTime - spentTime, 5000) + jitter);
             } else {
                 // When no crawl delay is specified, lean toward twice the fetch+process time,
                 // within sane limits. This means slower servers get slower crawling, and faster
@@ -71,17 +77,17 @@ public class CrawlDelayTimer {
                 if (spentTime > sleepTime)
                     return;
 
-                Thread.sleep(sleepTime - spentTime);
+                Thread.sleep(sleepTime - spentTime + jitter);
             }
 
             if (slowDown) {
                 // Additional delay when the server is signalling it wants slower requests
-                Thread.sleep(DEFAULT_CRAWL_DELAY_MIN_MS);
+                Thread.sleep(DEFAULT_CRAWL_DELAY_MIN_MS + jitter);
             }
         }
         catch (InterruptedException e) {
             Thread.currentThread().interrupt();
-            throw new RuntimeException();
+            throw new RuntimeException("Interrupted", e);
         }
     }
 }

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerConnectionThrottle.java

@@ -0,0 +1,42 @@
+package nu.marginalia.crawl.retreival;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This class is used to stagger the rate at which connections are created.
+ * <p></p>
+ * It is used to ensure that we do not create too many connections at once,
+ * which can lead to network congestion and other issues.  Since the connections
+ * tend to be very long-lived, we can afford to wait a bit before creating the next
+ * even if it adds a bit of build-up time when the crawl starts.
+ */
+public class CrawlerConnectionThrottle {
+    private Instant lastCrawlStart = Instant.EPOCH;
+    private final Semaphore launchSemaphore = new Semaphore(1);
+
+    private final Duration launchInterval;
+
+    public CrawlerConnectionThrottle(Duration launchInterval) {
+        this.launchInterval = launchInterval;
+    }
+
+    public void waitForConnectionPermission() throws InterruptedException {
+        try {
+            launchSemaphore.acquire();
+            Instant nextPermittedLaunch = lastCrawlStart.plus(launchInterval);
+
+            if (nextPermittedLaunch.isAfter(Instant.now())) {
+                long waitTime = Duration.between(Instant.now(), nextPermittedLaunch).toMillis();
+                TimeUnit.MILLISECONDS.sleep(waitTime);
+            }
+
+            lastCrawlStart = Instant.now();
+        }
+        finally {
+            launchSemaphore.release();
+        }
+    }
+}

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java

@@ -6,8 +6,8 @@ import nu.marginalia.contenttype.ContentType;
 import nu.marginalia.crawl.CrawlerMain;
 import nu.marginalia.crawl.DomainStateDb;
 import nu.marginalia.crawl.fetcher.ContentTags;
+import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.HttpFetcher;
-import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
 import nu.marginalia.crawl.logic.LinkFilterSelector;
 import nu.marginalia.crawl.retreival.revisit.CrawlerRevisitor;
@@ -19,7 +19,6 @@ import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.body.DocumentBodyExtractor;
 import nu.marginalia.model.body.HttpFetchResult;
 import nu.marginalia.model.crawldata.CrawlerDomainStatus;
-import org.jsoup.Jsoup;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,14 +26,16 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 public class CrawlerRetreiver implements AutoCloseable {
 
     private static final int MAX_ERRORS = 20;
-    private static final int HTTP_429_RETRY_LIMIT = 1; // Retry 429s once
 
     private final HttpFetcher fetcher;
 
@@ -51,6 +52,11 @@ public class CrawlerRetreiver implements AutoCloseable {
     private final DomainStateDb domainStateDb;
     private final WarcRecorder warcRecorder;
     private final CrawlerRevisitor crawlerRevisitor;
+    private final DomainCookies cookies = new DomainCookies();
+
+    private static final CrawlerConnectionThrottle connectionThrottle = new CrawlerConnectionThrottle(
+            Duration.ofSeconds(1) // pace the connections to avoid network congestion at startup
+    );
 
     int errorCount = 0;
 
@@ -90,30 +96,63 @@ public class CrawlerRetreiver implements AutoCloseable {
     }
 
     public int crawlDomain(DomainLinks domainLinks, CrawlDataReference oldCrawlData) {
-        try {
-            // Do an initial domain probe to determine the root URL
-            EdgeUrl rootUrl;
+        try (oldCrawlData) {
 
+            // Wait for permission to open a connection to avoid network congestion
+            // from hundreds/thousands of TCP handshakes
+            connectionThrottle.waitForConnectionPermission();
+
+            // Do an initial domain probe to determine the root URL
             var probeResult = probeRootUrl();
-            switch (probeResult) {
+
+            return switch (probeResult) {
                 case HttpFetcher.DomainProbeResult.Ok(EdgeUrl probedUrl) -> {
-                    rootUrl = probedUrl; // Good track
+
+                    // Sleep after the initial probe, we don't have access to the robots.txt yet
+                    // so we don't know the crawl delay
+                    TimeUnit.SECONDS.sleep(1);
+
+                    final SimpleRobotRules robotsRules = fetcher.fetchRobotRules(probedUrl.domain, warcRecorder);
+                    final CrawlDelayTimer delayTimer = new CrawlDelayTimer(robotsRules.getCrawlDelay());
+
+                    delayTimer.waitFetchDelay(0); // initial delay after robots.txt
+
+                    DomainStateDb.SummaryRecord summaryRecord = sniffRootDocument(probedUrl, delayTimer);
+                    domainStateDb.save(summaryRecord);
+
+                    if (Thread.interrupted()) {
+                        // There's a small chance we're interrupted during the sniffing portion
+                        throw new InterruptedException();
+                    }
+
+                    Instant recrawlStart = Instant.now();
+                    CrawlerRevisitor.RecrawlMetadata recrawlMetadata = crawlerRevisitor.recrawl(oldCrawlData, cookies, robotsRules, delayTimer);
+                    Duration recrawlTime = Duration.between(recrawlStart, Instant.now());
+
+                    // Play back the old crawl data (if present) and fetch the documents comparing etags and last-modified
+                    if (recrawlMetadata.size() > 0) {
+                        // If we have reference data, we will always grow the crawl depth a bit
+                        crawlFrontier.increaseDepth(1.5, 2500);
+                    }
+
+                    oldCrawlData.close(); // proactively close the crawl data reference here to not hold onto expensive resources
+
+                    yield crawlDomain(probedUrl, robotsRules, delayTimer, domainLinks, recrawlMetadata, recrawlTime);
                 }
                 case HttpFetcher.DomainProbeResult.Redirect(EdgeDomain domain1) -> {
                     domainStateDb.save(DomainStateDb.SummaryRecord.forError(domain, "Redirect", domain1.toString()));
-                    return 1;
+                    yield 1;
                 }
                 case HttpFetcher.DomainProbeResult.Error(CrawlerDomainStatus status, String desc) -> {
                     domainStateDb.save(DomainStateDb.SummaryRecord.forError(domain, status.toString(), desc));
-                    return 1;
+                    yield 1;
                 }
-            }
+                default -> {
+                    logger.error("Unexpected domain probe result {}", probeResult);
+                    yield 1;
+                }
+            };
 
-            // Sleep after the initial probe, we don't have access to the robots.txt yet
-            // so we don't know the crawl delay
-            TimeUnit.SECONDS.sleep(1);
-
-            return crawlDomain(oldCrawlData, rootUrl, domainLinks);
         }
         catch (Exception ex) {
             logger.error("Error crawling domain {}", domain, ex);
@@ -121,33 +160,32 @@ public class CrawlerRetreiver implements AutoCloseable {
         }
     }
 
-    private int crawlDomain(CrawlDataReference oldCrawlData,
-                            EdgeUrl rootUrl,
-                            DomainLinks domainLinks) throws InterruptedException {
+    private int crawlDomain(EdgeUrl rootUrl,
+                            SimpleRobotRules robotsRules,
+                            CrawlDelayTimer delayTimer,
+                            DomainLinks domainLinks,
+                            CrawlerRevisitor.RecrawlMetadata recrawlMetadata,
+                            Duration recrawlTime) {
 
-        final SimpleRobotRules robotsRules = fetcher.fetchRobotRules(rootUrl.domain, warcRecorder);
-        final CrawlDelayTimer delayTimer = new CrawlDelayTimer(robotsRules.getCrawlDelay());
-
-        delayTimer.waitFetchDelay(0); // initial delay after robots.txt
-
-        DomainStateDb.SummaryRecord summaryRecord = sniffRootDocument(rootUrl, delayTimer);
-        domainStateDb.save(summaryRecord);
-
-        // Play back the old crawl data (if present) and fetch the documents comparing etags and last-modified
-        if (crawlerRevisitor.recrawl(oldCrawlData, robotsRules, delayTimer) > 0) {
-            // If we have reference data, we will always grow the crawl depth a bit
-            crawlFrontier.increaseDepth(1.5, 2500);
-        }
+        Instant crawlStart = Instant.now();
 
         // Add external links to the crawl frontier
         crawlFrontier.addAllToQueue(domainLinks.getUrls(rootUrl.proto));
 
-
         // Fetch sitemaps
         for (var sitemap : robotsRules.getSitemaps()) {
-            crawlFrontier.addAllToQueue(fetcher.fetchSitemapUrls(sitemap, delayTimer));
+
+            // Validate the sitemap URL and check if it belongs to the domain as the root URL
+            if (EdgeUrl.parse(sitemap)
+                    .map(url -> url.getDomain().equals(rootUrl.domain))
+                    .orElse(false)) {
+
+                crawlFrontier.addAllToQueue(fetcher.fetchSitemapUrls(sitemap, delayTimer));
+            }
         }
 
+        int crawlerAdditions = 0;
+
         while (!crawlFrontier.isEmpty()
             && !crawlFrontier.isCrawlDepthReached()
             && errorCount < MAX_ERRORS
@@ -179,7 +217,11 @@ public class CrawlerRetreiver implements AutoCloseable {
                 continue;
 
             try {
-                fetchContentWithReference(top, delayTimer, DocumentWithReference.empty());
+                var result = fetchContentWithReference(top, delayTimer, DocumentWithReference.empty());
+
+                if (result.isOk()) {
+                    crawlerAdditions++;
+                }
             }
             catch (InterruptedException ex) {
                 Thread.currentThread().interrupt();
@@ -187,6 +229,17 @@ public class CrawlerRetreiver implements AutoCloseable {
             }
         }
 
+        Duration crawlTime = Duration.between(crawlStart, Instant.now());
+        domainStateDb.save(new DomainStateDb.CrawlMeta(
+                domain,
+                Instant.now(),
+                recrawlTime,
+                crawlTime,
+                recrawlMetadata.errors(),
+                crawlerAdditions,
+                recrawlMetadata.size() + crawlerAdditions
+        ));
+
         return crawlFrontier.visitedSize();
     }
 
@@ -215,17 +268,29 @@ public class CrawlerRetreiver implements AutoCloseable {
         return domainProbeResult;
     }
 
+
+
     private DomainStateDb.SummaryRecord sniffRootDocument(EdgeUrl rootUrl, CrawlDelayTimer timer) {
         Optional<String> feedLink = Optional.empty();
 
         try {
             var url = rootUrl.withPathAndParam("/", null);
 
-            HttpFetchResult result = fetchWithRetry(url, timer, HttpFetcher.ProbeType.DISABLED, ContentTags.empty());
+            HttpFetchResult result = fetcher.fetchContent(url, warcRecorder, cookies, timer, ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
             timer.waitFetchDelay(0);
 
-            if (!(result instanceof HttpFetchResult.ResultOk ok))
+            if (result instanceof HttpFetchResult.ResultRedirect(EdgeUrl location)) {
+                if (Objects.equals(location.domain, url.domain)) {
+                    // TODO: Follow the redirect to the new location and sniff the document
+                    crawlFrontier.addFirst(location);
+                }
+
                 return DomainStateDb.SummaryRecord.forSuccess(domain);
+            }
+
+            if (!(result instanceof HttpFetchResult.ResultOk ok)) {
+                return DomainStateDb.SummaryRecord.forSuccess(domain);
+            }
 
             var optDoc = ok.parseDocument();
             if (optDoc.isEmpty())
@@ -273,12 +338,25 @@ public class CrawlerRetreiver implements AutoCloseable {
             feedLink.ifPresent(s -> fetcher.fetchSitemapUrls(s, timer));
 
             // Grab the favicon if it exists
-            fetchWithRetry(faviconUrl, timer, HttpFetcher.ProbeType.DISABLED, ContentTags.empty());
+
+            if (fetcher.fetchContent(faviconUrl, warcRecorder, cookies, timer, ContentTags.empty(), HttpFetcher.ProbeType.DISABLED) instanceof HttpFetchResult.ResultOk iconResult) {
+                String contentType = iconResult.header("Content-Type");
+                byte[] iconData = iconResult.getBodyBytes();
+
+                domainStateDb.saveIcon(
+                        domain,
+                        new DomainStateDb.FaviconRecord(contentType, iconData)
+                );
+            }
             timer.waitFetchDelay(0);
 
         }
         catch (Exception ex) {
             logger.error("Error configuring link filter", ex);
+            if (Thread.interrupted()) {
+                Thread.currentThread().interrupt();
+                return DomainStateDb.SummaryRecord.forError(domain, "Crawler Interrupted", ex.getMessage());
+            }
         }
         finally {
             crawlFrontier.addVisited(rootUrl);
@@ -306,7 +384,7 @@ public class CrawlerRetreiver implements AutoCloseable {
     );
 
     private Optional<String> guessFeedUrl(CrawlDelayTimer timer) throws InterruptedException {
-        var oldDomainStateRecord = domainStateDb.get(domain);
+        var oldDomainStateRecord = domainStateDb.getSummary(domain);
 
         // If we are already aware of an old feed URL, then we can just revalidate it
         if (oldDomainStateRecord.isPresent()) {
@@ -331,7 +409,7 @@ public class CrawlerRetreiver implements AutoCloseable {
         if (parsedOpt.isEmpty())
             return false;
 
-        HttpFetchResult result = fetchWithRetry(parsedOpt.get(), timer, HttpFetcher.ProbeType.DISABLED, ContentTags.empty());
+        HttpFetchResult result = fetcher.fetchContent(parsedOpt.get(), warcRecorder, cookies, timer, ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
         timer.waitFetchDelay(0);
 
         if (!(result instanceof HttpFetchResult.ResultOk ok)) {
@@ -357,110 +435,63 @@ public class CrawlerRetreiver implements AutoCloseable {
                                                      CrawlDelayTimer timer,
                                                      DocumentWithReference reference) throws InterruptedException
     {
-        logger.debug("Fetching {}", top);
-
-        long startTime = System.currentTimeMillis();
         var contentTags = reference.getContentTags();
 
-        HttpFetchResult fetchedDoc = fetchWithRetry(top, timer, HttpFetcher.ProbeType.FULL, contentTags);
+        HttpFetchResult fetchedDoc = fetcher.fetchContent(top, warcRecorder, cookies, timer, contentTags, HttpFetcher.ProbeType.FULL);
+        timer.waitFetchDelay();
+
+        if (Thread.interrupted()) {
+            Thread.currentThread().interrupt();
+            throw new InterruptedException();
+        }
 
         // Parse the document and enqueue links
         try {
-            if (fetchedDoc instanceof HttpFetchResult.ResultOk ok) {
-                var docOpt = ok.parseDocument();
-                if (docOpt.isPresent()) {
-                    var doc = docOpt.get();
+            switch (fetchedDoc) {
+                case HttpFetchResult.ResultOk ok -> {
+                    var docOpt = ok.parseDocument();
+                    if (docOpt.isPresent()) {
+                        var doc = docOpt.get();
 
-                    crawlFrontier.enqueueLinksFromDocument(top, doc);
-                    crawlFrontier.addVisited(new EdgeUrl(ok.uri()));
+                        var responseUrl = new EdgeUrl(ok.uri());
+
+                        crawlFrontier.enqueueLinksFromDocument(responseUrl, doc);
+                        crawlFrontier.addVisited(responseUrl);
+                    }
                 }
-            }
-            else if (fetchedDoc instanceof HttpFetchResult.Result304Raw && reference.doc() != null) {
-                var doc = reference.doc();
+                case HttpFetchResult.Result304Raw ref when reference.doc() != null ->
+                {
+                    var doc = reference.doc();
 
-                warcRecorder.writeReferenceCopy(top, doc.contentType, doc.httpStatus, doc.documentBodyBytes, doc.headers, contentTags);
+                    warcRecorder.writeReferenceCopy(top, cookies, doc.contentType, doc.httpStatus, doc.documentBodyBytes, doc.headers, contentTags);
 
-                fetchedDoc = new HttpFetchResult.Result304ReplacedWithReference(doc.url,
-                        new ContentType(doc.contentType, "UTF-8"),
-                        doc.documentBodyBytes);
+                    fetchedDoc = new HttpFetchResult.Result304ReplacedWithReference(doc.url,
+                            new ContentType(doc.contentType, "UTF-8"),
+                            doc.documentBodyBytes);
 
-                if (doc.documentBodyBytes != null) {
-                    var parsed = doc.parseBody();
+                    if (doc.documentBodyBytes != null) {
+                        var parsed = doc.parseBody();
 
-                    crawlFrontier.enqueueLinksFromDocument(top, parsed);
-                    crawlFrontier.addVisited(top);
+                        crawlFrontier.enqueueLinksFromDocument(top, parsed);
+                        crawlFrontier.addVisited(top);
+                    }
                 }
-            }
-            else if (fetchedDoc instanceof HttpFetchResult.ResultException) {
-                errorCount ++;
+                case HttpFetchResult.ResultRedirect(EdgeUrl location) -> {
+                    if (Objects.equals(location.domain, top.domain)) {
+                        crawlFrontier.addFirst(location);
+                    }
+                }
+                case HttpFetchResult.ResultException ex -> errorCount++;
+                default -> {} // Ignore other types
             }
         }
         catch (Exception ex) {
             logger.error("Error parsing document {}", top, ex);
         }
 
-        timer.waitFetchDelay(System.currentTimeMillis() - startTime);
-
         return fetchedDoc;
     }
 
-    /** Fetch a document and retry on 429s */
-    private HttpFetchResult fetchWithRetry(EdgeUrl url,
-                                           CrawlDelayTimer timer,
-                                           HttpFetcher.ProbeType probeType,
-                                           ContentTags contentTags) throws InterruptedException {
-
-        long probeStart = System.currentTimeMillis();
-
-        if (probeType == HttpFetcher.ProbeType.FULL) {
-            retryLoop:
-            for (int i = 0; i <= HTTP_429_RETRY_LIMIT; i++) {
-                try {
-                    var probeResult = fetcher.probeContentType(url, warcRecorder, contentTags);
-
-                    switch (probeResult) {
-                        case HttpFetcher.ContentTypeProbeResult.Ok(EdgeUrl resolvedUrl):
-                            url = resolvedUrl; // If we were redirected while probing, use the final URL for fetching
-                            break retryLoop;
-                        case HttpFetcher.ContentTypeProbeResult.BadContentType badContentType:
-                            return new HttpFetchResult.ResultNone();
-                        case HttpFetcher.ContentTypeProbeResult.BadContentType.Timeout timeout:
-                            return new HttpFetchResult.ResultException(timeout.ex());
-                        case HttpFetcher.ContentTypeProbeResult.Exception exception:
-                            return new HttpFetchResult.ResultException(exception.ex());
-                        default:  // should be unreachable
-                            throw new IllegalStateException("Unknown probe result");
-                    }
-                }
-                catch (HttpFetcherImpl.RateLimitException ex) {
-                    timer.waitRetryDelay(ex);
-                }
-                catch (Exception ex) {
-                    logger.warn("Failed to fetch {}", url, ex);
-                    return new HttpFetchResult.ResultException(ex);
-                }
-            }
-
-            timer.waitFetchDelay(System.currentTimeMillis() - probeStart);
-        }
-
-
-        for (int i = 0; i <= HTTP_429_RETRY_LIMIT; i++) {
-            try {
-                return fetcher.fetchContent(url, warcRecorder, contentTags, probeType);
-            }
-            catch (HttpFetcherImpl.RateLimitException ex) {
-                timer.waitRetryDelay(ex);
-            }
-            catch (Exception ex) {
-                logger.warn("Failed to fetch {}", url, ex);
-                return new HttpFetchResult.ResultException(ex);
-            }
-        }
-
-        return new HttpFetchResult.ResultNone();
-    }
-
     private boolean isAllowedProtocol(String proto) {
         return proto.equalsIgnoreCase("http")
                 || proto.equalsIgnoreCase("https");

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/DomainCrawlFrontier.java

@@ -55,6 +55,9 @@ public class DomainCrawlFrontier {
         }
     }
 
+    public EdgeDomain getDomain() {
+        return thisDomain;
+    }
     /** Increase the depth of the crawl by a factor.  If the current depth is smaller
      * than the number of already visited documents, the base depth will be adjusted
      * to the visited count first.

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java

@@ -2,6 +2,7 @@ package nu.marginalia.crawl.retreival.revisit;
 
 import crawlercommons.robots.SimpleRobotRules;
 import nu.marginalia.crawl.fetcher.ContentTags;
+import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
 import nu.marginalia.crawl.retreival.CrawlDataReference;
 import nu.marginalia.crawl.retreival.CrawlDelayTimer;
@@ -10,6 +11,8 @@ import nu.marginalia.crawl.retreival.DomainCrawlFrontier;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.body.HttpFetchResult;
 import nu.marginalia.model.crawldata.CrawledDocument;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
@@ -18,10 +21,13 @@ import java.io.IOException;
  *  E-Tag and Last-Modified headers.
  */
 public class CrawlerRevisitor {
+
     private final DomainCrawlFrontier crawlFrontier;
     private final CrawlerRetreiver crawlerRetreiver;
     private final WarcRecorder warcRecorder;
 
+    private static final Logger logger = LoggerFactory.getLogger(CrawlerRevisitor.class);
+
     public CrawlerRevisitor(DomainCrawlFrontier crawlFrontier,
                             CrawlerRetreiver crawlerRetreiver,
                             WarcRecorder warcRecorder) {
@@ -31,7 +37,8 @@ public class CrawlerRevisitor {
     }
 
     /** Performs a re-crawl of old documents, comparing etags and last-modified */
-    public int recrawl(CrawlDataReference oldCrawlData,
+    public RecrawlMetadata recrawl(CrawlDataReference oldCrawlData,
+                       DomainCookies cookies,
                        SimpleRobotRules robotsRules,
                        CrawlDelayTimer delayTimer)
     throws InterruptedException {
@@ -39,19 +46,18 @@ public class CrawlerRevisitor {
         int retained = 0;
         int errors = 0;
         int skipped = 0;
+        int size = 0;
 
-        for (;;) {
+        for (CrawledDocument doc : oldCrawlData) {
             if (errors > 20) {
                 // If we've had too many errors, we'll stop trying to recrawl
                 break;
             }
 
-            CrawledDocument doc = oldCrawlData.nextDocument();
+            if (Thread.interrupted()) {
+                throw new InterruptedException();
+            }
 
-            if (doc == null)
-                break;
-
-            // This Shouldn't Happen (TM)
             var urlMaybe = EdgeUrl.parse(doc.url);
             if (urlMaybe.isEmpty())
                 continue;
@@ -84,6 +90,7 @@ public class CrawlerRevisitor {
                 continue;
             }
 
+            size++;
 
             double skipProb;
 
@@ -127,6 +134,7 @@ public class CrawlerRevisitor {
                 }
                 // Add a WARC record so we don't repeat this
                 warcRecorder.writeReferenceCopy(url,
+                        cookies,
                         doc.contentType,
                         doc.httpStatus,
                         doc.documentBodyBytes,
@@ -151,11 +159,15 @@ public class CrawlerRevisitor {
                 else if (result instanceof HttpFetchResult.ResultException) {
                     errors++;
                 }
-
                 recrawled++;
             }
         }
 
-        return recrawled;
+        logger.info("Recrawl summary {}: {} recrawled, {} retained, {} errors, {} skipped",
+                crawlFrontier.getDomain(), recrawled, retained, errors, skipped);
+
+        return new RecrawlMetadata(size, errors, skipped);
     }
+
+    public record RecrawlMetadata(int size, int errors, int skipped) {}
 }

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java

@@ -6,6 +6,7 @@ import nu.marginalia.model.body.HttpFetchResult;
 import nu.marginalia.model.crawldata.CrawledDocument;
 
 import javax.annotation.Nullable;
+import java.util.Objects;
 
 public record DocumentWithReference(
         @Nullable CrawledDocument doc,
@@ -33,8 +34,22 @@ public record DocumentWithReference(
             return false;
         if (doc == null)
             return false;
-        if (doc.documentBodyBytes.length == 0)
-            return false;
+        if (doc.documentBodyBytes.length == 0) {
+            if (doc.httpStatus < 300) {
+                return resultOk.bytesLength() == 0;
+            }
+            else if (doc.httpStatus == 301 || doc.httpStatus == 302 || doc.httpStatus == 307) {
+                @Nullable
+                String docLocation = doc.getHeader("Location");
+                @Nullable
+                String resultLocation = resultOk.header("Location");
+
+                return Objects.equals(docLocation, resultLocation);
+            }
+            else {
+                return doc.httpStatus == resultOk.statusCode();
+            }
+        }
 
         return CrawlDataReference.isContentBodySame(doc.documentBodyBytes, resultOk.bytesRaw());
     }

code/processes/crawling-process/model/build.gradle

@@ -41,6 +41,8 @@ dependencies {
     implementation libs.snakeyaml
     implementation libs.zstd
 
+    implementation libs.bundles.httpcomponents
+
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit
     testImplementation libs.mockito

code/processes/crawling-process/model/java/nu/marginalia/ContentTypes.java

@@ -6,6 +6,7 @@ public class ContentTypes {
     public static final Set<String> acceptedContentTypes = Set.of("application/xhtml+xml",
             "application/xhtml",
             "text/html",
+            "application/pdf",
             "image/x-icon",
             "text/plain");
 
@@ -19,4 +20,9 @@ public class ContentTypes {
         return false;
     }
 
+    public static boolean isBinary(String contentTypeHeader) {
+        String lcHeader = contentTypeHeader.toLowerCase();
+        return lcHeader.startsWith("application/pdf");
+    }
+
 }

code/processes/crawling-process/model/java/nu/marginalia/io/CrawledDomainReader.java

@@ -1,55 +0,0 @@
-package nu.marginalia.io;
-
-import nu.marginalia.io.crawldata.format.ParquetSerializableCrawlDataStream;
-import nu.marginalia.io.crawldata.format.SlopSerializableCrawlDataStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-public class CrawledDomainReader {
-    private static final Logger logger = LoggerFactory.getLogger(CrawledDomainReader.class);
-
-    /** An iterator-like access to domain data  This must be closed otherwise it will leak off-heap memory! */
-    public static SerializableCrawlDataStream createDataStream(Path fullPath) throws IOException
-    {
-
-        String fileName = fullPath.getFileName().toString();
-        if (fileName.endsWith(".parquet")) {
-            try {
-                return new ParquetSerializableCrawlDataStream(fullPath);
-            } catch (Exception ex) {
-                logger.error("Error reading domain data from " + fullPath, ex);
-                return SerializableCrawlDataStream.empty();
-            }
-        }
-        else if (fileName.endsWith(".slop.zip")) {
-            try {
-                return new SlopSerializableCrawlDataStream(fullPath);
-            } catch (Exception ex) {
-                logger.error("Error reading domain data from " + fullPath, ex);
-                return SerializableCrawlDataStream.empty();
-            }
-        }
-        else {
-            logger.error("Unknown file type: {}", fullPath);
-            return SerializableCrawlDataStream.empty();
-        }
-    }
-
-    /** An iterator-like access to domain data. This must be closed otherwise it will leak off-heap memory! */
-    public static SerializableCrawlDataStream createDataStream(Path basePath, String domain, String id) throws IOException {
-        Path parquetPath = CrawlerOutputFile.getParquetPath(basePath, id, domain);
-
-        if (Files.exists(parquetPath)) {
-            return createDataStream(parquetPath);
-        }
-        else {
-            throw new FileNotFoundException("No such file: " + parquetPath);
-        }
-    }
-
-}

code/processes/crawling-process/model/java/nu/marginalia/io/CrawlerOutputFile.java

@@ -35,19 +35,6 @@ public class CrawlerOutputFile {
         return destDir.resolve(id + "-" + filesystemSafeName(domain) + "-" + version.suffix + ".warc.gz");
     }
 
-    public static Path createParquetPath(Path basePath, String id, String domain) throws IOException {
-        id = padId(id);
-
-        String first = id.substring(0, 2);
-        String second = id.substring(2, 4);
-
-        Path destDir = basePath.resolve(first).resolve(second);
-        if (!Files.exists(destDir)) {
-            Files.createDirectories(destDir);
-        }
-        return destDir.resolve(id + "-" + filesystemSafeName(domain) + ".parquet");
-    }
-
     public static Path createSlopPath(Path basePath, String id, String domain) throws IOException {
         id = padId(id);
 
@@ -71,16 +58,17 @@ public class CrawlerOutputFile {
         return destDir.resolve(id + "-" + filesystemSafeName(domain) + ".parquet");
     }
 
-    public static Path getWarcPath(Path basePath, String id, String domain, WarcFileVersion version) {
+    public static Path getSlopPath(Path basePath, String id, String domain) {
         id = padId(id);
 
         String first = id.substring(0, 2);
         String second = id.substring(2, 4);
 
         Path destDir = basePath.resolve(first).resolve(second);
-        return destDir.resolve(id + "-" + filesystemSafeName(domain) + ".warc" + version.suffix);
+        return destDir.resolve(id + "-" + filesystemSafeName(domain) + ".slop.zip");
     }
 
+
     /**
      * Pads the given ID with leading zeros to ensure it has a length of 4 characters.
      */

code/processes/crawling-process/model/java/nu/marginalia/io/SerializableCrawlDataStream.java

@@ -1,5 +1,7 @@
 package nu.marginalia.io;
 
+import nu.marginalia.io.crawldata.format.ParquetSerializableCrawlDataStream;
+import nu.marginalia.io.crawldata.format.SlopSerializableCrawlDataStream;
 import nu.marginalia.model.crawldata.CrawledDocument;
 import nu.marginalia.model.crawldata.CrawledDomain;
 import nu.marginalia.model.crawldata.SerializableCrawlData;
@@ -18,7 +20,6 @@ import java.util.function.Function;
 /** Closable iterator exceptional over serialized crawl data
  * The data may appear in any order, and the iterator must be closed.
  *
- * @see CrawledDomainReader
  * */
 public interface SerializableCrawlDataStream extends AutoCloseable {
     Logger logger = LoggerFactory.getLogger(SerializableCrawlDataStream.class);
@@ -27,13 +28,60 @@ public interface SerializableCrawlDataStream extends AutoCloseable {
 
     /** Return a size hint for the stream.  0 is returned if the hint is not available,
      * or if the file is seemed too small to bother */
-    default int sizeHint() { return 0; }
+    default int getSizeHint() { return 0; }
 
     boolean hasNext() throws IOException;
 
     @Nullable
     default Path path() { return null; }
 
+    void close() throws IOException;
+
+    /** An iterator-like access to domain data  This must be closed otherwise it will leak off-heap memory! */
+    static SerializableCrawlDataStream openDataStream(Path fullPath) throws IOException
+    {
+
+        String fileName = fullPath.getFileName().toString();
+
+        if (fileName.endsWith(".slop.zip")) {
+            try {
+                return new SlopSerializableCrawlDataStream(fullPath);
+            } catch (Exception ex) {
+                logger.error("Error reading domain data from " + fullPath, ex);
+                return SerializableCrawlDataStream.empty();
+            }
+        }
+
+        else if (fileName.endsWith(".parquet")) {
+            logger.error("Opening deprecated parquet-style crawl data stream", new Exception());
+            try {
+                return new ParquetSerializableCrawlDataStream(fullPath);
+            } catch (Exception ex) {
+                logger.error("Error reading domain data from " + fullPath, ex);
+                return SerializableCrawlDataStream.empty();
+            }
+        }
+
+        logger.error("Unknown file type: {}", fullPath);
+        return SerializableCrawlDataStream.empty();
+    }
+
+    /** Get an idication of the size of the stream.  This is used to determine whether to
+     * load the stream into memory or not.  0 is returned if the hint is not available,
+     * or if the file is seemed too small to bother */
+    static int getSizeHint(Path fullPath) {
+        String fileName = fullPath.getFileName().toString();
+        if (fileName.endsWith(".parquet")) {
+            return ParquetSerializableCrawlDataStream.sizeHint(fullPath);
+        }
+        else if (fileName.endsWith(".slop.zip")) {
+            return SlopSerializableCrawlDataStream.sizeHint(fullPath);
+        }
+        else {
+            return 0;
+        }
+    }
+
     default <T>  Iterator<T> map(Function<SerializableCrawlData, Optional<T>> mapper) {
         return new Iterator<>() {
             T next = null;

code/processes/crawling-process/model/java/nu/marginalia/io/crawldata/format/ParquetSerializableCrawlDataStream.java

@@ -40,7 +40,7 @@ public class ParquetSerializableCrawlDataStream implements AutoCloseable, Serial
         return path;
     }
 
-    public int sizeHint() {
+    public static int sizeHint(Path path) {
         // Only calculate size hint for large files
         // (the reason we calculate them in the first place is to assess whether it is large
         // because it has many documents, or because it is a small number of large documents)

code/processes/crawling-process/model/java/nu/marginalia/io/crawldata/format/SlopSerializableCrawlDataStream.java

@@ -37,8 +37,12 @@ public class SlopSerializableCrawlDataStream implements AutoCloseable, Serializa
             public boolean filter(String url, int status, String contentType) {
                 String ctLc = contentType.toLowerCase();
 
+                // Permit all plain text content types
                 if (ctLc.startsWith("text/"))
                     return true;
+                // PDF
+                else if (ctLc.startsWith("application/pdf"))
+                    return true;
                 else if (ctLc.startsWith("x-marginalia/"))
                     return true;
 
@@ -52,7 +56,7 @@ public class SlopSerializableCrawlDataStream implements AutoCloseable, Serializa
         return path;
     }
 
-    public int sizeHint() {
+    public static int sizeHint(Path path) {
         // Only calculate size hint for large files
         // (the reason we calculate them in the first place is to assess whether it is large
         // because it has many documents, or because it is a small number of large documents)

code/processes/crawling-process/model/java/nu/marginalia/model/body/ContentTypeLogic.java

@@ -10,7 +10,7 @@ import java.util.regex.Pattern;
 
 public class ContentTypeLogic {
 
-    private static final Predicate<String> probableHtmlPattern = Pattern.compile("^.*\\.(htm|html|php|txt|md)$").asMatchPredicate();
+    private static final Predicate<String> probableGoodPattern = Pattern.compile("^.*\\.(htm|html|php|txt|md|pdf)$").asMatchPredicate();
     private static final Predicate<String> probableBinaryPattern = Pattern.compile("^.*\\.[a-z]+$").asMatchPredicate();
     private static final Set<String> blockedContentTypes = Set.of("text/css", "text/javascript");
     private static final List<String> acceptedContentTypePrefixes = List.of(
@@ -22,6 +22,7 @@ public class ContentTypeLogic {
             "application/rss+xml",
             "application/x-rss+xml",
             "application/rdf+xml",
+            "application/pdf",
             "x-rss+xml"
     );
     private boolean allowAllContentTypes = false;
@@ -34,7 +35,7 @@ public class ContentTypeLogic {
     public boolean isUrlLikeBinary(EdgeUrl url) {
         String pathLowerCase = url.path.toLowerCase();
 
-        if (probableHtmlPattern.test(pathLowerCase))
+        if (probableGoodPattern.test(pathLowerCase))
             return false;
 
         return probableBinaryPattern.test(pathLowerCase);

code/processes/crawling-process/model/java/nu/marginalia/model/body/HttpFetchResult.java

@@ -1,6 +1,9 @@
 package nu.marginalia.model.body;
 
 import nu.marginalia.contenttype.ContentType;
+import nu.marginalia.model.EdgeUrl;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.jetbrains.annotations.Nullable;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -11,7 +14,9 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.URI;
-import java.net.http.HttpHeaders;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Optional;
 
 /* FIXME:  This interface has a very unfortunate name that is not very descriptive.
@@ -56,15 +61,30 @@ public sealed interface HttpFetchResult {
      */
     record ResultOk(URI uri,
                     int statusCode,
-                    HttpHeaders headers,
+                    Header[] headers,
                     String ipAddress,
-                    byte[] bytesRaw,
+                    byte[] bytesRaw, // raw data for the entire response including headers
                     int bytesStart,
                     int bytesLength
     ) implements HttpFetchResult {
 
         public ResultOk(URI uri, int status, MessageHeaders headers, String ipAddress, byte[] bytes, int bytesStart, int length) {
-            this(uri, status, HttpHeaders.of(headers.map(), (k,v) -> true), ipAddress, bytes, bytesStart, length);
+            this(uri, status, convertHeaders(headers), ipAddress, bytes, bytesStart, length);
+        }
+
+        private static Header[] convertHeaders(MessageHeaders messageHeaders) {
+            List<Header> headers = new ArrayList<>(12);
+
+            messageHeaders.map().forEach((k, v) -> {
+                if (k.isBlank()) return;
+                if (!Character.isAlphabetic(k.charAt(0))) return;
+
+                for (var value : v) {
+                    headers.add(new BasicHeader(k, value));
+                }
+            });
+
+            return headers.toArray(new Header[0]);
         }
 
         public boolean isOk() {
@@ -75,6 +95,12 @@ public sealed interface HttpFetchResult {
             return new ByteArrayInputStream(bytesRaw, bytesStart, bytesLength);
         }
 
+        /** Copy the byte range corresponding to the payload of the response,
+            Warning:  Copies the data, use getInputStream() for zero copy access */
+        public byte[] getBodyBytes() {
+            return Arrays.copyOfRange(bytesRaw, bytesStart, bytesStart + bytesLength);
+        }
+
         public Optional<Document> parseDocument() {
             return DocumentBodyExtractor.asString(this).flatMapOpt((contentType, body) -> {
                 if (contentType.is("text/html")) {
@@ -88,7 +114,13 @@ public sealed interface HttpFetchResult {
 
         @Nullable
         public String header(String name) {
-            return headers.firstValue(name).orElse(null);
+            for (var header : headers) {
+                if (header.getName().equalsIgnoreCase(name)) {
+                    String headerValue = header.getValue();
+                    return headerValue;
+                }
+            }
+            return null;
         }
 
     }
@@ -112,6 +144,12 @@ public sealed interface HttpFetchResult {
         }
     }
 
+    record ResultRedirect(EdgeUrl url) implements HttpFetchResult {
+        public boolean isOk() {
+            return true;
+        }
+    }
+
     /** Fetching resulted in a HTTP 304, the remote content is identical to
      * our reference copy.  This will be replaced with a Result304ReplacedWithReference
      * at a later stage.

code/processes/crawling-process/model/java/nu/marginalia/model/crawldata/CrawledDocument.java

@@ -59,9 +59,12 @@ public final class CrawledDocument implements SerializableCrawlData {
     }
 
     public Document parseBody() throws IOException {
+        // Prevent stalls from parsing excessively large documents
+
         return DocumentBodyToString.getParsedData(
                 ContentType.parse(contentType),
                 documentBodyBytes,
+                200_000,
                 url);
     }
 
@@ -99,7 +102,7 @@ public final class CrawledDocument implements SerializableCrawlData {
     }
 
     @Nullable
-    private String getHeader(String header) {
+    public String getHeader(String header) {
         if (headers == null) {
             return null;
         }

code/processes/crawling-process/model/java/nu/marginalia/parquet/crawldata/CrawledDocumentParquetRecordFileWriter.java

@@ -165,12 +165,26 @@ public class CrawledDocumentParquetRecordFileWriter implements AutoCloseable {
             contentType = "";
         }
 
+        boolean hasCookies = false;
+        String etag = null;
+        String lastModified = null;
+
         StringJoiner headersStrBuilder = new StringJoiner("\n");
-        for (var header : headers.map().entrySet()) {
-            for (var value : header.getValue()) {
-                headersStrBuilder.add(header.getKey() + ": " + value);
+        for (var header : headers) {
+            if (header.getName().equalsIgnoreCase("X-Has-Cookies")) {
+                hasCookies = hasCookies || header.getValue().equals("1");
+            }
+            else if (header.getName().equalsIgnoreCase("ETag")) {
+                etag = header.getValue();
+            }
+            else if (header.getName().equalsIgnoreCase("Last-Modified")) {
+                lastModified = header.getValue();
+            }
+            else {
+                headersStrBuilder.add(header.getName() + ": " + header.getValue());
             }
         }
+
         String headersStr = headersStrBuilder.toString();
 
 
@@ -178,14 +192,14 @@ public class CrawledDocumentParquetRecordFileWriter implements AutoCloseable {
                 domain,
                 response.target(),
                 fetchOk.ipAddress(),
-                headers.firstValue("X-Has-Cookies").orElse("0").equals("1"),
+                hasCookies,
                 fetchOk.statusCode(),
                 response.date(),
                 contentType,
                 bodyBytes,
                 headersStr,
-                headers.firstValue("ETag").orElse(null),
-                headers.firstValue("Last-Modified").orElse(null)
+                etag,
+                lastModified
         ));
     }
 

code/processes/crawling-process/model/java/nu/marginalia/slop/SlopCrawlDataRecord.java

@@ -108,15 +108,17 @@ public record SlopCrawlDataRecord(String domain,
     public static void convertFromParquet(Path parquetInput, Path slopOutput) throws IOException {
         Path tempDir = Files.createTempDirectory(slopOutput.getParent(), "conversion");
 
-        try (var writer = new Writer(tempDir)) {
-            CrawledDocumentParquetRecordFileReader.stream(parquetInput).forEach(
-                    parquetRecord -> {
-                        try {
-                            writer.write(new SlopCrawlDataRecord(parquetRecord));
-                        } catch (IOException e) {
-                            throw new RuntimeException(e);
-                        }
-                    });
+        try (var writer = new Writer(tempDir);
+             var stream = CrawledDocumentParquetRecordFileReader.stream(parquetInput))
+        {
+            stream.forEach(
+                parquetRecord -> {
+                    try {
+                        writer.write(new SlopCrawlDataRecord(parquetRecord));
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
         }
         catch (IOException ex) {
             FileUtils.deleteDirectory(tempDir.toFile());
@@ -214,6 +216,11 @@ public record SlopCrawlDataRecord(String domain,
             return false;
         }
 
+        // If the format is binary, we don't want to translate it if the response is truncated
+        if (response.truncated() != WarcTruncationReason.NOT_TRUNCATED && ContentTypes.isBinary(contentType)) {
+            return false;
+        }
+
         return true;
     }
 
@@ -339,12 +346,15 @@ public record SlopCrawlDataRecord(String domain,
                 contentType = "";
             }
 
+            boolean hasCookies = false;
+
             String headersStr;
             StringJoiner headersStrBuilder = new StringJoiner("\n");
-            for (var header : headers.map().entrySet()) {
-                for (var value : header.getValue()) {
-                    headersStrBuilder.add(header.getKey() + ": " + value);
+            for (var header : headers) {
+                if (header.getName().equalsIgnoreCase("X-Cookies") && "1".equals(header.getValue())) {
+                    hasCookies = true;
                 }
+                headersStrBuilder.add(header.getName() + ": " + header.getValue());
             }
             headersStr = headersStrBuilder.toString();
 
@@ -353,7 +363,7 @@ public record SlopCrawlDataRecord(String domain,
                     domain,
                     response.target(),
                     fetchOk.ipAddress(),
-                    "1".equals(headers.firstValue("X-Cookies").orElse("0")),
+                    hasCookies,
                     fetchOk.statusCode(),
                     response.date().toEpochMilli(),
                     contentType,

code/processes/crawling-process/test/nu/marginalia/crawl/DomainStateDbTest.java

@@ -8,9 +8,10 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.sql.SQLException;
+import java.time.Duration;
 import java.time.Instant;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.*;
 
 class DomainStateDbTest {
 
@@ -26,7 +27,7 @@ class DomainStateDbTest {
     }
 
     @Test
-    public void testSunnyDay() throws SQLException {
+    public void testSummaryRecord() throws SQLException {
         try (var db = new DomainStateDb(tempFile)) {
             var allFields = new DomainStateDb.SummaryRecord(
                     "all.marginalia.nu",
@@ -47,8 +48,8 @@ class DomainStateDbTest {
             db.save(allFields);
             db.save(minFields);
 
-            assertEquals(allFields, db.get("all.marginalia.nu").orElseThrow());
-            assertEquals(minFields, db.get("min.marginalia.nu").orElseThrow());
+            assertEquals(allFields, db.getSummary("all.marginalia.nu").orElseThrow());
+            assertEquals(minFields, db.getSummary("min.marginalia.nu").orElseThrow());
 
             var updatedAllFields = new DomainStateDb.SummaryRecord(
                     "all.marginalia.nu",
@@ -59,7 +60,36 @@ class DomainStateDbTest {
             );
 
             db.save(updatedAllFields);
-            assertEquals(updatedAllFields, db.get("all.marginalia.nu").orElseThrow());
+            assertEquals(updatedAllFields, db.getSummary("all.marginalia.nu").orElseThrow());
+        }
+    }
+
+    @Test
+    public void testMetadata() throws SQLException {
+        try (var db = new DomainStateDb(tempFile)) {
+            var original = new DomainStateDb.CrawlMeta("example.com", Instant.ofEpochMilli(12345), Duration.ofMillis(30), Duration.ofMillis(300), 1, 2, 3);
+            db.save(original);
+
+            var maybeMeta = db.getMeta("example.com");
+            assertTrue(maybeMeta.isPresent());
+            assertEquals(original, maybeMeta.get());
+        }
+    }
+
+    @Test
+    public void testFavicon() throws SQLException {
+        try (var db = new DomainStateDb(tempFile)) {
+            db.saveIcon("www.marginalia.nu", new DomainStateDb.FaviconRecord("text/plain", "hello world".getBytes()));
+
+            var maybeData = db.getIcon("www.marginalia.nu");
+            assertTrue(maybeData.isPresent());
+            var actualData = maybeData.get();
+
+            assertEquals("text/plain", actualData.contentType());
+            assertArrayEquals("hello world".getBytes(), actualData.imageData());
+
+            maybeData = db.getIcon("foobar");
+            assertTrue(maybeData.isEmpty());
         }
     }
 

code/processes/crawling-process/test/nu/marginalia/crawl/fetcher/HttpFetcherImplContentTypeProbeTest.java

@@ -0,0 +1,146 @@
+package nu.marginalia.crawl.fetcher;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import nu.marginalia.UserAgent;
+import nu.marginalia.crawl.retreival.CrawlDelayTimer;
+import nu.marginalia.model.EdgeUrl;
+import org.junit.jupiter.api.*;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("slow")
+class HttpFetcherImplContentTypeProbeTest {
+
+    private HttpFetcherImpl fetcher;
+    private  static WireMockServer wireMockServer;
+
+    private static EdgeUrl timeoutUrl;
+    private static EdgeUrl contentTypeHtmlUrl;
+    private static EdgeUrl contentTypeBinaryUrl;
+    private static EdgeUrl redirectUrl;
+    private static EdgeUrl badHttpStatusUrl;
+    private static EdgeUrl onlyGetAllowedUrl;
+
+    @BeforeAll
+    public static void setupAll() throws URISyntaxException {
+        wireMockServer =
+                new WireMockServer(WireMockConfiguration.wireMockConfig()
+                        .port(18089));
+
+        timeoutUrl = new EdgeUrl("http://localhost:18089/timeout.bin");
+
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(timeoutUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withFixedDelay(15000))); // 10 seconds delay to simulate timeout
+
+        contentTypeHtmlUrl = new EdgeUrl("http://localhost:18089/test.html.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(contentTypeHtmlUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(200)));
+
+        contentTypeBinaryUrl = new EdgeUrl("http://localhost:18089/test.bad.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(contentTypeBinaryUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "application/octet-stream")
+                        .withStatus(200)));
+
+        redirectUrl = new EdgeUrl("http://localhost:18089/redirect.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(redirectUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Location", "http://localhost:18089/test.html.bin")
+                        .withStatus(301)));
+
+        badHttpStatusUrl = new EdgeUrl("http://localhost:18089/badstatus.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(badHttpStatusUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(500)));
+
+        onlyGetAllowedUrl = new EdgeUrl("http://localhost:18089/onlyget.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(onlyGetAllowedUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withStatus(405))); // Method Not Allowed
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(onlyGetAllowedUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(200)));
+
+        wireMockServer.start();
+
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        wireMockServer.stop();
+    }
+
+    @BeforeEach
+    public void setUp() {
+        fetcher = new HttpFetcherImpl(new UserAgent("test.marginalia.nu", "test.marginalia.nu"));
+    }
+
+    @AfterEach
+    public void tearDown() throws IOException {
+        var stats = fetcher.getPoolStats();
+        assertEquals(0, stats.getLeased());
+        assertEquals(0, stats.getPending());
+
+        fetcher.close();
+    }
+
+    @Test
+    public void testProbeContentTypeHtmlShortcircuitPath() throws URISyntaxException {
+        var result = fetcher.probeContentType(new EdgeUrl("https://localhost/test.html"), new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertInstanceOf(HttpFetcher.ContentTypeProbeResult.NoOp.class, result);
+    }
+
+
+    @Test
+    public void testProbeContentTypeHtmlShortcircuitTags() {
+        var result = fetcher.probeContentType(contentTypeBinaryUrl, new DomainCookies(), new CrawlDelayTimer(50), new ContentTags("a", "b"));
+        Assertions.assertInstanceOf(HttpFetcher.ContentTypeProbeResult.NoOp.class, result);
+    }
+
+    @Test
+    public void testProbeContentTypeHtml() {
+        var result = fetcher.probeContentType(contentTypeHtmlUrl, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertEquals(new HttpFetcher.ContentTypeProbeResult.Ok(contentTypeHtmlUrl), result);
+    }
+
+    @Test
+    public void testProbeContentTypeBinary() {
+        var result = fetcher.probeContentType(contentTypeBinaryUrl, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertEquals(new HttpFetcher.ContentTypeProbeResult.BadContentType("application/octet-stream", 200), result);
+    }
+
+    @Test
+    public void testProbeContentTypeRedirect() {
+        var result = fetcher.probeContentType(redirectUrl, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertEquals(new HttpFetcher.ContentTypeProbeResult.Redirect(contentTypeHtmlUrl), result);
+    }
+
+    @Test
+    public void testProbeContentTypeBadHttpStatus() {
+        var result = fetcher.probeContentType(badHttpStatusUrl, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertEquals(new HttpFetcher.ContentTypeProbeResult.HttpError(500, "Bad status code"), result);
+    }
+
+    @Test
+    public void testOnlyGetAllowed() {
+        var result = fetcher.probeContentType(onlyGetAllowedUrl, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertEquals(new HttpFetcher.ContentTypeProbeResult.Ok(onlyGetAllowedUrl), result);
+    }
+
+    @Test
+    public void testTimeout() {
+        var result = fetcher.probeContentType(timeoutUrl, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
+        Assertions.assertInstanceOf(HttpFetcher.ContentTypeProbeResult.Timeout.class, result);
+    }
+
+}

code/processes/crawling-process/test/nu/marginalia/crawl/fetcher/HttpFetcherImplDomainProbeTest.java

@@ -0,0 +1,95 @@
+package nu.marginalia.crawl.fetcher;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import nu.marginalia.UserAgent;
+import nu.marginalia.model.EdgeDomain;
+import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.model.crawldata.CrawlerDomainStatus;
+import org.junit.jupiter.api.*;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("slow")
+class HttpFetcherImplDomainProbeTest {
+
+    private HttpFetcherImpl fetcher;
+    private  static WireMockServer wireMockServer;
+
+    private static EdgeUrl timeoutUrl;
+
+    @BeforeAll
+    public static void setupAll() throws URISyntaxException {
+        wireMockServer =
+                new WireMockServer(WireMockConfiguration.wireMockConfig()
+                        .port(18089));
+
+
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo("/timeout"))
+                .willReturn(WireMock.aResponse()
+                        .withFixedDelay(15000))); // 10 seconds delay to simulate timeout
+
+        wireMockServer.start();
+        timeoutUrl = new EdgeUrl("http://localhost:18089/timeout");
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        wireMockServer.stop();
+    }
+
+    @BeforeEach
+    public void setUp() {
+        fetcher = new HttpFetcherImpl(new UserAgent("test.marginalia.nu", "test.marginalia.nu"));
+    }
+
+    @AfterEach
+    public void tearDown() throws IOException {
+        var stats = fetcher.getPoolStats();
+        assertEquals(0, stats.getLeased());
+        assertEquals(0, stats.getPending());
+
+        fetcher.close();
+    }
+
+    @Test
+    public void testProbeDomain() throws URISyntaxException {
+        var result = fetcher.probeDomain(new EdgeUrl("https://www.marginalia.nu/"));
+        Assertions.assertEquals(new HttpFetcher.DomainProbeResult.Ok(new EdgeUrl("https://www.marginalia.nu/")), result);
+    }
+
+    @Test
+    public void testProbeDomainProtoUpgrade() throws URISyntaxException {
+        var result = fetcher.probeDomain(new EdgeUrl("http://www.marginalia.nu/"));
+        Assertions.assertEquals(new HttpFetcher.DomainProbeResult.Ok(new EdgeUrl("https://www.marginalia.nu/")), result);
+    }
+
+    @Test
+    public void testProbeDomainRedirect() throws URISyntaxException {
+        var result = fetcher.probeDomain(new EdgeUrl("http://search.marginalia.nu/"));
+        Assertions.assertEquals(new HttpFetcher.DomainProbeResult.Redirect(new EdgeDomain("marginalia-search.com")), result);
+    }
+
+    @Test
+    public void testProbeDomainOnlyGET() throws URISyntaxException {
+        // This test is to check if the domain probe only allows GET requests
+        var result = fetcher.probeDomain(new EdgeUrl("https://marginalia-search.com/"));
+        Assertions.assertEquals(new HttpFetcher.DomainProbeResult.Ok(new EdgeUrl("https://marginalia-search.com/")), result);
+    }
+
+    @Test
+    public void testProbeDomainError() throws URISyntaxException {
+        var result = fetcher.probeDomain(new EdgeUrl("https://invalid.example.com/"));
+        Assertions.assertEquals(new HttpFetcher.DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Error during domain probe"), result);
+    }
+
+    @Test
+    public void testProbeDomainTimeout() throws URISyntaxException {
+        var result = fetcher.probeDomain(timeoutUrl);
+        Assertions.assertEquals(new HttpFetcher.DomainProbeResult.Error(CrawlerDomainStatus.ERROR, "Timeout during domain probe"), result);
+    }
+}

code/processes/crawling-process/test/nu/marginalia/crawl/fetcher/HttpFetcherImplFetchTest.java

@@ -0,0 +1,398 @@
+package nu.marginalia.crawl.fetcher;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import nu.marginalia.UserAgent;
+import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
+import nu.marginalia.crawl.retreival.CrawlDelayTimer;
+import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.model.body.HttpFetchResult;
+import org.junit.jupiter.api.*;
+import org.netpreserve.jwarc.*;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("slow")
+class HttpFetcherImplFetchTest {
+
+    private HttpFetcherImpl fetcher;
+    private static WireMockServer wireMockServer;
+
+    private static String etag = "etag";
+    private static String lastModified = "Wed, 21 Oct 2024 07:28:00 GMT";
+
+    private static EdgeUrl okUrl;
+    private static EdgeUrl okUrlSetsCookie;
+    private static EdgeUrl okRangeResponseUrl;
+    private static EdgeUrl okUrlWith304;
+
+    private static EdgeUrl timeoutUrl;
+    private static EdgeUrl redirectUrl;
+    private static EdgeUrl badHttpStatusUrl;
+    private static EdgeUrl keepAliveUrl;
+
+    private static EdgeUrl pdfUrl;
+
+    @BeforeAll
+    public static void setupAll() throws URISyntaxException {
+        wireMockServer =
+                new WireMockServer(WireMockConfiguration.wireMockConfig()
+                        .port(18089));
+
+        timeoutUrl = new EdgeUrl("http://localhost:18089/timeout.bin");
+
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(timeoutUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withFixedDelay(15000)
+                )); // 15 seconds delay to simulate timeout
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(timeoutUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withFixedDelay(15000)
+                        .withBody("Hello World")
+                )); // 15 seconds delay to simulate timeout
+
+        redirectUrl = new EdgeUrl("http://localhost:18089/redirect.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(redirectUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Location", "http://localhost:18089/test.html.bin")
+                        .withStatus(301)));
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(redirectUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Location", "http://localhost:18089/test.html.bin")
+                        .withStatus(301)));
+
+        badHttpStatusUrl = new EdgeUrl("http://localhost:18089/badstatus");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(badHttpStatusUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(500)));
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(badHttpStatusUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(500)));
+
+        okUrl = new EdgeUrl("http://localhost:18089/ok.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(okUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(200)));
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(okUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(200)
+                        .withBody("Hello World")));
+
+        okUrlSetsCookie = new EdgeUrl("http://localhost:18089/okSetCookie.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(okUrlSetsCookie.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withHeader("Set-Cookie", "test=1")
+                        .withStatus(200)));
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(okUrlSetsCookie.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withHeader("Set-Cookie", "test=1")
+                        .withStatus(200)
+                        .withBody("Hello World")));
+
+        okUrlWith304 = new EdgeUrl("http://localhost:18089/ok304.bin");
+        wireMockServer.stubFor(WireMock.head(WireMock.urlEqualTo(okUrlWith304.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withHeader("ETag", etag)
+                        .withHeader("Last-Modified", lastModified)
+                        .withStatus(304)));
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(okUrlWith304.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withHeader("ETag", etag)
+                        .withHeader("Last-Modified", lastModified)
+                        .withStatus(304)));
+
+        okRangeResponseUrl = new EdgeUrl("http://localhost:18089/okRangeResponse.bin");
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(okRangeResponseUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Range", "bytes 0-100/200")
+                        .withBody("Hello World")
+                        .withStatus(206)));
+
+        keepAliveUrl = new EdgeUrl("http://localhost:18089/keepalive.bin");
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(keepAliveUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "text/html")
+                        .withStatus(200)
+                        .withHeader("Keep-Alive", "max=4, timeout=30")
+                        .withBody("Hello")
+                        ));
+
+
+        pdfUrl = new EdgeUrl("http://localhost:18089/test.pdf");
+        wireMockServer.stubFor(WireMock.get(WireMock.urlEqualTo(pdfUrl.path))
+                .willReturn(WireMock.aResponse()
+                        .withHeader("Content-Type", "application/pdf")
+                        .withStatus(200)
+                        .withBody("Hello World")));
+
+        wireMockServer.start();
+
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        wireMockServer.stop();
+    }
+
+
+    WarcRecorder warcRecorder;
+    Path warcFile;
+
+    @BeforeEach
+    public void setUp() throws IOException {
+        fetcher = new HttpFetcherImpl(new UserAgent("test.marginalia.nu", "test.marginalia.nu"));
+        warcFile = Files.createTempFile(getClass().getSimpleName(), ".warc");
+        warcRecorder = new WarcRecorder(warcFile);
+    }
+
+    @AfterEach
+    public void tearDown() throws IOException {
+        var stats = fetcher.getPoolStats();
+        assertEquals(0, stats.getLeased());
+        assertEquals(0, stats.getPending());
+
+        System.out.println(stats);
+
+        fetcher.close();
+        warcRecorder.close();
+        Files.deleteIfExists(warcFile);
+    }
+
+
+    @Test
+    public void testFoo() {
+        fetcher.fetchSitemapUrls("https://www.marginalia.nu/sitemap.xml", new CrawlDelayTimer(100));
+    }
+
+    @Test
+    public void testOk_NoProbe() throws IOException {
+        var result = fetcher.fetchContent(okUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+
+        List<WarcRecord> warcRecords = getWarcRecords();
+        assertEquals(2, warcRecords.size());
+        Assertions.assertInstanceOf(WarcRequest.class, warcRecords.get(0));
+        Assertions.assertInstanceOf(WarcResponse.class, warcRecords.get(1));
+
+        WarcResponse response = (WarcResponse) warcRecords.get(1);
+        assertEquals("0", response.http().headers().first("X-Has-Cookies").orElse("0"));
+    }
+
+    @Test
+    public void testOkSetsCookie() throws IOException {
+        var cookies = new DomainCookies();
+        var result = fetcher.fetchContent(okUrlSetsCookie, warcRecorder, cookies, new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+
+        List<WarcRecord> warcRecords = getWarcRecords();
+        assertEquals(2, warcRecords.size());
+        Assertions.assertInstanceOf(WarcRequest.class, warcRecords.get(0));
+        Assertions.assertInstanceOf(WarcResponse.class, warcRecords.get(1));
+
+        WarcResponse response = (WarcResponse) warcRecords.get(1);
+        assertEquals("1", response.http().headers().first("X-Has-Cookies").orElse("0"));
+    }
+
+    @Test
+    public void testOk_FullProbe() {
+        var result = fetcher.fetchContent(okUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+    }
+
+    @Test
+    public void testOk304_NoProbe() {
+        var result = fetcher.fetchContent(okUrlWith304, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), new ContentTags(etag, lastModified), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.Result304Raw.class, result);
+        System.out.println(result);
+
+    }
+
+    @Test
+    public void testOk304_FullProbe() {
+        var result = fetcher.fetchContent(okUrlWith304, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), new ContentTags(etag, lastModified), HttpFetcher.ProbeType.FULL);
+
+        Assertions.assertInstanceOf(HttpFetchResult.Result304Raw.class, result);
+        System.out.println(result);
+    }
+
+    @Test
+    public void testBadStatus_NoProbe() throws IOException {
+        var result = fetcher.fetchContent(badHttpStatusUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertFalse(result.isOk());
+
+
+        List<WarcRecord> warcRecords = getWarcRecords();
+        assertEquals(2, warcRecords.size());
+        Assertions.assertInstanceOf(WarcRequest.class, warcRecords.get(0));
+        Assertions.assertInstanceOf(WarcResponse.class, warcRecords.get(1));
+    }
+
+    @Test
+    public void testBadStatus_FullProbe() {
+        var result = fetcher.fetchContent(badHttpStatusUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertFalse(result.isOk());
+
+        System.out.println(result);
+    }
+
+    @Test
+    public void testRedirect_NoProbe() throws URISyntaxException, IOException {
+        var result = fetcher.fetchContent(redirectUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultRedirect.class, result);
+        assertEquals(new EdgeUrl("http://localhost:18089/test.html.bin"), ((HttpFetchResult.ResultRedirect) result).url());
+
+        List<WarcRecord> warcRecords = getWarcRecords();
+        assertEquals(2, warcRecords.size());
+        Assertions.assertInstanceOf(WarcRequest.class, warcRecords.get(0));
+        Assertions.assertInstanceOf(WarcResponse.class, warcRecords.get(1));
+    }
+
+    @Test
+    public void testRedirect_FullProbe() throws URISyntaxException {
+        var result = fetcher.fetchContent(redirectUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultRedirect.class, result);
+        assertEquals(new EdgeUrl("http://localhost:18089/test.html.bin"), ((HttpFetchResult.ResultRedirect) result).url());
+
+        System.out.println(result);
+    }
+
+
+    @Test
+    public void testFetchTimeout_NoProbe() throws IOException, URISyntaxException {
+        Instant requestStart = Instant.now();
+
+        var result = fetcher.fetchContent(timeoutUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultException.class, result);
+
+        Instant requestEnd = Instant.now();
+
+        System.out.println(result);
+
+        // Verify that we are actually timing out, and not blocking on the request until it finishes (which would be a bug),
+        // the request will take 15 seconds to complete, so we should be able to timeout before that, something like 10 seconds and change;
+        // but we'll verify that it is less than 15 seconds to make the test less fragile.
+
+        Assertions.assertTrue(requestEnd.isBefore(requestStart.plusSeconds(15)), "Request should have taken less than 15 seconds");
+
+        var records = getWarcRecords();
+        Assertions.assertEquals(1, records.size());
+        Assertions.assertInstanceOf(WarcXEntityRefused.class, records.getFirst());
+        WarcXEntityRefused entity = (WarcXEntityRefused) records.getFirst();
+        assertEquals(WarcXEntityRefused.documentProbeTimeout, entity.profile());
+        assertEquals(timeoutUrl.asURI(), entity.targetURI());
+    }
+
+    @Test
+    public void testRangeResponse() throws IOException {
+        var result = fetcher.fetchContent(okRangeResponseUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+
+        List<WarcRecord> warcRecords = getWarcRecords();
+        assertEquals(2, warcRecords.size());
+        Assertions.assertInstanceOf(WarcRequest.class, warcRecords.get(0));
+        Assertions.assertInstanceOf(WarcResponse.class, warcRecords.get(1));
+
+        var response = (WarcResponse) warcRecords.get(1);
+        assertEquals("length", response.headers().first("WARC-Truncated").orElse(""));
+    }
+
+    @Test
+    public void testFetchTimeout_Probe() throws IOException, URISyntaxException {
+        Instant requestStart = Instant.now();
+        var result = fetcher.fetchContent(timeoutUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+        Instant requestEnd = Instant.now();
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultException.class, result);
+
+
+        // Verify that we are actually timing out, and not blocking on the request until it finishes (which would be a bug),
+        // the request will take 15 seconds to complete, so we should be able to timeout before that, something like 10 seconds and change;
+        // but we'll verify that it is less than 15 seconds to make the test less fragile.
+
+        Assertions.assertTrue(requestEnd.isBefore(requestStart.plusSeconds(15)), "Request should have taken less than 15 seconds");
+
+        var records = getWarcRecords();
+        Assertions.assertEquals(1, records.size());
+        Assertions.assertInstanceOf(WarcXEntityRefused.class, records.getFirst());
+        WarcXEntityRefused entity = (WarcXEntityRefused) records.getFirst();
+        assertEquals(WarcXEntityRefused.documentProbeTimeout, entity.profile());
+        assertEquals(timeoutUrl.asURI(), entity.targetURI());
+    }
+
+    @Test
+    public void testKeepaliveUrl() {
+        // mostly for smoke testing and debugger utility
+        var result = fetcher.fetchContent(keepAliveUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.DISABLED);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+    }
+
+    @Test
+    public void testPdf() {
+        var result = fetcher.fetchContent(pdfUrl, warcRecorder, new DomainCookies(), new CrawlDelayTimer(1000), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+
+        Assertions.assertInstanceOf(HttpFetchResult.ResultOk.class, result);
+        Assertions.assertTrue(result.isOk());
+    }
+
+
+    private List<WarcRecord> getWarcRecords() throws IOException {
+        List<WarcRecord> records = new ArrayList<>();
+
+        System.out.println(Files.readString(warcFile));
+
+        try (var reader = new WarcReader(warcFile)) {
+            WarcXResponseReference.register(reader);
+            WarcXEntityRefused.register(reader);
+
+            for (var record : reader) {
+                // Load the body, we need to do this before we close the reader to have access to the content.
+                if (record instanceof WarcRequest req) {
+                    req.http();
+                } else if (record instanceof WarcResponse rsp) {
+                    rsp.http();
+                }
+
+                records.add(record);
+            }
+        }
+
+        return records;
+    }
+
+
+}

code/processes/crawling-process/test/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java

@@ -1,9 +1,12 @@
 package nu.marginalia.crawl.retreival;
 
-import nu.marginalia.crawl.fetcher.Cookies;
+import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -13,8 +16,6 @@ import org.netpreserve.jwarc.WarcResponse;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
@@ -30,8 +31,7 @@ class CrawlerWarcResynchronizerTest {
     HttpClient httpClient;
     @BeforeEach
     public void setUp() throws Exception {
-        httpClient = HttpClient.newBuilder()
-                .build();
+        httpClient = HttpClients.createDefault();
 
         fileName = Files.createTempFile("test", ".warc.gz");
         outputFile = Files.createTempFile("test", ".warc.gz");
@@ -45,7 +45,7 @@ class CrawlerWarcResynchronizerTest {
 
     @Test
     void run() throws IOException, URISyntaxException {
-        try (var oldRecorder = new WarcRecorder(fileName, new Cookies())) {
+        try (var oldRecorder = new WarcRecorder(fileName)) {
             fetchUrl(oldRecorder, "https://www.marginalia.nu/");
             fetchUrl(oldRecorder, "https://www.marginalia.nu/log/");
             fetchUrl(oldRecorder, "https://www.marginalia.nu/feed/");
@@ -55,7 +55,7 @@ class CrawlerWarcResynchronizerTest {
 
         var crawlFrontier = new DomainCrawlFrontier(new EdgeDomain("www.marginalia.nu"), List.of(), 100);
 
-        try (var newRecorder = new WarcRecorder(outputFile, new Cookies())) {
+        try (var newRecorder = new WarcRecorder(outputFile)) {
             new CrawlerWarcResynchronizer(crawlFrontier, newRecorder).run(fileName);
         }
 
@@ -78,11 +78,10 @@ class CrawlerWarcResynchronizerTest {
     }
 
     void fetchUrl(WarcRecorder recorder, String url) throws NoSuchAlgorithmException, IOException, URISyntaxException, InterruptedException {
-        var req = HttpRequest.newBuilder()
-                .uri(new java.net.URI(url))
-                .header("User-agent", "test.marginalia.nu")
-                .header("Accept-Encoding", "gzip")
-                .GET().build();
-        recorder.fetch(httpClient, req);
+        HttpGet request = new HttpGet(url);
+        request.addHeader("User-agent", "test.marginalia.nu");
+        request.addHeader("Accept-Encoding", "gzip");
+
+        recorder.fetch(httpClient, new DomainCookies(), request);
     }
 }

code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java

@@ -2,10 +2,10 @@ package nu.marginalia.crawl.retreival.fetcher;
 
 import com.sun.net.httpserver.HttpServer;
 import nu.marginalia.crawl.fetcher.ContentTags;
-import nu.marginalia.crawl.fetcher.Cookies;
+import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.HttpFetcher;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
-import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
+import nu.marginalia.crawl.retreival.CrawlDelayTimer;
 import nu.marginalia.model.EdgeUrl;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -32,7 +32,6 @@ class ContentTypeProberTest {
     static EdgeUrl timeoutEndpoint;
 
     static Path warcFile;
-    static WarcRecorder recorder;
 
     @BeforeEach
     void setUp() throws IOException  {
@@ -80,21 +79,17 @@ class ContentTypeProberTest {
         htmlRedirEndpoint = EdgeUrl.parse("http://localhost:" + port + "/redir.gz").get();
 
         fetcher = new HttpFetcherImpl("test");
-        recorder = new WarcRecorder(warcFile, new Cookies());
     }
 
     @AfterEach
     void tearDown() throws IOException {
         server.stop(0);
         fetcher.close();
-        recorder.close();
-
-        Files.deleteIfExists(warcFile);
     }
 
     @Test
     void probeContentTypeOk() throws Exception {
-        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(htmlEndpoint, recorder, ContentTags.empty());
+        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(htmlEndpoint, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
 
         System.out.println(result);
 
@@ -103,16 +98,16 @@ class ContentTypeProberTest {
 
     @Test
     void probeContentTypeRedir() throws Exception {
-        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(htmlRedirEndpoint, recorder, ContentTags.empty());
+        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(htmlRedirEndpoint, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
 
         System.out.println(result);
 
-        assertEquals(result, new HttpFetcher.ContentTypeProbeResult.Ok(htmlEndpoint));
+        assertEquals(result, new HttpFetcher.ContentTypeProbeResult.Redirect(htmlEndpoint));
     }
 
     @Test
     void probeContentTypeBad() throws Exception {
-        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(binaryEndpoint, recorder, ContentTags.empty());
+        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(binaryEndpoint, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
 
         System.out.println(result);
 
@@ -121,7 +116,7 @@ class ContentTypeProberTest {
 
     @Test
     void probeContentTypeTimeout() throws Exception {
-        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(timeoutEndpoint, recorder, ContentTags.empty());
+        HttpFetcher.ContentTypeProbeResult result = fetcher.probeContentType(timeoutEndpoint, new DomainCookies(), new CrawlDelayTimer(50), ContentTags.empty());
 
         System.out.println(result);
 

code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/WarcRecorderFakeServerTest.java

@@ -0,0 +1,162 @@
+package nu.marginalia.crawl.retreival.fetcher;
+
+import com.sun.net.httpserver.HttpServer;
+import nu.marginalia.crawl.fetcher.DomainCookies;
+import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.junit.jupiter.api.*;
+import org.netpreserve.jwarc.WarcReader;
+import org.netpreserve.jwarc.WarcRequest;
+import org.netpreserve.jwarc.WarcResponse;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+@Tag("slow")
+class WarcRecorderFakeServerTest {
+    static HttpServer server;
+
+    @BeforeAll
+    public static void setUpAll() throws IOException {
+        server = HttpServer.create(new InetSocketAddress("127.0.0.1", 14510), 10);
+
+        // This endpoint will finish sending the response immediately
+        server.createContext("/fast", exchange -> {
+            exchange.getResponseHeaders().add("Content-Type", "text/html");
+            exchange.sendResponseHeaders(200, "<html><body>hello</body></html>".length());
+
+            try (var os = exchange.getResponseBody()) {
+                os.write("<html><body>hello</body></html>".getBytes());
+                os.flush();
+            }
+            exchange.close();
+        });
+
+        // This endpoint will take 10 seconds to finish sending the response,
+        // which should trigger a timeout in the client
+        server.createContext("/slow", exchange -> {
+            exchange.getResponseHeaders().add("Content-Type", "text/html");
+            exchange.sendResponseHeaders(200, "<html><body>hello</body></html>:D".length());
+
+            try (var os = exchange.getResponseBody()) {
+                os.write("<html><body>hello</body></html>".getBytes());
+                os.flush();
+                try {
+                    TimeUnit.SECONDS.sleep(2);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                os.write(":".getBytes());
+                os.flush();
+                try {
+                    TimeUnit.SECONDS.sleep(2);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+
+                os.write("D".getBytes());
+                os.flush();
+            }
+            exchange.close();
+        });
+
+        server.start();
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        server.stop(0);
+    }
+
+    Path fileNameWarc;
+    Path fileNameParquet;
+    WarcRecorder client;
+
+    HttpClient httpClient;
+    @BeforeEach
+    public void setUp() throws Exception {
+        httpClient = HttpClients.createDefault();
+
+        fileNameWarc = Files.createTempFile("test", ".warc");
+        fileNameParquet = Files.createTempFile("test", ".parquet");
+
+        client = new WarcRecorder(fileNameWarc);
+    }
+
+    @AfterEach
+    public void tearDown() throws Exception {
+
+        client.close();
+        Files.delete(fileNameWarc);
+    }
+
+    @Test
+    public void fetchFast() throws Exception {
+        HttpGet request = new HttpGet("http://localhost:14510/fast");
+        request.addHeader("User-agent", "test.marginalia.nu");
+        request.addHeader("Accept-Encoding", "gzip");
+        client.fetch(httpClient, new DomainCookies(), request);
+
+        Map<String, String> sampleData = new HashMap<>();
+        try (var warcReader = new WarcReader(fileNameWarc)) {
+            warcReader.forEach(record -> {
+                if (record instanceof WarcRequest req) {
+                    sampleData.put(record.type(), req.target());
+                }
+                if (record instanceof WarcResponse rsp) {
+                    sampleData.put(record.type(), rsp.target());
+                }
+            });
+        }
+
+        System.out.println(sampleData);
+    }
+
+    @Test
+    public void fetchSlow() throws Exception {
+        Instant start = Instant.now();
+
+        HttpGet request = new HttpGet("http://localhost:14510/slow");
+        request.addHeader("User-agent", "test.marginalia.nu");
+        request.addHeader("Accept-Encoding", "gzip");
+
+        client.fetch(httpClient,
+                new DomainCookies(),
+                request,
+                Duration.ofSeconds(1)
+        );
+        Instant end = Instant.now();
+
+        Map<String, String> sampleData = new HashMap<>();
+        try (var warcReader = new WarcReader(fileNameWarc)) {
+            warcReader.forEach(record -> {
+                if (record instanceof WarcRequest req) {
+                    sampleData.put(record.type(), req.target());
+                }
+                if (record instanceof WarcResponse rsp) {
+                    sampleData.put(record.type(), rsp.target());
+                    System.out.println(rsp.target());
+                }
+            });
+        }
+
+        System.out.println(
+                Files.readString(fileNameWarc));
+        System.out.println(sampleData);
+
+        // Timeout is set to 1 second, but the server will take 5 seconds to respond,
+        // so we expect the request to take 1s and change before it times out.
+
+        Assertions.assertTrue(Duration.between(start, end).toMillis() < 3000);
+    }
+
+}

code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java

@@ -2,11 +2,14 @@ package nu.marginalia.crawl.retreival.fetcher;
 
 import nu.marginalia.UserAgent;
 import nu.marginalia.crawl.fetcher.ContentTags;
-import nu.marginalia.crawl.fetcher.Cookies;
+import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
+import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.model.EdgeUrl;
-import nu.marginalia.parquet.crawldata.CrawledDocumentParquetRecordFileReader;
-import nu.marginalia.parquet.crawldata.CrawledDocumentParquetRecordFileWriter;
+import nu.marginalia.slop.SlopCrawlDataRecord;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -17,30 +20,29 @@ import org.netpreserve.jwarc.WarcXResponseReference;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 class WarcRecorderTest {
     Path fileNameWarc;
-    Path fileNameParquet;
+    Path fileNameSlop;
     WarcRecorder client;
 
     HttpClient httpClient;
     @BeforeEach
     public void setUp() throws Exception {
-        httpClient = HttpClient.newBuilder().build();
+        httpClient = HttpClients.createDefault();
 
         fileNameWarc = Files.createTempFile("test", ".warc");
-        fileNameParquet = Files.createTempFile("test", ".parquet");
+        fileNameSlop = Files.createTempFile("test", ".slop.zip");
 
-        client = new WarcRecorder(fileNameWarc, new Cookies());
+        client = new WarcRecorder(fileNameWarc);
     }
 
     @AfterEach
@@ -51,13 +53,12 @@ class WarcRecorderTest {
 
     @Test
     void fetch() throws NoSuchAlgorithmException, IOException, URISyntaxException, InterruptedException {
-        client.fetch(httpClient,
-                HttpRequest.newBuilder()
-                        .uri(new java.net.URI("https://www.marginalia.nu/"))
-                        .header("User-agent", "test.marginalia.nu")
-                        .header("Accept-Encoding", "gzip")
-                        .GET().build()
-        );
+
+        HttpGet request = new HttpGet("https://www.marginalia.nu/");
+        request.addHeader("User-agent", "test.marginalia.nu");
+        request.addHeader("Accept-Encoding", "gzip");
+
+        client.fetch(httpClient, new DomainCookies(), request);
 
         Map<String, String> sampleData = new HashMap<>();
         try (var warcReader = new WarcReader(fileNameWarc)) {
@@ -78,8 +79,9 @@ class WarcRecorderTest {
     @Test
     public void flagAsSkipped() throws IOException, URISyntaxException {
 
-        try (var recorder = new WarcRecorder(fileNameWarc, new Cookies())) {
+        try (var recorder = new WarcRecorder(fileNameWarc)) {
             recorder.writeReferenceCopy(new EdgeUrl("https://www.marginalia.nu/"),
+                    new DomainCookies(),
                     "text/html",
                     200,
                     "<?doctype html><html><body>test</body></html>".getBytes(),
@@ -102,8 +104,9 @@ class WarcRecorderTest {
     @Test
     public void flagAsSkippedNullBody() throws IOException, URISyntaxException {
 
-        try (var recorder = new WarcRecorder(fileNameWarc, new Cookies())) {
+        try (var recorder = new WarcRecorder(fileNameWarc)) {
             recorder.writeReferenceCopy(new EdgeUrl("https://www.marginalia.nu/"),
+                    new DomainCookies(),
                     "text/html",
                     200,
                     null,
@@ -114,8 +117,9 @@ class WarcRecorderTest {
 
     @Test
     public void testSaveImport() throws URISyntaxException, IOException {
-        try (var recorder = new WarcRecorder(fileNameWarc, new Cookies())) {
+        try (var recorder = new WarcRecorder(fileNameWarc)) {
             recorder.writeReferenceCopy(new EdgeUrl("https://www.marginalia.nu/"),
+                    new DomainCookies(),
                     "text/html",
                     200,
                     "<?doctype html><html><body>test</body></html>".getBytes(),
@@ -138,35 +142,46 @@ class WarcRecorderTest {
 
     @Test
     public void testConvertToParquet() throws NoSuchAlgorithmException, IOException, URISyntaxException, InterruptedException {
-        client.fetch(httpClient, HttpRequest.newBuilder()
-                .uri(new java.net.URI("https://www.marginalia.nu/"))
-                .header("User-agent", "test.marginalia.nu")
-                .header("Accept-Encoding", "gzip")
-                .GET().build());
+        HttpGet request1 = new HttpGet("https://www.marginalia.nu/");
+        request1.addHeader("User-agent", "test.marginalia.nu");
+        request1.addHeader("Accept-Encoding", "gzip");
 
-        client.fetch(httpClient, HttpRequest.newBuilder()
-                .uri(new java.net.URI("https://www.marginalia.nu/log/"))
-                .header("User-agent", "test.marginalia.nu")
-                .header("Accept-Encoding", "gzip")
-                .GET().build());
+        client.fetch(httpClient, new DomainCookies(), request1);
 
-        client.fetch(httpClient, HttpRequest.newBuilder()
-                .uri(new java.net.URI("https://www.marginalia.nu/sanic.png"))
-                .header("User-agent", "test.marginalia.nu")
-                .header("Accept-Encoding", "gzip")
-                .GET().build());
+        HttpGet request2 = new HttpGet("https://www.marginalia.nu/log/");
+        request2.addHeader("User-agent", "test.marginalia.nu");
+        request2.addHeader("Accept-Encoding", "gzip");
 
-        CrawledDocumentParquetRecordFileWriter.convertWarc(
+        client.fetch(httpClient, new DomainCookies(), request2);
+
+        HttpGet request3 = new HttpGet("https://www.marginalia.nu/sanic.png");
+        request3.addHeader("User-agent", "test.marginalia.nu");
+        request3.addHeader("Accept-Encoding", "gzip");
+
+        client.fetch(httpClient, new DomainCookies(), request3);
+
+        HttpGet request4 = new HttpGet("https://downloads.marginalia.nu/test.pdf");
+        request4.addHeader("User-agent", "test.marginalia.nu");
+        request4.addHeader("Accept-Encoding", "gzip");
+
+        client.fetch(httpClient, new DomainCookies(), request4);
+
+        SlopCrawlDataRecord.convertWarc(
                 "www.marginalia.nu",
                 new UserAgent("test", "test"),
                 fileNameWarc,
-                fileNameParquet);
+                fileNameSlop);
 
-        var urls = CrawledDocumentParquetRecordFileReader.stream(fileNameParquet).map(doc -> doc.url).toList();
-        assertEquals(2, urls.size());
+        List<String> urls;
+        try (var stream = SerializableCrawlDataStream.openDataStream(fileNameSlop)) {
+            urls = stream.docsAsList().stream().map(doc -> doc.url.toString()).toList();
+        }
+
+        assertEquals(3, urls.size());
         assertEquals("https://www.marginalia.nu/", urls.get(0));
         assertEquals("https://www.marginalia.nu/log/", urls.get(1));
         // sanic.jpg gets filtered out for its bad mime type
+        assertEquals("https://downloads.marginalia.nu/test.pdf", urls.get(2));
 
     }
 

code/processes/crawling-process/test/nu/marginalia/crawling/HttpFetcherTest.java

@@ -1,6 +1,7 @@
 package nu.marginalia.crawling;
 
 import nu.marginalia.crawl.fetcher.ContentTags;
+import nu.marginalia.crawl.fetcher.DomainCookies;
 import nu.marginalia.crawl.fetcher.HttpFetcher;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
@@ -31,7 +32,7 @@ class HttpFetcherTest {
     void fetchUTF8() throws Exception {
         var fetcher = new HttpFetcherImpl("nu.marginalia.edge-crawler");
         try (var recorder = new WarcRecorder()) {
-            var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu"), recorder, ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+            var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu"), recorder, new DomainCookies(), new CrawlDelayTimer(100), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
             if (DocumentBodyExtractor.asString(result) instanceof DocumentBodyResult.Ok bodyOk) {
                 System.out.println(bodyOk.contentType());
             }
@@ -49,7 +50,7 @@ class HttpFetcherTest {
         var fetcher = new HttpFetcherImpl("nu.marginalia.edge-crawler");
 
         try (var recorder = new WarcRecorder()) {
-            var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu/robots.txt"), recorder, ContentTags.empty(), HttpFetcher.ProbeType.FULL);
+            var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu/robots.txt"), recorder, new DomainCookies(), new CrawlDelayTimer(100), ContentTags.empty(), HttpFetcher.ProbeType.FULL);
             if (DocumentBodyExtractor.asString(result) instanceof DocumentBodyResult.Ok bodyOk) {
                 System.out.println(bodyOk.contentType());
             }

code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerMockFetcherTest.java

@@ -15,6 +15,9 @@ import nu.marginalia.model.crawldata.CrawledDocument;
 import nu.marginalia.model.crawldata.CrawlerDocumentStatus;
 import nu.marginalia.model.crawldata.SerializableCrawlData;
 import nu.marginalia.test.CommonTestData;
+import org.apache.hc.client5.http.cookie.BasicCookieStore;
+import org.apache.hc.client5.http.cookie.CookieStore;
+import org.apache.hc.core5.http.Header;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -24,7 +27,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.net.http.HttpHeaders;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.sql.SQLException;
@@ -120,7 +122,7 @@ public class CrawlerMockFetcherTest {
         public void setAllowAllContentTypes(boolean allowAllContentTypes) {}
 
         @Override
-        public Cookies getCookies() { return new Cookies();}
+        public CookieStore getCookies() { return new BasicCookieStore();}
 
         @Override
         public void clearCookies() {}
@@ -132,13 +134,7 @@ public class CrawlerMockFetcherTest {
         }
 
         @Override
-        public ContentTypeProbeResult probeContentType(EdgeUrl url, WarcRecorder recorder, ContentTags tags) {
-            logger.info("Probing {}", url);
-            return new HttpFetcher.ContentTypeProbeResult.Ok(url);
-        }
-
-        @Override
-        public HttpFetchResult fetchContent(EdgeUrl url, WarcRecorder recorder, ContentTags tags, ProbeType probeType) {
+        public HttpFetchResult fetchContent(EdgeUrl url, WarcRecorder recorder, DomainCookies cookies, CrawlDelayTimer timer, ContentTags tags, ProbeType probeType) {
             logger.info("Fetching {}", url);
             if (mockData.containsKey(url)) {
                 byte[] bodyBytes = mockData.get(url).documentBodyBytes;
@@ -147,7 +143,7 @@ public class CrawlerMockFetcherTest {
                     return new HttpFetchResult.ResultOk(
                             url.asURI(),
                             200,
-                            HttpHeaders.of(Map.of(), (k,v)->true),
+                            new Header[0],
                             "127.0.0.1",
                             bodyBytes,
                             0,

code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java

@@ -5,19 +5,17 @@ import nu.marginalia.WmsaHome;
 import nu.marginalia.atags.model.DomainLinks;
 import nu.marginalia.crawl.CrawlerMain;
 import nu.marginalia.crawl.DomainStateDb;
-import nu.marginalia.crawl.fetcher.Cookies;
 import nu.marginalia.crawl.fetcher.HttpFetcher;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
 import nu.marginalia.crawl.retreival.*;
-import nu.marginalia.io.CrawledDomainReader;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawldata.CrawledDocument;
 import nu.marginalia.model.crawldata.CrawledDomain;
 import nu.marginalia.model.crawldata.SerializableCrawlData;
-import nu.marginalia.parquet.crawldata.CrawledDocumentParquetRecordFileWriter;
+import nu.marginalia.slop.SlopCrawlDataRecord;
 import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.*;
 import org.netpreserve.jwarc.*;
@@ -38,16 +36,16 @@ class CrawlerRetreiverTest {
     private HttpFetcher httpFetcher;
 
     Path tempFileWarc1;
-    Path tempFileParquet1;
+    Path tempFileSlop1;
     Path tempFileWarc2;
-    Path tempFileParquet2;
+    Path tempFileSlop2;
     Path tempFileWarc3;
     Path tempFileDb;
     @BeforeEach
     public void setUp() throws IOException {
         httpFetcher = new HttpFetcherImpl("search.marginalia.nu; testing a bit :D");
-        tempFileParquet1 = Files.createTempFile("crawling-process", ".parquet");
-        tempFileParquet2 = Files.createTempFile("crawling-process", ".parquet");
+        tempFileSlop1 = Files.createTempFile("crawling-process", ".slop.zip");
+        tempFileSlop2 = Files.createTempFile("crawling-process", ".slop.zip");
         tempFileDb = Files.createTempFile("crawling-process", ".db");
 
     }
@@ -63,14 +61,14 @@ class CrawlerRetreiverTest {
         if (tempFileWarc1 != null) {
             Files.deleteIfExists(tempFileWarc1);
         }
-        if (tempFileParquet1 != null) {
-            Files.deleteIfExists(tempFileParquet1);
+        if (tempFileSlop1 != null) {
+            Files.deleteIfExists(tempFileSlop1);
         }
         if (tempFileWarc2 != null) {
             Files.deleteIfExists(tempFileWarc2);
         }
-        if (tempFileParquet2 != null) {
-            Files.deleteIfExists(tempFileParquet2);
+        if (tempFileSlop2 != null) {
+            Files.deleteIfExists(tempFileSlop2);
         }
         if (tempFileWarc3 != null) {
             Files.deleteIfExists(tempFileWarc3);
@@ -181,7 +179,7 @@ class CrawlerRetreiverTest {
                 new EdgeDomain("www.marginalia.nu"),
                 List.of(), 100);
         var resync = new CrawlerWarcResynchronizer(revisitCrawlFrontier,
-                new WarcRecorder(tempFileWarc2, new Cookies())
+                new WarcRecorder(tempFileWarc2)
         );
 
         // truncate the size of the file to simulate a crash
@@ -225,9 +223,9 @@ class CrawlerRetreiverTest {
 
         doCrawl(tempFileWarc1, specs);
 
-        convertToParquet(tempFileWarc1, tempFileParquet1);
+        convertToSlop(tempFileWarc1, tempFileSlop1);
 
-        try (var stream = CrawledDomainReader.createDataStream(tempFileParquet1)) {
+        try (var stream = SerializableCrawlDataStream.openDataStream(tempFileSlop1)) {
             while (stream.hasNext()) {
                 if (stream.next() instanceof CrawledDocument doc) {
                     data.add(doc);
@@ -278,9 +276,9 @@ class CrawlerRetreiverTest {
         assertFalse(frontier.isVisited(new EdgeUrl("https://www.marginalia.nu/log/06-optimization/")));
         assertTrue(frontier.isKnown(new EdgeUrl("https://www.marginalia.nu/log/06-optimization/")));
 
-        convertToParquet(tempFileWarc1, tempFileParquet1);
+        convertToSlop(tempFileWarc1, tempFileSlop1);
 
-        try (var stream = CrawledDomainReader.createDataStream(tempFileParquet1)) {
+        try (var stream = SerializableCrawlDataStream.openDataStream(tempFileSlop1)) {
             while (stream.hasNext()) {
                 if (stream.next() instanceof CrawledDocument doc) {
                     data.add(doc);
@@ -294,7 +292,7 @@ class CrawlerRetreiverTest {
         // redirects to https://www.marginalia.nu/log/06-optimization.gmi/  (note the trailing slash)
         //
         // Ensure that the redirect is followed, and that the trailing slash is added
-        // to the url as reported in the parquet file.
+        // to the url as reported in the Slop file.
 
         var fetchedUrls =
                 data.stream()
@@ -327,9 +325,9 @@ class CrawlerRetreiverTest {
         tempFileWarc1 = Files.createTempFile("crawling-process", ".warc");
 
         doCrawl(tempFileWarc1, specs);
-        convertToParquet(tempFileWarc1, tempFileParquet1);
+        convertToSlop(tempFileWarc1, tempFileSlop1);
 
-        try (var stream = CrawledDomainReader.createDataStream(tempFileParquet1)) {
+        try (var stream = SerializableCrawlDataStream.openDataStream(tempFileSlop1)) {
             while (stream.hasNext()) {
                 if (stream.next() instanceof CrawledDocument doc) {
                     data.add(doc);
@@ -374,11 +372,11 @@ class CrawlerRetreiverTest {
         tempFileWarc2 = Files.createTempFile("crawling-process", ".warc.gz");
 
         doCrawl(tempFileWarc1, specs);
-        convertToParquet(tempFileWarc1, tempFileParquet1);
+        convertToSlop(tempFileWarc1, tempFileSlop1);
         doCrawlWithReferenceStream(specs,
-                CrawledDomainReader.createDataStream(tempFileParquet1)
+                new CrawlDataReference(tempFileSlop1)
         );
-        convertToParquet(tempFileWarc2, tempFileParquet2);
+        convertToSlop(tempFileWarc2, tempFileSlop2);
 
         try (var reader = new WarcReader(tempFileWarc2)) {
             WarcXResponseReference.register(reader);
@@ -397,7 +395,7 @@ class CrawlerRetreiverTest {
             });
         }
 
-        try (var ds = CrawledDomainReader.createDataStream(tempFileParquet2)) {
+        try (var ds = SerializableCrawlDataStream.openDataStream(tempFileSlop2)) {
             while (ds.hasNext()) {
                 var doc = ds.next();
                 if (doc instanceof CrawledDomain dr) {
@@ -412,9 +410,9 @@ class CrawlerRetreiverTest {
         }
     }
 
-    private void convertToParquet(Path tempFileWarc2, Path tempFileParquet2) {
-        CrawledDocumentParquetRecordFileWriter.convertWarc("www.marginalia.nu",
-                new UserAgent("test", "test"), tempFileWarc2, tempFileParquet2);
+    private void convertToSlop(Path tempFileWarc2, Path tempFileSlop2) throws IOException {
+        SlopCrawlDataRecord.convertWarc("www.marginalia.nu",
+                new UserAgent("test", "test"), tempFileWarc2, tempFileSlop2);
     }
 
 
@@ -437,9 +435,9 @@ class CrawlerRetreiverTest {
 
         doCrawl(tempFileWarc1, specs);
 
-        convertToParquet(tempFileWarc1, tempFileParquet1);
+        convertToSlop(tempFileWarc1, tempFileSlop1);
 
-        try (var stream = CrawledDomainReader.createDataStream(tempFileParquet1)) {
+        try (var stream = SerializableCrawlDataStream.openDataStream(tempFileSlop1)) {
             while (stream.hasNext()) {
                 var doc = stream.next();
                 data.computeIfAbsent(doc.getClass(), c -> new ArrayList<>()).add(doc);
@@ -448,18 +446,16 @@ class CrawlerRetreiverTest {
             throw new RuntimeException(e);
         }
 
-        var stream = CrawledDomainReader.createDataStream(tempFileParquet1);
-
         System.out.println("---");
 
-        doCrawlWithReferenceStream(specs, stream);
+        doCrawlWithReferenceStream(specs, new CrawlDataReference(tempFileSlop1));
 
         var revisitCrawlFrontier = new DomainCrawlFrontier(
                 new EdgeDomain("www.marginalia.nu"),
                 List.of(), 100);
 
         var resync = new CrawlerWarcResynchronizer(revisitCrawlFrontier,
-                new WarcRecorder(tempFileWarc3, new Cookies())
+                new WarcRecorder(tempFileWarc3)
         );
 
         // truncate the size of the file to simulate a crash
@@ -468,7 +464,7 @@ class CrawlerRetreiverTest {
         resync.run(tempFileWarc2);
 
         assertTrue(revisitCrawlFrontier.addKnown(new EdgeUrl("https://www.marginalia.nu/")));
-        convertToParquet(tempFileWarc3, tempFileParquet2);
+        convertToSlop(tempFileWarc3, tempFileSlop2);
 
 
         try (var reader = new WarcReader(tempFileWarc3)) {
@@ -488,7 +484,7 @@ class CrawlerRetreiverTest {
             });
         }
 
-        try (var ds = CrawledDomainReader.createDataStream(tempFileParquet2)) {
+        try (var ds = SerializableCrawlDataStream.openDataStream(tempFileSlop2)) {
             while (ds.hasNext()) {
                 var doc = ds.next();
                 if (doc instanceof CrawledDomain dr) {
@@ -509,12 +505,11 @@ class CrawlerRetreiverTest {
         }
     }
 
-    private void doCrawlWithReferenceStream(CrawlerMain.CrawlSpecRecord specs, SerializableCrawlDataStream stream) {
-        try (var recorder = new WarcRecorder(tempFileWarc2, new Cookies());
+    private void doCrawlWithReferenceStream(CrawlerMain.CrawlSpecRecord specs, CrawlDataReference reference) {
+        try (var recorder = new WarcRecorder(tempFileWarc2);
              var db = new DomainStateDb(tempFileDb)
         ) {
-            new CrawlerRetreiver(httpFetcher, new DomainProber(d -> true), specs, db, recorder).crawlDomain(new DomainLinks(),
-                    new CrawlDataReference(stream));
+            new CrawlerRetreiver(httpFetcher, new DomainProber(d -> true), specs, db, recorder).crawlDomain(new DomainLinks(), reference);
         }
         catch (IOException | SQLException ex) {
             Assertions.fail(ex);
@@ -523,7 +518,7 @@ class CrawlerRetreiverTest {
 
     @NotNull
     private DomainCrawlFrontier doCrawl(Path tempFileWarc1, CrawlerMain.CrawlSpecRecord specs) {
-        try (var recorder = new WarcRecorder(tempFileWarc1, new Cookies());
+        try (var recorder = new WarcRecorder(tempFileWarc1);
              var db = new DomainStateDb(tempFileDb)
         ) {
             var crawler = new CrawlerRetreiver(httpFetcher, new DomainProber(d -> true), specs, db, recorder);

code/processes/export-task-process/build.gradle

@@ -53,6 +53,8 @@ dependencies {
     implementation libs.commons.compress
     implementation libs.commons.codec
     implementation libs.jsoup
+    implementation libs.slop
+    implementation libs.jwarc
 
 
 

code/processes/export-task-process/java/nu/marginalia/extractor/AtagExporter.java

@@ -3,7 +3,6 @@ package nu.marginalia.extractor;
 import com.google.inject.Inject;
 import gnu.trove.set.hash.TLongHashSet;
 import nu.marginalia.hash.MurmurHash3_128;
-import nu.marginalia.io.CrawledDomainReader;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.link_parser.LinkParser;
 import nu.marginalia.model.EdgeDomain;
@@ -59,7 +58,7 @@ public class AtagExporter implements ExporterIf {
                 }
 
                 Path crawlDataPath = inputDir.resolve(item.relPath());
-                try (var stream = CrawledDomainReader.createDataStream(crawlDataPath)) {
+                try (var stream = SerializableCrawlDataStream.openDataStream(crawlDataPath)) {
                     exportLinks(tagWriter, stream);
                 }
                 catch (Exception ex) {

code/processes/export-task-process/java/nu/marginalia/extractor/FeedExporter.java

@@ -1,7 +1,6 @@
 package nu.marginalia.extractor;
 
 import com.google.inject.Inject;
-import nu.marginalia.io.CrawledDomainReader;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.link_parser.FeedExtractor;
 import nu.marginalia.link_parser.LinkParser;
@@ -56,7 +55,7 @@ public class FeedExporter implements ExporterIf {
                 }
 
                 Path crawlDataPath = inputDir.resolve(item.relPath());
-                try (var stream = CrawledDomainReader.createDataStream(crawlDataPath)) {
+                try (var stream = SerializableCrawlDataStream.openDataStream(crawlDataPath)) {
                     exportFeeds(tagWriter, stream);
                 }
                 catch (Exception ex) {
@@ -75,7 +74,7 @@ public class FeedExporter implements ExporterIf {
     private boolean exportFeeds(FeedCsvWriter exporter, SerializableCrawlDataStream stream) throws IOException, URISyntaxException {
         FeedExtractor feedExtractor = new FeedExtractor(new LinkParser());
 
-        int size = stream.sizeHint();
+        int size = stream.getSizeHint();
 
         while (stream.hasNext()) {
             if (!(stream.next() instanceof CrawledDocument doc))

code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java

@@ -3,11 +3,15 @@ package nu.marginalia.extractor;
 import com.google.inject.Inject;
 import nu.marginalia.process.log.WorkLog;
 import nu.marginalia.process.log.WorkLogEntry;
+import nu.marginalia.slop.SlopCrawlDataRecord;
+import nu.marginalia.slop.SlopTablePacker;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorage;
 import nu.marginalia.storage.model.FileStorageId;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -27,7 +31,7 @@ public class SampleDataExporter {
     public SampleDataExporter(FileStorageService storageService) {
         this.storageService = storageService;
     }
-    public void export(FileStorageId crawlId, FileStorageId destId, int size, String name) throws SQLException, IOException {
+    public void export(FileStorageId crawlId, FileStorageId destId, int size, String ctFilter, String name) throws SQLException, IOException {
         FileStorage destStorage = storageService.getStorage(destId);
         Path inputDir = storageService.getStorage(crawlId).asPath();
 
@@ -54,6 +58,7 @@ public class SampleDataExporter {
 
         Path newCrawlerLogFile = Files.createTempFile(destStorage.asPath(), "crawler", ".log",
                 PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
+
         try (var bw = Files.newBufferedWriter(newCrawlerLogFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
             for (var item : entriesAll) {
                 bw.write(item.id() + " " + item.ts() + " " + item.relPath() + " " + item.cnt() + "\n");
@@ -72,7 +77,22 @@ public class SampleDataExporter {
                 Path crawlDataPath = inputDir.resolve(item.relPath());
                 if (!Files.exists(crawlDataPath)) continue;
 
-                addFileToTar(stream, crawlDataPath, item.relPath());
+                if (StringUtils.isBlank(ctFilter)) {
+                    addFileToTar(stream, crawlDataPath, item.relPath());
+                }
+                else /* filter != null */ {
+                    boolean didFilterData = false;
+                    try {
+                        crawlDataPath = filterEntries(crawlDataPath, ctFilter);
+                        didFilterData = true;
+                        addFileToTar(stream, crawlDataPath, item.relPath());
+                    }
+                    finally {
+                        if (didFilterData) {
+                            Files.deleteIfExists(crawlDataPath);
+                        }
+                    }
+                }
             }
 
             addFileToTar(stream, newCrawlerLogFile, "crawler.log");
@@ -86,6 +106,46 @@ public class SampleDataExporter {
         Files.move(tmpTarFile, destStorage.asPath().resolve("crawl-data.tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
     }
 
+    /** Filters the entries in the crawl data file based on the content type.
+     * @param crawlDataPath The path to the crawl data file.
+     * @param contentTypeFilter The content type to filter by.
+     * @return The path to the filtered crawl data file, or null if an error occurred.
+     */
+    private Path filterEntries(Path crawlDataPath, String contentTypeFilter) throws IOException {
+        Path tempDir = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered");
+        Path tempFile = crawlDataPath.resolveSibling(crawlDataPath.getFileName() + ".filtered.slop.zip");
+
+        Files.createDirectory(tempDir);
+
+        try (var writer = new SlopCrawlDataRecord.Writer(tempDir);
+             var reader = new SlopCrawlDataRecord.FilteringReader(crawlDataPath) {
+                 @Override
+                 public boolean filter(String url, int status, String contentType) {
+                     if (contentTypeFilter.equals(contentType))
+                         return true;
+                     else if (contentType.startsWith("x-marginalia/"))
+                         // This is a metadata entry, typically domain or redirect information
+                         // let's keep those to not confuse the consumer of the data, which might
+                         // expect at least the domain summary
+                         return true;
+                     return false;
+                 }
+             }
+        ) {
+            while (reader.hasRemaining()) {
+                writer.write(reader.get());
+            }
+
+            SlopTablePacker.packToSlopZip(tempDir, tempFile);
+        }
+        finally {
+            FileUtils.deleteDirectory(tempDir.toFile());
+        }
+
+
+        return tempFile;
+    }
+
     private void addFileToTar(TarArchiveOutputStream outputStream, Path file, String fileName) throws IOException {
         var entry = outputStream.createArchiveEntry(file.toFile(), fileName);
         entry.setSize(Files.size(file));

code/processes/export-task-process/java/nu/marginalia/extractor/TermFrequencyExporter.java

@@ -5,7 +5,7 @@ import gnu.trove.map.hash.TLongIntHashMap;
 import gnu.trove.set.hash.TLongHashSet;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.converting.processor.logic.dom.DomPruningFilter;
-import nu.marginalia.io.CrawledDomainReader;
+import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.language.filter.LanguageFilter;
 import nu.marginalia.language.model.DocumentLanguageData;
 import nu.marginalia.language.sentence.SentenceExtractor;
@@ -103,7 +103,7 @@ public class TermFrequencyExporter implements ExporterIf {
     {
         TLongHashSet words = new TLongHashSet(1000);
 
-        try (var stream = CrawledDomainReader.createDataStream(crawlDataPath)) {
+        try (var stream = SerializableCrawlDataStream.openDataStream(crawlDataPath)) {
             while (stream.hasNext()) {
                 if (Thread.interrupted())
                     return;

code/processes/export-task-process/java/nu/marginalia/task/ExportTasksMain.java

@@ -92,7 +92,7 @@ public class ExportTasksMain extends ProcessMainClass {
                     termFrequencyExporter.export(request.crawlId, request.destId);
                     break;
                 case SAMPLE_DATA:
-                    sampleDataExporter.export(request.crawlId, request.destId, request.size, request.name);
+                    sampleDataExporter.export(request.crawlId, request.destId, request.size, request.ctFilter, request.name);
                     break;
                 case ADJACENCIES:
                     websiteAdjacenciesCalculator.export();

code/processes/live-crawling-process/java/nu/marginalia/livecrawler/LiveCrawlDataSet.java

@@ -228,7 +228,7 @@ public class LiveCrawlDataSet implements AutoCloseable {
         }
 
         @Override
-        public boolean hasNext() throws IOException {
+        public boolean hasNext() {
             if (dataStack == null) {
                 query();
             }
@@ -236,7 +236,7 @@ public class LiveCrawlDataSet implements AutoCloseable {
         }
 
         @Override
-        public void close() throws Exception {
+        public void close() {
             dataStack.clear();
         }
     }

code/processes/process-mq-api/java/nu/marginalia/mqapi/tasks/ExportTaskRequest.java

@@ -16,6 +16,7 @@ public class ExportTaskRequest {
     public FileStorageId destId;
     public int size;
     public String name;
+    public String ctFilter;
 
     public ExportTaskRequest(Task task) {
         this.task = task;
@@ -42,12 +43,13 @@ public class ExportTaskRequest {
         return request;
     }
 
-    public static ExportTaskRequest sampleData(FileStorageId crawlId, FileStorageId destId, int size, String name) {
+    public static ExportTaskRequest sampleData(FileStorageId crawlId, FileStorageId destId, String ctFilter, int size, String name) {
         ExportTaskRequest request = new ExportTaskRequest(Task.SAMPLE_DATA);
         request.crawlId = crawlId;
         request.destId = destId;
         request.size = size;
         request.name = name;
+        request.ctFilter = ctFilter;
         return request;
     }
 

code/services-application/api-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.3'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 java {

code/services-application/dating-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.3'
+    id 'com.google.cloud.tools.jib' version '3.4.5'
 }
 
 application {