(sample) Diagnosing compression errors

This reverts commit b7a5219ed3.

ROADMAP.md

@@ -48,10 +48,6 @@ filter for any API consumer.
 
 I've talked to the stract dev and he does not think it's a good idea to mimic their optics language, which is quite ad-hoc, but instead to work together to find some new common description language for this. 
 
-## Show favicons next to search results
-
-This is expected from search engines.  Basic proof of concept sketch of fetching this data has been done, but the feature is some way from being reality. 
-
 ## Specialized crawler for github
 
 One of the search engine's biggest limitations right now is that it does not index github at all.   A specialized crawler that fetches at least the readme.md would go a long way toward providing search capabilities in this domain.
@@ -66,6 +62,10 @@ The documents database probably should have some sort of flag indicating it's a
 PDF parsing is known to be a bit of a security liability so some thought needs to be put in
 that direction as well.
 
+## Show favicons next to search results (COMPLETED 2025-03)
+
+This is expected from search engines.  Basic proof of concept sketch of fetching this data has been done, but the feature is some way from being reality.
+
 ## Web Design Overhaul (COMPLETED 2025-01)
 
 The design is kinda clunky and hard to maintain, and needlessly outdated-looking.  

code/common/model/java/nu/marginalia/model/crawl/HtmlFeature.java

@@ -5,13 +5,15 @@ import java.util.Collection;
 public enum HtmlFeature {
     // Note, the first 32 of these features are bit encoded in the database
     // so be sure to keep anything that's potentially important toward the top
-    // of the list
+    // of the list; but adding new values will shift the encoded values and break
+    // binary compatibility!  Scroll down for a marker where you should add new values
+    // if they need to be accessible from IndexResultScoreCalculator!
 
     MEDIA( "special:media"),
     JS("special:scripts"),
     AFFILIATE_LINK( "special:affiliate"),
     TRACKING("special:tracking"),
-    TRACKING_ADTECH("special:ads"), // We'll call this ads for now
+    TRACKING_ADTECH("special:adtech"),
 
     KEBAB_CASE_URL("special:kcurl"), // https://www.example.com/urls-that-look-like-this/
     LONG_URL("special:longurl"),
@@ -30,6 +32,15 @@ public enum HtmlFeature {
 
     PDF("format:pdf"),
 
+    POPOVER("special:popover"),
+    CONSENT("special:consent"),
+    SHORT_DOCUMENT("special:shorty"),
+    THIRD_PARTY_REQUESTS("special:3pr"),
+
+    // Here!  It is generally safe to add additional values here without
+    // disrupting the encoded values used by the DocumentValuator
+    // class in the index!
+
     /** For fingerprinting and ranking */
     OPENGRAPH("special:opengraph"),
     OPENGRAPH_IMAGE("special:opengraph:image"),
@@ -67,6 +78,7 @@ public enum HtmlFeature {
 
     S3_FEATURE("special:s3"),
 
+    MISSING_DOM_SAMPLE("special:nosample"),
     UNKNOWN("special:uncategorized");
 
 
@@ -93,6 +105,8 @@ public enum HtmlFeature {
     }
 
     public int getFeatureBit() {
+        if (getClass().desiredAssertionStatus() && ordinal() >= 32)
+            throw new IllegalStateException("Attempting to extract feature bit of " + name() + ", with ordinal " + ordinal());
         return (1<< ordinal());
     }
 }

code/common/service/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java

@@ -13,6 +13,7 @@ import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.util.NamedExecutorFactory;
 
 import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
 import java.util.function.Function;
 
 @Singleton
@@ -20,10 +21,15 @@ public class GrpcChannelPoolFactory {
 
     private final NodeConfigurationWatcher nodeConfigurationWatcher;
     private final ServiceRegistryIf serviceRegistryIf;
-    private static final Executor executor = NamedExecutorFactory.createFixed("gRPC-Channel-Pool",
-            Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 32));
-    private static final Executor offloadExecutor = NamedExecutorFactory.createFixed("gRPC-Offload-Pool",
-            Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 32));
+
+    private static final boolean useLoom = Boolean.getBoolean("system.experimentalUseLoom");
+
+    private static final Executor executor = useLoom
+            ? Executors.newVirtualThreadPerTaskExecutor()
+            : NamedExecutorFactory.createFixed("gRPC-Channel-Pool", Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 32));
+    private static final Executor offloadExecutor =  useLoom
+            ? Executors.newVirtualThreadPerTaskExecutor()
+            : NamedExecutorFactory.createFixed("gRPC-Offload-Pool", Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 32));
 
     @Inject
     public GrpcChannelPoolFactory(NodeConfigurationWatcher nodeConfigurationWatcher,

code/common/service/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java

@@ -2,6 +2,7 @@ package nu.marginalia.service.client;
 
 import com.google.common.collect.Sets;
 import io.grpc.ManagedChannel;
+import io.grpc.StatusRuntimeException;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor;
 import nu.marginalia.service.discovery.property.PartitionTraits;
@@ -206,6 +207,11 @@ public class GrpcSingleNodeChannelPool<STUB> extends ServiceChangeMonitor {
         }
 
         for (var e : exceptions) {
+            if (e instanceof StatusRuntimeException se) {
+                throw se; // Re-throw SRE as-is
+            }
+
+            // If there are other exceptions, log them
             logger.error(grpcMarker, "Failed to call service {}", serviceKey, e);
         }
 

code/common/service/java/nu/marginalia/service/server/GrpcServer.java

@@ -1,9 +1,9 @@
 package nu.marginalia.service.server;
 
 import io.grpc.Server;
-import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
-import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup;
-import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.grpc.netty.NettyServerBuilder;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.discovery.property.ServiceKey;
 import nu.marginalia.service.discovery.property.ServicePartition;
@@ -13,9 +13,14 @@ import nu.marginalia.util.NamedExecutorFactory;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 public class GrpcServer {
     private final Server server;
+
+    private static final boolean useLoom = Boolean.getBoolean("system.experimentalUseLoom");
+
     public GrpcServer(ServiceConfiguration config,
                       ServiceRegistryIf serviceRegistry,
                       ServicePartition partition,
@@ -26,13 +31,19 @@ public class GrpcServer {
         int nThreads = Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 16);
 
         // Start the gRPC server
+
+        ExecutorService workExecutor = useLoom ?
+                Executors.newVirtualThreadPerTaskExecutor() :
+                NamedExecutorFactory.createFixed("nettyExecutor", nThreads);
+
         var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port))
-                .executor(NamedExecutorFactory.createFixed("nettyExecutor", nThreads))
+                .executor(workExecutor)
                 .workerEventLoopGroup(new NioEventLoopGroup(nThreads, NamedExecutorFactory.createFixed("Worker-ELG", nThreads)))
                 .bossEventLoopGroup(new NioEventLoopGroup(nThreads, NamedExecutorFactory.createFixed("Boss-ELG", nThreads)))
                 .channelType(NioServerSocketChannel.class);
 
         for (var grpcService : grpcServices) {
+
             if (!grpcService.shouldRegisterService()) {
                 continue;
             }

code/common/service/java/nu/marginalia/service/server/JoobyService.java

@@ -125,8 +125,7 @@ public class JoobyService {
         // Set a cap on the number of worker threads, as Jooby's default value does not seem to consider
         // multi-tenant servers with high thread counts, and spins up an exorbitant number of threads in that
         // scenario
-        options.setWorkerThreads(Math.min(128, options.getWorkerThreads()));
-
+        options.setWorkerThreads(Math.min(16, options.getWorkerThreads()));
 
         jooby.setServerOptions(options);
 

code/common/service/resources/log4j2-json.xml

@@ -7,6 +7,7 @@
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <Console name="ProcessConsole" target="SYSTEM_OUT">
@@ -23,6 +24,7 @@
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
             <SizeBasedTriggeringPolicy size="10MB" />
         </RollingFile>
@@ -36,6 +38,16 @@
                 <MarkerFilter marker="CRAWLER" onMatch="ALLOW" onMismatch="DENY" />
             </Filters>
         </RollingFile>
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/converter-audit-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/converter-audit-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+                     ignoreExceptions="false">
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS}: %msg{nolookups}%n</Pattern>
+            </PatternLayout>
+            <SizeBasedTriggeringPolicy size="100MB" />
+            <Filters>
+                <MarkerFilter marker="CONVERTER" onMatch="ALLOW" onMismatch="DENY" />
+            </Filters>
+        </RollingFile>
     </Appenders>
     <Loggers>
         <Logger name="org.apache.zookeeper" level="WARN" />

code/common/service/resources/log4j2-prod.xml

@@ -8,6 +8,7 @@
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <Console name="ConsoleWarn" target="SYSTEM_OUT">
@@ -18,6 +19,7 @@
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <Console name="ConsoleError" target="SYSTEM_OUT">
@@ -28,6 +30,7 @@
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <Console name="ConsoleFatal" target="SYSTEM_OUT">
@@ -38,6 +41,7 @@
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
         <Console name="ProcessConsole" target="SYSTEM_OUT">
@@ -57,6 +61,7 @@
                 <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                 <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CONVERTER" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </RollingFile>
         <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/crawler-audit-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/crawler-audit-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
@@ -69,6 +74,16 @@
                 <MarkerFilter marker="CRAWLER" onMatch="ALLOW" onMismatch="DENY" />
             </Filters>
         </RollingFile>
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/converter-audit-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/converter-audit-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+                     ignoreExceptions="false">
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS}: %msg{nolookups}%n</Pattern>
+            </PatternLayout>
+            <SizeBasedTriggeringPolicy size="100MB" />
+            <Filters>
+                <MarkerFilter marker="CONVERTER" onMatch="ALLOW" onMismatch="DENY" />
+            </Filters>
+        </RollingFile>
     </Appenders>
     <Loggers>
         <Logger name="org.apache.zookeeper" level="WARN" />

code/execution/api/java/nu/marginalia/executor/client/ExecutorClient.java

@@ -9,6 +9,7 @@ import nu.marginalia.executor.storage.FileStorageFile;
 import nu.marginalia.executor.upload.UploadDirContents;
 import nu.marginalia.executor.upload.UploadDirItem;
 import nu.marginalia.functions.execution.api.*;
+import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.client.GrpcChannelPoolFactory;
 import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
@@ -25,27 +26,37 @@ import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.List;
 
 import static nu.marginalia.functions.execution.api.ExecutorApiGrpc.ExecutorApiBlockingStub;
 
 @Singleton
 public class ExecutorClient {
+    private final MqPersistence persistence;
     private final GrpcMultiNodeChannelPool<ExecutorApiBlockingStub> channelPool;
     private static final Logger logger = LoggerFactory.getLogger(ExecutorClient.class);
     private final ServiceRegistryIf registry;
 
     @Inject
     public ExecutorClient(ServiceRegistryIf registry,
+                          MqPersistence persistence,
                           GrpcChannelPoolFactory grpcChannelPoolFactory)
     {
         this.registry = registry;
+        this.persistence = persistence;
         this.channelPool = grpcChannelPoolFactory
                 .createMulti(
                         ServiceKey.forGrpcApi(ExecutorApiGrpc.class, ServicePartition.multi()),
                         ExecutorApiGrpc::newBlockingStub);
     }
 
+    private long createTrackingTokenMsg(String task, int node, Duration ttl) throws Exception {
+        return persistence.sendNewMessage("task-tracking[" + node + "]", "export-client", null, task, "", ttl);
+    }
+
+
+
     public void startFsm(int node, String actorName) {
         channelPool.call(ExecutorApiBlockingStub::startFsm)
                 .forNode(node)
@@ -96,6 +107,16 @@ public class ExecutorClient {
                         .build());
     }
 
+    public long updateNsfwFilters() throws Exception {
+        long msgId = createTrackingTokenMsg("nsfw-filters", 1, Duration.ofHours(6));
+
+        channelPool.call(ExecutorApiBlockingStub::updateNsfwFilters)
+                .forNode(1)
+                .run(RpcUpdateNsfwFilters.newBuilder().setMsgId(msgId).build());
+
+        return msgId;
+    }
+
     public ActorRunStates getActorStates(int node) {
         try {
             var rs = channelPool.call(ExecutorApiBlockingStub::getActorStates)

code/execution/api/src/main/protobuf/executor-api.proto

@@ -18,6 +18,8 @@ service ExecutorApi {
   rpc calculateAdjacencies(Empty) returns (Empty) {}
   rpc restoreBackup(RpcFileStorageId) returns (Empty) {}
 
+  rpc updateNsfwFilters(RpcUpdateNsfwFilters) returns (Empty) {}
+
   rpc restartExecutorService(Empty) returns (Empty) {}
 }
 
@@ -66,6 +68,9 @@ message RpcExportRequest {
   int64 fileStorageId = 1;
   int64 msgId = 2;
 }
+message RpcUpdateNsfwFilters {
+  int64 msgId = 1;
+}
 message RpcFileStorageIdWithDomainName {
   int64 fileStorageId = 1;
   string targetDomainName = 2;

code/execution/java/nu/marginalia/actor/ActorApi.java

@@ -2,10 +2,11 @@ package nu.marginalia.actor;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
-import nu.marginalia.functions.execution.api.*;
+import nu.marginalia.functions.execution.api.RpcFsmName;
+import nu.marginalia.functions.execution.api.RpcProcessId;
 import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.persistence.MqPersistence;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -14,18 +15,18 @@ import spark.Spark;
 @Singleton
 public class ActorApi {
     private final ExecutorActorControlService actors;
-    private final ProcessService processService;
+    private final ProcessSpawnerService processSpawnerService;
     private final MqPersistence mqPersistence;
     private final ServiceConfiguration serviceConfiguration;
     private final Logger logger = LoggerFactory.getLogger(getClass());
     @Inject
     public ActorApi(ExecutorActorControlService actors,
-                    ProcessService processService,
+                    ProcessSpawnerService processSpawnerService,
                     MqPersistence mqPersistence,
                     ServiceConfiguration serviceConfiguration)
     {
         this.actors = actors;
-        this.processService = processService;
+        this.processSpawnerService = processSpawnerService;
         this.mqPersistence = mqPersistence;
         this.serviceConfiguration = serviceConfiguration;
     }
@@ -43,7 +44,7 @@ public class ActorApi {
     }
 
     public Object stopProcess(RpcProcessId processId) {
-        ProcessService.ProcessId id = ProcessService.translateExternalIdBase(processId.getProcessId());
+        ProcessSpawnerService.ProcessId id = ProcessSpawnerService.translateExternalIdBase(processId.getProcessId());
 
         try {
             String inbox = id.name().toLowerCase() + ":" + serviceConfiguration.node();
@@ -60,7 +61,7 @@ public class ActorApi {
                 }
 
             }
-            processService.kill(id);
+            processSpawnerService.kill(id);
         }
         catch (Exception ex) {
             logger.error("Failed to stop process {}", id, ex);

code/execution/java/nu/marginalia/actor/ExecutorActor.java

@@ -6,7 +6,7 @@ import java.util.Set;
 
 public enum ExecutorActor {
     PREC_EXPORT_ALL(NodeProfile.BATCH_CRAWL, NodeProfile.MIXED),
-    SYNC_NSFW_LISTS(NodeProfile.BATCH_CRAWL, NodeProfile.MIXED),
+    UPDATE_NSFW_LISTS(NodeProfile.BATCH_CRAWL, NodeProfile.MIXED, NodeProfile.SIDELOAD, NodeProfile.REALTIME),
 
     CRAWL(NodeProfile.BATCH_CRAWL, NodeProfile.MIXED),
     RECRAWL(NodeProfile.BATCH_CRAWL, NodeProfile.MIXED),

code/execution/java/nu/marginalia/actor/ExecutorActorControlService.java

@@ -113,7 +113,7 @@ public class ExecutorActorControlService {
         register(ExecutorActor.UPDATE_RSS, updateRssActor);
 
         register(ExecutorActor.MIGRATE_CRAWL_DATA, migrateCrawlDataActor);
-        register(ExecutorActor.SYNC_NSFW_LISTS, updateNsfwFiltersActor);
+        register(ExecutorActor.UPDATE_NSFW_LISTS, updateNsfwFiltersActor);
 
         if (serviceConfiguration.node() == 1) {
             register(ExecutorActor.PREC_EXPORT_ALL, exportAllPrecessionActor);

code/execution/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java

@@ -4,11 +4,14 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.prototype.RecordActorPrototype;
-import nu.marginalia.actor.state.*;
-import nu.marginalia.mq.persistence.MqMessageHandlerRegistry;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.actor.state.ActorResumeBehavior;
+import nu.marginalia.actor.state.ActorStep;
+import nu.marginalia.actor.state.Resume;
+import nu.marginalia.actor.state.Terminal;
 import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqMessageHandlerRegistry;
 import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -24,13 +27,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class AbstractProcessSpawnerActor extends RecordActorPrototype {
 
     private final MqPersistence persistence;
-    private final ProcessService processService;
+    private final ProcessSpawnerService processSpawnerService;
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     public static final int MAX_ATTEMPTS = 3;
     private final String inboxName;
-    private final ProcessService.ProcessId processId;
+    private final ProcessSpawnerService.ProcessId processId;
     private final ExecutorService executorService = Executors.newSingleThreadExecutor();
     private final int node;
 
@@ -50,7 +53,7 @@ public class AbstractProcessSpawnerActor extends RecordActorPrototype {
                 for (;;) {
                     var messages = persistence.eavesdrop(inboxName, 1);
 
-                    if (messages.isEmpty() && !processService.isRunning(processId)) {
+                    if (messages.isEmpty() && !processSpawnerService.isRunning(processId)) {
                         synchronized (processId) {
                             processId.wait(5000);
                         }
@@ -92,7 +95,7 @@ public class AbstractProcessSpawnerActor extends RecordActorPrototype {
                 catch (InterruptedException ex) {
                     // We get this exception when the process is cancelled by the user
 
-                    processService.kill(processId);
+                    processSpawnerService.kill(processId);
                     setCurrentMessageToDead();
 
                     yield new Aborted();
@@ -112,13 +115,13 @@ public class AbstractProcessSpawnerActor extends RecordActorPrototype {
     public AbstractProcessSpawnerActor(Gson gson,
                                        ServiceConfiguration configuration,
                                        MqPersistence persistence,
-                                       ProcessService processService,
+                                       ProcessSpawnerService processSpawnerService,
                                        String inboxName,
-                                       ProcessService.ProcessId processId) {
+                                       ProcessSpawnerService.ProcessId processId) {
         super(gson);
         this.node = configuration.node();
         this.persistence = persistence;
-        this.processService = processService;
+        this.processSpawnerService = processSpawnerService;
         this.inboxName = inboxName + ":" + node;
         this.processId = processId;
     }
@@ -149,7 +152,7 @@ public class AbstractProcessSpawnerActor extends RecordActorPrototype {
             // Run this call in a separate thread so that this thread can be interrupted waiting for it
             executorService.submit(() -> {
                 try {
-                    processService.trigger(processId);
+                    processSpawnerService.trigger(processId);
                 } catch (Exception e) {
                     logger.warn("Error in triggering process", e);
                     error.set(true);

code/execution/java/nu/marginalia/actor/proc/ConverterMonitorActor.java

@@ -4,9 +4,9 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
-import nu.marginalia.process.ProcessService;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -17,13 +17,13 @@ public class ConverterMonitorActor extends AbstractProcessSpawnerActor {
     public ConverterMonitorActor(Gson gson,
                                  ServiceConfiguration configuration,
                                  MqPersistence persistence,
-                                 ProcessService processService) {
+                                 ProcessSpawnerService processSpawnerService) {
         super(gson,
                 configuration,
                 persistence,
-                processService,
+                processSpawnerService,
                 ProcessInboxNames.CONVERTER_INBOX,
-                ProcessService.ProcessId.CONVERTER);
+                ProcessSpawnerService.ProcessId.CONVERTER);
     }
 
 

code/execution/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java

@@ -4,9 +4,9 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
-import nu.marginalia.process.ProcessService;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -16,13 +16,13 @@ public class CrawlerMonitorActor extends AbstractProcessSpawnerActor {
     public CrawlerMonitorActor(Gson gson,
                                ServiceConfiguration configuration,
                                MqPersistence persistence,
-                               ProcessService processService) {
+                               ProcessSpawnerService processSpawnerService) {
         super(gson,
                 configuration,
                 persistence,
-                processService,
+                processSpawnerService,
                 ProcessInboxNames.CRAWLER_INBOX,
-                ProcessService.ProcessId.CRAWLER);
+                ProcessSpawnerService.ProcessId.CRAWLER);
     }
 
 

code/execution/java/nu/marginalia/actor/proc/ExportTaskMonitorActor.java

@@ -6,7 +6,7 @@ import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -16,13 +16,13 @@ public class ExportTaskMonitorActor extends AbstractProcessSpawnerActor {
     public ExportTaskMonitorActor(Gson gson,
                                   ServiceConfiguration configuration,
                                   MqPersistence persistence,
-                                  ProcessService processService) {
+                                  ProcessSpawnerService processSpawnerService) {
         super(gson,
                 configuration,
                 persistence,
-                processService,
+                processSpawnerService,
                 ProcessInboxNames.EXPORT_TASK_INBOX,
-                ProcessService.ProcessId.EXPORT_TASKS);
+                ProcessSpawnerService.ProcessId.EXPORT_TASKS);
     }
 
 

code/execution/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java

@@ -4,9 +4,9 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
-import nu.marginalia.process.ProcessService;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -17,13 +17,13 @@ public class IndexConstructorMonitorActor extends AbstractProcessSpawnerActor {
     public IndexConstructorMonitorActor(Gson gson,
                                         ServiceConfiguration configuration,
                                         MqPersistence persistence,
-                                        ProcessService processService) {
+                                        ProcessSpawnerService processSpawnerService) {
         super(gson,
                 configuration,
                 persistence,
-                processService,
+                processSpawnerService,
                 ProcessInboxNames.INDEX_CONSTRUCTOR_INBOX,
-                ProcessService.ProcessId.INDEX_CONSTRUCTOR);
+                ProcessSpawnerService.ProcessId.INDEX_CONSTRUCTOR);
     }
 
 

code/execution/java/nu/marginalia/actor/proc/LiveCrawlerMonitorActor.java

@@ -6,7 +6,7 @@ import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -16,13 +16,13 @@ public class LiveCrawlerMonitorActor extends AbstractProcessSpawnerActor {
     public LiveCrawlerMonitorActor(Gson gson,
                                    ServiceConfiguration configuration,
                                    MqPersistence persistence,
-                                   ProcessService processService) {
+                                   ProcessSpawnerService processSpawnerService) {
         super(gson,
                 configuration,
                 persistence,
-                processService,
+                processSpawnerService,
                 ProcessInboxNames.LIVE_CRAWLER_INBOX,
-                ProcessService.ProcessId.LIVE_CRAWLER);
+                ProcessSpawnerService.ProcessId.LIVE_CRAWLER);
     }
 
 

code/execution/java/nu/marginalia/actor/proc/LoaderMonitorActor.java

@@ -4,9 +4,9 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
-import nu.marginalia.process.ProcessService;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -17,13 +17,13 @@ public class LoaderMonitorActor extends AbstractProcessSpawnerActor {
     public LoaderMonitorActor(Gson gson,
                               ServiceConfiguration configuration,
                               MqPersistence persistence,
-                              ProcessService processService) {
+                              ProcessSpawnerService processSpawnerService) {
 
         super(gson,
                 configuration,
-                persistence, processService,
+                persistence, processSpawnerService,
                 ProcessInboxNames.LOADER_INBOX,
-                ProcessService.ProcessId.LOADER);
+                ProcessSpawnerService.ProcessId.LOADER);
     }
 
 }

code/execution/java/nu/marginalia/actor/proc/NdpMonitorActor.java

@@ -6,7 +6,7 @@ import com.google.inject.Singleton;
 import nu.marginalia.actor.monitor.AbstractProcessSpawnerActor;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 @Singleton
@@ -16,13 +16,13 @@ public class NdpMonitorActor extends AbstractProcessSpawnerActor {
     public NdpMonitorActor(Gson gson,
                                    ServiceConfiguration configuration,
                                    MqPersistence persistence,
-                                   ProcessService processService) {
+                                   ProcessSpawnerService processSpawnerService) {
         super(gson,
                 configuration,
                 persistence,
-                processService,
+                processSpawnerService,
                 ProcessInboxNames.NDP_INBOX,
-                ProcessService.ProcessId.NDP);
+                ProcessSpawnerService.ProcessId.NDP);
     }
 
 

code/execution/java/nu/marginalia/actor/proc/PingMonitorActor.java

@@ -13,7 +13,7 @@ import nu.marginalia.mq.persistence.MqMessageHandlerRegistry;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.ProcessInboxNames;
 import nu.marginalia.mqapi.ping.PingRequest;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,17 +25,21 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+
+// Unlike other monitor actors, the ping monitor will not merely wait for a request
+// to be sent, but send one itself,  hence we can't extend AbstractProcessSpawnerActor
+// but have to reimplement a lot of the same logic ourselves.
 @Singleton
 public class PingMonitorActor extends RecordActorPrototype {
 
     private final MqPersistence persistence;
-    private final ProcessService processService;
+    private final ProcessSpawnerService processSpawnerService;
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     public static final int MAX_ATTEMPTS = 3;
     private final String inboxName;
-    private final ProcessService.ProcessId processId;
+    private final ProcessSpawnerService.ProcessId processId;
     private final ExecutorService executorService = Executors.newSingleThreadExecutor();
     private final int node;
     private final Gson gson;
@@ -53,7 +57,6 @@ public class PingMonitorActor extends RecordActorPrototype {
         return switch (self) {
             case Initial i -> {
                 PingRequest request = new PingRequest();
-
                 persistence.sendNewMessage(inboxName, null, null,
                         "PingRequest",
                         gson.toJson(request),
@@ -65,7 +68,7 @@ public class PingMonitorActor extends RecordActorPrototype {
                 for (;;) {
                     var messages = persistence.eavesdrop(inboxName, 1);
 
-                    if (messages.isEmpty() && !processService.isRunning(processId)) {
+                    if (messages.isEmpty() && !processSpawnerService.isRunning(processId)) {
                         synchronized (processId) {
                             processId.wait(5000);
                         }
@@ -107,7 +110,7 @@ public class PingMonitorActor extends RecordActorPrototype {
                 catch (InterruptedException ex) {
                     // We get this exception when the process is cancelled by the user
 
-                    processService.kill(processId);
+                    processSpawnerService.kill(processId);
                     setCurrentMessageToDead();
 
                     yield new Aborted();
@@ -127,14 +130,14 @@ public class PingMonitorActor extends RecordActorPrototype {
     public PingMonitorActor(Gson gson,
                                        ServiceConfiguration configuration,
                                        MqPersistence persistence,
-                                       ProcessService processService) throws SQLException {
+                                       ProcessSpawnerService processSpawnerService) throws SQLException {
         super(gson);
         this.gson = gson;
         this.node = configuration.node();
         this.persistence = persistence;
-        this.processService = processService;
+        this.processSpawnerService = processSpawnerService;
         this.inboxName = ProcessInboxNames.PING_INBOX + ":" + node;
-        this.processId = ProcessService.ProcessId.PING;
+        this.processId = ProcessSpawnerService.ProcessId.PING;
     }
 
     /** Sets the message to dead in the database to avoid
@@ -163,7 +166,7 @@ public class PingMonitorActor extends RecordActorPrototype {
             // Run this call in a separate thread so that this thread can be interrupted waiting for it
             executorService.submit(() -> {
                 try {
-                    processService.trigger(processId);
+                    processSpawnerService.trigger(processId);
                 } catch (Exception e) {
                     logger.warn("Error in triggering process", e);
                     error.set(true);

code/execution/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java

@@ -8,7 +8,7 @@ import nu.marginalia.actor.prototype.RecordActorPrototype;
 import nu.marginalia.actor.state.ActorResumeBehavior;
 import nu.marginalia.actor.state.ActorStep;
 import nu.marginalia.actor.state.Resume;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.control.ServiceEventLog;
 import nu.marginalia.service.module.ServiceConfiguration;
 
@@ -21,7 +21,7 @@ import java.util.concurrent.TimeUnit;
 public class ProcessLivenessMonitorActor extends RecordActorPrototype {
 
     private final ServiceEventLog eventLogService;
-    private final ProcessService processService;
+    private final ProcessSpawnerService processSpawnerService;
     private final HikariDataSource dataSource;
 
     private final int node;
@@ -49,7 +49,7 @@ public class ProcessLivenessMonitorActor extends RecordActorPrototype {
                         var processId = heartbeat.getProcessId();
                         if (null == processId) continue;
 
-                        if (processService.isRunning(processId) && heartbeat.lastSeenMillis() < 10_000)
+                        if (processSpawnerService.isRunning(processId) && heartbeat.lastSeenMillis() < 10_000)
                             continue;
 
                         flagProcessAsStopped(heartbeat);
@@ -72,12 +72,12 @@ public class ProcessLivenessMonitorActor extends RecordActorPrototype {
     public ProcessLivenessMonitorActor(Gson gson,
                                        ServiceEventLog eventLogService,
                                        ServiceConfiguration configuration,
-                                       ProcessService processService,
+                                       ProcessSpawnerService processSpawnerService,
                                        HikariDataSource dataSource) {
         super(gson);
         this.node = configuration.node();
         this.eventLogService = eventLogService;
-        this.processService = processService;
+        this.processSpawnerService = processSpawnerService;
         this.dataSource = dataSource;
     }
 
@@ -208,8 +208,8 @@ public class ProcessLivenessMonitorActor extends RecordActorPrototype {
         public boolean isRunning() {
             return "RUNNING".equals(status);
         }
-        public ProcessService.ProcessId getProcessId() {
-            return ProcessService.translateExternalIdBase(processBase);
+        public ProcessSpawnerService.ProcessId getProcessId() {
+            return ProcessSpawnerService.translateExternalIdBase(processBase);
         }
     }
 

code/execution/java/nu/marginalia/actor/proc/ScrapeFeedsActor.java

@@ -47,6 +47,8 @@ public class ScrapeFeedsActor extends RecordActorPrototype {
 
     private final Path feedPath = WmsaHome.getHomePath().resolve("data/scrape-urls.txt");
 
+    private static boolean insertFoundDomains = Boolean.getBoolean("loader.insertFoundDomains");
+
     public record Initial() implements ActorStep {}
     @Resume(behavior = ActorResumeBehavior.RETRY)
     public record Wait(String ts) implements ActorStep {}
@@ -57,6 +59,8 @@ public class ScrapeFeedsActor extends RecordActorPrototype {
     public ActorStep transition(ActorStep self) throws Exception {
         return switch(self) {
             case Initial() -> {
+                if (!insertFoundDomains) yield new Error("Domain insertion prohibited, aborting");
+
                 if (nodeConfigurationService.get(nodeId).profile() != NodeProfile.REALTIME) {
                     yield new Error("Invalid node profile for RSS update");
                 }

code/execution/java/nu/marginalia/actor/task/ActorProcessWatcher.java

@@ -3,11 +3,11 @@ package nu.marginalia.actor.task;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.state.ActorControlFlowException;
-import nu.marginalia.mq.MqMessageState;
-import nu.marginalia.mq.persistence.MqPersistence;
-import nu.marginalia.process.ProcessService;
 import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
+import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.process.ProcessSpawnerService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -20,13 +20,13 @@ public class ActorProcessWatcher {
 
     private static final Logger logger = LoggerFactory.getLogger(ActorProcessWatcher.class);
     private final MqPersistence persistence;
-    private final ProcessService processService;
+    private final ProcessSpawnerService processSpawnerService;
 
     @Inject
     public ActorProcessWatcher(MqPersistence persistence,
-                               ProcessService processService) {
+                               ProcessSpawnerService processSpawnerService) {
         this.persistence = persistence;
-        this.processService = processService;
+        this.processSpawnerService = processSpawnerService;
     }
 
     /** Wait for a process to start, and then wait for a response from the process,
@@ -36,7 +36,7 @@ public class ActorProcessWatcher {
      * <p>
      * When interrupted, the process is killed and the message is marked as dead.
      */
-    public MqMessage waitResponse(MqOutbox outbox, ProcessService.ProcessId processId, long msgId)
+    public MqMessage waitResponse(MqOutbox outbox, ProcessSpawnerService.ProcessId processId, long msgId)
             throws ActorControlFlowException, InterruptedException, SQLException
     {
         // enums values only have a single instance,
@@ -65,7 +65,7 @@ public class ActorProcessWatcher {
                 // This will prevent the monitor process from attempting to respawn the process as we kill it
 
                 outbox.flagAsDead(msgId);
-                processService.kill(processId);
+                processSpawnerService.kill(processId);
 
                 logger.info("Process {} killed due to interrupt", processId);
             }
@@ -94,12 +94,12 @@ public class ActorProcessWatcher {
     }
 
     /** Wait the specified time for the specified process to start running (does not start the process) */
-    private boolean waitForProcess(ProcessService.ProcessId processId, TimeUnit unit, int duration) throws InterruptedException {
+    private boolean waitForProcess(ProcessSpawnerService.ProcessId processId, TimeUnit unit, int duration) throws InterruptedException {
 
         // Wait for process to start
         long deadline = System.currentTimeMillis() + unit.toMillis(duration);
         while (System.currentTimeMillis() < deadline) {
-            if (processService.isRunning(processId))
+            if (processSpawnerService.isRunning(processId))
                 return true;
 
             TimeUnit.MILLISECONDS.sleep(100);

code/execution/java/nu/marginalia/actor/task/ConvertActor.java

@@ -12,7 +12,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mqapi.converting.ConvertRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.sideload.RedditSideloadHelper;
 import nu.marginalia.sideload.SideloadHelper;
 import nu.marginalia.sideload.StackExchangeSideloadHelper;
@@ -218,7 +218,7 @@ public class ConvertActor extends RecordActorPrototype {
                 );
             }
             case ConvertWait(FileStorageId destFid, long msgId) -> {
-                var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessService.ProcessId.CONVERTER, msgId);
+                var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessSpawnerService.ProcessId.CONVERTER, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     yield new Error("Converter failed");

code/execution/java/nu/marginalia/actor/task/ConvertAndLoadActor.java

@@ -18,7 +18,7 @@ import nu.marginalia.mqapi.index.IndexName;
 import nu.marginalia.mqapi.loading.LoadRequest;
 import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
@@ -95,7 +95,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype {
             case Convert(FileStorageId crawlId, FileStorageId processedId, long msgId) when msgId < 0 ->
                     new Convert(crawlId, processedId, mqConverterOutbox.sendAsync(ConvertRequest.forCrawlData(crawlId, processedId)));
             case Convert(FileStorageId crawlId, FileStorageId processedId, long msgId) -> {
-                var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessService.ProcessId.CONVERTER, msgId);
+                var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessSpawnerService.ProcessId.CONVERTER, msgId);
 
                 if (rsp.state() != MqMessageState.OK)
                     yield new Error("Converter failed");
@@ -129,7 +129,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype {
                 yield new Load(processedIds, id);
             }
             case Load(List<FileStorageId> processedIds, long msgId) -> {
-                var rsp = processWatcher.waitResponse(mqLoaderOutbox, ProcessService.ProcessId.LOADER, msgId);
+                var rsp = processWatcher.waitResponse(mqLoaderOutbox, ProcessSpawnerService.ProcessId.LOADER, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     yield new Error("Loader failed");
@@ -165,7 +165,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype {
             }
             case ReindexFwd(long id) when id < 0 -> new ReindexFwd(createIndex(IndexName.FORWARD));
             case ReindexFwd(long id) -> {
-                var rsp = processWatcher.waitResponse(mqIndexConstructorOutbox, ProcessService.ProcessId.INDEX_CONSTRUCTOR, id);
+                var rsp = processWatcher.waitResponse(mqIndexConstructorOutbox, ProcessSpawnerService.ProcessId.INDEX_CONSTRUCTOR, id);
 
                 if (rsp.state() != MqMessageState.OK)
                     yield new Error("Forward index construction failed");
@@ -174,7 +174,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype {
             }
             case ReindexFull(long id) when id < 0 -> new ReindexFull(createIndex(IndexName.REVERSE_FULL));
             case ReindexFull(long id) -> {
-                var rsp = processWatcher.waitResponse(mqIndexConstructorOutbox, ProcessService.ProcessId.INDEX_CONSTRUCTOR, id);
+                var rsp = processWatcher.waitResponse(mqIndexConstructorOutbox, ProcessSpawnerService.ProcessId.INDEX_CONSTRUCTOR, id);
 
                 if (rsp.state() != MqMessageState.OK)
                     yield new Error("Full index construction failed");
@@ -183,7 +183,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype {
             }
             case ReindexPrio(long id) when id < 0 -> new ReindexPrio(createIndex(IndexName.REVERSE_PRIO));
             case ReindexPrio(long id) -> {
-                var rsp = processWatcher.waitResponse(mqIndexConstructorOutbox, ProcessService.ProcessId.INDEX_CONSTRUCTOR, id);
+                var rsp = processWatcher.waitResponse(mqIndexConstructorOutbox, ProcessSpawnerService.ProcessId.INDEX_CONSTRUCTOR, id);
 
                 if (rsp.state() != MqMessageState.OK)
                     yield new Error("Prio index construction failed");

code/execution/java/nu/marginalia/actor/task/CrawlActor.java

@@ -13,7 +13,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mqapi.crawling.CrawlRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.storage.model.FileStorageType;
@@ -76,7 +76,7 @@ public class CrawlActor extends RecordActorPrototype {
             case Crawl (long msgId, FileStorageId fid, boolean cascadeLoad) -> {
                 var rsp = processWatcher.waitResponse(
                         mqCrawlerOutbox,
-                        ProcessService.ProcessId.CRAWLER,
+                        ProcessSpawnerService.ProcessId.CRAWLER,
                         msgId);
 
                 if (rsp.state() != MqMessageState.OK) {

code/execution/java/nu/marginalia/actor/task/ExportAtagsActor.java

@@ -10,7 +10,7 @@ import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.tasks.ExportTaskRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.storage.model.FileStorageState;
@@ -55,7 +55,7 @@ public class ExportAtagsActor extends RecordActorPrototype {
                 yield new Run(responseMsgId, crawlId, destId, newMsgId);
             }
             case Run(long responseMsgId, FileStorageId crawlId, FileStorageId destId, long msgId) -> {
-                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
+                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessSpawnerService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     storageService.flagFileForDeletion(destId);

code/execution/java/nu/marginalia/actor/task/ExportFeedsActor.java

@@ -10,7 +10,7 @@ import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.tasks.ExportTaskRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.storage.model.FileStorageState;
@@ -54,7 +54,7 @@ public class ExportFeedsActor extends RecordActorPrototype {
                 yield new Run(responseMsgId, crawlId, destId, newMsgId);
             }
             case Run(long responseMsgId, _, FileStorageId destId, long msgId) -> {
-                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
+                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessSpawnerService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     storageService.flagFileForDeletion(destId);

code/execution/java/nu/marginalia/actor/task/ExportSampleDataActor.java

@@ -9,7 +9,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mqapi.tasks.ExportTaskRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.storage.model.FileStorageState;
@@ -52,7 +52,7 @@ public class ExportSampleDataActor extends RecordActorPrototype {
                 yield new Run(crawlId, destId, size, ctFilter, name, newMsgId);
             }
             case Run(_, FileStorageId destId, _, _, _, long msgId) -> {
-                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
+                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessSpawnerService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     storageService.flagFileForDeletion(destId);

code/execution/java/nu/marginalia/actor/task/ExportTermFreqActor.java

@@ -10,7 +10,7 @@ import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.mqapi.tasks.ExportTaskRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.storage.model.FileStorageState;
@@ -52,7 +52,7 @@ public class ExportTermFreqActor extends RecordActorPrototype {
                 yield new Run(responseMsgId, crawlId, destId, newMsgId);
             }
             case Run(long responseMsgId, _, FileStorageId destId, long msgId) -> {
-                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
+                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessSpawnerService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     storageService.flagFileForDeletion(destId);

code/execution/java/nu/marginalia/actor/task/LiveCrawlActor.java

@@ -13,7 +13,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mqapi.crawling.LiveCrawlRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
@@ -44,7 +44,6 @@ public class LiveCrawlActor extends RecordActorPrototype {
 
     @Override
     public ActorStep transition(ActorStep self) throws Exception {
-        logger.info("{}", self);
         return switch (self) {
             case Initial() -> {
                 yield new Monitor("-");
@@ -75,7 +74,7 @@ public class LiveCrawlActor extends RecordActorPrototype {
                 yield new LiveCrawl(feedsHash, id);
             }
             case LiveCrawl(String feedsHash, long msgId) -> {
-                var rsp = processWatcher.waitResponse(mqLiveCrawlerOutbox, ProcessService.ProcessId.LIVE_CRAWLER, msgId);
+                var rsp = processWatcher.waitResponse(mqLiveCrawlerOutbox, ProcessSpawnerService.ProcessId.LIVE_CRAWLER, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     yield new Error("Crawler failed");

code/execution/java/nu/marginalia/actor/task/RecrawlSingleDomainActor.java

@@ -11,7 +11,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mqapi.crawling.CrawlRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.storage.model.FileStorageType;
@@ -51,7 +51,7 @@ public class RecrawlSingleDomainActor extends RecordActorPrototype {
             case Crawl (long msgId) -> {
                 var rsp = processWatcher.waitResponse(
                         mqCrawlerOutbox,
-                        ProcessService.ProcessId.CRAWLER,
+                        ProcessSpawnerService.ProcessId.CRAWLER,
                         msgId);
 
                 if (rsp.state() != MqMessageState.OK) {

code/execution/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java

@@ -9,7 +9,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.mqapi.tasks.ExportTaskRequest;
 import nu.marginalia.process.ProcessOutboxes;
-import nu.marginalia.process.ProcessService;
+import nu.marginalia.process.ProcessSpawnerService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,7 +34,7 @@ public class TriggerAdjacencyCalculationActor extends RecordActorPrototype {
                 yield new Run(newMsgId);
             }
             case Run(long msgId) -> {
-                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessService.ProcessId.EXPORT_TASKS, msgId);
+                var rsp = processWatcher.waitResponse(exportTasksOutbox, ProcessSpawnerService.ProcessId.EXPORT_TASKS, msgId);
 
                 if (rsp.state() != MqMessageState.OK) {
                     yield new Error("Exporter failed");

code/execution/java/nu/marginalia/actor/task/UpdateNsfwFiltersActor.java

@@ -5,6 +5,8 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.actor.prototype.RecordActorPrototype;
 import nu.marginalia.actor.state.ActorStep;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.nsfw.NsfwDomainFilter;
 import nu.marginalia.service.module.ServiceConfiguration;
 
@@ -12,23 +14,26 @@ import nu.marginalia.service.module.ServiceConfiguration;
 public class UpdateNsfwFiltersActor extends RecordActorPrototype {
     private final ServiceConfiguration serviceConfiguration;
     private final NsfwDomainFilter nsfwDomainFilter;
+    private final MqPersistence persistence;
 
-    public record Initial() implements ActorStep {}
-    public record Run() implements ActorStep {}
+    public record Initial(long respondMsgId) implements ActorStep {}
+    public record Run(long respondMsgId) implements ActorStep {}
 
     @Override
     public ActorStep transition(ActorStep self) throws Exception {
         return switch(self) {
-            case Initial() -> {
+            case Initial(long respondMsgId) -> {
                 if (serviceConfiguration.node() != 1) {
+                    persistence.updateMessageState(respondMsgId, MqMessageState.ERR);
                     yield new Error("This actor can only run on node 1");
                 }
                 else {
-                    yield new Run();
+                    yield new Run(respondMsgId);
                 }
             }
-            case Run() -> {
+            case Run(long respondMsgId) -> {
                 nsfwDomainFilter.fetchLists();
+                persistence.updateMessageState(respondMsgId, MqMessageState.OK);
                 yield new End();
             }
             default -> new Error();
@@ -43,11 +48,13 @@ public class UpdateNsfwFiltersActor extends RecordActorPrototype {
     @Inject
     public UpdateNsfwFiltersActor(Gson gson,
                                   ServiceConfiguration serviceConfiguration,
-                                  NsfwDomainFilter nsfwDomainFilter)
+                                  NsfwDomainFilter nsfwDomainFilter,
+                                  MqPersistence persistence)
     {
         super(gson);
         this.serviceConfiguration = serviceConfiguration;
         this.nsfwDomainFilter = nsfwDomainFilter;
+        this.persistence = persistence;
     }
 
 }

code/execution/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java

@@ -1,6 +1,7 @@
 package nu.marginalia.execution;
 
 import com.google.inject.Inject;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import nu.marginalia.actor.ExecutorActor;
 import nu.marginalia.actor.ExecutorActorControlService;
@@ -36,7 +37,7 @@ public class ExecutorCrawlGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -52,7 +53,7 @@ public class ExecutorCrawlGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -66,7 +67,7 @@ public class ExecutorCrawlGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -80,7 +81,7 @@ public class ExecutorCrawlGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -98,7 +99,7 @@ public class ExecutorCrawlGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 

code/execution/java/nu/marginalia/execution/ExecutorExportGrpcService.java

@@ -2,6 +2,7 @@ package nu.marginalia.execution;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import nu.marginalia.actor.ExecutorActor;
 import nu.marginalia.actor.ExecutorActorControlService;
@@ -38,7 +39,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -57,7 +58,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -73,7 +74,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -87,7 +88,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -99,7 +100,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -114,14 +115,14 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
     @Override
     public void exportAllAtags(Empty request, StreamObserver<Empty> responseObserver) {
         if (serviceConfiguration.node() != 1) {
-            responseObserver.onError(new IllegalArgumentException("Export all atags is only available on node 1"));
+            responseObserver.onError(Status.UNAVAILABLE.withDescription("Export all atags is only available on node 1").asRuntimeException());
         }
         try {
             actorControlService.startFrom(ExecutorActor.PREC_EXPORT_ALL,
@@ -131,7 +132,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -145,7 +146,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -159,7 +160,7 @@ public class ExecutorExportGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 }

code/execution/java/nu/marginalia/execution/ExecutorGrpcService.java

@@ -1,6 +1,7 @@
 package nu.marginalia.execution;
 
 import com.google.inject.Inject;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.actor.ActorApi;
@@ -10,6 +11,7 @@ import nu.marginalia.actor.state.ActorStateInstance;
 import nu.marginalia.actor.task.DownloadSampleActor;
 import nu.marginalia.actor.task.RestoreBackupActor;
 import nu.marginalia.actor.task.TriggerAdjacencyCalculationActor;
+import nu.marginalia.actor.task.UpdateNsfwFiltersActor;
 import nu.marginalia.functions.execution.api.*;
 import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.server.DiscoverableService;
@@ -57,7 +59,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -69,7 +71,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -81,7 +83,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -95,7 +97,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -111,7 +113,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -127,7 +129,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -202,7 +204,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -228,7 +230,7 @@ public class ExecutorGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -263,4 +265,19 @@ public class ExecutorGrpcService
         System.exit(0);
     }
 
+    @Override
+    public void updateNsfwFilters(RpcUpdateNsfwFilters request, StreamObserver<Empty> responseObserver) {
+        logger.info("Got request {}", request);
+        try {
+            actorControlService.startFrom(ExecutorActor.UPDATE_NSFW_LISTS,
+                    new UpdateNsfwFiltersActor.Initial(request.getMsgId()));
+
+            responseObserver.onNext(Empty.getDefaultInstance());
+            responseObserver.onCompleted();
+        }
+        catch (Exception e) {
+            logger.error("Failed to update nsfw filters", e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
+        }
+    }
 }

code/execution/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java

@@ -1,6 +1,7 @@
 package nu.marginalia.execution;
 
 import com.google.inject.Inject;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import nu.marginalia.actor.ExecutorActor;
 import nu.marginalia.actor.ExecutorActorControlService;
@@ -33,7 +34,7 @@ public class ExecutorSideloadGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -48,7 +49,7 @@ public class ExecutorSideloadGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -63,7 +64,7 @@ public class ExecutorSideloadGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -78,7 +79,7 @@ public class ExecutorSideloadGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -93,7 +94,7 @@ public class ExecutorSideloadGrpcService
             responseObserver.onCompleted();
         }
         catch (Exception e) {
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 

code/execution/java/nu/marginalia/process/ProcessSpawnerService.java

@@ -29,7 +29,7 @@ import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 
 @Singleton
-public class ProcessService {
+public class ProcessSpawnerService {
     private final Logger logger = LoggerFactory.getLogger(getClass());
     private final Marker processMarker = MarkerFactory.getMarker("PROCESS");
 
@@ -88,7 +88,7 @@ public class ProcessService {
     }
 
     @Inject
-    public ProcessService(BaseServiceParams params) {
+    public ProcessSpawnerService(BaseServiceParams params) {
         this.eventLog = params.eventLog;
         this.node = params.configuration.node();
     }

code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java

@@ -2,6 +2,8 @@ package nu.marginalia.api.domains;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import nu.marginalia.api.domains.model.DomainInformation;
+import nu.marginalia.api.domains.model.SimilarDomain;
 import nu.marginalia.service.client.GrpcChannelPoolFactory;
 import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
 import nu.marginalia.service.discovery.property.ServiceKey;
@@ -10,16 +12,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
-import java.util.concurrent.*;
-
-import nu.marginalia.api.domains.model.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 @Singleton
 public class DomainInfoClient {
     private static final Logger logger = LoggerFactory.getLogger(DomainInfoClient.class);
 
     private final GrpcSingleNodeChannelPool<DomainInfoAPIGrpc.DomainInfoAPIBlockingStub> channelPool;
-    private final ExecutorService executor = Executors.newWorkStealingPool(8);
+
+
+    private static final boolean useLoom = Boolean.getBoolean("system.experimentalUseLoom");
+    private static final ExecutorService executor = useLoom ? Executors.newVirtualThreadPerTaskExecutor() : Executors.newWorkStealingPool(8);
 
     @Inject
     public DomainInfoClient(GrpcChannelPoolFactory factory) {

code/functions/live-capture/api/java/nu/marginalia/api/domsample/DomSampleClient.java

@@ -0,0 +1,114 @@
+package nu.marginalia.api.domsample;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import nu.marginalia.service.client.GrpcChannelPoolFactory;
+import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+
+@Singleton
+public class DomSampleClient {
+    private final GrpcSingleNodeChannelPool<DomSampleApiGrpc.DomSampleApiBlockingStub> channelPool;
+    private static final Logger logger = LoggerFactory.getLogger(DomSampleClient.class);
+
+    @Inject
+    public DomSampleClient(GrpcChannelPoolFactory factory) {
+
+        // The client is only interested in the primary node
+        var key = ServiceKey.forGrpcApi(DomSampleApiGrpc.class, ServicePartition.any());
+        this.channelPool = factory.createSingle(key, DomSampleApiGrpc::newBlockingStub);
+    }
+
+    public Optional<RpcDomainSample> getSample(String domainName) {
+        try {
+            var val = channelPool.call(DomSampleApiGrpc.DomSampleApiBlockingStub::getSample)
+                    .run(RpcDomainName.newBuilder().setDomainName(domainName).build());
+
+            return Optional.of(val);
+        }
+        catch (StatusRuntimeException sre) {
+            if (sre.getStatus() != Status.NOT_FOUND) {
+                logger.error("Failed to fetch DOM sample", sre);
+            }
+            return Optional.empty();
+        }
+    }
+
+    public Optional<RpcDomainSampleRequests> getSampleRequests(String domainName) {
+        try {
+            var val = channelPool.call(DomSampleApiGrpc.DomSampleApiBlockingStub::getSampleRequests)
+                    .run(RpcDomainName.newBuilder().setDomainName(domainName).build());
+
+            return Optional.of(val);
+        }
+        catch (StatusRuntimeException sre) {
+            if (sre.getStatus() != Status.NOT_FOUND) {
+                logger.error("Failed to fetch DOM sample", sre);
+            }
+            return Optional.empty();
+        }
+    }
+
+    public boolean hasSample(String domainName) {
+        try {
+            return channelPool.call(DomSampleApiGrpc.DomSampleApiBlockingStub::hasSample)
+                    .run(RpcDomainName.newBuilder().setDomainName(domainName).build())
+                    .getAnswer();
+        }
+        catch (StatusRuntimeException sre) {
+            return false;
+        }
+    }
+
+    public CompletableFuture<Boolean> hasSample(String domainName, ExecutorService executor) {
+        try {
+            return channelPool.call(DomSampleApiGrpc.DomSampleApiBlockingStub::hasSample)
+                    .async(executor)
+                    .run(RpcDomainName.newBuilder().setDomainName(domainName).build())
+                    .thenApply(RpcBooleanRsp::getAnswer);
+        }
+        catch (StatusRuntimeException sre) {
+            return CompletableFuture.completedFuture(false);
+        }
+    }
+
+    public CompletableFuture<RpcDomainSample> getSampleAsync(String domainName, ExecutorService executorService) {
+        return channelPool.call(DomSampleApiGrpc.DomSampleApiBlockingStub::getSample)
+                .async(executorService)
+                .run(RpcDomainName.newBuilder().setDomainName(domainName).build());
+    }
+
+    public List<RpcDomainSample> getAllSamples(String domainName) {
+        try {
+            Iterator<RpcDomainSample> val = channelPool.call(DomSampleApiGrpc.DomSampleApiBlockingStub::getAllSamples)
+                    .run(RpcDomainName.newBuilder().setDomainName(domainName).build());
+
+            List<RpcDomainSample> ret = new ArrayList<>();
+            val.forEachRemaining(ret::add);
+            return ret;
+        }
+        catch (StatusRuntimeException sre) {
+            logger.error("Failed to fetch DOM sample");
+            return List.of();
+        }
+    }
+
+    public boolean waitReady(Duration duration) throws InterruptedException {
+        return channelPool.awaitChannel(duration);
+    }
+
+
+}

code/functions/live-capture/api/java/nu/marginalia/api/feeds/FeedsClient.java

@@ -11,6 +11,7 @@ import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.module.ServiceConfiguration;
 
 import javax.annotation.CheckReturnValue;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -23,7 +24,9 @@ import java.util.function.BiConsumer;
 
 @Singleton
 public class FeedsClient {
-    private final ExecutorService executorService = Executors.newCachedThreadPool();
+    private static final boolean useLoom = Boolean.getBoolean("system.experimentalUseLoom");
+    private static final ExecutorService executorService = useLoom ? Executors.newVirtualThreadPerTaskExecutor() : Executors.newCachedThreadPool();
+
     private final GrpcSingleNodeChannelPool<FeedApiGrpc.FeedApiBlockingStub> channelPool;
     private final MqOutbox updateFeedsOutbox;
 
@@ -59,6 +62,11 @@ public class FeedsClient {
                 .forEachRemaining(rsp -> consumer.accept(rsp.getDomain(), new ArrayList<>(rsp.getUrlList())));
     }
 
+    public boolean waitReady(Duration duration) throws InterruptedException {
+        return channelPool.awaitChannel(duration);
+    }
+
+
     /** Get the hash of the feed data, for identifying when the data has been updated */
     public String getFeedDataHash() {
         return channelPool.call(FeedApiGrpc.FeedApiBlockingStub::getFeedDataHash)

code/functions/live-capture/api/src/main/protobuf/dom-sample.proto

@@ -0,0 +1,47 @@
+syntax="proto3";
+package nu.marginalia.api.domsample;
+
+option java_package="nu.marginalia.api.domsample";
+option java_multiple_files=true;
+
+
+service DomSampleApi {
+  rpc getSample(RpcDomainName) returns (RpcDomainSample) {}
+  rpc getSampleRequests(RpcDomainName) returns (RpcDomainSampleRequests) {}
+  rpc hasSample(RpcDomainName) returns (RpcBooleanRsp) {}
+  rpc getAllSamples(RpcDomainName) returns (stream RpcDomainSample) {}
+}
+
+message RpcDomainName {
+  string domainName = 1;
+}
+
+message RpcBooleanRsp {
+  bool answer = 1;
+}
+
+message RpcDomainSampleRequests {
+  string domainName = 1;
+  string url = 2;
+  repeated RpcOutgoingRequest outgoingRequests = 5;
+}
+
+message RpcDomainSample {
+  string domainName = 1;
+  string url = 2;
+  bytes htmlSampleZstd = 3;
+  bool accepted_popover = 4;
+  repeated RpcOutgoingRequest outgoingRequests = 5;
+}
+
+message RpcOutgoingRequest {
+  RequestMethod method = 1;
+  int64 timestamp = 2;
+  string url = 3;
+
+  enum RequestMethod {
+    GET = 0;
+    POST = 1;
+    OTHER = 2;
+  };
+}

code/functions/live-capture/build.gradle

@@ -31,6 +31,7 @@ dependencies {
     implementation libs.jsoup
     implementation libs.opencsv
     implementation libs.slop
+    implementation libs.zstd
     implementation libs.sqlite
     implementation libs.bundles.slf4j
     implementation libs.commons.lang3

code/functions/live-capture/java/nu/marginalia/domsample/DomSampleGrpcService.java

@@ -0,0 +1,176 @@
+package nu.marginalia.domsample;
+
+import com.github.luben.zstd.Zstd;
+import com.google.inject.Inject;
+import com.google.protobuf.ByteString;
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+import nu.marginalia.api.domsample.*;
+import nu.marginalia.domsample.db.DomSampleDb;
+import nu.marginalia.service.server.DiscoverableService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+public class DomSampleGrpcService
+        extends DomSampleApiGrpc.DomSampleApiImplBase
+        implements DiscoverableService
+{
+    private static final Logger logger = LoggerFactory.getLogger(DomSampleGrpcService.class);
+
+    private final DomSampleDb domSampleDb;
+
+    @Inject
+    public DomSampleGrpcService(DomSampleDb domSampleDb) {
+        this.domSampleDb = domSampleDb;
+    }
+
+    @Override
+    public void getSample(RpcDomainName request, StreamObserver<RpcDomainSample> responseObserver) {
+        String domainName = request.getDomainName();
+        if (domainName.isBlank()) {
+            responseObserver.onError(Status.INVALID_ARGUMENT
+                            .withDescription("Invalid domain name")
+                            .asRuntimeException());
+            return;
+        }
+
+        try {
+            List<DomSampleDb.Sample> dbRecords = domSampleDb.getSamples(domainName);
+            if (dbRecords.isEmpty()) {
+                responseObserver.onError(Status.NOT_FOUND.withDescription("No sample found").asRuntimeException());
+                return;
+            }
+
+            // Grab the first sample
+            RpcDomainSample.Builder response = convertFullSample(dbRecords.getFirst());
+
+            responseObserver.onNext(response.build());
+            responseObserver.onCompleted();
+        }
+        catch (Exception e) {
+            logger.error("Error in getSample()", e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
+        }
+    }
+
+    @Override
+    public void getSampleRequests(RpcDomainName request, StreamObserver<RpcDomainSampleRequests> responseObserver) {
+        String domainName = request.getDomainName();
+        if (domainName.isBlank()) {
+            responseObserver.onError(Status.INVALID_ARGUMENT
+                    .withDescription("Invalid domain name")
+                    .asRuntimeException());
+            return;
+        }
+
+        try {
+            List<DomSampleDb.Sample> dbRecords = domSampleDb.getSamples(domainName);
+            if (dbRecords.isEmpty()) {
+                responseObserver.onError(Status.NOT_FOUND.withDescription("No sample found").asRuntimeException());
+                return;
+            }
+
+            // Grab the first sample
+            RpcDomainSampleRequests.Builder response = convertRequestData(dbRecords.getFirst());
+
+            responseObserver.onNext(response.build());
+            responseObserver.onCompleted();
+        }
+        catch (Exception e) {
+            logger.error("Error in getSample()", e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
+        }
+    }
+
+    @Override
+    public void hasSample(RpcDomainName request, StreamObserver<RpcBooleanRsp> responseObserver) {
+        String domainName = request.getDomainName();
+        if (domainName.isBlank()) {
+            responseObserver.onError(Status.INVALID_ARGUMENT
+                    .withDescription("Invalid domain name")
+                    .asRuntimeException());
+            return;
+        }
+
+        try {
+            responseObserver.onNext(RpcBooleanRsp.newBuilder()
+                    .setAnswer(domSampleDb.hasSample(domainName)).build());
+            responseObserver.onCompleted();
+        }
+        catch (Exception e) {
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
+        }
+    }
+
+    @Override
+    public void getAllSamples(RpcDomainName request, StreamObserver<RpcDomainSample> responseObserver) {
+        String domainName = request.getDomainName();
+        if (domainName.isBlank()) {
+            responseObserver.onError(Status.INVALID_ARGUMENT
+                    .withDescription("Invalid domain name")
+                    .asRuntimeException());
+            return;
+        }
+
+        try {
+            List<DomSampleDb.Sample> dbRecords = domSampleDb.getSamples(domainName);
+
+            for (var record : dbRecords) {
+                responseObserver.onNext(convertFullSample(record).build());
+            }
+
+            responseObserver.onCompleted();
+        }
+        catch (Exception e) {
+            logger.error("Error in getSample()", e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
+        }
+    }
+
+    private RpcDomainSample.Builder convertFullSample(DomSampleDb.Sample dbSample) {
+
+        ByteString htmlZstd = ByteString.copyFrom(Zstd.compress(dbSample.sample().getBytes(StandardCharsets.UTF_8)));
+
+        var sampleBuilder = RpcDomainSample.newBuilder()
+                .setDomainName(dbSample.domain())
+                .setAcceptedPopover(dbSample.acceptedPopover())
+                .setHtmlSampleZstd(htmlZstd);
+
+        for (var req : dbSample.parseRequests()) {
+            sampleBuilder.addOutgoingRequestsBuilder()
+                    .setUrl(req.uri().toString())
+                    .setMethod(switch (req.method().toUpperCase())
+                    {
+                        case "GET" -> RpcOutgoingRequest.RequestMethod.GET;
+                        case "POST" -> RpcOutgoingRequest.RequestMethod.POST;
+                        default -> RpcOutgoingRequest.RequestMethod.OTHER;
+                    })
+                    .setTimestamp(req.timestamp());
+        }
+
+        return sampleBuilder;
+    }
+
+    private RpcDomainSampleRequests.Builder convertRequestData(DomSampleDb.Sample dbSample) {
+
+        var sampleBuilder = RpcDomainSampleRequests.newBuilder()
+                .setDomainName(dbSample.domain());
+
+        for (var req : dbSample.parseRequests()) {
+            sampleBuilder.addOutgoingRequestsBuilder()
+                    .setUrl(req.uri().toString())
+                    .setMethod(switch (req.method().toUpperCase())
+                    {
+                        case "GET" -> RpcOutgoingRequest.RequestMethod.GET;
+                        case "POST" -> RpcOutgoingRequest.RequestMethod.POST;
+                        default -> RpcOutgoingRequest.RequestMethod.OTHER;
+                    })
+                    .setTimestamp(req.timestamp());
+        }
+
+        return sampleBuilder;
+    }
+}

code/functions/live-capture/java/nu/marginalia/domsample/db/DomSampleDb.java

@@ -1,17 +1,28 @@
 package nu.marginalia.domsample.db;
 
 import nu.marginalia.WmsaHome;
+import nu.marginalia.model.EdgeUrl;
+import org.apache.commons.lang3.StringUtils;
 import org.jsoup.Jsoup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
 
 public class DomSampleDb implements AutoCloseable {
     private static final String dbFileName = "dom-sample.db";
     private final Connection connection;
+    private static final Logger logger = LoggerFactory.getLogger(DomSampleDb.class);
 
     public DomSampleDb() throws SQLException{
         this(WmsaHome.getDataPath().resolve(dbFileName));
@@ -88,7 +99,71 @@ public class DomSampleDb implements AutoCloseable {
     }
 
 
-    public record Sample(String url, String domain, String sample, String requests, boolean acceptedPopover) {}
+    public record Sample(String url, String domain, String sample, String requests, boolean acceptedPopover) {
+
+        public List<SampleRequest> parseRequests() {
+            List<SampleRequest> requests = new ArrayList<>();
+
+            // Request format is METHOD\tTIMESTAMP\tURI\n
+
+            for (var line : StringUtils.split(this.requests, '\n')) {
+                String[] parts = StringUtils.split(line, "\t", 3);
+                if (parts.length != 3) continue;
+
+                try {
+                    String method =  parts[0];
+                    long ts = Long.parseLong(parts[1]);
+                    String linkUrl = parts[2];
+
+                    URI uri = parseURI(linkUrl);
+
+                    requests.add(new SampleRequest(method, ts, uri));
+                }
+                catch (Exception e) {
+                    logger.warn("Failed to parse requests", e);
+                }
+            }
+
+            return requests;
+        }
+
+
+        private static URI parseURI(String uri) throws URISyntaxException {
+            try {
+                return new URI(uri);
+            }
+            catch (URISyntaxException ex) {
+                return new EdgeUrl(uri).asURI();
+            }
+        }
+    }
+
+    public record SampleRequest(String method, long timestamp, URI uri) {}
+
+    /**
+     * @param consumer - consume the sample, return true to continue consumption
+     * @throws SQLException
+     */
+    public void forEachSample(Predicate<Sample> consumer) throws SQLException {
+        try (var stmt = connection.prepareStatement("""
+                SELECT url, domain, sample, requests, accepted_popover
+                FROM samples
+                """))
+        {
+            var rs = stmt.executeQuery();
+            while (rs.next()) {
+                var sample = new Sample(
+                        rs.getString("url"),
+                        rs.getString("domain"),
+                        rs.getString("sample"),
+                        rs.getString("requests"),
+                        rs.getBoolean("accepted_popover")
+                );
+
+                if (!consumer.test(sample)) break;
+            }
+        }
+    }
 
     public List<Sample> getSamples(String domain) throws SQLException {
         List<Sample> samples = new ArrayList<>();
@@ -116,6 +191,21 @@ public class DomSampleDb implements AutoCloseable {
         return samples;
     }
 
+
+    public boolean hasSample(String domain) throws SQLException {
+
+        try (var stmt = connection.prepareStatement("""
+                SELECT 1
+                FROM samples
+                WHERE domain = ?
+                """))
+        {
+            stmt.setString(1, domain);
+            var rs = stmt.executeQuery();
+            return rs.next();
+        }
+    }
+
     public void saveSample(String domain, String url, String rawContent) throws SQLException {
         var doc = Jsoup.parse(rawContent);
 

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedsGrpcService.java

@@ -1,6 +1,7 @@
 package nu.marginalia.rss.svc;
 
 import com.google.inject.Inject;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import nu.marginalia.api.feeds.*;
 import nu.marginalia.db.DbDomainQueries;
@@ -69,7 +70,7 @@ public class FeedsGrpcService extends FeedApiGrpc.FeedApiImplBase implements Dis
     @Override
     public void getFeedDataHash(Empty request, StreamObserver<RpcFeedDataHash> responseObserver) {
         if (!feedDb.isEnabled()) {
-            responseObserver.onError(new IllegalStateException("Feed database is disabled on this node"));
+            responseObserver.onError(Status.INTERNAL.withDescription("Feed database is disabled on this node").asRuntimeException());
             return;
         }
 
@@ -80,7 +81,7 @@ public class FeedsGrpcService extends FeedApiGrpc.FeedApiImplBase implements Dis
         }
         catch (Exception e) {
             logger.error("Error getting feed data hash", e);
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -101,7 +102,7 @@ public class FeedsGrpcService extends FeedApiGrpc.FeedApiImplBase implements Dis
         }
         catch (Exception e) {
             logger.error("Error getting updated links", e);
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 
@@ -109,13 +110,13 @@ public class FeedsGrpcService extends FeedApiGrpc.FeedApiImplBase implements Dis
     public void getFeed(RpcDomainId request,
                         StreamObserver<RpcFeed> responseObserver) {
         if (!feedDb.isEnabled()) {
-            responseObserver.onError(new IllegalStateException("Feed database is disabled on this node"));
+            responseObserver.onError(Status.INTERNAL.withDescription("Feed database is disabled on this node").asRuntimeException());
             return;
         }
 
         Optional<EdgeDomain> domainName = domainQueries.getDomain(request.getDomainId());
         if (domainName.isEmpty()) {
-            responseObserver.onError(new IllegalArgumentException("Domain not found"));
+            responseObserver.onError(Status.NOT_FOUND.withDescription("Domain not found").asRuntimeException());
             return;
         }
 

code/functions/math/api/java/nu/marginalia/api/math/MathClient.java

@@ -26,7 +26,9 @@ public class MathClient {
     private static final Logger logger = LoggerFactory.getLogger(MathClient.class);
 
     private final GrpcSingleNodeChannelPool<MathApiGrpc.MathApiBlockingStub> channelPool;
-    private final ExecutorService executor = Executors.newWorkStealingPool(8);
+
+    private static final boolean useLoom = Boolean.getBoolean("system.experimentalUseLoom");
+    private static final ExecutorService executor = useLoom ? Executors.newVirtualThreadPerTaskExecutor() : Executors.newWorkStealingPool(8);
 
     @Inject
     public MathClient(GrpcChannelPoolFactory factory) {

code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java

@@ -3,6 +3,7 @@ package nu.marginalia.functions.searchquery;
 import com.google.common.collect.Lists;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import io.prometheus.client.Histogram;
 import nu.marginalia.api.searchquery.*;
@@ -93,7 +94,7 @@ public class QueryGRPCService
             });
         } catch (Exception e) {
             logger.error("Exception", e);
-            responseObserver.onError(e);
+            responseObserver.onError(Status.INTERNAL.withCause(e).asRuntimeException());
         }
     }
 

code/index/api/java/nu/marginalia/index/api/IndexClient.java

@@ -38,7 +38,9 @@ public class IndexClient {
             .help("Count of results filtered by NSFW tier")
             .register();
 
-    private static final ExecutorService executor = Executors.newCachedThreadPool();
+
+    private static final boolean useLoom = Boolean.getBoolean("system.experimentalUseLoom");
+    private static final ExecutorService executor = useLoom ? Executors.newVirtualThreadPerTaskExecutor() : Executors.newCachedThreadPool();
 
     @Inject
     public IndexClient(GrpcChannelPoolFactory channelPoolFactory,

code/index/java/nu/marginalia/domainranking/PageRankDomainRanker.java

@@ -1,10 +1,10 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
 import gnu.trove.list.TIntList;
 import gnu.trove.list.array.TIntArrayList;
-import nu.marginalia.ranking.domains.accumulator.RankingResultAccumulator;
-import nu.marginalia.ranking.domains.data.GraphSource;
-import nu.marginalia.ranking.domains.jgrapht.PersonalizedPageRank;
+import nu.marginalia.domainranking.accumulator.RankingResultAccumulator;
+import nu.marginalia.domainranking.data.GraphSource;
+import nu.marginalia.domainranking.jgrapht.PersonalizedPageRank;
 import org.jgrapht.Graph;
 import org.jgrapht.alg.interfaces.VertexScoringAlgorithm;
 import org.jgrapht.alg.scoring.PageRank;

code/index/java/nu/marginalia/domainranking/RankingAlgorithm.java

@@ -1,6 +1,6 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
-import nu.marginalia.ranking.domains.accumulator.RankingResultAccumulator;
+import nu.marginalia.domainranking.accumulator.RankingResultAccumulator;
 
 import java.util.function.Supplier;
 

code/index/java/nu/marginalia/domainranking/accumulator/RankingResultAccumulator.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.accumulator;
+package nu.marginalia.domainranking.accumulator;
 
 public interface RankingResultAccumulator<T> {
     void add(int domainId, int rank);

code/index/java/nu/marginalia/domainranking/accumulator/RankingResultBitSetAccumulator.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.accumulator;
+package nu.marginalia.domainranking.accumulator;
 
 import org.roaringbitmap.RoaringBitmap;
 

code/index/java/nu/marginalia/domainranking/accumulator/RankingResultHashMapAccumulator.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.accumulator;
+package nu.marginalia.domainranking.accumulator;
 
 import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
 

code/index/java/nu/marginalia/domainranking/accumulator/RankingResultHashSetAccumulator.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.accumulator;
+package nu.marginalia.domainranking.accumulator;
 
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 

code/index/java/nu/marginalia/domainranking/accumulator/RankingResultListAccumulator.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.accumulator;
+package nu.marginalia.domainranking.accumulator;
 
 import gnu.trove.list.array.TIntArrayList;
 

code/index/java/nu/marginalia/domainranking/data/AbstractGraphSource.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.data;
+package nu.marginalia.domainranking.data;
 
 import com.zaxxer.hikari.HikariDataSource;
 import org.jgrapht.Graph;

code/index/java/nu/marginalia/domainranking/data/GraphSource.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.data;
+package nu.marginalia.domainranking.data;
 
 import org.jgrapht.Graph;
 

code/index/java/nu/marginalia/domainranking/data/InvertedLinkGraphSource.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.data;
+package nu.marginalia.domainranking.data;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;

code/index/java/nu/marginalia/domainranking/data/LinkGraphSource.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.data;
+package nu.marginalia.domainranking.data;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;

code/index/java/nu/marginalia/domainranking/data/SimilarityGraphSource.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.data;
+package nu.marginalia.domainranking.data;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;

code/index/java/nu/marginalia/domainranking/jgrapht/PersonalizedPageRank.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking.domains.jgrapht;
+package nu.marginalia.domainranking.jgrapht;
 
 /*
  * (C) Copyright 2016-2023, by Dimitrios Michail and Contributors.
@@ -21,8 +21,9 @@ package nu.marginalia.ranking.domains.jgrapht;
 
 /* (modified by @vlofgren to add personalization) */
 
-import org.jgrapht.*;
-import org.jgrapht.alg.interfaces.*;
+import org.jgrapht.Graph;
+import org.jgrapht.Graphs;
+import org.jgrapht.alg.interfaces.VertexScoringAlgorithm;
 
 import java.util.*;
 

code/index/java/nu/marginalia/index/IndexGrpcService.java

@@ -2,6 +2,7 @@ package nu.marginalia.index;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
 import io.prometheus.client.Counter;
 import io.prometheus.client.Gauge;
@@ -148,7 +149,7 @@ public class IndexGrpcService
         }
         catch (Exception ex) {
             logger.error("Error in handling request", ex);
-            responseObserver.onError(ex);
+            responseObserver.onError(Status.INTERNAL.withCause(ex).asRuntimeException());
         }
     }
 

code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java

@@ -551,9 +551,18 @@ public class IndexResultScoreCalculator {
             largeSiteFactor = 2;
         }
 
-        if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.TRACKING_ADTECH.getFeatureBit()))
+        if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.ADVERTISEMENT.getFeatureBit()))
             penalty += 7.5 * largeSiteFactor;
 
+        if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.CONSENT.getFeatureBit()))
+            penalty += 2.5 * largeSiteFactor;
+
+        if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.POPOVER.getFeatureBit()))
+            penalty += 2.5 * largeSiteFactor;
+
+        if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.TRACKING_ADTECH.getFeatureBit()))
+            penalty += 5.0 * largeSiteFactor;
+
         if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.AFFILIATE_LINK.getFeatureBit()))
             penalty += 5.0 * largeSiteFactor;
 
@@ -563,6 +572,9 @@ public class IndexResultScoreCalculator {
         if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.TRACKING.getFeatureBit()))
             penalty += 2.5 * largeSiteFactor;
 
+        if (DocumentMetadata.hasFlags(featureFlags, HtmlFeature.SHORT_DOCUMENT.getFeatureBit()))
+            penalty += 2.5  * largeSiteFactor;
+
         if (isForum || isWiki) {
             penalty = Math.min(0, penalty - 2);
         }

code/index/java/nu/marginalia/index/searchset/SearchSetsService.java

@@ -6,14 +6,14 @@ import gnu.trove.list.TIntList;
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 import nu.marginalia.db.DomainRankingSetsService;
 import nu.marginalia.db.DomainTypes;
+import nu.marginalia.domainranking.PageRankDomainRanker;
+import nu.marginalia.domainranking.accumulator.RankingResultHashMapAccumulator;
+import nu.marginalia.domainranking.accumulator.RankingResultHashSetAccumulator;
+import nu.marginalia.domainranking.data.GraphSource;
+import nu.marginalia.domainranking.data.LinkGraphSource;
+import nu.marginalia.domainranking.data.SimilarityGraphSource;
 import nu.marginalia.index.IndexFactory;
 import nu.marginalia.index.domainrankings.DomainRankings;
-import nu.marginalia.ranking.domains.PageRankDomainRanker;
-import nu.marginalia.ranking.domains.accumulator.RankingResultHashMapAccumulator;
-import nu.marginalia.ranking.domains.accumulator.RankingResultHashSetAccumulator;
-import nu.marginalia.ranking.domains.data.GraphSource;
-import nu.marginalia.ranking.domains.data.LinkGraphSource;
-import nu.marginalia.ranking.domains.data.SimilarityGraphSource;
 import nu.marginalia.service.control.ServiceEventLog;
 import nu.marginalia.service.module.ServiceConfiguration;
 import org.slf4j.Logger;

code/index/test/nu/marginalia/domainranking/RankingAlgorithmWithRealDataTest.java

@@ -1,6 +1,6 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
-import nu.marginalia.ranking.domains.accumulator.RankingResultListAccumulator;
+import nu.marginalia.domainranking.accumulator.RankingResultListAccumulator;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 

code/index/test/nu/marginalia/domainranking/RankingAlgorithmsContainerTest.java

@@ -1,12 +1,12 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
 
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.api.linkgraph.AggregateLinkGraphClient;
-import nu.marginalia.ranking.domains.data.InvertedLinkGraphSource;
-import nu.marginalia.ranking.domains.data.LinkGraphSource;
-import nu.marginalia.ranking.domains.data.SimilarityGraphSource;
+import nu.marginalia.domainranking.data.InvertedLinkGraphSource;
+import nu.marginalia.domainranking.data.LinkGraphSource;
+import nu.marginalia.domainranking.data.SimilarityGraphSource;
 import nu.marginalia.test.TestMigrationLoader;
 import org.jgrapht.Graph;
 import org.jgrapht.graph.DefaultWeightedEdge;

code/index/test/nu/marginalia/domainranking/TestGraphSourceForInvertedLinkData.java

@@ -1,7 +1,7 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
 import nu.marginalia.array.LongArrayFactory;
-import nu.marginalia.ranking.domains.data.GraphSource;
+import nu.marginalia.domainranking.data.GraphSource;
 import org.apache.commons.lang3.StringUtils;
 import org.jgrapht.Graph;
 import org.jgrapht.graph.DefaultDirectedGraph;

code/index/test/nu/marginalia/domainranking/TestGraphSourceForLinkData.java

@@ -1,7 +1,7 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
 import nu.marginalia.array.LongArrayFactory;
-import nu.marginalia.ranking.domains.data.GraphSource;
+import nu.marginalia.domainranking.data.GraphSource;
 import org.apache.commons.lang3.StringUtils;
 import org.jgrapht.Graph;
 import org.jgrapht.graph.DefaultDirectedGraph;

code/index/test/nu/marginalia/domainranking/TestGraphSourceForSimilarityData.java

@@ -1,6 +1,6 @@
-package nu.marginalia.ranking.domains;
+package nu.marginalia.domainranking;
 
-import nu.marginalia.ranking.domains.data.GraphSource;
+import nu.marginalia.domainranking.data.GraphSource;
 import org.apache.commons.lang3.StringUtils;
 import org.jgrapht.Graph;
 import org.jgrapht.graph.DefaultUndirectedWeightedGraph;

code/processes/converting-process/build.gradle

@@ -47,11 +47,14 @@ dependencies {
 
     implementation project(':code:processes:converting-process:ft-anchor-keywords')
     implementation project(':code:processes:converting-process:ft-keyword-extraction')
+    implementation project(':code:processes:converting-process:ft-dom-classifier')
 
     implementation project(':code:processes:crawling-process:ft-crawl-blocklist')
     implementation project(':code:processes:crawling-process:ft-link-parser')
     implementation project(':code:processes:crawling-process:ft-content-type')
 
+    implementation project(':code:functions:live-capture:api')
+
     testImplementation project(':code:libraries:term-frequency-dict')
     testImplementation project(':code:processes:crawling-process:model')
 
@@ -87,6 +90,7 @@ dependencies {
     implementation libs.commons.lang3
     implementation libs.commons.compress
     implementation libs.sqlite
+    implementation libs.bundles.grpc
 
     implementation libs.bundles.httpcomponents
 

code/processes/converting-process/ft-dom-classifier/build.gradle

@@ -0,0 +1,41 @@
+plugins {
+    id 'java'
+    id "de.undercouch.download" version "5.1.0"
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
+    }
+}
+
+apply from: "$rootProject.projectDir/srcsets.gradle"
+
+dependencies {
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+    implementation project(':code:common:model')
+    implementation project(':code:common:db')
+    implementation project(':code:functions:live-capture:api')
+
+
+    implementation libs.bundles.slf4j
+    implementation libs.guava
+    implementation libs.zstd
+    implementation dependencies.create(libs.guice.get()) {
+        exclude group: 'com.google.guava'
+    }
+    implementation libs.trove
+    implementation libs.gson
+    implementation libs.bundles.protobuf
+    implementation libs.bundles.mariadb
+    implementation libs.duckdb
+    implementation libs.notnull
+    implementation libs.jsoup
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+}
+

code/processes/converting-process/ft-dom-classifier/java/nu/marginalia/ddtrackergradar/DDGTrackerData.java

@@ -0,0 +1,99 @@
+package nu.marginalia.ddtrackergradar;
+
+import com.google.gson.Gson;
+import nu.marginalia.WmsaHome;
+import nu.marginalia.ddtrackergradar.model.DDGTDomain;
+import nu.marginalia.model.gson.GsonFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+
+/** Holds tracker metadata from DuckDuckGo's Tracker Radar
+ *  data itself CC-BY-NC-SA 4.0
+ * */
+public class DDGTrackerData {
+    private final Map<String, DDGTDomain> topDomains = new HashMap<>();
+    private final Map<String, DDGTDomain> domains = new HashMap<>();
+
+    private final Gson gson = GsonFactory.get();
+
+    private static final Logger logger = LoggerFactory.getLogger(DDGTrackerData.class);
+
+    public DDGTrackerData() {
+
+        // Data is assumed to be in ${WMSA_HOME}/data/tracker-radar
+        // ... do a shallow clone of the repo
+        // https://github.com/duckduckgo/tracker-radar/
+
+        Path dataDir = WmsaHome.getDataPath().resolve("tracker-radar");
+        if (!Files.exists(dataDir)) {
+            logger.info("tracker-radar data absent from expected path {}, loading nothing", dataDir);
+            return;
+        }
+
+        try (var sources = Files.list(dataDir.resolve("domains"))) {
+            sources.filter(Files::isDirectory).forEach(this::loadDomainDir);
+        }
+        catch (IOException e) {
+            logger.error("Failed to read tracker radar data dir", e);
+        }
+    }
+
+    /** Tries to fetch available information about tracking coming from the specified domain
+     */
+    public Optional<DDGTDomain> getDomainInfo(String domain) {
+        return Optional
+                .ofNullable(topDomains.get(domain))
+                .or(() -> Optional.ofNullable(domains.get(domain)));
+    }
+
+    /** public for testing */
+    public void loadDomainDir(Path dir) {
+        try (var dirContent = Files.list(dir)) {
+            dirContent
+                    .filter(Files::isRegularFile)
+                    .filter(path -> path.toString().endsWith(".json"))
+                    .forEach(this::loadDomainModel);
+        }
+        catch (IOException e) {
+            logger.error("Error while loading DDGT tracker data", e);
+        }
+    }
+
+    void loadDomainModel(Path jsonFile) {
+        try {
+            var model = gson.fromJson(Files.readString(jsonFile), DDGTDomain.class);
+
+            if (model.domain() == null)
+                return;
+            if ((model.owner() == null || model.owner().isEmpty())
+             && (model.categories() == null || model.categories().isEmpty()))
+                return;
+
+            topDomains.put(model.domain(), model);
+            domains.put(model.domain(), model);
+
+            if (model.subdomains() != null) {
+                for (String subdomain : model.subdomains()) {
+                    domains.put(subdomain + "." + model.domain(), model);
+                }
+            }
+        }
+        catch (Exception e) {
+            logger.error("Error while loading DDGT tracker data", e);
+        }
+    }
+
+    // Export all classifications in the data set
+    public Set<String> getAllClassifications() {
+        Set<String> ret = new HashSet<>();
+        for (var domain: domains.values()) {
+            ret.addAll(domain.categories());
+        }
+        return ret;
+    }
+}

code/processes/converting-process/ft-dom-classifier/java/nu/marginalia/ddtrackergradar/model/DDGTDomain.java

@@ -0,0 +1,12 @@
+package nu.marginalia.ddtrackergradar.model;
+
+import java.util.List;
+
+public record DDGTDomain(
+        String domain,
+        DDGTOwner owner,
+        List<String> categories,
+        List<String> subdomains
+)
+{
+}

code/processes/converting-process/ft-dom-classifier/java/nu/marginalia/ddtrackergradar/model/DDGTOwner.java

@@ -0,0 +1,10 @@
+package nu.marginalia.ddtrackergradar.model;
+
+public record DDGTOwner(String name, String displayName, String privacyPolicy, String url) {
+    public boolean isEmpty() {
+        return name == null
+                && displayName == null
+                && privacyPolicy == null
+                && url == null;
+    }
+}

code/processes/converting-process/ft-dom-classifier/java/nu/marginalia/domclassifier/DomSampleClassification.java

@@ -0,0 +1,25 @@
+package nu.marginalia.domclassifier;
+
+import nu.marginalia.model.crawl.HtmlFeature;
+
+import javax.annotation.Nullable;
+
+/**
+ * Feature classifications for the DOM sample
+ */
+public enum DomSampleClassification {
+    ADS(HtmlFeature.ADVERTISEMENT),
+    TRACKING(HtmlFeature.TRACKING_ADTECH),
+    CONSENT(HtmlFeature.CONSENT),
+    POPOVER(HtmlFeature.POPOVER),
+    THIRD_PARTY_REQUESTS(HtmlFeature.THIRD_PARTY_REQUESTS),
+    UNCLASSIFIED(HtmlFeature.MISSING_DOM_SAMPLE),
+    IGNORE(null);
+
+    @Nullable
+    public final HtmlFeature htmlFeature;
+
+    DomSampleClassification(@Nullable HtmlFeature feature) {
+        this.htmlFeature = feature;
+    }
+}

code/processes/converting-process/ft-dom-classifier/java/nu/marginalia/domclassifier/DomSampleClassifier.java

@@ -0,0 +1,185 @@
+package nu.marginalia.domclassifier;
+
+import com.github.luben.zstd.ZstdInputStream;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.api.domsample.RpcDomainSample;
+import nu.marginalia.model.EdgeDomain;
+import nu.marginalia.model.EdgeUrl;
+import org.jsoup.Jsoup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
+
+@Singleton
+public class DomSampleClassifier {
+
+    private static final Logger logger = LoggerFactory.getLogger(DomSampleClassifier.class);
+
+    private final List<Map.Entry<Predicate<String>, DomSampleClassification>> regexClassification = new ArrayList<>();
+    private final Map<String, DomSampleClassification> urlClassification = new HashMap<>();
+    private final Map<String, DomSampleClassification> topDomainClassification = new HashMap<>();
+    private final Map<String, DomSampleClassification> fullDomainClassification = new HashMap<>();
+
+    @Inject
+    public DomSampleClassifier() throws ParserConfigurationException, IOException, SAXException {
+        this(ClassLoader.getSystemResourceAsStream("request-classifier.xml"));
+    }
+
+    public DomSampleClassifier(InputStream specificationXmlData) throws ParserConfigurationException, IOException, SAXException {
+        Objects.requireNonNull(specificationXmlData, "specificationXmlData is null");
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        DocumentBuilder builder = factory.newDocumentBuilder();
+        Document doc = builder.parse(specificationXmlData);
+
+        NodeList classifierNodes = doc.getElementsByTagName("classifier");
+
+        for (int i = 0; i < classifierNodes.getLength(); i++) {
+            Element classifier = (Element) classifierNodes.item(i);
+
+            String target = classifier.getAttribute("target");
+            String rule = classifier.getAttribute("rule");
+            String content = classifier.getTextContent().trim();
+
+            // Convert rule to Classification enum
+            DomSampleClassification classification = DomSampleClassification.valueOf(rule.toUpperCase());
+
+            // Add to appropriate map based on target
+            switch (target) {
+                case "url":
+                    urlClassification.put(content, classification);
+                    break;
+                case "url-regex":
+                    regexClassification.add(Map.entry(Pattern.compile(content).asPredicate(), classification));
+                    break;
+                case "top":
+                    topDomainClassification.put(content, classification);
+                    break;
+                case "domain":
+                    fullDomainClassification.put(content, classification);
+                    break;
+                default:
+                    throw new IllegalArgumentException("Unknown target type: " + target);
+            }
+        }
+
+    }
+
+    public Set<DomSampleClassification> classifySample(RpcDomainSample sample) {
+        Set<DomSampleClassification> classifications = new HashSet<>();
+
+        // Look at DOM
+
+        EdgeDomain sampleDomain = new EdgeDomain(sample.getDomainName());
+
+        byte[] sampleBytes = sample.getHtmlSampleZstd().toByteArray();
+        if (sampleBytes.length >= 8) {
+            logger.info("sampleBytes magic fingerprint: {}", Arrays.toString(Arrays.copyOf(sampleBytes, 8)));
+        }
+        else {
+            logger.info("sampleBytes too short! Was {}", Arrays.toString(sampleBytes));
+        }
+        try (var compressedStream = new ZstdInputStream(new ByteArrayInputStream(sampleBytes))) {
+            String html = new String(compressedStream.readAllBytes(), StandardCharsets.UTF_8);
+            var parsedDoc = Jsoup.parse(html);
+            var fixedElements = parsedDoc.select("*[data-position=fixed]");
+
+            if (sample.getAcceptedPopover()) {
+                classifications.add(DomSampleClassification.POPOVER);
+            }
+            else if (!fixedElements.isEmpty()) {
+                String fixedText = fixedElements.text().toLowerCase();
+                if (fixedText.contains("cookie") ||
+                        fixedText.contains("subscribe") ||
+                        fixedText.contains("consent") ||
+                        fixedText.contains("newsletter") ||
+                        fixedText.contains("gdpr"))
+                {
+                    classifications.add(DomSampleClassification.POPOVER);
+                }
+            }
+        }
+        catch (Exception ex) {
+            logger.warn("Error when parsing DOM HTML sample for size" + sample.getHtmlSampleZstd().size(), ex);
+        }
+
+        // Classify outgoing requests
+        for (var req : sample.getOutgoingRequestsList()) {
+            EdgeUrl url;
+
+            try {
+                url = new EdgeUrl(req.getUrl());
+            }
+            catch (URISyntaxException ex) {
+                continue;
+            }
+
+            if (!url.domain.hasSameTopDomain(sampleDomain)) {
+                classifications.add(DomSampleClassification.THIRD_PARTY_REQUESTS);
+            }
+
+            var clazz = classifyRequest(url);
+            if (clazz != DomSampleClassification.IGNORE && clazz != DomSampleClassification.UNCLASSIFIED) {
+                classifications.add(clazz);
+            }
+        }
+
+        return classifications;
+    }
+
+    public DomSampleClassification classifyRequest(EdgeUrl edgeUrl) {
+        StringBuilder pathSb = new StringBuilder(edgeUrl.path);
+        if (edgeUrl.param != null) {
+            pathSb.append("?").append(edgeUrl.param);
+        }
+        String pathMatchString = pathSb.toString();
+        String urlDisplayString = edgeUrl.toDisplayString();
+
+        for (Map.Entry<Predicate<String>, DomSampleClassification> regexMatcher : regexClassification) {
+            var matcher =  regexMatcher.getKey();
+            if (matcher.test(pathMatchString) || matcher.test(urlDisplayString)) {
+                var clazz = regexMatcher.getValue();
+
+                if (clazz != DomSampleClassification.IGNORE) {
+                    return clazz;
+                }
+            }
+        }
+
+        DomSampleClassification clazz = urlClassification.get(edgeUrl.toDisplayString());
+
+        if (clazz != null && clazz != DomSampleClassification.IGNORE) {
+            return clazz;
+        }
+
+        clazz = fullDomainClassification.get(edgeUrl.domain.toString());
+
+        if (clazz != null && clazz != DomSampleClassification.IGNORE) {
+            return clazz;
+        }
+
+        clazz = topDomainClassification.get(edgeUrl.domain.topDomain);
+
+        if (clazz != null && clazz != DomSampleClassification.IGNORE) {
+            return clazz;
+        }
+
+        return DomSampleClassification.UNCLASSIFIED;
+    }
+
+}

code/processes/converting-process/ft-dom-classifier/readme.md

@@ -0,0 +1,8 @@
+Holds a classification model for rendered DOM data and exported network traffic generated by
+[functions/live-capture](../../../functions/live-capture).  
+
+The model is primarily used in the [converting-process](../../converting-process) but also run in the search UI for inspection purposes.
+
+The traffic classification model is found in [resources/request-classifier.xml](resources/request-classifier.xml).
+
+The code evaluating the model is in [DomSampleClassifier.java](java/nu/marginalia/domclassifier/DomSampleClassifier.java).

code/processes/converting-process/ft-dom-classifier/resources/request-classifier.xml

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE rules [
+        <!ELEMENT rules (classifier*)>
+        <!ELEMENT classifier (#PCDATA)>
+        <!ATTLIST classifier
+                target (url-regex|url|domain|top) #REQUIRED
+                rule (ads|tracking|consent|ignore) #REQUIRED>
+        ]>
+
+<!-- Contains rules for mapping outgoing requests during DOM Sampling to website classification -->
+<rules>
+    <!-- Regex rules -->
+    <classifier target="url-regex" rule="tracking">/ads/ga-audiences</classifier>
+    <classifier target="url-regex" rule="tracking">/google_top_exp.js$</classifier>
+    <classifier target="url-regex" rule="tracking">/ccm/collect$</classifier>
+    <classifier target="url-regex" rule="tracking">^/[0-9]+\.js$</classifier>
+    <classifier target="url-regex" rule="tracking">^/[a-z0-9]\.gif$</classifier>
+    <classifier target="url-regex" rule="tracking">^/pixel\.gif$</classifier>
+    <classifier target="url-regex" rule="ads">/pagead/</classifier>
+    <classifier target="url-regex" rule="ads">/google-ads/</classifier>
+
+    <!-- URL classifications TRACKING -->
+    <classifier target="url" rule="tracking">https://googleads.g.doubleclick.net/pagead/id</classifier>
+    <classifier target="url" rule="tracking">https://securepubads.g.doubleclick.net/tag/js/gpt.js</classifier>
+    <classifier target="url" rule="tracking">https://pagead2.googlesyndication.com/ccm/collect</classifier>
+    <classifier target="url" rule="tracking">https://z-na.amazon-adsystem.com/widgets/onejs</classifier>
+
+    <!-- Full domain classifications ADS -->
+
+    <classifier target="domain" rule="ads">securepubads.g.doubleclick.net</classifier>
+    <classifier target="domain" rule="ads">googleads.g.doubleclick.net</classifier>
+
+    <!-- Full domain classifications TRACKING -->
+    <classifier target="domain" rule="tracking">stats.g.doubleclick.net</classifier>
+    <classifier target="domain" rule="tracking">insight.adsrvr.org</classifier>
+
+    <classifier target="domain" rule="tracking">pixel.wp.com</classifier>
+    <classifier target="domain" rule="tracking">connect.facebook.net</classifier>
+    <classifier target="domain" rule="tracking">stats.wp.com</classifier>
+    <classifier target="domain" rule="tracking">track.hubspot.com</classifier>
+    <classifier target="domain" rule="tracking">analytics.tiktok.com</classifier>
+    <classifier target="domain" rule="tracking">analytics-ipv6.tiktokw.us</classifier>
+    <classifier target="domain" rule="tracking">tr6.snapchat.com</classifier>
+    <classifier target="domain" rule="tracking">tr.snapchat.com</classifier>
+    <classifier target="domain" rule="tracking">geo-location.prebid.cloud</classifier>
+    <classifier target="domain" rule="tracking">px.ads.linkedin.com</classifier>
+    <classifier target="domain" rule="tracking">region1.analytics.google.com</classifier>
+    <classifier target="domain" rule="tracking">api.hubapi.com</classifier>
+    <classifier target="domain" rule="tracking">bat.bing.com</classifier>
+    <classifier target="domain" rule="tracking">bat.bing.net</classifier>
+    <classifier target="domain" rule="tracking">c.bing.com</classifier>
+    <classifier target="domain" rule="tracking">c.bing.net</classifier>
+    <classifier target="domain" rule="tracking">analytics.twitter.com</classifier>
+    <classifier target="domain" rule="tracking">play.google.com</classifier>
+    <classifier target="domain" rule="tracking">www.youtube.com</classifier>
+
+    <!-- Full domain classifications CONSENT -->
+    <classifier target="domain" rule="consent">cdnconsents.websitepolicies.com</classifier>
+
+    <!-- Top-level domain classifications - ADS -->
+    <classifier target="top" rule="ads">googlesyndication.com</classifier>
+    <classifier target="top" rule="ads">amazon-adsystem.com</classifier>
+    <classifier target="top" rule="ads">smartadserver.com</classifier>
+    <classifier target="top" rule="ads">googleadservices.com</classifier>
+
+    <classifier target="top" rule="ads">prebid.cloud</classifier>
+    <classifier target="top" rule="ads">pubmine.com</classifier>
+    <classifier target="top" rule="ads">adtrafficquality.google</classifier>
+    <classifier target="top" rule="ads">syndicatedsearch.goog</classifier>
+    <classifier target="top" rule="ads">adsrvr.org</classifier>
+    <classifier target="top" rule="ads">adnxs.net</classifier>
+    <classifier target="top" rule="ads">aditude.io</classifier>
+    <classifier target="top" rule="ads">buysellads.net</classifier>
+
+    <!-- Top-level domain classifications - TRACKING -->
+    <classifier target="top" rule="tracking">plausible.io</classifier>
+    <classifier target="top" rule="tracking">amplitude.com</classifier>
+    <classifier target="top" rule="tracking">hsadspixel.net</classifier>
+    <classifier target="top" rule="tracking">demdex.net</classifier>
+    <classifier target="top" rule="tracking">omtrdc.net</classifier>
+    <classifier target="top" rule="tracking">ggpht.com</classifier>
+
+    <classifier target="top" rule="tracking">doubleclick.net</classifier>
+    <classifier target="top" rule="tracking">google.com</classifier>
+    <classifier target="top" rule="tracking">google.se</classifier>
+    <classifier target="top" rule="tracking">google-analytics.com</classifier>
+    <classifier target="top" rule="tracking">googletagmanager.com</classifier>
+    <classifier target="top" rule="tracking">cloudflareinsights.com</classifier>
+    <classifier target="top" rule="tracking">branch.io</classifier>
+    <classifier target="top" rule="tracking">clarity.ms</classifier>
+    <classifier target="top" rule="tracking">hotjar.com</classifier>
+    <classifier target="top" rule="tracking">hotjar.io</classifier>
+    <classifier target="top" rule="tracking">nr-data.net</classifier>
+    <classifier target="top" rule="tracking">newrelic.com</classifier>
+    <classifier target="top" rule="tracking">siteimproveanalytics.com</classifier>
+    <classifier target="top" rule="tracking">siteimproveanalytics.io</classifier>
+    <classifier target="top" rule="tracking">hs-analytics.net</classifier>
+    <classifier target="top" rule="tracking">sentry.io</classifier>
+    <classifier target="top" rule="tracking">hs-scripts.com</classifier>
+    <classifier target="top" rule="tracking">addtoany.com</classifier>
+    <classifier target="top" rule="tracking">facebook.com</classifier>
+    <classifier target="top" rule="tracking">scorecardresearch.com</classifier>
+
+    <!-- Top-level domain classifications - CONSENT -->
+    <classifier target="top" rule="consent">trustarc.com</classifier>
+    <classifier target="top" rule="consent">truste.com</classifier>
+    <classifier target="top" rule="consent">onetrust.com</classifier>
+    <classifier target="top" rule="consent">cookielaw.org</classifier>
+    <classifier target="top" rule="consent">hs-banner.com</classifier>
+    <classifier target="top" rule="consent">fundingchoicesmessages.google.com</classifier>
+
+</rules>

code/processes/converting-process/ft-dom-classifier/test/nu/marginalia/ddtrackergradar/DDGTrackerDataTest.java

@@ -0,0 +1,15 @@
+package nu.marginalia.ddtrackergradar;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Path;
+
+class DDGTrackerDataTest {
+    @Test
+    public void testLoad() {
+        DDGTrackerData data = new DDGTrackerData();
+        data.loadDomainDir(Path.of("/home/vlofgren/Work/tracker-radar/domains/US/"));
+        data.getDomainInfo("hotjar.com").ifPresent(System.out::println);
+        data.getAllClassifications().forEach(System.out::println);
+    }
+}

code/processes/converting-process/ft-keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywordsBuilder.java

@@ -113,6 +113,13 @@ public class DocumentKeywordsBuilder {
         newWords.forEach(word -> wordToMeta.putIfAbsent(word, meta));
     }
 
+    public void addSyntheticTerm(String newWord) {
+        byte meta = WordFlags.Synthetic.asBit();
+
+        wordToMeta.putIfAbsent(newWord, meta);
+    }
+
+
     public List<String> getWordsWithAnyFlag(long flags) {
         List<String> ret = new ArrayList<>();
 

code/processes/converting-process/java/nu/marginalia/converting/ConverterMain.java

@@ -23,6 +23,7 @@ import nu.marginalia.process.control.ProcessHeartbeatImpl;
 import nu.marginalia.process.log.WorkLog;
 import nu.marginalia.process.log.WorkLogEntry;
 import nu.marginalia.service.module.DatabaseModule;
+import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.util.SimpleBlockingThreadPool;
 import nu.marginalia.worklog.BatchingWorkLog;
@@ -59,6 +60,7 @@ public class ConverterMain extends ProcessMainClass {
             Injector injector = Guice.createInjector(
                     new ConverterModule(),
                     new ProcessConfigurationModule("converter"),
+                    new ServiceDiscoveryModule(),
                     new DatabaseModule(false)
             );
 

code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentProcessor.java

@@ -5,10 +5,12 @@ import nu.marginalia.atags.AnchorTextKeywords;
 import nu.marginalia.atags.model.DomainLinks;
 import nu.marginalia.converting.model.DisqualifiedException;
 import nu.marginalia.converting.model.ProcessedDocument;
+import nu.marginalia.converting.processor.classifier.AcceptableAds;
 import nu.marginalia.converting.processor.plugin.AbstractDocumentProcessorPlugin;
 import nu.marginalia.converting.processor.plugin.HtmlDocumentProcessorPlugin;
 import nu.marginalia.converting.processor.plugin.PdfDocumentProcessorPlugin;
 import nu.marginalia.converting.processor.plugin.PlainTextDocumentProcessorPlugin;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
@@ -19,8 +21,9 @@ import nu.marginalia.model.crawldata.CrawlerDocumentStatus;
 import nu.marginalia.model.idx.WordFlags;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
 
-import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
@@ -37,6 +40,7 @@ public class DocumentProcessor {
             "text/plain",
             "application/pdf");
 
+    private final Marker converterAuditMarker = MarkerFactory.getMarker("CONVERTER");
 
     private final List<AbstractDocumentProcessorPlugin> processorPlugins = new ArrayList<>();
     private final AnchorTextKeywords anchorTextKeywords;
@@ -57,6 +61,7 @@ public class DocumentProcessor {
     public ProcessedDocument process(CrawledDocument crawledDocument,
                                      EdgeDomain domain,
                                      DomainLinks externalDomainLinks,
+                                     Set<DomSampleClassification> domSampleClassifications,
                                      DocumentDecorator documentDecorator) {
         ProcessedDocument ret = new ProcessedDocument();
 
@@ -76,55 +81,27 @@ public class DocumentProcessor {
                 default -> DocumentClass.EXTERNALLY_LINKED_MULTI;
             };
 
-            processDocument(crawledDocument, documentClass, documentDecorator, externalDomainLinks, ret);
-        }
-        catch (DisqualifiedException ex) {
-            ret.state = UrlIndexingState.DISQUALIFIED;
-            ret.stateReason = ex.reason.toString();
-            logger.debug("Disqualified {}: {}", ret.url, ex.reason);
-        }
-        catch (Exception ex) {
-            ret.state = UrlIndexingState.DISQUALIFIED;
-            ret.stateReason = DisqualifiedException.DisqualificationReason.PROCESSING_EXCEPTION.toString();
-            logger.info("Failed to convert " + crawledDocument.url, ex);
-        }
-
-        return ret;
-    }
-
-    private void processDocument(CrawledDocument crawledDocument,
-                                 DocumentClass documentClass,
-                                 DocumentDecorator documentDecorator,
-                                 DomainLinks externalDomainLinks,
-                                 ProcessedDocument ret) throws URISyntaxException, IOException, DisqualifiedException
-    {
-
             var crawlerStatus = CrawlerDocumentStatus.valueOf(crawledDocument.crawlerStatus);
-        if (crawlerStatus != CrawlerDocumentStatus.OK) {
+
+            if (crawlerStatus != CrawlerDocumentStatus.OK)
                 throw new DisqualifiedException(crawlerStatus);
-        }
-
-        if (AcceptableAds.hasAcceptableAdsHeader(crawledDocument)) {
+            if (AcceptableAds.hasAcceptableAdsHeader(crawledDocument))
                 throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.ACCEPTABLE_ADS);
-        }
-
-        if (!isAcceptedContentType(crawledDocument)) {
+            if (!isAcceptedContentType(crawledDocument))
                 throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.CONTENT_TYPE);
-        }
 
             ret.state = crawlerStatusToUrlState(crawledDocument.crawlerStatus, crawledDocument.httpStatus);
 
-        AbstractDocumentProcessorPlugin plugin = findPlugin(crawledDocument);
+            LinkTexts linkTexts = anchorTextKeywords.getAnchorTextKeywords(externalDomainLinks, ret.url);
 
-        EdgeUrl url = new EdgeUrl(crawledDocument.url);
-        LinkTexts linkTexts = anchorTextKeywords.getAnchorTextKeywords(externalDomainLinks, url);
-
-        AbstractDocumentProcessorPlugin.DetailsWithWords detailsWithWords = plugin.createDetails(crawledDocument, linkTexts, documentClass);
+            var detailsWithWords =
+                    findPlugin(crawledDocument)
+                            .createDetails(crawledDocument, linkTexts, domSampleClassifications, documentClass);
 
             ret.details = detailsWithWords.details();
             ret.words = detailsWithWords.words();
 
-        if (url.path.equals("/")) {
+            if (ret.url.path.equals("/")) {
                 ret.words.addMeta("special:root", WordFlags.Synthetic.asBit());
             }
 
@@ -136,7 +113,19 @@ public class DocumentProcessor {
             {
                 ret.details.features.add(HtmlFeature.COOKIES);
             }
+        }
+        catch (DisqualifiedException ex) {
+            ret.state = UrlIndexingState.DISQUALIFIED;
+            ret.stateReason = ex.reason.toString();
+            logger.info(converterAuditMarker, "Disqualified {}: {}", ret.url, ex.reason);
+        }
+        catch (Exception ex) {
+            ret.state = UrlIndexingState.DISQUALIFIED;
+            ret.stateReason = DisqualifiedException.DisqualificationReason.PROCESSING_EXCEPTION.toString();
+            logger.warn(converterAuditMarker, "Failed to convert {}: {}", crawledDocument.url, ex.getClass().getSimpleName());
+        }
 
+        return ret;
     }
 
     private AbstractDocumentProcessorPlugin findPlugin(CrawledDocument crawledDocument) throws DisqualifiedException {

code/processes/converting-process/java/nu/marginalia/converting/processor/DomainProcessor.java

@@ -1,6 +1,9 @@
 package nu.marginalia.converting.processor;
 
 import com.google.inject.Inject;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import nu.marginalia.api.domsample.DomSampleClient;
 import nu.marginalia.atags.model.DomainLinks;
 import nu.marginalia.atags.source.AnchorTagsSource;
 import nu.marginalia.atags.source.AnchorTagsSourceFactory;
@@ -12,11 +15,14 @@ import nu.marginalia.converting.processor.logic.links.TopKeywords;
 import nu.marginalia.converting.sideload.SideloadSource;
 import nu.marginalia.converting.writer.ConverterBatchWritableIf;
 import nu.marginalia.converting.writer.ConverterBatchWriter;
+import nu.marginalia.domclassifier.DomSampleClassification;
+import nu.marginalia.domclassifier.DomSampleClassifier;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.geoip.sources.AsnTable;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.crawl.DomainIndexingState;
+import nu.marginalia.model.crawl.UrlIndexingState;
 import nu.marginalia.model.crawldata.CrawledDocument;
 import nu.marginalia.model.crawldata.CrawledDomain;
 import nu.marginalia.model.crawldata.CrawlerDomainStatus;
@@ -28,7 +34,11 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.sql.SQLException;
+import java.time.Duration;
 import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.regex.Pattern;
 
 public class DomainProcessor {
@@ -36,21 +46,29 @@ public class DomainProcessor {
     private final SiteWords siteWords;
     private final AnchorTagsSource anchorTagsSource;
     private final GeoIpDictionary geoIpDictionary;
+    private final DomSampleClient domSampleClient;
+    private final DomSampleClassifier domSampleClassifier;
+    private final ExecutorService domSampleExecutor = Executors.newCachedThreadPool();
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
+    private final boolean hasDomSamples;
 
     @Inject
     public DomainProcessor(DocumentProcessor documentProcessor,
                            SiteWords siteWords,
                            AnchorTagsSourceFactory anchorTagsSourceFactory,
-                           GeoIpDictionary geoIpDictionary) throws SQLException
-    {
+                           DomSampleClient domSampleClient,
+                           GeoIpDictionary geoIpDictionary,
+                           DomSampleClassifier domSampleClassifier) throws SQLException, InterruptedException {
         this.documentProcessor = documentProcessor;
         this.siteWords = siteWords;
         this.anchorTagsSource = anchorTagsSourceFactory.create();
         this.geoIpDictionary = geoIpDictionary;
+        this.domSampleClient = domSampleClient;
+        this.domSampleClassifier = domSampleClassifier;
 
         geoIpDictionary.waitReady();
+        hasDomSamples = !Boolean.getBoolean("converter.ignoreDomSampleData") && domSampleClient.waitReady(Duration.ofSeconds(15));
     }
 
     public SimpleProcessing simpleProcessing(SerializableCrawlDataStream dataStream, int sizeHint, Collection<String> extraKeywords) {
@@ -73,6 +91,27 @@ public class DomainProcessor {
         }
     }
 
+    /** Fetch and process the DOM sample and extract classifications */
+    private Set<DomSampleClassification> getDomainClassifications(String domainName) throws ExecutionException, InterruptedException {
+        if (!hasDomSamples) {
+            return EnumSet.of(DomSampleClassification.UNCLASSIFIED);
+        }
+
+        return domSampleClient
+                .getSampleAsync(domainName, domSampleExecutor)
+                .thenApply(domSampleClassifier::classifySample)
+                .handle((a,b) -> {
+                    if (b != null) {
+                        var cause = b.getCause();
+                        if (!(cause instanceof StatusRuntimeException sre && sre.getStatus() != Status.NOT_FOUND)) {
+                            logger.warn("Exception when fetching sample data", b);
+                        }
+                        return EnumSet.of(DomSampleClassification.UNCLASSIFIED);
+                    }
+                    return a;
+                }).get();
+    }
+
     @Nullable
     public ProcessedDomain fullProcessing(SerializableCrawlDataStream dataStream) {
         try {
@@ -80,7 +119,6 @@ public class DomainProcessor {
                 return null;
             }
 
-            List<ProcessedDocument> docs = new ArrayList<>();
             Set<String> processedUrls = new HashSet<>();
 
             if (!(dataStream.next() instanceof CrawledDomain crawledDomain)) {
@@ -94,10 +132,12 @@ public class DomainProcessor {
 
             ProcessedDomain ret = new ProcessedDomain();
             processDomain(crawledDomain, ret, documentDecorator);
-            ret.documents = docs;
+            ret.documents = new ArrayList<>();
 
             // Process Documents
 
+            Set<DomSampleClassification> classifications = getDomainClassifications(crawledDomain.getDomain());
+
             try (var deduplicator = new LshDocumentDeduplicator()) {
                 while (dataStream.hasNext()) {
                     if (!(dataStream.next() instanceof CrawledDocument doc))
@@ -110,9 +150,23 @@ public class DomainProcessor {
                         continue;
 
                     try {
-                        var processedDoc = documentProcessor.process(doc, ret.domain, externalDomainLinks, documentDecorator);
-                        deduplicator.markIfDuplicate(processedDoc);
-                        docs.add(processedDoc);
+                        var processedDoc = documentProcessor.process(doc, ret.domain, externalDomainLinks, classifications, documentDecorator);
+
+                        if (deduplicator.isDocumentDuplicate(processedDoc)) {
+                            processedDoc.state = UrlIndexingState.DISQUALIFIED;
+                            processedDoc.stateReason = "Duplicate";
+                        }
+
+                        if (processedDoc.isOk() && processedDoc.words != null && processedDoc.details != null) {
+                            classifications.forEach(classification -> {
+                                if (classification.htmlFeature == null) return;
+
+                                processedDoc.words.addSyntheticTerm(classification.htmlFeature.getKeyword());
+                                processedDoc.details.features.add(classification.htmlFeature);
+                            });
+                        }
+
+                        ret.documents.add(processedDoc);
                     } catch (Exception ex) {
                         logger.warn("Failed to process " + doc.url, ex);
                     }
@@ -142,15 +196,17 @@ public class DomainProcessor {
         private final DomainLinks externalDomainLinks;
         private final LshDocumentDeduplicator deduplicator = new LshDocumentDeduplicator();
 
+        Set<DomSampleClassification> classifications;
+
         private static final ProcessingIterator.Factory iteratorFactory = ProcessingIterator.factory(8,
                 Integer.getInteger("java.util.concurrent.ForkJoinPool.common.parallelism", Runtime.getRuntime().availableProcessors())
         );
 
-        SimpleProcessing(SerializableCrawlDataStream dataStream, int sizeHint) throws IOException {
+        SimpleProcessing(SerializableCrawlDataStream dataStream, int sizeHint) throws Exception {
             this(dataStream, sizeHint, List.of());
         }
 
-        SimpleProcessing(SerializableCrawlDataStream dataStream, int sizeHint, Collection<String> extraKeywords) throws IOException {
+        SimpleProcessing(SerializableCrawlDataStream dataStream, int sizeHint, Collection<String> extraKeywords) throws Exception {
             this.dataStream = dataStream;
 
             if (!dataStream.hasNext() || !(dataStream.next() instanceof CrawledDomain crawledDomain))
@@ -166,6 +222,8 @@ public class DomainProcessor {
 
             processDomain(crawledDomain, domain, documentDecorator);
 
+            classifications = getDomainClassifications(crawledDomain.getDomain());
+
             externalDomainLinks = anchorTagsSource.getAnchorTags(domain.domain);
         }
 
@@ -187,16 +245,31 @@ public class DomainProcessor {
 
 
                     taskConsumer.accept(() -> {
-                        var processedDoc = documentProcessor.process(doc, domain.domain, externalDomainLinks, documentDecorator);
+                        var processedDoc = documentProcessor.process(doc, domain.domain, externalDomainLinks, classifications, documentDecorator);
 
                         synchronized (deduplicator) {
-                            deduplicator.markIfDuplicate(processedDoc);
+                            if (deduplicator.isDocumentDuplicate(processedDoc)) {
+                                processedDoc.state = UrlIndexingState.DISQUALIFIED;
+                                processedDoc.stateReason = "Duplicate";
+                            }
                         }
 
                         if (processedDoc.isProcessedFully()) {
                             // This is a bit sketchy, but we need to set the size and topology to something
                             processedDoc.details.metadata = processedDoc.details.metadata.withSizeAndTopology(
                                     10_000, externalDomainLinks.countForUrl(processedDoc.url));
+
+                            // Apply classifications
+                            try {
+                                classifications.forEach(classification -> {
+                                    if (classification.htmlFeature == null) return;
+
+                                    processedDoc.words.addSyntheticTerm(classification.htmlFeature.getKeyword());
+                                    processedDoc.details.features.add(classification.htmlFeature);
+                                });
+                            }
+                            catch (Exception ex) {
+                            }
                         }
 
                         return processedDoc;

code/processes/converting-process/java/nu/marginalia/converting/processor/classifier/AcceptableAds.java

@@ -1,4 +1,4 @@
-package nu.marginalia.converting.processor;
+package nu.marginalia.converting.processor.classifier;
 
 import nu.marginalia.model.crawldata.CrawledDocument;
 import org.jsoup.nodes.Document;

code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentLengthLogic.java

@@ -3,7 +3,6 @@ package nu.marginalia.converting.processor.logic;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
-import nu.marginalia.converting.model.DisqualifiedException;
 import nu.marginalia.language.model.DocumentLanguageData;
 
 @Singleton
@@ -26,12 +25,9 @@ public class DocumentLengthLogic {
         return (int) Math.round((totalWords / (double) numSentences) / 4.);
     }
 
-    public void validateLength(DocumentLanguageData dld,
-                               double modifier) throws DisqualifiedException
+    public boolean validateLength(DocumentLanguageData dld, double modifier)
     {
-        if (modifier * dld.totalNumWords() < minDocumentLength) {
-            throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.LENGTH);
-        }
+        return modifier * dld.totalNumWords() >= minDocumentLength;
     }
 
 }

code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentValuator.java

@@ -2,6 +2,7 @@ package nu.marginalia.converting.processor.logic;
 
 import crawlercommons.utils.Strings;
 import nu.marginalia.converting.model.DisqualifiedException;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.crawl.HtmlFeature;
 import nu.marginalia.model.crawldata.CrawledDocument;
@@ -14,6 +15,8 @@ import org.jsoup.select.NodeVisitor;
 import java.util.List;
 import java.util.Set;
 
+import static nu.marginalia.domclassifier.DomSampleClassification.*;
+
 public class DocumentValuator {
 
     public double getQuality(CrawledDocument crawledDocument,
@@ -126,6 +129,25 @@ public class DocumentValuator {
         return quality + adjustment;
     }
 
+    public double getQuality(Set<DomSampleClassification> classifications) {
+        double quality = 0;
+        if (classifications.contains(ADS)) {
+            quality -= 6;
+        }
+        if (classifications.contains(TRACKING)) {
+            quality -= 4;
+        }
+
+        if (classifications.contains(CONSENT)) {
+            quality -= 4;
+        }
+        else if (classifications.contains(POPOVER)) {
+            quality -= 4;
+        }
+
+        return quality;
+    }
+
     public static class ScriptVisitor implements NodeVisitor {
         boolean hasBadScript = false;
         int scriptLength = 0;

code/processes/converting-process/java/nu/marginalia/converting/processor/logic/FeatureExtractor.java

@@ -3,7 +3,6 @@ package nu.marginalia.converting.processor.logic;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.converting.model.DocumentHeaders;
-import nu.marginalia.converting.processor.classifier.adblock.AdblockSimulator;
 import nu.marginalia.converting.processor.classifier.adblock.GoogleAnwersSpamDetector;
 import nu.marginalia.converting.processor.classifier.topic.RecipeDetector;
 import nu.marginalia.converting.processor.classifier.topic.TextileCraftDetector;
@@ -65,20 +64,17 @@ public class FeatureExtractor {
             "counter.yadro.ru"
     );
 
-    private final AdblockSimulator adblockSimulator;
     private final RecipeDetector recipeDetector;
     private final TextileCraftDetector textileCraftDetector;
     private final WoodworkingDetector woodworkingDetector;
     private final GoogleAnwersSpamDetector googleAnwersSpamDetector;
 
     @Inject
-    public FeatureExtractor(AdblockSimulator adblockSimulator,
-                            RecipeDetector recipeDetector,
+    public FeatureExtractor(RecipeDetector recipeDetector,
                             TextileCraftDetector textileCraftDetector,
                             WoodworkingDetector woodworkingDetector,
                             GoogleAnwersSpamDetector googleAnwersSpamDetector)
     {
-        this.adblockSimulator = adblockSimulator;
         this.recipeDetector = recipeDetector;
         this.textileCraftDetector = textileCraftDetector;
         this.woodworkingDetector = woodworkingDetector;
@@ -218,13 +214,6 @@ public class FeatureExtractor {
             }
         }
 
-        if (features.contains(HtmlFeature.JS)
-            // remove while disabled to get rid of expensive clone() call:
-            // adblockSimulator.hasAds(doc.clone())
-            ) {
-            features.add(HtmlFeature.ADVERTISEMENT);
-        }
-
         if (!doc.getElementsByTag("object").isEmpty()
                 || !doc.getElementsByTag("audio").isEmpty()
                 || !doc.getElementsByTag("video").isEmpty()) {

code/processes/converting-process/java/nu/marginalia/converting/processor/logic/LshDocumentDeduplicator.java

@@ -1,7 +1,6 @@
 package nu.marginalia.converting.processor.logic;
 
 import gnu.trove.list.array.TLongArrayList;
-import nu.marginalia.model.crawl.UrlIndexingState;
 import nu.marginalia.converting.model.ProcessedDocument;
 import nu.marginalia.lsh.EasyLSH;
 
@@ -14,26 +13,25 @@ public class LshDocumentDeduplicator implements AutoCloseable {
     private final TLongArrayList hashCodes = new TLongArrayList(1000);
     private static final int DISTANCE_THRESHOLD = 2;
 
-    public void markIfDuplicate(ProcessedDocument document) {
-        if (!document.isProcessedFully()) {
-            return;
-        }
+    public boolean isDocumentDuplicate(ProcessedDocument document) {
+        if (!document.isOk()) return false;
+        if (document.words == null) return false;
+        if (document.details == null) return false;
 
         if (document.words.size() < 100) {
-            return;
+            return false;
         }
 
         long hashCode = document.details.hashCode;
 
         for (int i = 0; i < hashCodes.size(); i++) {
             if (EasyLSH.hammingDistance(hashCode, hashCodes.get(i)) < DISTANCE_THRESHOLD) {
-                document.state = UrlIndexingState.DISQUALIFIED;
-                document.stateReason = "Duplicate";
-                return;
+                return true;
             }
         }
 
         hashCodes.add(hashCode);
+        return false;
     }
 
     @Override

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java

@@ -3,6 +3,7 @@ package nu.marginalia.converting.processor.plugin;
 import nu.marginalia.converting.model.DisqualifiedException;
 import nu.marginalia.converting.model.ProcessedDocumentDetails;
 import nu.marginalia.converting.processor.DocumentClass;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
 import nu.marginalia.language.filter.LanguageFilter;
@@ -26,7 +27,7 @@ public abstract class AbstractDocumentProcessorPlugin {
         this.languageFilter = languageFilter;
     }
 
-    public abstract DetailsWithWords createDetails(CrawledDocument crawledDocument, LinkTexts linkTexts, DocumentClass documentClass) throws DisqualifiedException, URISyntaxException, IOException;
+    public abstract DetailsWithWords createDetails(CrawledDocument crawledDocument, LinkTexts linkTexts, Set<DomSampleClassification> domSampleClassifications, DocumentClass documentClass) throws DisqualifiedException, URISyntaxException, IOException;
     public abstract boolean isApplicable(CrawledDocument doc);
 
     protected void checkDocumentLanguage(DocumentLanguageData dld) throws DisqualifiedException {

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java

@@ -6,15 +6,16 @@ import nu.marginalia.converting.model.DisqualifiedException;
 import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.model.GeneratorType;
 import nu.marginalia.converting.model.ProcessedDocumentDetails;
-import nu.marginalia.converting.processor.AcceptableAds;
 import nu.marginalia.converting.processor.DocumentClass;
 import nu.marginalia.converting.processor.MetaRobotsTag;
+import nu.marginalia.converting.processor.classifier.AcceptableAds;
 import nu.marginalia.converting.processor.logic.*;
 import nu.marginalia.converting.processor.logic.dom.MeasureLengthVisitor;
 import nu.marginalia.converting.processor.logic.links.FileLinks;
 import nu.marginalia.converting.processor.logic.links.LinkProcessor;
 import nu.marginalia.converting.processor.plugin.specialization.HtmlProcessorSpecializations;
 import nu.marginalia.converting.processor.pubdate.PubDateSniffer;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.gregex.GuardedRegex;
 import nu.marginalia.gregex.GuardedRegexFactory;
 import nu.marginalia.keyword.DocumentKeywordExtractor;
@@ -23,7 +24,6 @@ import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
 import nu.marginalia.language.filter.LanguageFilter;
 import nu.marginalia.language.model.DocumentLanguageData;
 import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
-import nu.marginalia.link_parser.FeedExtractor;
 import nu.marginalia.link_parser.LinkParser;
 import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeDomain;
@@ -62,12 +62,11 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
     private static final DocumentValuator documentValuator = new DocumentValuator();
 
     private static final LinkParser linkParser = new LinkParser();
-    private static final FeedExtractor feedExtractor = new FeedExtractor(linkParser);
 
     private final ThreadLocalSentenceExtractorProvider sentenceExtractorProvider;
     private final HtmlProcessorSpecializations htmlProcessorSpecializations;
 
-    private static final int MAX_DOCUMENT_LENGTH_BYTES = Integer.getInteger("converter.max-body-length",128_000);
+    private static boolean lenientProcessing = Boolean.getBoolean("converter.lenientProcessing");
 
     @Inject
     public HtmlDocumentProcessorPlugin(
@@ -105,16 +104,16 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
     @Override
     public DetailsWithWords createDetails(CrawledDocument crawledDocument,
                                           LinkTexts linkTexts,
-                                          DocumentClass documentClass)
+                                          Set<DomSampleClassification> domSampleClassifications, DocumentClass documentClass)
             throws DisqualifiedException, URISyntaxException, IOException {
 
-        if (languageFilter.isBlockedUnicodeRange(crawledDocument.documentBody(512))) {
+        if (!lenientProcessing && languageFilter.isBlockedUnicodeRange(crawledDocument.documentBody(512))) {
             throw new DisqualifiedException(DisqualificationReason.LANGUAGE);
         }
 
         Document doc = crawledDocument.parseBody();
 
-        if (AcceptableAds.hasAcceptableAdsTag(doc)) {
+        if (!lenientProcessing && AcceptableAds.hasAcceptableAdsTag(doc)) {
             throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.ACCEPTABLE_ADS);
         }
 
@@ -129,25 +128,30 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
 
         final var specialization = htmlProcessorSpecializations.select(generatorParts, url);
 
-        if (!specialization.shouldIndex(url)) {
+        if (!lenientProcessing && !specialization.shouldIndex(url)) {
             throw new DisqualifiedException(DisqualificationReason.IRRELEVANT);
         }
 
         var prunedDoc = specialization.prune(doc);
 
-
         final int length = getLength(doc);
         final DocumentFormat format = getDocumentFormat(doc);
-        final double quality = documentValuator.getQuality(crawledDocument, format, doc, length);
+        final double quality;
 
-        if (isDisqualified(documentClass, url, quality, doc.title())) {
+        if (domSampleClassifications.contains(DomSampleClassification.UNCLASSIFIED)) {
+            quality = documentValuator.getQuality(crawledDocument, format, doc, length);
+        }
+        else {
+            quality = documentValuator.getQuality(domSampleClassifications);
+        }
+
+        if (!lenientProcessing && isDisqualified(documentClass, url, quality, doc.title())) {
             throw new DisqualifiedException(DisqualificationReason.QUALITY);
         }
 
         DocumentLanguageData dld = sentenceExtractorProvider.get().extractSentences(prunedDoc);
 
         checkDocumentLanguage(dld);
-        documentLengthLogic.validateLength(dld, specialization.lengthModifier() * documentClass.lengthLimitModifier());
 
         var ret = new ProcessedDocumentDetails();
 
@@ -157,6 +161,11 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
 
         final Set<HtmlFeature> features = featureExtractor.getFeatures(url, doc, documentHeaders, dld);
 
+        if (!documentLengthLogic.validateLength(dld, specialization.lengthModifier() * documentClass.lengthLimitModifier())) {
+            features.add(HtmlFeature.SHORT_DOCUMENT);
+        }
+
+
         ret.features = features;
         ret.quality = documentValuator.adjustQuality(quality, features);
         ret.hashCode = dld.localitySensitiveHashCode();

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPlugin.java

@@ -7,6 +7,7 @@ import nu.marginalia.converting.model.ProcessedDocumentDetails;
 import nu.marginalia.converting.processor.DocumentClass;
 import nu.marginalia.converting.processor.logic.DocumentLengthLogic;
 import nu.marginalia.converting.processor.plugin.specialization.DefaultSpecialization;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.keyword.DocumentKeywordExtractor;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
@@ -43,6 +44,7 @@ public class PdfDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
     private final DefaultSpecialization defaultSpecialization;
 
     private static final Logger logger = LoggerFactory.getLogger(PdfDocumentProcessorPlugin.class);
+    private static boolean lenientProcessing = Boolean.getBoolean("converter.lenientProcessing");
 
     @Inject
     public PdfDocumentProcessorPlugin(@Named("max-title-length") Integer maxTitleLength,
@@ -76,12 +78,12 @@ public class PdfDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
     @Override
     public DetailsWithWords createDetails(CrawledDocument crawledDocument,
                                           LinkTexts linkTexts,
-                                          DocumentClass documentClass)
+                                          Set<DomSampleClassification> domSampleClassifications, DocumentClass documentClass)
             throws DisqualifiedException, URISyntaxException, IOException {
 
         String documentBody = crawledDocument.documentBody();
 
-        if (languageFilter.isBlockedUnicodeRange(documentBody)) {
+        if (!lenientProcessing && languageFilter.isBlockedUnicodeRange(documentBody)) {
             throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.LANGUAGE);
         }
 
@@ -100,7 +102,9 @@ public class PdfDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
 
         checkDocumentLanguage(dld);
 
-        documentLengthLogic.validateLength(dld, 1.0);
+        if (!lenientProcessing && !documentLengthLogic.validateLength(dld, 1.0)) {
+            throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.LENGTH);
+        }
 
         var ret = new ProcessedDocumentDetails();
 
@@ -111,7 +115,9 @@ public class PdfDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
 
         ret.quality = -5;
 
-        ret.features = Set.of(HtmlFeature.PDF);
+        ret.features = new HashSet<>(); // must be mutable!
+        ret.features.add(HtmlFeature.PDF);
+
         ret.description = getDescription(doc);
         ret.hashCode = dld.localitySensitiveHashCode();
 

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java

@@ -8,6 +8,7 @@ import nu.marginalia.converting.processor.DocumentClass;
 import nu.marginalia.converting.processor.logic.DocumentLengthLogic;
 import nu.marginalia.converting.processor.logic.PlainTextLogic;
 import nu.marginalia.converting.util.LineUtils;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.keyword.DocumentKeywordExtractor;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
@@ -23,10 +24,7 @@ import org.apache.commons.lang3.StringUtils;
 
 import java.net.URISyntaxException;
 import java.time.LocalDate;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
 
 
 public class PlainTextDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin {
@@ -37,6 +35,8 @@ public class PlainTextDocumentProcessorPlugin extends AbstractDocumentProcessorP
     private final ThreadLocalSentenceExtractorProvider sentenceExtractorProvider;
     private final DocumentLengthLogic documentLengthLogic;
 
+    private static boolean lenientProcessing = Boolean.getBoolean("converter.lenientProcessing");
+
 
     @Inject
     public PlainTextDocumentProcessorPlugin(@Named("max-title-length") Integer maxTitleLength,
@@ -68,12 +68,12 @@ public class PlainTextDocumentProcessorPlugin extends AbstractDocumentProcessorP
     @Override
     public DetailsWithWords createDetails(CrawledDocument crawledDocument,
                                           LinkTexts linkTexts,
-                                          DocumentClass documentClass)
+                                          Set<DomSampleClassification> domSampleClassifications, DocumentClass documentClass)
             throws DisqualifiedException, URISyntaxException {
 
         String documentBody = crawledDocument.documentBody();
 
-        if (languageFilter.isBlockedUnicodeRange(documentBody)) {
+        if (!lenientProcessing && languageFilter.isBlockedUnicodeRange(documentBody)) {
             throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.LANGUAGE);
         }
 
@@ -83,7 +83,9 @@ public class PlainTextDocumentProcessorPlugin extends AbstractDocumentProcessorP
 
         checkDocumentLanguage(dld);
 
-        documentLengthLogic.validateLength(dld, 1.0);
+        if (!lenientProcessing && !documentLengthLogic.validateLength(dld, 1.0)) {
+            throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.LENGTH);
+        }
 
         var ret = new ProcessedDocumentDetails();
 

code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloaderProcessing.java

@@ -7,6 +7,7 @@ import nu.marginalia.converting.model.GeneratorType;
 import nu.marginalia.converting.model.ProcessedDocument;
 import nu.marginalia.converting.processor.DocumentClass;
 import nu.marginalia.converting.processor.plugin.HtmlDocumentProcessorPlugin;
+import nu.marginalia.domclassifier.DomSampleClassification;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
@@ -64,7 +65,7 @@ public class SideloaderProcessing {
 
         var ret = new ProcessedDocument();
         try {
-            var details = htmlProcessorPlugin.createDetails(crawledDoc, linkTexts, documentClass);
+            var details = htmlProcessorPlugin.createDetails(crawledDoc, linkTexts, EnumSet.noneOf(DomSampleClassification.class), documentClass);
 
             ret.words = details.words();
 

code/processes/converting-process/test/nu/marginalia/converting/processor/classifier/adblock/DomSampleClassifierTest.java

@@ -0,0 +1,16 @@
+package nu.marginalia.converting.processor.classifier.adblock;
+
+import nu.marginalia.domclassifier.DomSampleClassifier;
+import org.junit.jupiter.api.Test;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+
+class DomSampleClassifierTest {
+
+    @Test
+    public void  testLoadSpecs() throws ParserConfigurationException, IOException, SAXException {
+        new DomSampleClassifier();
+    }
+}

code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPluginTest.java

@@ -25,6 +25,7 @@ import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
+import java.util.Set;
 
 @Tag("flaky")
 class PdfDocumentProcessorPluginTest {
@@ -51,7 +52,7 @@ class PdfDocumentProcessorPluginTest {
     }
     public AbstractDocumentProcessorPlugin.DetailsWithWords testPdfFile(byte[] pdfBytes) throws Exception {
         var doc = new CrawledDocument("test", "https://www.example.com/sample.pdf", "application/pdf", Instant.now().toString(), 200, "OK", "OK", "", pdfBytes, false, -1, null, null);
-        return plugin.createDetails(doc, new LinkTexts(), DocumentClass.NORMAL);
+        return plugin.createDetails(doc, new LinkTexts(), Set.of(), DocumentClass.NORMAL);
     }
 
     public AbstractDocumentProcessorPlugin.DetailsWithWords testPdfFile(Path file) throws Exception {

code/processes/crawling-process/model/java/nu/marginalia/model/crawldata/CrawledDocument.java

@@ -28,6 +28,8 @@ public final class CrawledDocument implements SerializableCrawlData {
     @Nullable
     public String headers;
 
+    private static int MAX_LENGTH_BYTES = 500_000;
+
     public String documentBody() {
         return DocumentBodyToString.getStringData(
                 ContentType.parse(contentType),
@@ -65,7 +67,7 @@ public final class CrawledDocument implements SerializableCrawlData {
         return DocumentBodyToString.getParsedData(
                 ContentType.parse(contentType),
                 documentBodyBytes,
-                200_000,
+                MAX_LENGTH_BYTES,
                 url);
     }