(search) Clean up the rate limiting

We fail quietly to make life harder for the bot farmers

code/common/db/resources/db/migration/V25_01_0_003__ping_domains__dsi_cert_details.sql

@@ -0,0 +1,7 @@
+-- Add additional summary columns to DOMAIN_SECURITY_INFORMATION table
+-- to make it easier to get more information about the SSL certificate's validity
+
+ALTER TABLE DOMAIN_SECURITY_INFORMATION ADD COLUMN SSL_CHAIN_VALID BOOLEAN DEFAULT NULL;
+ALTER TABLE DOMAIN_SECURITY_INFORMATION ADD COLUMN SSL_HOST_VALID BOOLEAN DEFAULT NULL;
+ALTER TABLE DOMAIN_SECURITY_INFORMATION ADD COLUMN SSL_DATE_VALID BOOLEAN DEFAULT NULL;
+OPTIMIZE TABLE DOMAIN_SECURITY_INFORMATION;

code/common/db/resources/db/migration/V25_01_0_004__ping_domains__schema_change.sql

@@ -0,0 +1,5 @@
+-- Add additional summary columns to DOMAIN_SECURITY_EVENTS table
+-- to make it easier to make sense of certificate changes
+
+ALTER TABLE DOMAIN_SECURITY_EVENTS ADD COLUMN CHANGE_SCHEMA ENUM('NONE', 'HTTP_TO_HTTPS', 'HTTPS_TO_HTTP', 'UNKNOWN') NOT NULL DEFAULT 'UNKNOWN';
+OPTIMIZE TABLE DOMAIN_SECURITY_EVENTS;

code/common/service/java/nu/marginalia/service/discovery/ServiceRegistryIf.java

@@ -5,12 +5,10 @@ import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor;
 import nu.marginalia.service.discovery.monitor.ServiceMonitorIf;
 import nu.marginalia.service.discovery.property.ServiceEndpoint;
 import nu.marginalia.service.discovery.property.ServiceKey;
+import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2;
 
-import java.util.Collection;
 import java.util.List;
 import java.util.UUID;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
 
 import static nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
 
@@ -66,6 +64,6 @@ public interface ServiceRegistryIf {
 
     void registerProcess(String processName, int nodeId);
     void deregisterProcess(String processName, int nodeId);
-    void watchProcess(String processName, int nodeId, Consumer<Boolean> callback) throws Exception;
+
-    void watchProcessAnyNode(String processName, Collection<Integer> nodes, BiConsumer<Boolean, Integer> callback) throws Exception;
+    InterProcessSemaphoreV2 getSemaphore(String name, int permits) throws Exception;
 }

code/common/service/java/nu/marginalia/service/discovery/ZkServiceRegistry.java

@@ -6,6 +6,7 @@ import nu.marginalia.service.discovery.monitor.ServiceMonitorIf;
 import nu.marginalia.service.discovery.property.ServiceEndpoint;
 import nu.marginalia.service.discovery.property.ServiceKey;
 import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2;
 import org.apache.curator.utils.ZKPaths;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.Watcher;
@@ -13,10 +14,11 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
 
 import static nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
 
@@ -283,60 +285,12 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
     }
 
     @Override
-    public void watchProcess(String processName, int nodeId, Consumer<Boolean> callback) throws Exception {
+    public InterProcessSemaphoreV2 getSemaphore(String name, int permits) {
-        String path = "/process-locks/" + processName + "/" + nodeId;
+        if (stopped)
+            throw new IllegalStateException("Service registry is stopped, cannot get semaphore " + name);
 
-        // first check if the path exists and call the callback accordingly
+        String path = "/semaphores/" + name;
-
+        return new InterProcessSemaphoreV2(curatorFramework, path, permits);
-        if (curatorFramework.checkExists().forPath(path) != null) {
-            callback.accept(true);
-        }
-        else {
-            callback.accept(false);
-        }
-
-        curatorFramework.watchers().add()
-                .usingWatcher((Watcher) change -> {
-                    Watcher.Event.EventType type = change.getType();
-
-                    if (type == Watcher.Event.EventType.NodeCreated) {
-                        callback.accept(true);
-                    }
-                    if (type == Watcher.Event.EventType.NodeDeleted) {
-                        callback.accept(false);
-                    }
-                })
-                .forPath(path);
-
-    }
-
-    @Override
-    public void watchProcessAnyNode(String processName, Collection<Integer> nodes, BiConsumer<Boolean, Integer> callback) throws Exception {
-
-        for (int node : nodes) {
-            String path = "/process-locks/" + processName + "/" + node;
-
-            // first check if the path exists and call the callback accordingly
-            if (curatorFramework.checkExists().forPath(path) != null) {
-                callback.accept(true, node);
-            }
-            else {
-                callback.accept(false, node);
-            }
-
-            curatorFramework.watchers().add()
-                    .usingWatcher((Watcher) change -> {
-                        Watcher.Event.EventType type = change.getType();
-
-                        if (type == Watcher.Event.EventType.NodeCreated) {
-                            callback.accept(true, node);
-                        }
-                        if (type == Watcher.Event.EventType.NodeDeleted) {
-                            callback.accept(false, node);
-                        }
-                    })
-                    .forPath(path);
-        }
     }
 
     /* Exposed for tests */

code/functions/live-capture/build.gradle

@@ -22,6 +22,7 @@ dependencies {
     implementation project(':code:common:db')
     implementation project(':code:libraries:blocking-thread-pool')
     implementation project(':code:libraries:message-queue')
+    implementation project(':code:libraries:domain-lock')
 
     implementation project(':code:execution:api')
     implementation project(':code:processes:crawling-process:ft-content-type')

code/functions/live-capture/java/nu/marginalia/rss/svc/DomainLocks.java

@@ -1,66 +0,0 @@
-package nu.marginalia.rss.svc;
-
-import nu.marginalia.model.EdgeDomain;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Semaphore;
-
-/** Holds lock objects for each domain, to prevent multiple threads from
- * crawling the same domain at the same time.
- */
-public class DomainLocks {
-    // The locks are stored in a map, with the domain name as the key.  This map will grow
-    // relatively big, but should be manageable since the number of domains is limited to
-    // a few hundred thousand typically.
-    private final Map<String, Semaphore> locks = new ConcurrentHashMap<>();
-
-    /** Returns a lock object corresponding to the given domain.  The object is returned as-is,
-     * and may be held by another thread.  The caller is responsible for locking and  releasing the lock.
-     */
-    public DomainLock lockDomain(EdgeDomain domain) throws InterruptedException {
-        return new DomainLock(domain.toString(),
-                locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits));
-    }
-
-    private Semaphore defaultPermits(String topDomain) {
-        if (topDomain.equals("wordpress.com"))
-            return new Semaphore(16);
-        if (topDomain.equals("blogspot.com"))
-            return new Semaphore(8);
-
-        if (topDomain.equals("neocities.org"))
-            return new Semaphore(4);
-        if (topDomain.equals("github.io"))
-            return new Semaphore(4);
-
-        if (topDomain.equals("substack.com")) {
-            return new Semaphore(1);
-        }
-        if (topDomain.endsWith(".edu")) {
-            return new Semaphore(1);
-        }
-
-        return new Semaphore(2);
-    }
-
-    public static class DomainLock implements AutoCloseable {
-        private final String domainName;
-        private final Semaphore semaphore;
-
-        DomainLock(String domainName, Semaphore semaphore) throws InterruptedException {
-            this.domainName = domainName;
-            this.semaphore = semaphore;
-
-            Thread.currentThread().setName("fetching:" + domainName + " [await domain lock]");
-            semaphore.acquire();
-            Thread.currentThread().setName("fetching:" + domainName);
-        }
-
-        @Override
-        public void close() {
-            semaphore.release();
-            Thread.currentThread().setName("fetching:" + domainName + " [wrapping up]");
-        }
-    }
-}

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedFetcherService.java

@@ -5,6 +5,8 @@ import com.opencsv.CSVReader;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.contenttype.ContentType;
 import nu.marginalia.contenttype.DocumentBodyToString;
+import nu.marginalia.coordination.DomainCoordinator;
+import nu.marginalia.coordination.DomainLock;
 import nu.marginalia.executor.client.ExecutorClient;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.nodecfg.NodeConfigurationService;
@@ -51,12 +53,13 @@ public class FeedFetcherService {
     private final ServiceHeartbeat serviceHeartbeat;
     private final ExecutorClient executorClient;
 
-    private final DomainLocks domainLocks = new DomainLocks();
+    private final DomainCoordinator domainCoordinator;
 
     private volatile boolean updating;
 
     @Inject
     public FeedFetcherService(FeedDb feedDb,
+                              DomainCoordinator domainCoordinator,
                               FileStorageService fileStorageService,
                               NodeConfigurationService nodeConfigurationService,
                               ServiceHeartbeat serviceHeartbeat,
@@ -67,6 +70,7 @@ public class FeedFetcherService {
         this.nodeConfigurationService = nodeConfigurationService;
         this.serviceHeartbeat = serviceHeartbeat;
         this.executorClient = executorClient;
+        this.domainCoordinator = domainCoordinator;
     }
 
     public enum UpdateMode {
@@ -132,7 +136,7 @@ public class FeedFetcherService {
                         };
 
                         FetchResult feedData;
-                        try (DomainLocks.DomainLock domainLock = domainLocks.lockDomain(new EdgeDomain(feed.domain()))) {
+                        try (DomainLock domainLock = domainCoordinator.lockDomain(new EdgeDomain(feed.domain()))) {
                             feedData = fetchFeedData(feed, client, fetchExecutor, ifModifiedSinceDate, ifNoneMatchTag);
                         } catch (Exception ex) {
                             feedData = new FetchResult.TransientError();

code/libraries/domain-lock/build.gradle

@@ -0,0 +1,32 @@
+plugins {
+    id 'java'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
+    }
+}
+
+apply from: "$rootProject.projectDir/srcsets.gradle"
+
+dependencies {
+    implementation libs.bundles.slf4j
+    implementation project(':code:common:model')
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+
+    implementation libs.bundles.curator
+
+    implementation libs.guava
+    implementation dependencies.create(libs.guice.get()) {
+        exclude group: 'com.google.guava'
+    }
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+}
+
+test {
+    useJUnitPlatform()
+}

code/libraries/domain-lock/java/nu/marginalia/coordination/DefaultDomainPermits.java

@@ -0,0 +1,32 @@
+package nu.marginalia.coordination;
+
+import nu.marginalia.model.EdgeDomain;
+
+public class DefaultDomainPermits {
+
+    public static int defaultPermits(EdgeDomain domain) {
+        return defaultPermits(domain.topDomain.toLowerCase());
+    }
+
+    public static int defaultPermits(String topDomain) {
+
+        if (topDomain.equals("wordpress.com"))
+            return 16;
+        if (topDomain.equals("blogspot.com"))
+            return 8;
+        if (topDomain.equals("tumblr.com"))
+            return 8;
+        if (topDomain.equals("neocities.org"))
+            return 8;
+        if (topDomain.equals("github.io"))
+            return 8;
+        // Substack really dislikes broad-scale crawlers, so we need to be careful
+        // to not get blocked.
+        if (topDomain.equals("substack.com")) {
+            return 1;
+        }
+
+        return 2;
+    }
+
+}

code/libraries/domain-lock/java/nu/marginalia/coordination/DomainCoordinationModule.java

@@ -0,0 +1,17 @@
+package nu.marginalia.coordination;
+
+import com.google.inject.AbstractModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DomainCoordinationModule extends AbstractModule {
+
+    private static final Logger logger = LoggerFactory.getLogger(DomainCoordinationModule.class);
+
+    public DomainCoordinationModule() {
+    }
+
+    public void configure() {
+        bind(DomainCoordinator.class).to(ZookeeperDomainCoordinator.class);
+    }
+}

code/libraries/domain-lock/java/nu/marginalia/coordination/DomainCoordinator.java

@@ -0,0 +1,13 @@
+package nu.marginalia.coordination;
+
+import nu.marginalia.model.EdgeDomain;
+
+import java.time.Duration;
+import java.util.Optional;
+
+public interface DomainCoordinator {
+    DomainLock lockDomain(EdgeDomain domain) throws InterruptedException;
+    Optional<DomainLock> tryLockDomain(EdgeDomain domain, Duration timeout) throws InterruptedException;
+    Optional<DomainLock> tryLockDomain(EdgeDomain domain)  throws InterruptedException;
+    boolean isLockableHint(EdgeDomain domain);
+}

code/libraries/domain-lock/java/nu/marginalia/coordination/DomainLock.java

@@ -0,0 +1,5 @@
+package nu.marginalia.coordination;
+
+public interface DomainLock extends AutoCloseable {
+    void close();
+}

code/libraries/domain-lock/java/nu/marginalia/coordination/LocalDomainCoordinator.java

@@ -1,16 +1,17 @@
-package nu.marginalia.crawl.logic;
+package nu.marginalia.coordination;
 
+import com.google.inject.Singleton;
 import nu.marginalia.model.EdgeDomain;
 
+import java.time.Duration;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
 
-/** Holds lock objects for each domain, to prevent multiple threads from
+@Singleton
- * crawling the same domain at the same time.
+public class LocalDomainCoordinator implements DomainCoordinator {
- */
-public class DomainLocks {
     // The locks are stored in a map, with the domain name as the key.  This map will grow
     // relatively big, but should be manageable since the number of domains is limited to
     // a few hundred thousand typically.
@@ -24,13 +25,25 @@ public class DomainLocks {
 
         sem.acquire();
 
-        return new DomainLock(sem);
+        return new LocalDomainLock(sem);
     }
 
     public Optional<DomainLock> tryLockDomain(EdgeDomain domain) {
         var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits);
         if (sem.tryAcquire(1)) {
-            return Optional.of(new DomainLock(sem));
+            return Optional.of(new LocalDomainLock(sem));
+        }
+        else {
+            // We don't have a lock, so we return an empty optional
+            return Optional.empty();
+        }
+    }
+
+
+    public Optional<DomainLock> tryLockDomain(EdgeDomain domain, Duration timeout) throws InterruptedException {
+        var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::defaultPermits);
+        if (sem.tryAcquire(1, timeout.toMillis(), TimeUnit.MILLISECONDS)) {
+            return Optional.of(new LocalDomainLock(sem));
         }
         else {
             // We don't have a lock, so we return an empty optional
@@ -39,24 +52,7 @@ public class DomainLocks {
     }
 
     private Semaphore defaultPermits(String topDomain) {
-        if (topDomain.equals("wordpress.com"))
+        return new Semaphore(DefaultDomainPermits.defaultPermits(topDomain));
-            return new Semaphore(16);
-        if (topDomain.equals("blogspot.com"))
-            return new Semaphore(8);
-        if (topDomain.equals("tumblr.com"))
-            return new Semaphore(8);
-        if (topDomain.equals("neocities.org"))
-            return new Semaphore(8);
-        if (topDomain.equals("github.io"))
-            return new Semaphore(8);
-
-        // Substack really dislikes broad-scale crawlers, so we need to be careful
-        // to not get blocked.
-        if (topDomain.equals("substack.com")) {
-            return new Semaphore(1);
-        }
-
-        return new Semaphore(2);
     }
 
     /** Returns true if the domain is lockable, i.e. if it is not already locked by another thread.
@@ -71,15 +67,15 @@ public class DomainLocks {
             return sem.availablePermits() > 0;
     }
 
-    public static class DomainLock implements AutoCloseable {
+    public static class LocalDomainLock implements DomainLock {
         private final Semaphore semaphore;
 
-        DomainLock(Semaphore semaphore) {
+        LocalDomainLock(Semaphore semaphore) {
             this.semaphore = semaphore;
         }
 
         @Override
-        public void close() throws Exception {
+        public void close() {
             semaphore.release();
             Thread.currentThread().setName("[idle]");
         }

code/libraries/domain-lock/java/nu/marginalia/coordination/ZookeeperDomainCoordinator.java

@@ -0,0 +1,116 @@
+package nu.marginalia.coordination;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+import nu.marginalia.model.EdgeDomain;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
+import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2;
+import org.apache.curator.framework.recipes.locks.Lease;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+@Singleton
+public class ZookeeperDomainCoordinator implements DomainCoordinator {
+    // The locks are stored in a map, with the domain name as the key.  This map will grow
+    // relatively big, but should be manageable since the number of domains is limited to
+    // a few hundred thousand typically.
+    private final Map<String, InterProcessSemaphoreV2> locks = new ConcurrentHashMap<>();
+    private final Map<String, Integer> waitCounts = new ConcurrentHashMap<>();
+
+    private final ServiceRegistryIf serviceRegistry;
+    private final int nodeId;
+
+    @Inject
+    public ZookeeperDomainCoordinator(ServiceRegistryIf serviceRegistry, @Named("wmsa-system-node") int nodeId) {
+        // Zookeeper-specific initialization can be done here if needed
+        this.serviceRegistry = serviceRegistry;
+        this.nodeId = nodeId;
+    }
+
+    /** Returns a lock object corresponding to the given domain.  The object is returned as-is,
+     * and may be held by another thread.  The caller is responsible for locking and  releasing the lock.
+     */
+    public DomainLock lockDomain(EdgeDomain domain) throws InterruptedException {
+        final String key = domain.topDomain.toLowerCase();
+        var sem = locks.computeIfAbsent(key, this::createSemapore);
+
+        // Increment or add a wait count for the domain
+        waitCounts.compute(key, (k,value) -> (value == null ? 1 : value + 1));
+        try {
+            return new ZkDomainLock(sem, sem.acquire());
+        }
+        catch (Exception e) {
+            throw new RuntimeException("Failed to acquire lock for domain: " + domain.topDomain, e);
+        }
+        finally {
+            // Decrement or remove the wait count for the domain
+            waitCounts.compute(key, (k,value) -> (value == null || value <= 1) ? null : value - 1);
+        }
+    }
+
+
+    public Optional<DomainLock> tryLockDomain(EdgeDomain domain) throws InterruptedException {
+        return tryLockDomain(domain, Duration.ofSeconds(1)); // Underlying semaphore doesn't have a tryLock method, so we use a short timeout
+    }
+
+
+    public Optional<DomainLock> tryLockDomain(EdgeDomain domain, Duration timeout) throws InterruptedException {
+        final String key = domain.topDomain.toLowerCase();
+        var sem = locks.computeIfAbsent(key, this::createSemapore);
+
+        // Increment or add a wait count for the domain
+        waitCounts.compute(key, (k,value) -> (value == null ? 1 : value + 1));
+        try {
+            var lease = sem.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS); // Acquire with timeout
+            if (lease != null) {
+                return Optional.of(new ZkDomainLock(sem, lease));
+            }
+            else {
+                return Optional.empty(); // If we fail to acquire the lease, we return an empty optional
+            }
+        }
+        catch (Exception e) {
+            return Optional.empty(); // If we fail to acquire the lock, we return an empty optional
+        }
+        finally {
+            waitCounts.compute(key, (k,value) -> (value == null || value <= 1) ? null : value - 1);
+        }
+    }
+
+    private InterProcessSemaphoreV2 createSemapore(String topDomain){
+        try {
+            return serviceRegistry.getSemaphore(topDomain + ":" + nodeId, DefaultDomainPermits.defaultPermits(topDomain));
+        }
+        catch (Exception e) {
+            throw new RuntimeException("Failed to get semaphore for domain: " + topDomain, e);
+        }
+    }
+
+    /** Returns true if the domain is lockable, i.e. if it is not already locked by another thread.
+     * (this is just a hint, and does not guarantee that the domain is actually lockable any time
+     * after this method returns true)
+     */
+    public boolean isLockableHint(EdgeDomain domain) {
+        return !waitCounts.containsKey(domain.topDomain.toLowerCase());
+    }
+
+    public static class ZkDomainLock implements DomainLock {
+        private final InterProcessSemaphoreV2 semaphore;
+        private final Lease lease;
+
+        ZkDomainLock(InterProcessSemaphoreV2 semaphore, Lease lease) {
+            this.semaphore = semaphore;
+            this.lease = lease;
+        }
+
+        @Override
+        public void close() {
+            semaphore.returnLease(lease);
+        }
+    }
+}

code/processes/crawling-process/build.gradle

@@ -32,6 +32,7 @@ dependencies {
     implementation project(':code:libraries:message-queue')
     implementation project(':code:libraries:language-processing')
     implementation project(':code:libraries:easy-lsh')
+    implementation project(':code:libraries:domain-lock')
     implementation project(':code:processes:crawling-process:model')
     implementation project(':code:processes:crawling-process:model')
 

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -10,9 +10,11 @@ import nu.marginalia.WmsaHome;
 import nu.marginalia.atags.model.DomainLinks;
 import nu.marginalia.atags.source.AnchorTagsSource;
 import nu.marginalia.atags.source.AnchorTagsSourceFactory;
+import nu.marginalia.coordination.DomainCoordinationModule;
+import nu.marginalia.coordination.DomainCoordinator;
+import nu.marginalia.coordination.DomainLock;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
 import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
-import nu.marginalia.crawl.logic.DomainLocks;
 import nu.marginalia.crawl.retreival.CrawlDataReference;
 import nu.marginalia.crawl.retreival.CrawlerRetreiver;
 import nu.marginalia.crawl.retreival.DomainProber;
@@ -68,7 +70,7 @@ public class CrawlerMain extends ProcessMainClass {
     private final ServiceRegistryIf serviceRegistry;
     private final SimpleBlockingThreadPool pool;
 
-    private final DomainLocks domainLocks = new DomainLocks();
+    private final DomainCoordinator domainCoordinator;
 
     private final Map<String, CrawlTask> pendingCrawlTasks = new ConcurrentHashMap<>();
 
@@ -97,6 +99,7 @@ public class CrawlerMain extends ProcessMainClass {
                        WarcArchiverFactory warcArchiverFactory,
                        HikariDataSource dataSource,
                        DomainBlacklist blacklist,
+                       DomainCoordinator domainCoordinator,
                        ServiceRegistryIf serviceRegistry,
                        Gson gson) throws InterruptedException {
 
@@ -114,6 +117,7 @@ public class CrawlerMain extends ProcessMainClass {
         this.blacklist = blacklist;
         this.node = processConfiguration.node();
         this.serviceRegistry = serviceRegistry;
+        this.domainCoordinator = domainCoordinator;
 
         SimpleBlockingThreadPool.ThreadType threadType;
         if (Boolean.getBoolean("crawler.useVirtualThreads")) {
@@ -157,6 +161,7 @@ public class CrawlerMain extends ProcessMainClass {
                     new CrawlerModule(),
                     new ProcessConfigurationModule("crawler"),
                     new ServiceDiscoveryModule(),
+                    new DomainCoordinationModule(),
                     new DatabaseModule(false)
             );
             var crawler = injector.getInstance(CrawlerMain.class);
@@ -451,7 +456,7 @@ public class CrawlerMain extends ProcessMainClass {
         /** Best effort indicator whether we could start this now without getting stuck in
          * DomainLocks purgatory */
         public boolean canRun() {
-            return domainLocks.isLockableHint(new EdgeDomain(domain));
+            return domainCoordinator.isLockableHint(new EdgeDomain(domain));
         }
 
         @Override
@@ -462,7 +467,7 @@ public class CrawlerMain extends ProcessMainClass {
                 return;
             }
 
-            Optional<DomainLocks.DomainLock> lock = domainLocks.tryLockDomain(new EdgeDomain(domain));
+            Optional<DomainLock> lock = domainCoordinator.tryLockDomain(new EdgeDomain(domain));
             // We don't have a lock, so we can't run this task
             // we return to avoid blocking the pool for too long
             if (lock.isEmpty()) {
@@ -470,7 +475,7 @@ public class CrawlerMain extends ProcessMainClass {
                 retryQueue.put(this);
                 return;
             }
-            DomainLocks.DomainLock domainLock = lock.get();
+            DomainLock domainLock = lock.get();
 
             try (domainLock) {
                 Thread.currentThread().setName("crawling:" + domain);

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -36,7 +36,6 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
 import org.apache.hc.core5.http.message.MessageSupport;
 import org.apache.hc.core5.http.protocol.HttpContext;
 import org.apache.hc.core5.pool.PoolStats;
-import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.apache.hc.core5.util.TimeValue;
 import org.apache.hc.core5.util.Timeout;
 import org.jsoup.Jsoup;
@@ -49,15 +48,12 @@ import org.slf4j.MarkerFactory;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.net.URISyntaxException;
 import java.net.UnknownHostException;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
-import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
@@ -99,42 +95,12 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
                 .setValidateAfterInactivity(TimeValue.ofSeconds(5))
                 .build();
 
-        // No-op up front validation of server certificates.
-        //
-        // We will validate certificates later, after the connection is established
-        // as we want to store the certificate chain and validation
-        // outcome to the database.
-
-        var trustMeBro = new X509TrustManager() {
-            private X509Certificate[] lastServerCertChain;
-
-            @Override
-            public void checkClientTrusted(X509Certificate[] chain, String authType) {
-            }
-
-            @Override
-            public void checkServerTrusted(X509Certificate[] chain, String authType) {
-                this.lastServerCertChain = chain.clone();
-            }
-
-            @Override
-            public X509Certificate[] getAcceptedIssuers() {
-                return new X509Certificate[0];
-            }
-
-            public X509Certificate[] getLastServerCertChain() {
-                return lastServerCertChain != null ? lastServerCertChain.clone() : null;
-            }
-        };
-
-        SSLContext sslContext = SSLContextBuilder.create().build();
-        sslContext.init(null, new TrustManager[]{trustMeBro}, null);
 
         connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
                 .setMaxConnPerRoute(2)
                 .setMaxConnTotal(5000)
                 .setDefaultConnectionConfig(connectionConfig)
-                .setTlsSocketStrategy(new DefaultClientTlsStrategy(sslContext))
+                .setTlsSocketStrategy(new DefaultClientTlsStrategy(SSLContext.getDefault()))
                 .build();
 
         connectionManager.setDefaultSocketConfig(SocketConfig.custom()

code/processes/live-crawling-process/build.gradle

@@ -32,6 +32,7 @@ dependencies {
     implementation project(':code:index:api')
     implementation project(':code:processes:process-mq-api')
     implementation project(':code:libraries:message-queue')
+    implementation project(':code:libraries:domain-lock')
     implementation project(':code:libraries:language-processing')
     implementation project(':code:libraries:easy-lsh')
     implementation project(':code:processes:crawling-process')

code/processes/live-crawling-process/java/nu/marginalia/livecrawler/LiveCrawlerMain.java

@@ -10,6 +10,8 @@ import nu.marginalia.api.feeds.FeedsClient;
 import nu.marginalia.converting.ConverterModule;
 import nu.marginalia.converting.processor.DomainProcessor;
 import nu.marginalia.converting.writer.ConverterBatchWriter;
+import nu.marginalia.coordination.DomainCoordinationModule;
+import nu.marginalia.coordination.DomainCoordinator;
 import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.db.DomainBlacklist;
 import nu.marginalia.io.SerializableCrawlDataStream;
@@ -58,6 +60,7 @@ public class LiveCrawlerMain extends ProcessMainClass {
     private final FileStorageService fileStorageService;
     private final KeywordLoaderService keywordLoaderService;
     private final DocumentLoaderService documentLoaderService;
+    private final DomainCoordinator domainCoordinator;
     private final HikariDataSource dataSource;
 
     @Inject
@@ -71,7 +74,7 @@ public class LiveCrawlerMain extends ProcessMainClass {
                            DomainProcessor domainProcessor,
                            FileStorageService fileStorageService,
                            KeywordLoaderService keywordLoaderService,
-                           DocumentLoaderService documentLoaderService, HikariDataSource dataSource)
+                           DocumentLoaderService documentLoaderService, DomainCoordinator domainCoordinator, HikariDataSource dataSource)
             throws Exception
     {
         super(messageQueueFactory, config, gson, LIVE_CRAWLER_INBOX);
@@ -84,6 +87,7 @@ public class LiveCrawlerMain extends ProcessMainClass {
         this.fileStorageService = fileStorageService;
         this.keywordLoaderService = keywordLoaderService;
         this.documentLoaderService = documentLoaderService;
+        this.domainCoordinator = domainCoordinator;
         this.dataSource = dataSource;
 
         domainBlacklist.waitUntilLoaded();
@@ -107,6 +111,7 @@ public class LiveCrawlerMain extends ProcessMainClass {
         try {
             Injector injector = Guice.createInjector(
                     new LiveCrawlerModule(),
+                    new DomainCoordinationModule(), // 2 hours lease timeout is enough for the live crawler
                     new ProcessConfigurationModule("crawler"),
                     new ConverterModule(),
                     new ServiceDiscoveryModule(),
@@ -172,7 +177,7 @@ public class LiveCrawlerMain extends ProcessMainClass {
 
             processHeartbeat.progress(LiveCrawlState.CRAWLING);
 
-            try (SimpleLinkScraper fetcher = new SimpleLinkScraper(dataSet, domainQueries, domainBlacklist);
+            try (SimpleLinkScraper fetcher = new SimpleLinkScraper(dataSet, domainCoordinator, domainQueries, domainBlacklist);
                  var hb = heartbeat.createAdHocTaskHeartbeat("Live Crawling"))
             {
                 for (Map.Entry<String, List<String>> entry : hb.wrap("Fetching", urlsPerDomain.entrySet())) {

code/processes/live-crawling-process/java/nu/marginalia/livecrawler/SimpleLinkScraper.java

@@ -5,8 +5,9 @@ import crawlercommons.robots.SimpleRobotRulesParser;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.contenttype.ContentType;
 import nu.marginalia.contenttype.DocumentBodyToString;
+import nu.marginalia.coordination.DomainCoordinator;
+import nu.marginalia.coordination.DomainLock;
 import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
-import nu.marginalia.crawl.logic.DomainLocks;
 import nu.marginalia.crawl.retreival.CrawlDelayTimer;
 import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.db.DomainBlacklist;
@@ -46,14 +47,16 @@ public class SimpleLinkScraper implements AutoCloseable {
     private final DomainBlacklist domainBlacklist;
     private final Duration connectTimeout = Duration.ofSeconds(10);
     private final Duration readTimeout = Duration.ofSeconds(10);
-    private final DomainLocks domainLocks = new DomainLocks();
+    private final DomainCoordinator domainCoordinator;
 
     private final static int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 10 * 1024 * 1024);
 
     public SimpleLinkScraper(LiveCrawlDataSet dataSet,
+                             DomainCoordinator domainCoordinator,
                              DbDomainQueries domainQueries,
                              DomainBlacklist domainBlacklist) {
         this.dataSet = dataSet;
+        this.domainCoordinator = domainCoordinator;
         this.domainQueries = domainQueries;
         this.domainBlacklist = domainBlacklist;
     }
@@ -98,7 +101,7 @@ public class SimpleLinkScraper implements AutoCloseable {
                 .version(HttpClient.Version.HTTP_2)
                 .build();
              // throttle concurrent access per domain; IDE will complain it's not used, but it holds a semaphore -- do not remove:
-             DomainLocks.DomainLock lock = domainLocks.lockDomain(domain)
+             DomainLock lock = domainCoordinator.lockDomain(domain)
         ) {
             SimpleRobotRules rules = fetchRobotsRules(rootUrl, client);
 

code/processes/live-crawling-process/test/nu/marginalia/livecrawler/SimpleLinkScraperTest.java

@@ -1,5 +1,6 @@
 package nu.marginalia.livecrawler;
 
+import nu.marginalia.coordination.LocalDomainCoordinator;
 import nu.marginalia.db.DomainBlacklistImpl;
 import nu.marginalia.io.SerializableCrawlDataStream;
 import nu.marginalia.model.EdgeDomain;
@@ -37,7 +38,7 @@ class SimpleLinkScraperTest {
 
     @Test
     public void testRetrieveNow() throws Exception {
-        var scraper = new SimpleLinkScraper(dataSet, null, Mockito.mock(DomainBlacklistImpl.class));
+        var scraper = new SimpleLinkScraper(dataSet, new LocalDomainCoordinator(),  null, Mockito.mock(DomainBlacklistImpl.class));
         int fetched = scraper.retrieveNow(new EdgeDomain("www.marginalia.nu"), 1, List.of("https://www.marginalia.nu/"));
         Assertions.assertEquals(1, fetched);
 
@@ -57,7 +58,7 @@ class SimpleLinkScraperTest {
     @Test
     public void testRetrieveNow_Redundant() throws Exception {
         dataSet.saveDocument(1, new EdgeUrl("https://www.marginalia.nu/"), "<html>", "", "127.0.0.1");
-        var scraper = new SimpleLinkScraper(dataSet, null, Mockito.mock(DomainBlacklistImpl.class));
+        var scraper = new SimpleLinkScraper(dataSet, new LocalDomainCoordinator(),null, Mockito.mock(DomainBlacklistImpl.class));
 
         // If the requested URL is already in the dataSet, we retrieveNow should shortcircuit and not fetch anything
         int fetched = scraper.retrieveNow(new EdgeDomain("www.marginalia.nu"), 1, List.of("https://www.marginalia.nu/"));

code/processes/ping-process/build.gradle

@@ -27,6 +27,7 @@ dependencies {
     implementation project(':code:common:config')
     implementation project(':code:common:service')
 
+    implementation project(':code:libraries:domain-lock')
     implementation project(':code:libraries:geo-ip')
     implementation project(':code:libraries:message-queue')
 

code/processes/ping-process/java/nu/marginalia/ping/PingDao.java

@@ -185,12 +185,12 @@ public class PingDao {
         return null;
     }
 
-    public List<UpdateSchedule.UpdateJob<Long, HistoricalAvailabilityData>> getDomainUpdateSchedule(int nodeId) {
+    public List<UpdateSchedule.UpdateJob<DomainReference, HistoricalAvailabilityData>> getDomainUpdateSchedule(int nodeId) {
-        List<UpdateSchedule.UpdateJob<Long, HistoricalAvailabilityData>> updateJobs = new ArrayList<>();
+        List<UpdateSchedule.UpdateJob<DomainReference, HistoricalAvailabilityData>> updateJobs = new ArrayList<>();
 
         try (var conn = dataSource.getConnection();
              var ps = conn.prepareStatement("""
-                SELECT ID, NEXT_SCHEDULED_UPDATE
+                SELECT ID, DOMAIN_NAME, NEXT_SCHEDULED_UPDATE
                 FROM EC_DOMAIN
                 LEFT JOIN DOMAIN_AVAILABILITY_INFORMATION
                 ON EC_DOMAIN.ID = DOMAIN_AVAILABILITY_INFORMATION.DOMAIN_ID
@@ -200,11 +200,13 @@ public class PingDao {
             ps.setInt(1, nodeId);
             ResultSet rs = ps.executeQuery();
             while (rs.next()) {
-                long domainId = rs.getLong("ID");
+                int domainId = rs.getInt("ID");
+                String domainName = rs.getString("DOMAIN_NAME");
                 var ts = rs.getTimestamp("NEXT_SCHEDULED_UPDATE");
                 Instant nextUpdate = ts == null ? Instant.now() : ts.toInstant();
 
-                updateJobs.add(new UpdateSchedule.UpdateJob<>(domainId, nextUpdate));
+                var ref = new DomainReference(domainId, nodeId, domainName.toLowerCase());
+                updateJobs.add(new UpdateSchedule.UpdateJob<>(ref, nextUpdate));
             }
         } catch (SQLException e) {
             throw new RuntimeException("Failed to retrieve domain update schedule", e);
@@ -241,7 +243,7 @@ public class PingDao {
                 else {
                     var record = new DomainDnsRecord(rs);
                     updateJobs.add(new UpdateSchedule.UpdateJob<>(
-                            new RootDomainReference.ById(dnsRootDomainId),
+                            new RootDomainReference.ByIdAndName(dnsRootDomainId, rootDomainName),
                             Objects.requireNonNullElseGet(record.tsNextScheduledUpdate(), Instant::now))
                     );
                 }

code/processes/ping-process/java/nu/marginalia/ping/PingJobScheduler.java

@@ -1,6 +1,8 @@
 package nu.marginalia.ping;
 
 import com.google.inject.Inject;
+import nu.marginalia.coordination.DomainCoordinator;
+import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.ping.model.*;
 import nu.marginalia.ping.svc.DnsPingService;
 import nu.marginalia.ping.svc.HttpPingService;
@@ -23,13 +25,14 @@ import java.util.concurrent.TimeUnit;
 public class PingJobScheduler {
     private final HttpPingService httpPingService;
     private final DnsPingService dnsPingService;
+    private final DomainCoordinator domainCoordinator;
     private final PingDao pingDao;
 
     private static final Logger logger = LoggerFactory.getLogger(PingJobScheduler.class);
 
     private static final UpdateSchedule<RootDomainReference, RootDomainReference> dnsUpdateSchedule
             = new UpdateSchedule<>(250_000);
-    private static final UpdateSchedule<Long, HistoricalAvailabilityData> availabilityUpdateSchedule
+    private static final UpdateSchedule<DomainReference, HistoricalAvailabilityData> availabilityUpdateSchedule
             = new UpdateSchedule<>(250_000);
 
     public volatile Instant dnsLastSync = Instant.now();
@@ -43,10 +46,12 @@ public class PingJobScheduler {
     @Inject
     public PingJobScheduler(HttpPingService httpPingService,
                             DnsPingService dnsPingService,
+                            DomainCoordinator domainCoordinator,
                             PingDao pingDao)
     {
         this.httpPingService = httpPingService;
         this.dnsPingService = dnsPingService;
+        this.domainCoordinator = domainCoordinator;
         this.pingDao = pingDao;
     }
 
@@ -134,7 +139,15 @@ public class PingJobScheduler {
                     continue;
                 }
 
-                long nextId = availabilityUpdateSchedule.next();
+                DomainReference ref = availabilityUpdateSchedule.nextIf(domain -> {
+                    EdgeDomain domainObj = new EdgeDomain(domain.domainName());
+                    if (!domainCoordinator.isLockableHint(domainObj)) {
+                        return false; // Skip locked domains
+                    }
+                    return true; // Process this domain
+                });
+
+                long nextId = ref.domainId();
                 var data = pingDao.getHistoricalAvailabilityData(nextId);
                 if (data == null) {
                     logger.warn("No availability data found for ID: {}", nextId);
@@ -159,7 +172,7 @@ public class PingJobScheduler {
                     for (var object : objects) {
                         var ts = object.nextUpdateTime();
                         if (ts != null) {
-                            availabilityUpdateSchedule.add(nextId, ts);
+                            availabilityUpdateSchedule.add(ref, ts);
                             break;
                         }
                     }
@@ -190,7 +203,7 @@ public class PingJobScheduler {
 
                 try {
                     List<WritableModel> objects = switch(ref) {
-                        case RootDomainReference.ById(long id) -> {
+                        case RootDomainReference.ByIdAndName(long id, String name) -> {
                             var oldRecord = Objects.requireNonNull(pingDao.getDomainDnsRecord(id));
                             yield dnsPingService.pingDomain(oldRecord.rootDomainName(), oldRecord);
                         }

code/processes/ping-process/java/nu/marginalia/ping/PingMain.java

@@ -5,15 +5,14 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.WmsaHome;
+import nu.marginalia.coordination.DomainCoordinationModule;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.mq.MessageQueueFactory;
 import nu.marginalia.mqapi.ProcessInboxNames;
 import nu.marginalia.mqapi.ping.PingRequest;
-import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.process.ProcessConfiguration;
 import nu.marginalia.process.ProcessConfigurationModule;
 import nu.marginalia.process.ProcessMainClass;
-import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.DatabaseModule;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import org.slf4j.Logger;
@@ -25,8 +24,6 @@ public class PingMain extends ProcessMainClass {
     private static final Logger log = LoggerFactory.getLogger(PingMain.class);
 
     private final PingJobScheduler pingJobScheduler;
-    private final ServiceRegistryIf serviceRegistry;
-    private final NodeConfigurationService nodeConfigurationService;
     private final int node;
 
     private static final Logger logger = LoggerFactory.getLogger(PingMain.class);
@@ -36,15 +33,11 @@ public class PingMain extends ProcessMainClass {
                     ProcessConfiguration config,
                     Gson gson,
                     PingJobScheduler pingJobScheduler,
-                    ServiceRegistryIf serviceRegistry,
-                    NodeConfigurationService nodeConfigurationService,
                     ProcessConfiguration processConfiguration
                     ) {
         super(messageQueueFactory, config, gson, ProcessInboxNames.PING_INBOX);
 
         this.pingJobScheduler = pingJobScheduler;
-        this.serviceRegistry = serviceRegistry;
-        this.nodeConfigurationService = nodeConfigurationService;
         this.node = processConfiguration.node();
     }
 
@@ -80,6 +73,7 @@ public class PingMain extends ProcessMainClass {
         Injector injector = Guice.createInjector(
                 new PingModule(),
                 new ServiceDiscoveryModule(),
+                new DomainCoordinationModule(),
                 new ProcessConfigurationModule("ping"),
                 new DatabaseModule(false)
         );

code/processes/ping-process/java/nu/marginalia/ping/UpdateSchedule.java

@@ -2,9 +2,8 @@ package nu.marginalia.ping;
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collection;
+import java.util.*;
-import java.util.Comparator;
+import java.util.function.Predicate;
-import java.util.PriorityQueue;
 
 /** In-memory schedule for updates, allowing jobs to be added and processed in order of their scheduled time.
  * This is not a particularly high-performance implementation, but exists to take contention off the database's
@@ -23,6 +22,9 @@ public class UpdateSchedule<T, T2> {
         notifyAll();
     }
 
+    /** Returns the next job in the queue that is due to be processed.
+     * If no jobs are due, it will block until a job is added or a job becomes due.
+     * */
     public synchronized T next() throws InterruptedException {
         while (true) {
             if (updateQueue.isEmpty()) {
@@ -44,6 +46,56 @@ public class UpdateSchedule<T, T2> {
         }
     }
 
+
+    /** Returns the first job in the queue matching the predicate that is not scheduled into the future,
+     * blocking until a job is added or a job becomes due.
+     */
+    public synchronized T nextIf(Predicate<T> predicate) throws InterruptedException {
+        List<UpdateJob<T, T2>> rejectedJobs = new ArrayList<>();
+
+        try {
+            while (true) {
+                if (updateQueue.isEmpty()) {
+                    wait(); // Wait for a new job to be added
+                    continue;
+                }
+
+                UpdateJob<T, T2> job = updateQueue.peek();
+                Instant now = Instant.now();
+
+                if (job.updateTime.isAfter(now)) {
+                    Duration toWait = Duration.between(now, job.updateTime);
+
+                    // Return the rejected jobs to the queue for other threads to process
+                    updateQueue.addAll(rejectedJobs);
+                    if (!rejectedJobs.isEmpty())
+                        notifyAll();
+                    rejectedJobs.clear();
+
+                    wait(Math.max(1, toWait.toMillis()));
+                } else {
+                    var candidate = updateQueue.poll(); // Remove the job from the queue since it's due
+
+                    assert candidate != null : "Update job should not be null at this point, since we just peeked it in a synchronized block";
+
+                    if (!predicate.test(candidate.key())) {
+                        rejectedJobs.add(candidate);
+                    }
+                    else {
+                        return candidate.key();
+                    }
+                }
+            }
+        }
+        finally {
+            // Return the rejected jobs to the queue for other threads to process
+            updateQueue.addAll(rejectedJobs);
+            if (!rejectedJobs.isEmpty())
+                notifyAll();
+        }
+
+    }
+
     public synchronized void clear() {
         updateQueue.clear();
         notifyAll();

code/processes/ping-process/java/nu/marginalia/ping/fetcher/PingHttpFetcher.java

@@ -4,12 +4,14 @@ import com.google.inject.Inject;
 import nu.marginalia.UserAgent;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.ping.fetcher.response.*;
+import org.apache.hc.client5.http.HttpHostConnectException;
 import org.apache.hc.client5.http.classic.HttpClient;
 import org.apache.hc.client5.http.protocol.HttpClientContext;
 import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
 
+import javax.net.ssl.SSLHandshakeException;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.time.Duration;
@@ -82,9 +84,12 @@ public class PingHttpFetcher {
             });
         } catch (SocketTimeoutException ex) {
             return new TimeoutResponse(ex.getMessage());
-        } catch (IOException e) {
+        } catch (HttpHostConnectException | SSLHandshakeException e) {
             return new ConnectionError(e.getClass().getSimpleName());
+        } catch (IOException e) {
+            return new ProtocolError(e.getClass().getSimpleName());
         }
+
     }
 
 }

code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java

@@ -18,13 +18,18 @@ import org.apache.hc.core5.http.HttpResponse;
 import org.apache.hc.core5.http.io.SocketConfig;
 import org.apache.hc.core5.http.message.MessageSupport;
 import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.apache.hc.core5.util.TimeValue;
 import org.apache.hc.core5.util.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
 import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 
@@ -37,24 +42,55 @@ public class HttpClientProvider implements Provider<HttpClient> {
     static {
         try {
             client = createClient();
-        } catch (NoSuchAlgorithmException e) {
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
-    private static CloseableHttpClient createClient() throws NoSuchAlgorithmException {
+    private static CloseableHttpClient createClient() throws NoSuchAlgorithmException, KeyManagementException {
         final ConnectionConfig connectionConfig = ConnectionConfig.custom()
                 .setSocketTimeout(15, TimeUnit.SECONDS)
                 .setConnectTimeout(15, TimeUnit.SECONDS)
                 .setValidateAfterInactivity(TimeValue.ofSeconds(5))
                 .build();
 
+        // No-op up front validation of server certificates.
+        //
+        // We will validate certificates later, after the connection is established
+        // as we want to store the certificate chain and validation
+        // outcome to the database.
+
+        var trustMeBro = new X509TrustManager() {
+            private X509Certificate[] lastServerCertChain;
+
+            @Override
+            public void checkClientTrusted(X509Certificate[] chain, String authType) {
+            }
+
+            @Override
+            public void checkServerTrusted(X509Certificate[] chain, String authType) {
+                this.lastServerCertChain = chain.clone();
+            }
+
+            @Override
+            public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+
+            public X509Certificate[] getLastServerCertChain() {
+                return lastServerCertChain != null ? lastServerCertChain.clone() : null;
+            }
+        };
+
+        SSLContext sslContext = SSLContextBuilder.create().build();
+        sslContext.init(null, new TrustManager[]{trustMeBro}, null);
+
         connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
                 .setMaxConnPerRoute(2)
                 .setMaxConnTotal(50)
                 .setDefaultConnectionConfig(connectionConfig)
                 .setTlsSocketStrategy(
-                        new DefaultClientTlsStrategy(SSLContext.getDefault(), NoopHostnameVerifier.INSTANCE))
+                        new DefaultClientTlsStrategy(sslContext, NoopHostnameVerifier.INSTANCE))
                 .build();
 
         connectionManager.setDefaultSocketConfig(SocketConfig.custom()
@@ -76,7 +112,7 @@ public class HttpClientProvider implements Provider<HttpClient> {
         });
 
         final RequestConfig defaultRequestConfig = RequestConfig.custom()
-                .setCookieSpec(StandardCookieSpec.RELAXED)
+                .setCookieSpec(StandardCookieSpec.IGNORE)
                 .setResponseTimeout(10, TimeUnit.SECONDS)
                 .setConnectionRequestTimeout(5, TimeUnit.MINUTES)
                 .build();

code/processes/ping-process/java/nu/marginalia/ping/io/RetryStrategy.java

@@ -1,5 +1,6 @@
 package nu.marginalia.ping.io;
 
+import org.apache.hc.client5.http.HttpHostConnectException;
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.core5.http.HttpRequest;
 import org.apache.hc.core5.http.HttpResponse;
@@ -10,6 +11,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLException;
 import java.io.IOException;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
 
@@ -22,6 +24,8 @@ public class RetryStrategy implements HttpRequestRetryStrategy {
             case SocketTimeoutException ste -> false;
             case SSLException ssle -> false;
             case UnknownHostException uhe -> false;
+            case HttpHostConnectException ex -> executionCount < 2;
+            case SocketException ex -> executionCount < 2;
             default -> executionCount <= 3;
         };
     }
@@ -50,7 +54,12 @@ public class RetryStrategy implements HttpRequestRetryStrategy {
 
         if (statusCode == 429) {
             // get the Retry-After header
-            String retryAfter = response.getFirstHeader("Retry-After").getValue();
+            var retryAfterHeader = response.getFirstHeader("Retry-After");
+            if (retryAfterHeader == null) {
+                return TimeValue.ofSeconds(3);
+            }
+
+            String retryAfter = retryAfterHeader.getValue();
             if (retryAfter == null) {
                 return TimeValue.ofSeconds(2);
             }

code/processes/ping-process/java/nu/marginalia/ping/model/DomainAvailabilityRecord.java

@@ -1,5 +1,7 @@
 package nu.marginalia.ping.model;
 
+import org.apache.commons.lang3.StringUtils;
+
 import javax.annotation.Nullable;
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -279,7 +281,7 @@ implements WritableModel
         }
 
         public Builder httpLocation(String httpLocation) {
-            this.httpLocation = httpLocation;
+            this.httpLocation = StringUtils.abbreviate(httpLocation, "...",255);
             return this;
         }
 

code/processes/ping-process/java/nu/marginalia/ping/model/DomainReference.java

@@ -1,3 +1,10 @@
 package nu.marginalia.ping.model;
 
-public record DomainReference(int domainId, int nodeId, String domainName) { }
+import nu.marginalia.model.EdgeDomain;
+
+public record DomainReference(int domainId, int nodeId, String domainName) {
+    public EdgeDomain asEdgeDomain() {
+        return new EdgeDomain(domainName);
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/model/DomainSecurityEvent.java

@@ -18,6 +18,7 @@ public record DomainSecurityEvent(
         boolean certificatePublicKeyChanged,
         boolean certificateSerialNumberChanged,
         boolean certificateIssuerChanged,
+        SchemaChange schemaChange,
         Duration oldCertificateTimeToExpiry,
         boolean securityHeadersChanged,
         boolean ipChanged,
@@ -45,8 +46,9 @@ public record DomainSecurityEvent(
                 security_signature_before,
                 security_signature_after,
                 change_certificate_serial_number,
-                change_certificate_issuer
+                change_certificate_issuer,
-                ) VALUES    (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                change_schema
+                ) VALUES    (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
             """))
         {
 
@@ -81,6 +83,7 @@ public record DomainSecurityEvent(
 
             ps.setBoolean(15, certificateSerialNumberChanged());
             ps.setBoolean(16, certificateIssuerChanged());
+            ps.setString(17, schemaChange.name());
 
             ps.executeUpdate();
         }

code/processes/ping-process/java/nu/marginalia/ping/model/DomainSecurityRecord.java

@@ -1,6 +1,7 @@
 package nu.marginalia.ping.model;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
 
 import javax.annotation.Nullable;
 import java.sql.Connection;
@@ -42,7 +43,10 @@ public record DomainSecurityRecord(
         @Nullable String headerXXssProtection,
         @Nullable String headerServer,
         @Nullable String headerXPoweredBy,
-        @Nullable Instant tsLastUpdate
+        @Nullable Instant tsLastUpdate,
+        @Nullable Boolean sslChainValid,
+        @Nullable Boolean sslHostValid,
+        @Nullable Boolean sslDateValid
         )
     implements WritableModel
 {
@@ -102,7 +106,11 @@ public record DomainSecurityRecord(
              rs.getString("DOMAIN_SECURITY_INFORMATION.HEADER_X_XSS_PROTECTION"),
              rs.getString("DOMAIN_SECURITY_INFORMATION.HEADER_SERVER"),
              rs.getString("DOMAIN_SECURITY_INFORMATION.HEADER_X_POWERED_BY"),
-             rs.getObject("DOMAIN_SECURITY_INFORMATION.TS_LAST_UPDATE", Instant.class));
+             rs.getObject("DOMAIN_SECURITY_INFORMATION.TS_LAST_UPDATE", Instant.class),
+             rs.getObject("SSL_CHAIN_VALID", Boolean.class),
+             rs.getObject("SSL_HOST_VALID", Boolean.class),
+             rs.getObject("SSL_DATE_VALID", Boolean.class)
+                );
     }
 
     private static HttpSchema httpSchemaFromString(@Nullable String schema) {
@@ -149,8 +157,11 @@ public record DomainSecurityRecord(
                              header_x_powered_by,
                              ssl_cert_public_key_hash,
                              asn,
-                             ts_last_update)
+                             ts_last_update,
-                         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                             ssl_chain_valid,
+                             ssl_host_valid,
+                             ssl_date_valid)
+                         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                         """))
         {
             ps.setInt(1, domainId());
@@ -294,6 +305,25 @@ public record DomainSecurityRecord(
             } else {
                 ps.setTimestamp(32, java.sql.Timestamp.from(tsLastUpdate()));
             }
+
+            if (sslChainValid() == null) {
+                ps.setNull(33, java.sql.Types.BOOLEAN);
+            } else {
+                ps.setBoolean(33, sslChainValid());
+            }
+
+            if (sslHostValid() == null) {
+                ps.setNull(34, java.sql.Types.BOOLEAN);
+            } else {
+                ps.setBoolean(34, sslHostValid());
+            }
+
+            if (sslDateValid() == null) {
+                ps.setNull(35, java.sql.Types.BOOLEAN);
+            } else {
+                ps.setBoolean(35, sslDateValid());
+            }
+
             ps.executeUpdate();
         }
     }
@@ -332,6 +362,13 @@ public record DomainSecurityRecord(
         private String headerXPoweredBy;
         private Instant tsLastUpdate;
 
+        private Boolean isCertChainValid;
+        private Boolean isCertHostValid;
+        private Boolean isCertDateValid;
+
+
+        private static Instant MAX_UNIX_TIMESTAMP = Instant.ofEpochSecond(Integer.MAX_VALUE);
+
         public Builder() {
             // Default values for boolean fields
             this.sslCertWildcard = false;
@@ -374,12 +411,18 @@ public record DomainSecurityRecord(
             return this;
         }
 
-        public Builder sslCertNotBefore(Instant sslCertNotBefore) {
+        public Builder sslCertNotBefore(@NotNull Instant sslCertNotBefore) {
+            if (sslCertNotBefore.isAfter(MAX_UNIX_TIMESTAMP)) {
+                sslCertNotBefore = MAX_UNIX_TIMESTAMP;
+            }
             this.sslCertNotBefore = sslCertNotBefore;
             return this;
         }
 
-        public Builder sslCertNotAfter(Instant sslCertNotAfter) {
+        public Builder sslCertNotAfter(@NotNull Instant sslCertNotAfter) {
+            if (sslCertNotAfter.isAfter(MAX_UNIX_TIMESTAMP)) {
+                sslCertNotAfter = MAX_UNIX_TIMESTAMP;
+            }
             this.sslCertNotAfter = sslCertNotAfter;
             return this;
         }
@@ -499,6 +542,21 @@ public record DomainSecurityRecord(
             return this;
         }
 
+        public Builder sslChainValid(@Nullable Boolean isCertChainValid) {
+            this.isCertChainValid = isCertChainValid;
+            return this;
+        }
+
+        public Builder sslHostValid(@Nullable Boolean isCertHostValid) {
+            this.isCertHostValid = isCertHostValid;
+            return this;
+        }
+
+        public Builder sslDateValid(@Nullable Boolean isCertDateValid) {
+            this.isCertDateValid = isCertDateValid;
+            return this;
+        }
+
         public DomainSecurityRecord build() {
             return new DomainSecurityRecord(
                     domainId,
@@ -532,7 +590,10 @@ public record DomainSecurityRecord(
                     headerXXssProtection,
                     headerServer,
                     headerXPoweredBy,
-                    tsLastUpdate
+                    tsLastUpdate,
+                    isCertChainValid,
+                    isCertHostValid,
+                    isCertDateValid
             );
         }
     }

code/processes/ping-process/java/nu/marginalia/ping/model/RootDomainReference.java

@@ -1,6 +1,6 @@
 package nu.marginalia.ping.model;
 
 public sealed interface RootDomainReference {
-    record ById(long id) implements RootDomainReference { }
+    record ByIdAndName(long id, String name) implements RootDomainReference { }
     record ByName(String name) implements RootDomainReference { }
 }

code/processes/ping-process/java/nu/marginalia/ping/model/SchemaChange.java

@@ -0,0 +1,12 @@
+package nu.marginalia.ping.model;
+
+public enum SchemaChange {
+    UNKNOWN,
+    NONE,
+    HTTP_TO_HTTPS,
+    HTTPS_TO_HTTP;
+
+    public boolean isSignificant() {
+        return this != NONE && this != UNKNOWN;
+    }
+}

code/processes/ping-process/java/nu/marginalia/ping/model/comparison/SecurityInformationChange.java

@@ -2,6 +2,9 @@ package nu.marginalia.ping.model.comparison;
 
 import nu.marginalia.ping.model.DomainAvailabilityRecord;
 import nu.marginalia.ping.model.DomainSecurityRecord;
+import nu.marginalia.ping.model.HttpSchema;
+import nu.marginalia.ping.model.SchemaChange;
+import org.jetbrains.annotations.NotNull;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -20,7 +23,8 @@ public record SecurityInformationChange(
         Duration oldCertificateTimeToExpiry,
         boolean isSecurityHeadersChanged,
         boolean isIpAddressChanged,
-        boolean isSoftwareHeaderChanged
+        boolean isSoftwareHeaderChanged,
+        SchemaChange schemaChange
 ) {
     public static SecurityInformationChange between(
             DomainSecurityRecord before, DomainAvailabilityRecord availabilityBefore,
@@ -43,9 +47,10 @@ public record SecurityInformationChange(
         );
 
         boolean securityHeadersChanged = before.securityHeadersHash() != after.securityHeadersHash();
-
         boolean softwareChanged = !Objects.equals(before.headerServer(), after.headerServer());
 
+        SchemaChange schemaChange = getSchemaChange(before, after);
+
         // Note we don't include IP address changes in the overall change status,
         // as this is not alone considered a change in security information; we may have
         // multiple IP addresses for a domain, and the IP address may change frequently
@@ -55,7 +60,8 @@ public record SecurityInformationChange(
                 || certificateFingerprintChanged
                 || securityHeadersChanged
                 || certificateProfileChanged
-                || softwareChanged;
+                || softwareChanged
+                || schemaChange.isSignificant();
 
         return new SecurityInformationChange(
                 isChanged,
@@ -69,9 +75,36 @@ public record SecurityInformationChange(
                 oldCertificateTimeToExpiry,
                 securityHeadersChanged,
                 ipChanged,
-                softwareChanged
+                softwareChanged,
+                schemaChange
         );
     }
 
+    private static @NotNull SchemaChange getSchemaChange(DomainSecurityRecord before, DomainSecurityRecord after) {
+        if (before.httpSchema() == null || after.httpSchema() == null) {
+            return SchemaChange.UNKNOWN;
+        }
+
+        boolean beforeIsHttp = before.httpSchema() == HttpSchema.HTTP;
+        boolean afterIsHttp = after.httpSchema() == HttpSchema.HTTP;
+        boolean beforeIsHttps = before.httpSchema() == HttpSchema.HTTPS;
+        boolean afterIsHttps = after.httpSchema() == HttpSchema.HTTPS;
+
+        SchemaChange schemaChange;
+
+        if (beforeIsHttp && afterIsHttp) {
+            schemaChange = SchemaChange.NONE;
+        } else if (beforeIsHttps && afterIsHttps) {
+            schemaChange = SchemaChange.NONE;
+        } else if (beforeIsHttp && afterIsHttps) {
+            schemaChange = SchemaChange.HTTP_TO_HTTPS;
+        } else if (beforeIsHttps && afterIsHttp) {
+            schemaChange = SchemaChange.HTTPS_TO_HTTP;
+        } else {
+            schemaChange = SchemaChange.UNKNOWN;
+        }
+        return schemaChange;
+    }
+
 
 }

code/processes/ping-process/java/nu/marginalia/ping/ssl/AIAExtractor.java

@@ -0,0 +1,59 @@
+package nu.marginalia.ping.ssl;
+
+import org.bouncycastle.asn1.ASN1OctetString;
+import org.bouncycastle.asn1.ASN1Primitive;
+import org.bouncycastle.asn1.x509.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AIAExtractor {
+
+    private static final Logger logger = LoggerFactory.getLogger(AIAExtractor.class);
+
+    public static List<String> getCaIssuerUrls(X509Certificate certificate) {
+        List<String> caIssuerUrls = new ArrayList<>();
+
+        try {
+            // Get the AIA extension value
+            byte[] aiaExtensionValue = certificate.getExtensionValue(Extension.authorityInfoAccess.getId());
+            if (aiaExtensionValue == null) {
+                logger.warn("No AIA extension found");
+                return caIssuerUrls;
+            }
+
+            // Parse the extension - first unwrap the OCTET STRING
+            ASN1OctetString octetString = ASN1OctetString.getInstance(aiaExtensionValue);
+            ASN1Primitive aiaObj = ASN1Primitive.fromByteArray(octetString.getOctets());
+
+            // Parse as AuthorityInformationAccess
+            AuthorityInformationAccess aia = AuthorityInformationAccess.getInstance(aiaObj);
+
+            if (aia != null) {
+                AccessDescription[] accessDescriptions = aia.getAccessDescriptions();
+
+                for (AccessDescription accessDesc : accessDescriptions) {
+                    // Check if this is a CA Issuers access method
+                    if (X509ObjectIdentifiers.id_ad_caIssuers.equals(accessDesc.getAccessMethod())) {
+                        GeneralName accessLocation = accessDesc.getAccessLocation();
+
+                        // Check if it's a URI
+                        if (accessLocation.getTagNo() == GeneralName.uniformResourceIdentifier) {
+                            String url = accessLocation.getName().toString();
+                            caIssuerUrls.add(url);
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            logger.error("Error parsing AIA extension: {}", e.getMessage());
+        }
+
+        return caIssuerUrls;
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/ssl/CertificateFetcher.java

@@ -0,0 +1,273 @@
+package nu.marginalia.ping.ssl;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import nu.marginalia.WmsaHome;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
+import org.bouncycastle.asn1.ASN1OctetString;
+import org.bouncycastle.asn1.ASN1Primitive;
+import org.bouncycastle.asn1.x509.*;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cms.CMSSignedData;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.util.Store;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.CertificateFactory;
+import java.security.cert.TrustAnchor;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public class CertificateFetcher {
+
+    private static final Logger logger = LoggerFactory.getLogger(CertificateFetcher.class);
+
+    private static HttpClient client = HttpClientBuilder.create()
+            .build();
+
+    private static Cache<String, X509Certificate> cache = CacheBuilder
+            .newBuilder()
+            .expireAfterAccess(Duration.ofHours(6))
+            .maximumSize(10_000)
+            .build();
+
+
+    public static List<X509Certificate> fetchMissingIntermediates(X509Certificate leafCert) {
+        List<X509Certificate> intermediates = new ArrayList<>();
+
+        // Get CA Issuer URLs from AIA extension
+        List<String> caIssuerUrls = AIAExtractor.getCaIssuerUrls(leafCert);
+
+        for (String url : caIssuerUrls) {
+            try {
+                // Check cache first
+                X509Certificate cached = cache.getIfPresent(url);
+                if (cached != null) {
+                    intermediates.add(cached);
+                    continue;
+                }
+
+                // Download certificate
+                X509Certificate downloaded = downloadCertificate(url);
+                if (downloaded != null) {
+                    // Verify this certificate can actually sign the leaf
+                    if (canSign(downloaded, leafCert)) {
+                        intermediates.add(downloaded);
+                        cache.put(url, downloaded);
+                        logger.info("Downloaded certificate for url: {}", url);
+                    } else {
+                        logger.warn("Downloaded certificate cannot sign leaf cert from: {}", url);
+                    }
+                }
+
+            } catch (Exception e) {
+                logger.error("Failed to fetch certificate from {}: {}", url, e.getMessage());
+            }
+        }
+
+        return intermediates;
+    }
+    private static X509Certificate downloadCertificate(String urlString) {
+        try {
+            ClassicHttpRequest request =  ClassicRequestBuilder.create("GET")
+                    .addHeader("User-Agent", WmsaHome.getUserAgent() + " (Certificate Fetcher)")
+                    .setUri(urlString)
+                    .build();
+
+            byte[] data = client.execute(request, rsp -> {
+                var entity = rsp.getEntity();
+                if (entity == null) {
+                    logger.warn("GET request returned no content for {}", urlString);
+                    return null;
+                }
+                return entity.getContent().readAllBytes();
+            });
+
+            if (data.length == 0) {
+                logger.warn("Empty response from {}", urlString);
+                return null;
+            }
+
+            // Try different formats based on file extension
+            if (urlString.toLowerCase().endsWith(".p7c") || urlString.toLowerCase().endsWith(".p7b")) {
+                return parsePKCS7(data);
+            } else {
+                return parseX509(data);
+            }
+
+        } catch (Exception e) {
+            logger.warn("Failed to fetch certificate from {}: {}", urlString, e.getMessage());
+            return null;
+        }
+    }
+
+    private static List<X509Certificate> parseMultiplePEM(byte[] data) throws Exception {
+        List<X509Certificate> certificates = new ArrayList<>();
+
+        try (StringReader stringReader = new StringReader(new String(data, StandardCharsets.UTF_8));
+             PEMParser pemParser = new PEMParser(stringReader)) {
+
+            JcaX509CertificateConverter converter = new JcaX509CertificateConverter();
+            Object object;
+
+            while ((object = pemParser.readObject()) != null) {
+                if (object instanceof X509CertificateHolder) {
+                    X509CertificateHolder certHolder = (X509CertificateHolder) object;
+                    certificates.add(converter.getCertificate(certHolder));
+                } else if (object instanceof X509Certificate) {
+                    certificates.add((X509Certificate) object);
+                }
+            }
+        }
+
+        return certificates;
+    }
+    private static X509Certificate parseX509(byte[] data) throws Exception {
+        CertificateFactory cf = CertificateFactory.getInstance("X.509");
+        return (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(data));
+    }
+
+    private static X509Certificate parsePKCS7(byte[] data) throws Exception {
+        try {
+            // Parse PKCS#7/CMS structure
+            CMSSignedData cmsData = new CMSSignedData(data);
+            Store<X509CertificateHolder> certStore = cmsData.getCertificates();
+
+            JcaX509CertificateConverter converter = new JcaX509CertificateConverter();
+
+            // Get the first certificate from the store
+            for (X509CertificateHolder certHolder : certStore.getMatches(null)) {
+                X509Certificate cert = converter.getCertificate(certHolder);
+                return cert;
+            }
+
+            logger.warn("No certificates found in PKCS#7 structure");
+            return null;
+
+        } catch (Exception e) {
+            logger.error("Failed to parse PKCS#7 structure from {}: {}", data.length, e.getMessage());
+            return parseX509(data);
+        }
+    }
+
+    private static boolean canSign(X509Certificate issuerCert, X509Certificate subjectCert) {
+        try {
+            // Check if the issuer DN matches
+            if (!issuerCert.getSubjectDN().equals(subjectCert.getIssuerDN())) {
+                return false;
+            }
+
+            // Try to verify the signature
+            subjectCert.verify(issuerCert.getPublicKey());
+            return true;
+
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    // Recursive fetching for complete chains
+    public static List<X509Certificate> buildCompleteChain(X509Certificate leafCert) {
+        List<X509Certificate> completeChain = new ArrayList<>();
+        completeChain.add(leafCert);
+
+        X509Certificate currentCert = leafCert;
+        int maxDepth = 10; // Prevent infinite loops
+
+        while (maxDepth-- > 0) {
+            // If current cert is self-signed (root), we're done
+            if (currentCert.getSubjectDN().equals(currentCert.getIssuerDN())) {
+                break;
+            }
+
+            // Try to find the issuer
+            List<X509Certificate> intermediates = fetchMissingIntermediates(currentCert);
+            if (intermediates.isEmpty()) {
+                logger.error("Could not find issuer for: {}", currentCert.getSubjectDN());
+                break;
+            }
+
+            // Add the first valid intermediate
+            X509Certificate intermediate = intermediates.get(0);
+            completeChain.add(intermediate);
+            currentCert = intermediate;
+        }
+
+        return completeChain;
+    }
+
+    // Add this to your AIAExtractor class if not already present
+    public static List<String> getOCSPUrls(X509Certificate certificate) {
+        List<String> ocspUrls = new ArrayList<>();
+
+        try {
+            byte[] aiaExtensionValue = certificate.getExtensionValue(Extension.authorityInfoAccess.getId());
+            if (aiaExtensionValue == null) {
+                return ocspUrls;
+            }
+
+            ASN1OctetString octetString = ASN1OctetString.getInstance(aiaExtensionValue);
+            ASN1Primitive aiaObj = ASN1Primitive.fromByteArray(octetString.getOctets());
+            AuthorityInformationAccess aia = AuthorityInformationAccess.getInstance(aiaObj);
+
+            if (aia != null) {
+                AccessDescription[] accessDescriptions = aia.getAccessDescriptions();
+
+                for (AccessDescription accessDesc : accessDescriptions) {
+                    if (X509ObjectIdentifiers.id_ad_ocsp.equals(accessDesc.getAccessMethod())) {
+                        GeneralName accessLocation = accessDesc.getAccessLocation();
+                        if (accessLocation.getTagNo() == GeneralName.uniformResourceIdentifier) {
+                            String url = accessLocation.getName().toString();
+                            ocspUrls.add(url);
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            logger.error("Error parsing AIA extension for OCSP: {}", e.getMessage());
+        }
+
+        return ocspUrls;
+    }
+
+    public static Set<TrustAnchor> getRootCerts(String bundleUrl) throws Exception {
+        ClassicHttpRequest request =  ClassicRequestBuilder.create("GET")
+                .addHeader("User-Agent", WmsaHome.getUserAgent() + " (Certificate Fetcher)")
+                .setUri(bundleUrl)
+                .build();
+
+        byte[] data = client.execute(request, rsp -> {
+            var entity = rsp.getEntity();
+            if (entity == null) {
+                logger.warn("GET request returned no content for {}", bundleUrl);
+                return null;
+            }
+            return entity.getContent().readAllBytes();
+        });
+
+        List<TrustAnchor> anchors = new ArrayList<>();
+        for (var cert : parseMultiplePEM(data)) {
+            try {
+                anchors.add(new TrustAnchor(cert, null));
+            } catch (Exception e) {
+                logger.warn("Failed to create TrustAnchor for certificate: {}", e.getMessage());
+            }
+        }
+
+        logger.info("Loaded {} root certificates from {}", anchors.size(), bundleUrl);
+
+        return Set.copyOf(anchors);
+    }
+}

code/processes/ping-process/java/nu/marginalia/ping/ssl/CertificateValidator.java

@@ -0,0 +1,493 @@
+package nu.marginalia.ping.ssl;
+
+import org.bouncycastle.asn1.ASN1OctetString;
+import org.bouncycastle.asn1.ASN1Primitive;
+import org.bouncycastle.asn1.x509.*;
+
+import javax.security.auth.x500.X500Principal;
+import java.security.cert.TrustAnchor;
+import java.security.cert.X509Certificate;
+import java.util.*;
+
+/** Utility class for validating X.509 certificates.
+ * This class provides methods to validate certificate chains, check expiration,
+ * hostname validity, and revocation status.
+ * <p></p>
+ * This is extremely unsuitable for actual SSL/TLS validation,
+ * and is only to be used in analyzing certificates for fingerprinting
+ * and diagnosing servers!
+ */
+public class CertificateValidator {
+    // If true, will attempt to fetch missing intermediate certificates via AIA urls.
+    private static final boolean TRY_FETCH_MISSING_CERTS = false;
+
+    public static class ValidationResult {
+        public boolean chainValid = false;
+        public boolean certificateExpired = false;
+        public boolean certificateRevoked = false;
+        public boolean selfSigned = false;
+        public boolean hostnameValid = false;
+
+        public boolean isValid() {
+            return !selfSigned && !certificateExpired && !certificateRevoked && hostnameValid;
+        }
+
+        public List<String> errors = new ArrayList<>();
+        public List<String> warnings = new ArrayList<>();
+        public Map<String, Object> details = new HashMap<>();
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append("=== Certificate Validation Result ===\n");
+            sb.append("Chain Valid: ").append(chainValid ? "✓" : "✗").append("\n");
+            sb.append("Not Expired: ").append(!certificateExpired ? "✓" : "✗").append("\n");
+            sb.append("Not Revoked: ").append(!certificateRevoked ? "✓" : "✗").append("\n");
+            sb.append("Hostname Valid: ").append(hostnameValid ? "✓" : "✗").append("\n");
+            sb.append("Self-Signed: ").append(selfSigned ? "✓" : "✗").append("\n");
+
+            if (!errors.isEmpty()) {
+                sb.append("\nErrors:\n");
+                for (String error : errors) {
+                    sb.append("  ✗ ").append(error).append("\n");
+                }
+            }
+
+            if (!warnings.isEmpty()) {
+                sb.append("\nWarnings:\n");
+                for (String warning : warnings) {
+                    sb.append("  ⚠ ").append(warning).append("\n");
+                }
+            }
+
+            if (!details.isEmpty()) {
+                sb.append("\nDetails:\n");
+                for (Map.Entry<String, Object> entry : details.entrySet()) {
+                    sb.append("  ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
+                }
+            }
+
+            return sb.toString();
+        }
+    }
+
+    public static ValidationResult validateCertificate(X509Certificate[] certChain,
+                                                       String hostname) {
+        return validateCertificate(certChain, hostname, false);
+    }
+
+    public static ValidationResult validateCertificate(X509Certificate[] certChain,
+                                                       String hostname,
+                                                       boolean autoTrustFetchedRoots) {
+        ValidationResult result = new ValidationResult();
+
+        if (certChain == null || certChain.length == 0) {
+            result.errors.add("No certificates provided");
+            return result;
+        }
+
+        X509Certificate leafCert = certChain[0];
+
+        // 1. Check certificate expiration
+        result.certificateExpired = checkExpiration(leafCert, result);
+
+        // 2. Check hostname validity
+        result.hostnameValid = checkHostname(leafCert, hostname, result);
+
+        // 3. Not really checking if it's self-signed, but if the chain is incomplete (and likely self-signed)
+        result.selfSigned = certChain.length <= 1;
+
+        // 4. Check certificate chain validity (optionally with AIA fetching)
+        result.chainValid = checkChainValidity(certChain, RootCerts.getTrustAnchors(), result, autoTrustFetchedRoots);
+
+        // 5. Check revocation status
+        result.certificateRevoked = false; // not implemented
+        // checkRevocation(leafCert, result);
+
+        return result;
+    }
+
+    private static boolean checkExpiration(X509Certificate cert, ValidationResult result) {
+        try {
+            cert.checkValidity();
+            result.details.put("validFrom", cert.getNotBefore());
+            result.details.put("validTo", cert.getNotAfter());
+
+            // Warn if expires soon (30 days)
+            long daysUntilExpiry = (cert.getNotAfter().getTime() - System.currentTimeMillis()) / (1000 * 60 * 60 * 24);
+            if (daysUntilExpiry < 30) {
+                result.warnings.add("Certificate expires in " + daysUntilExpiry + " days");
+            }
+
+            return false; // Not expired
+        } catch (Exception e) {
+            result.errors.add("Certificate expired or not yet valid: " + e.getMessage());
+            return true; // Expired
+        }
+    }
+
+    private static boolean checkHostname(X509Certificate cert, String hostname, ValidationResult result) {
+        if (hostname == null || hostname.isEmpty()) {
+            result.warnings.add("No hostname provided for validation");
+            return false;
+        }
+
+        try {
+            // Check Subject CN
+            String subjectCN = getCommonName(cert.getSubjectX500Principal());
+            if (subjectCN != null && matchesHostname(subjectCN, hostname)) {
+                result.details.put("hostnameMatchedBy", "Subject CN: " + subjectCN);
+                return true;
+            }
+
+            // Check Subject Alternative Names
+            Collection<List<?>> subjectAltNames = cert.getSubjectAlternativeNames();
+            if (subjectAltNames != null) {
+                for (List<?> altName : subjectAltNames) {
+                    if (altName.size() >= 2) {
+                        Integer type = (Integer) altName.get(0);
+                        if (type == 2) { // DNS name
+                            String dnsName = (String) altName.get(1);
+                            if (matchesHostname(dnsName, hostname)) {
+                                result.details.put("hostnameMatchedBy", "SAN DNS: " + dnsName);
+                                return true;
+                            }
+                        }
+                    }
+                }
+            }
+
+            result.errors.add("Hostname '" + hostname + "' does not match certificate");
+            result.details.put("subjectCN", subjectCN);
+            result.details.put("subjectAltNames", subjectAltNames);
+            return false;
+
+        } catch (Exception e) {
+            result.errors.add("Error checking hostname: " + e.getMessage());
+            return false;
+        }
+    }
+
+    private static boolean checkChainValidity(X509Certificate[] originalChain,
+                                              Set<TrustAnchor> trustAnchors,
+                                              ValidationResult result,
+                                              boolean autoTrustFetchedRoots) {
+        try {
+            // First try with the original chain
+            ChainValidationResult originalResult = validateChain(originalChain, trustAnchors);
+
+            if (originalResult.isValid) {
+                result.details.put("chainLength", originalChain.length);
+                result.details.put("chainExtended", false);
+                return true;
+            }
+
+            else if (!TRY_FETCH_MISSING_CERTS) {
+                result.errors.addAll(originalResult.issues);
+                result.details.put("chainLength", originalChain.length);
+                result.details.put("chainExtended", false);
+                return false;
+            }
+
+            try {
+                List<X509Certificate> repairedChain = CertificateFetcher.buildCompleteChain(originalChain[0]);
+
+                if (!repairedChain.isEmpty()) {
+
+                    X509Certificate[] extendedArray = repairedChain.toArray(new X509Certificate[0]);
+
+                    // Create a copy of trust anchors for potential modification
+                    Set<TrustAnchor> workingTrustAnchors = new HashSet<>(trustAnchors);
+
+                    // If auto-trust is enabled, add any self-signed certs as trusted roots
+                    if (autoTrustFetchedRoots) {
+                        for (X509Certificate cert : extendedArray) {
+                            if (cert.getSubjectX500Principal().equals(cert.getIssuerX500Principal())) {
+                                // Self-signed certificate - add to trust anchors if not already there
+                                boolean alreadyTrusted = false;
+                                for (TrustAnchor anchor : workingTrustAnchors) {
+                                    if (anchor.getTrustedCert().equals(cert)) {
+                                        alreadyTrusted = true;
+                                        break;
+                                    }
+                                }
+                                if (!alreadyTrusted) {
+                                    workingTrustAnchors.add(new TrustAnchor(cert, null));
+                                    result.warnings.add("Auto-trusted fetched root: " + cert.getSubjectX500Principal().getName());
+                                }
+                            }
+                        }
+                    }
+
+                    ChainValidationResult extendedResult = validateChain(extendedArray, workingTrustAnchors);
+
+                    result.details.put("chainLength", extendedArray.length);
+                    result.details.put("originalChainLength", originalChain.length);
+                    result.details.put("chainExtended", true);
+                    result.details.put("fetchedIntermediates", extendedArray.length);
+                    result.details.put("autoTrustedRoots", autoTrustFetchedRoots);
+
+                    if (extendedResult.isValid) {
+                        result.warnings.add("Extended certificate chain with " + extendedArray.length + " fetched intermediates");
+                        return true;
+                    } else {
+                        result.errors.addAll(extendedResult.issues);
+                        return false;
+                    }
+                } else {
+                    result.warnings.add("Could not fetch missing intermediate certificates");
+                    result.details.put("chainLength", originalChain.length);
+                    result.details.put("chainExtended", false);
+                    result.errors.addAll(originalResult.issues);
+                    return false;
+                }
+
+            } catch (Exception e) {
+                result.warnings.add("Failed to fetch intermediates: " + e.getMessage());
+                result.details.put("chainLength", originalChain.length);
+                result.details.put("chainExtended", false);
+                result.errors.addAll(originalResult.issues);
+                return false;
+            }
+
+        } catch (Exception e) {
+            result.errors.add("Error validating chain: " + e.getMessage());
+            return false;
+        }
+    }
+
+
+    private static void debugCertificateChain(List<X509Certificate> certs, Set<TrustAnchor> trustAnchors) {
+        System.out.println("=== Certificate Chain Analysis ===");
+
+        int length = certs.size();
+        System.out.println("Chain length: " + length);
+
+        int i = 0;
+        for (var x509cert : certs) {
+            System.out.println("\nCertificate " + i++ + ":");
+            System.out.println("  Subject: " + x509cert.getSubjectDN().getName());
+            System.out.println("  Issuer:  " + x509cert.getIssuerDN().getName());
+            System.out.println("  Serial:  " + x509cert.getSerialNumber().toString(16));
+            System.out.println("  Valid:   " + x509cert.getNotBefore() + " to " + x509cert.getNotAfter());
+            System.out.println("  Self-signed: " + x509cert.getSubjectDN().equals(x509cert.getIssuerDN()));
+
+            // Check if we have the issuer in our trust anchors
+            boolean issuerFound = false;
+            for (TrustAnchor anchor : trustAnchors) {
+                if (anchor.getTrustedCert().getSubjectDN().equals(x509cert.getIssuerDN())) {
+                    issuerFound = true;
+                    System.out.println("  Issuer found in trust anchors: " + anchor.getTrustedCert().getSubjectDN().getName());
+                    break;
+                }
+            }
+            if (!issuerFound && i == length) {
+                System.out.println("  *** MISSING ISSUER: " + x509cert.getIssuerDN().getName());
+            }
+        }
+    }
+
+    private static class ChainValidationResult {
+        boolean isValid = false;
+        List<String> issues = new ArrayList<>();
+    }
+
+    private static ChainValidationResult validateChain(X509Certificate[] certChain, Set<TrustAnchor> trustAnchors) {
+        ChainValidationResult result = new ChainValidationResult();
+
+        // Check each certificate in the chain
+        for (int i = 0; i < certChain.length; i++) {
+            X509Certificate cert = certChain[i];
+
+            // Check certificate validity dates
+            try {
+                cert.checkValidity();
+            } catch (Exception e) {
+                result.issues.add("Certificate " + i + " expired: " + cert.getSubjectDN());
+            }
+
+            // Check signature (except for self-signed root)
+            if (i < certChain.length - 1) {
+                X509Certificate issuer = certChain[i + 1];
+                try {
+                    cert.verify(issuer.getPublicKey());
+                } catch (Exception e) {
+                    result.issues.add("Certificate " + i + " signature invalid: " + e.getMessage());
+                }
+
+                // Check issuer/subject relationship
+                if (!cert.getIssuerX500Principal().equals(issuer.getSubjectX500Principal())) {
+                    result.issues.add("Certificate " + i + " issuer does not match certificate " + (i + 1) + " subject");
+                }
+            }
+        }
+
+        // Check if chain ends with a trusted root
+        X509Certificate rootCert = certChain[certChain.length - 1];
+        boolean trustedRootFound = false;
+
+        if (rootCert.getSubjectX500Principal().equals(rootCert.getIssuerX500Principal())) {
+            // Self-signed root - check if it's in trust anchors
+            for (TrustAnchor anchor : trustAnchors) {
+                if (anchor.getTrustedCert().equals(rootCert)) {
+                    trustedRootFound = true;
+                    break;
+                }
+            }
+
+            if (!trustedRootFound) {
+                // Check if we trust the root's subject even if the certificate is different
+                for (TrustAnchor anchor : trustAnchors) {
+                    if (anchor.getTrustedCert().getSubjectX500Principal().equals(rootCert.getSubjectX500Principal())) {
+                        trustedRootFound = true;
+                        // Note: we'll add this as a warning in the main result
+                        break;
+                    }
+                }
+            }
+        } else {
+            // Chain doesn't end with self-signed cert - check if issuer is trusted
+            for (TrustAnchor anchor : trustAnchors) {
+                if (anchor.getTrustedCert().getSubjectX500Principal().equals(rootCert.getIssuerX500Principal())) {
+                    trustedRootFound = true;
+                    break;
+                }
+            }
+        }
+
+        if (!trustedRootFound) {
+            result.issues.add("Chain does not end with a trusted root");
+        }
+
+        result.isValid = result.issues.isEmpty();
+        return result;
+    }
+
+    private static boolean checkRevocation(X509Certificate cert, ValidationResult result) {
+        try {
+            // Try OCSP first
+            if (checkOCSP(cert, result)) {
+                return true; // Revoked
+            }
+
+            // Fallback to CRL
+            if (checkCRL(cert, result)) {
+                return true; // Revoked
+            }
+
+            result.warnings.add("Could not check revocation status");
+            return false; // Assume not revoked if we can't check
+
+        } catch (Exception e) {
+            result.warnings.add("Error checking revocation: " + e.getMessage());
+            return false;
+        }
+    }
+
+    private static boolean checkOCSP(X509Certificate cert, ValidationResult result) {
+        // For now, just extract OCSP URL and note that we found it
+        try {
+            List<String> ocspUrls = CertificateFetcher.getOCSPUrls(cert);
+            if (!ocspUrls.isEmpty()) {
+                result.details.put("ocspUrls", ocspUrls);
+                result.warnings.add("OCSP checking not implemented - found OCSP URLs: " + ocspUrls);
+            }
+            return false;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    private static boolean checkCRL(X509Certificate cert, ValidationResult result) {
+        // Basic CRL URL extraction
+        try {
+            List<String> crlUrls = getCRLUrls(cert);
+            if (!crlUrls.isEmpty()) {
+                result.details.put("crlUrls", crlUrls);
+                result.warnings.add("CRL checking not implemented - found CRL URLs: " + crlUrls);
+            }
+            return false;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    // Helper methods
+    private static String getCommonName(X500Principal principal) {
+        String name = principal.getName();
+        String[] parts = name.split(",");
+        for (String part : parts) {
+            part = part.trim();
+            if (part.startsWith("CN=")) {
+                return part.substring(3);
+            }
+        }
+        return null;
+    }
+
+    private static boolean matchesHostname(String certName, String hostname) {
+        if (certName == null || hostname == null) {
+            return false;
+        }
+
+        // Exact match
+        if (certName.equalsIgnoreCase(hostname)) {
+            return true;
+        }
+
+        // Wildcard match
+        if (certName.startsWith("*.")) {
+            String certDomain = certName.substring(2);
+            String hostDomain = hostname;
+            int firstDot = hostname.indexOf('.');
+            if (firstDot > 0) {
+                hostDomain = hostname.substring(firstDot + 1);
+            }
+            return certDomain.equalsIgnoreCase(hostDomain);
+        }
+
+        return false;
+    }
+
+    private static List<String> getCRLUrls(X509Certificate cert) {
+        // This would need to parse the CRL Distribution Points extension
+        // For now, return empty list
+        return new ArrayList<>();
+    }
+
+    // Add this to your AIAExtractor class if not already present
+    public static List<String> getOCSPUrls(X509Certificate certificate) {
+        List<String> ocspUrls = new ArrayList<>();
+
+        try {
+            byte[] aiaExtensionValue = certificate.getExtensionValue(Extension.authorityInfoAccess.getId());
+            if (aiaExtensionValue == null) {
+                return ocspUrls;
+            }
+
+            ASN1OctetString octetString = ASN1OctetString.getInstance(aiaExtensionValue);
+            ASN1Primitive aiaObj = ASN1Primitive.fromByteArray(octetString.getOctets());
+            AuthorityInformationAccess aia = AuthorityInformationAccess.getInstance(aiaObj);
+
+            if (aia != null) {
+                AccessDescription[] accessDescriptions = aia.getAccessDescriptions();
+
+                for (AccessDescription accessDesc : accessDescriptions) {
+                    if (X509ObjectIdentifiers.id_ad_ocsp.equals(accessDesc.getAccessMethod())) {
+                        GeneralName accessLocation = accessDesc.getAccessLocation();
+                        if (accessLocation.getTagNo() == GeneralName.uniformResourceIdentifier) {
+                            String url = accessLocation.getName().toString();
+                            ocspUrls.add(url);
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            System.err.println("Error parsing AIA extension for OCSP: " + e.getMessage());
+        }
+
+        return ocspUrls;
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/ssl/CustomPKIXValidator.java

@@ -1,491 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-import javax.net.ssl.*;
-import java.io.FileInputStream;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.*;
-import java.util.*;
-
-/**
- * Custom PKIX validator for validating X.509 certificate chains with verbose output
- * for db export (i.e. not just SSLException).
- */
-public class CustomPKIXValidator {
-
-    private final Set<TrustAnchor> trustAnchors;
-    private final boolean revocationEnabled;
-    private final boolean anyPolicyInhibited;
-    private final boolean explicitPolicyRequired;
-    private final boolean policyMappingInhibited;
-    private final Set<String> initialPolicies;
-
-    private static final Set<String> EV_POLICY_OIDS = Set.of(
-            "1.3.6.1.4.1.17326.10.14.2.1.2", // Entrust
-            "1.3.6.1.4.1.17326.10.8.12.1.2",  // Entrust
-            "2.16.840.1.114028.10.1.2",       // Entrust/AffirmTrust
-            "1.3.6.1.4.1.6449.1.2.1.5.1",     // Comodo
-            "1.3.6.1.4.1.8024.0.2.100.1.2",   // QuoVadis
-            "2.16.840.1.114404.1.1.2.4.1",    // GoDaddy
-            "2.16.840.1.114413.1.7.23.3",     // DigiCert
-            "2.16.840.1.114414.1.7.23.3",     // DigiCert
-            "1.3.6.1.4.1.14370.1.6",          // GlobalSign
-            "2.16.756.1.89.1.2.1.1",          // SwissSign
-            "1.3.6.1.4.1.4146.1.1"            // GlobalSign
-    );
-
-    // Constructor with default settings
-    public CustomPKIXValidator() throws Exception {
-        this(true, false, false, false, null);
-    }
-
-    // Constructor with custom settings
-    public CustomPKIXValidator(boolean revocationEnabled,
-                               boolean anyPolicyInhibited,
-                               boolean explicitPolicyRequired,
-                               boolean policyMappingInhibited,
-                               Set<String> initialPolicies) throws Exception {
-        this.trustAnchors = loadDefaultTrustAnchors();
-        this.revocationEnabled = revocationEnabled;
-        this.anyPolicyInhibited = anyPolicyInhibited;
-        this.explicitPolicyRequired = explicitPolicyRequired;
-        this.policyMappingInhibited = policyMappingInhibited;
-        this.initialPolicies = initialPolicies;
-    }
-
-    // Constructor with custom trust anchors
-    public CustomPKIXValidator(Set<TrustAnchor> customTrustAnchors,
-                               boolean revocationEnabled) {
-        this.trustAnchors = new HashSet<>(customTrustAnchors);
-        this.revocationEnabled = revocationEnabled;
-        this.anyPolicyInhibited = false;
-        this.explicitPolicyRequired = false;
-        this.policyMappingInhibited = false;
-        this.initialPolicies = null;
-    }
-
-    /**
-     * Validates certificate chain using PKIX algorithm
-     */
-    public PKIXValidationResult validateCertificateChain(String hostname, X509Certificate[] certChain) {
-        EnumSet<PkixValidationError> errors = EnumSet.noneOf(PkixValidationError.class);
-        try {
-            // 1. Basic input validation
-            if (certChain == null || certChain.length == 0) {
-                return new PKIXValidationResult(false, "Certificate chain is empty", errors,
-                        null, null, null, false);
-            }
-
-            if (hostname == null || hostname.trim().isEmpty()) {
-                return new PKIXValidationResult(false, "Hostname is null or empty", errors,
-                        null, null, null, false);
-            }
-
-            // 2. Create certificate path
-            CertPath certPath = createCertificatePath(certChain);
-            if (certPath == null) {
-                return new PKIXValidationResult(false, "Failed to create certificate path", errors,
-                        null, null, null, false);
-            }
-
-            // 3. Build and validate certificate path using PKIX
-            PKIXCertPathValidatorResult pkixResult = performPKIXValidation(certPath, errors);
-
-            // 4. Validate hostname
-            boolean hostnameValid = validateHostname(hostname, certChain[0], errors);
-
-            // 5. Extract critical extensions information
-            Set<String> criticalExtensions = extractCriticalExtensions(certChain);
-
-            boolean overallValid = (pkixResult != null) && hostnameValid;
-            String errorMessage = null;
-
-            if (pkixResult == null) {
-                errorMessage = "PKIX path validation failed";
-            } else if (!hostnameValid) {
-                errorMessage = "Hostname validation failed";
-            }
-
-            return new PKIXValidationResult(overallValid, errorMessage, errors,
-                    pkixResult, certPath, criticalExtensions, hostnameValid);
-
-        } catch (Exception e) {
-            return new PKIXValidationResult(false, "Validation exception: " + e.getMessage(),
-                    errors, null, null, null, false);
-        }
-    }
-
-    /**
-     * Creates a certificate path from the certificate chain
-     */
-    private CertPath createCertificatePath(X509Certificate[] certChain) throws CertificateException {
-        CertificateFactory cf = CertificateFactory.getInstance("X.509");
-        List<Certificate> certList = Arrays.asList(certChain);
-        return cf.generateCertPath(certList);
-    }
-
-    /**
-     * Performs PKIX validation
-     */
-    private PKIXCertPathValidatorResult performPKIXValidation(CertPath certPath, Set<PkixValidationError> warnings) {
-        try {
-            // Create PKIX parameters
-            PKIXParameters params = new PKIXParameters(trustAnchors);
-
-            // Configure PKIX parameters
-            params.setRevocationEnabled(revocationEnabled);
-            params.setAnyPolicyInhibited(anyPolicyInhibited);
-            params.setExplicitPolicyRequired(explicitPolicyRequired);
-            params.setPolicyMappingInhibited(policyMappingInhibited);
-
-            if (initialPolicies != null && !initialPolicies.isEmpty()) {
-                params.setInitialPolicies(initialPolicies);
-            }
-
-            // Set up certificate stores for intermediate certificates if needed
-            // This helps with path building when intermediate certs are missing
-            List<Certificate> intermediateCerts = extractIntermediateCertificates(certPath);
-            if (!intermediateCerts.isEmpty()) {
-                CertStore certStore = CertStore.getInstance("Collection",
-                        new CollectionCertStoreParameters(intermediateCerts));
-                params.addCertStore(certStore);
-            }
-
-            // Configure revocation checking if enabled
-            if (revocationEnabled) {
-                configureRevocationChecking(params);
-            }
-
-            // Create and run validator
-            CertPathValidator validator = CertPathValidator.getInstance("PKIX");
-            PKIXCertPathValidatorResult result = (PKIXCertPathValidatorResult)
-                    validator.validate(certPath, params);
-
-            return result;
-
-        } catch (CertPathValidatorException e) {
-            warnings.add(PkixValidationError.PATH_VALIDATION_FAILED);
-            return null;
-        } catch (InvalidAlgorithmParameterException e) {
-            warnings.add(PkixValidationError.INVALID_PKIX_PARAMETERS);
-            return null;
-        } catch (Exception e) {
-            warnings.add(PkixValidationError.UNKNOWN);
-            return null;
-        }
-    }
-
-    /**
-     * Extracts intermediate certificates from the path
-     */
-    private List<Certificate> extractIntermediateCertificates(CertPath certPath) {
-        List<Certificate> certs = (List<Certificate>) certPath.getCertificates();
-        if (certs.size() <= 2) {
-            return new ArrayList<>(); // Only leaf and root, no intermediates
-        }
-        // Return all but the first (leaf) and potentially last (root)
-        return new ArrayList<>(certs.subList(1, certs.size()));
-    }
-
-    /**
-     * Configures revocation checking (CRL/OCSP)
-     */
-    private void configureRevocationChecking(PKIXParameters params) throws NoSuchAlgorithmException {
-        // Create PKIX revocation checker
-        PKIXRevocationChecker revocationChecker = (PKIXRevocationChecker)
-                CertPathValidator.getInstance("PKIX").getRevocationChecker();
-
-        // Configure revocation checker options
-        Set<PKIXRevocationChecker.Option> options = EnumSet.of(
-                PKIXRevocationChecker.Option.PREFER_CRLS,
-                PKIXRevocationChecker.Option.SOFT_FAIL  // Don't fail if revocation info unavailable
-        );
-        revocationChecker.setOptions(options);
-
-        params.addCertPathChecker(revocationChecker);
-    }
-
-    /**
-     * Comprehensive hostname validation including SAN and CN
-     */
-    private boolean validateHostname(String hostname, X509Certificate cert, Set<PkixValidationError> warnings) {
-        try {
-            // Use Java's built-in hostname verifier as a starting point
-            HostnameVerifier defaultVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
-
-            // Create a mock SSL session for the hostname verifier
-            MockSSLSession mockSession = new MockSSLSession(cert);
-
-            boolean defaultResult = defaultVerifier.verify(hostname, mockSession);
-
-            if (defaultResult) {
-                return true;
-            }
-
-            // If default fails, do manual validation
-            return performManualHostnameValidation(hostname, cert, warnings);
-
-        } catch (Exception e) {
-            warnings.add(PkixValidationError.UNSPECIFIED_HOST_ERROR);
-            return false;
-        }
-    }
-
-    /**
-     * Manual hostname validation implementation
-     */
-    private boolean performManualHostnameValidation(String hostname, X509Certificate cert, Set<PkixValidationError> warnings) {
-        try {
-            // 1. Check Subject Alternative Names (SAN) - preferred method
-            Collection<List<?>> sanEntries = cert.getSubjectAlternativeNames();
-            if (sanEntries != null) {
-                for (List<?> sanEntry : sanEntries) {
-                    if (sanEntry.size() >= 2) {
-                        Integer type = (Integer) sanEntry.get(0);
-                        if (type == 2) { // DNS name
-                            String dnsName = (String) sanEntry.get(1);
-                            if (matchesHostname(hostname, dnsName)) {
-                                return true;
-                            }
-                        } else if (type == 7) { // IP address
-                            String ipAddress = (String) sanEntry.get(1);
-                            if (hostname.equals(ipAddress)) {
-                                return true;
-                            }
-                        }
-                    }
-                }
-                // If SAN is present but no match found, don't check CN (RFC 6125)
-                warnings.add(PkixValidationError.SAN_MISMATCH);
-                return false;
-            }
-
-            // 2. Fallback to Common Name (CN) in subject if no SAN present
-            String subjectDN = cert.getSubjectDN().getName();
-            String cn = extractCommonName(subjectDN);
-            if (cn != null) {
-                if (matchesHostname(hostname, cn)) {
-                    return true;
-                }
-            }
-
-            warnings.add(PkixValidationError.SAN_MISMATCH);
-            return false;
-
-        } catch (Exception e) {
-            warnings.add(PkixValidationError.UNKNOWN);
-            return false;
-        }
-    }
-
-    /**
-     * Checks if hostname matches certificate name (handles wildcards)
-     */
-    private boolean matchesHostname(String hostname, String certName) {
-        if (hostname == null || certName == null) {
-            return false;
-        }
-
-        hostname = hostname.toLowerCase();
-        certName = certName.toLowerCase();
-
-        // Exact match
-        if (hostname.equals(certName)) {
-            return true;
-        }
-
-        // Wildcard matching (*.example.com)
-        if (certName.startsWith("*.")) {
-            String domain = certName.substring(2);
-
-            // Wildcard must match exactly one level
-            if (hostname.endsWith("." + domain)) {
-                String prefix = hostname.substring(0, hostname.length() - domain.length() - 1);
-                // Ensure wildcard doesn't match multiple levels (no dots in prefix)
-                return !prefix.contains(".");
-            }
-        }
-
-        return false;
-    }
-
-    /**
-     * Extracts Common Name from Subject DN
-     */
-    private String extractCommonName(String subjectDN) {
-        if (subjectDN == null) {
-            return null;
-        }
-
-        // Parse DN components
-        String[] components = subjectDN.split(",");
-        for (String component : components) {
-            component = component.trim();
-            if (component.startsWith("CN=")) {
-                return component.substring(3).trim();
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Extracts critical extensions from all certificates in the chain
-     */
-    private Set<String> extractCriticalExtensions(X509Certificate[] certChain) {
-        Set<String> allCriticalExtensions = new HashSet<>();
-
-        for (X509Certificate cert : certChain) {
-            Set<String> criticalExtensions = cert.getCriticalExtensionOIDs();
-            if (criticalExtensions != null) {
-                allCriticalExtensions.addAll(criticalExtensions);
-            }
-        }
-
-        return allCriticalExtensions;
-    }
-
-
-    /**
-     * Gets the key length from a certificate
-     */
-    private int getKeyLength(X509Certificate cert) {
-        try {
-            java.security.PublicKey publicKey = cert.getPublicKey();
-            if (publicKey instanceof java.security.interfaces.RSAPublicKey) {
-                return ((java.security.interfaces.RSAPublicKey) publicKey).getModulus().bitLength();
-            } else if (publicKey instanceof java.security.interfaces.DSAPublicKey) {
-                return ((java.security.interfaces.DSAPublicKey) publicKey).getParams().getP().bitLength();
-            } else if (publicKey instanceof java.security.interfaces.ECPublicKey) {
-                return ((java.security.interfaces.ECPublicKey) publicKey).getParams().getOrder().bitLength();
-            }
-        } catch (Exception e) {
-            // Ignore
-        }
-        return -1;
-    }
-
-    /**
-     * Checks if signature algorithm is considered weak
-     */
-    private boolean isWeakSignatureAlgorithm(String sigAlg) {
-        if (sigAlg == null) return false;
-
-        sigAlg = sigAlg.toLowerCase();
-        return sigAlg.contains("md5") ||
-                sigAlg.contains("sha1") ||
-                sigAlg.equals("md2withrsa") ||
-                sigAlg.equals("md4withrsa");
-    }
-
-    /**
-     * Checks for deprecated or problematic extensions
-     */
-    private void checkDeprecatedExtensions(X509Certificate cert, int index, List<String> warnings) {
-        // Check for Netscape extensions (deprecated)
-        if (cert.getNonCriticalExtensionOIDs() != null) {
-            for (String oid : cert.getNonCriticalExtensionOIDs()) {
-                if (oid.startsWith("2.16.840.1.113730")) { // Netscape OID space
-                    warnings.add("Certificate " + index + " contains deprecated Netscape extension: " + oid);
-                }
-            }
-        }
-
-        // Additional extension checks can be added here
-    }
-
-    /**
-     * Loads default trust anchors from Java's cacerts keystore
-     */
-    private Set<TrustAnchor> loadDefaultTrustAnchors() throws Exception {
-        Set<TrustAnchor> trustAnchors = new HashSet<>();
-
-        // Try to load from default locations
-        String[] keystorePaths = {
-                System.getProperty("javax.net.ssl.trustStore"),
-                System.getProperty("java.home") + "/lib/security/cacerts",
-                System.getProperty("java.home") + "/jre/lib/security/cacerts"
-        };
-
-        String[] keystorePasswords = {
-                System.getProperty("javax.net.ssl.trustStorePassword"),
-                "changeit",
-                ""
-        };
-
-        for (String keystorePath : keystorePaths) {
-            if (keystorePath != null) {
-                for (String password : keystorePasswords) {
-                    try {
-                        KeyStore trustStore = loadKeyStore(keystorePath, password);
-                        if (trustStore != null) {
-                            trustAnchors.addAll(extractTrustAnchors(trustStore));
-                            if (!trustAnchors.isEmpty()) {
-                                return trustAnchors;
-                            }
-                        }
-                    } catch (Exception e) {
-                        // Try next combination
-                    }
-                }
-            }
-        }
-
-        // Fallback: try to get from default trust manager
-        try {
-            TrustManagerFactory tmf = TrustManagerFactory.getInstance(
-                    TrustManagerFactory.getDefaultAlgorithm());
-            tmf.init((KeyStore) null);
-
-            for (TrustManager tm : tmf.getTrustManagers()) {
-                if (tm instanceof X509TrustManager) {
-                    X509TrustManager x509tm = (X509TrustManager) tm;
-                    for (X509Certificate cert : x509tm.getAcceptedIssuers()) {
-                        trustAnchors.add(new TrustAnchor(cert, null));
-                    }
-                }
-            }
-        } catch (Exception e) {
-            throw new Exception("Failed to load any trust anchors", e);
-        }
-
-        if (trustAnchors.isEmpty()) {
-            throw new Exception("No trust anchors could be loaded");
-        }
-
-        return trustAnchors;
-    }
-
-    /**
-     * Loads a keystore from file
-     */
-    private KeyStore loadKeyStore(String keystorePath, String password) throws Exception {
-        KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType());
-        try (FileInputStream fis = new FileInputStream(keystorePath)) {
-            keystore.load(fis, password != null ? password.toCharArray() : null);
-            return keystore;
-        }
-    }
-
-    /**
-     * Extracts trust anchors from a keystore
-     */
-    private Set<TrustAnchor> extractTrustAnchors(KeyStore trustStore) throws KeyStoreException {
-        Set<TrustAnchor> trustAnchors = new HashSet<>();
-
-        Enumeration<String> aliases = trustStore.aliases();
-        while (aliases.hasMoreElements()) {
-            String alias = aliases.nextElement();
-            if (trustStore.isCertificateEntry(alias)) {
-                Certificate cert = trustStore.getCertificate(alias);
-                if (cert instanceof X509Certificate) {
-                    trustAnchors.add(new TrustAnchor((X509Certificate) cert, null));
-                }
-            }
-        }
-
-        return trustAnchors;
-    }
-
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/MockSSLSession.java

@@ -1,116 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSessionContext;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-
-/**
- * Mock SSL session for hostname verification
- */
-public class MockSSLSession implements SSLSession {
-    private final X509Certificate[] peerCertificates;
-
-    public MockSSLSession(X509Certificate cert) {
-        this.peerCertificates = new X509Certificate[]{cert};
-    }
-
-    @Override
-    public Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException {
-        return peerCertificates;
-    }
-
-    // All other methods return default/empty values as they're not used by hostname verification
-    @Override
-    public byte[] getId() {
-        return new byte[0];
-    }
-
-    @Override
-    public SSLSessionContext getSessionContext() {
-        return null;
-    }
-
-    @Override
-    public long getCreationTime() {
-        return 0;
-    }
-
-    @Override
-    public long getLastAccessedTime() {
-        return 0;
-    }
-
-    @Override
-    public void invalidate() {
-    }
-
-    @Override
-    public boolean isValid() {
-        return true;
-    }
-
-    @Override
-    public void putValue(String name, Object value) {
-    }
-
-    @Override
-    public Object getValue(String name) {
-        return null;
-    }
-
-    @Override
-    public void removeValue(String name) {
-    }
-
-    @Override
-    public String[] getValueNames() {
-        return new String[0];
-    }
-
-    @Override
-    public java.security.Principal getPeerPrincipal() throws SSLPeerUnverifiedException {
-        return null;
-    }
-
-    @Override
-    public java.security.Principal getLocalPrincipal() {
-        return null;
-    }
-
-    @Override
-    public String getCipherSuite() {
-        return "";
-    }
-
-    @Override
-    public String getProtocol() {
-        return "";
-    }
-
-    @Override
-    public String getPeerHost() {
-        return "";
-    }
-
-    @Override
-    public int getPeerPort() {
-        return 0;
-    }
-
-    @Override
-    public int getPacketBufferSize() {
-        return 0;
-    }
-
-    @Override
-    public int getApplicationBufferSize() {
-        return 0;
-    }
-
-    @Override
-    public Certificate[] getLocalCertificates() {
-        return new Certificate[0];
-    }
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/PKIXValidationResult.java

@@ -1,14 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-import java.security.cert.CertPath;
-import java.security.cert.PKIXCertPathValidatorResult;
-import java.util.Set;
-
-public record PKIXValidationResult(boolean isValid, String errorMessage,
-                                   Set<PkixValidationError> errors,
-                                   PKIXCertPathValidatorResult pkixResult,
-                                   CertPath validatedPath,
-                                   Set<String> criticalExtensions,
-                                   boolean hostnameValid)
-{
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/PkixValidationError.java

@@ -1,11 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-public enum PkixValidationError {
-    SAN_MISMATCH,
-    EXPIRED,
-    NOT_YET_VALID,
-    PATH_VALIDATION_FAILED,
-    INVALID_PKIX_PARAMETERS,
-    UNKNOWN,
-    UNSPECIFIED_HOST_ERROR;
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/RootCerts.java

@@ -0,0 +1,57 @@
+package nu.marginalia.ping.ssl;
+
+import java.security.cert.TrustAnchor;
+import java.time.Duration;
+import java.util.Set;
+
+public class RootCerts {
+    private static final String MOZILLA_CA_BUNDLE_URL = "https://curl.se/ca/cacert.pem";
+
+    volatile static boolean initialized = false;
+    volatile static Set<TrustAnchor> trustAnchors;
+
+    public static Set<TrustAnchor> getTrustAnchors() {
+        if (!initialized) {
+            try {
+                synchronized (RootCerts.class) {
+                    while (!initialized) {
+                        RootCerts.class.wait(100);
+                    }
+                }
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException("RootCerts initialization interrupted", e);
+            }
+        }
+        return trustAnchors;
+    }
+
+    static {
+        Thread.ofPlatform()
+                .name("RootCertsUpdater")
+                .daemon()
+                .unstarted(RootCerts::updateTrustAnchors)
+                .start();
+    }
+
+    private static void updateTrustAnchors() {
+        while (true) {
+            try {
+                trustAnchors = CertificateFetcher.getRootCerts(MOZILLA_CA_BUNDLE_URL);
+                synchronized (RootCerts.class) {
+                    initialized = true;
+                    RootCerts.class.notifyAll(); // Notify any waiting threads
+                }
+                Thread.sleep(Duration.ofHours(24));
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break; // Exit if interrupted
+            } catch (Exception e) {
+                // Log the exception and continue to retry
+                System.err.println("Failed to update trust anchors: " + e.getMessage());
+            }
+        }
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/svc/DomainAvailabilityInformationFactory.java

@@ -9,7 +9,7 @@ import nu.marginalia.ping.fetcher.response.HttpsResponse;
 import nu.marginalia.ping.model.DomainAvailabilityRecord;
 import nu.marginalia.ping.model.ErrorClassification;
 import nu.marginalia.ping.model.HttpSchema;
-import nu.marginalia.ping.ssl.PKIXValidationResult;
+import nu.marginalia.ping.ssl.CertificateValidator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -96,6 +96,7 @@ public class DomainAvailabilityInformationFactory {
                 .serverIp(address != null ? address.getAddress() : null)
                 .serverIpAsn(getAsn(address))
                 .httpSchema(HttpSchema.HTTP)
+                .httpLocation(rsp.headers().getFirst("Location"))
                 .httpStatus(rsp.httpStatus())
                 .errorClassification(errorClassification)
                 .httpResponseTime(rsp.httpResponseTime())
@@ -123,7 +124,7 @@ public class DomainAvailabilityInformationFactory {
                                                         int nodeId,
                                                         @Nullable InetAddress address,
                                                         @Nullable DomainAvailabilityRecord previousRecord,
-                                                        PKIXValidationResult validationResult,
+                                                        CertificateValidator.ValidationResult validationResult,
                                                         HttpsResponse rsp) {
         Instant updateTime;
 
@@ -164,6 +165,7 @@ public class DomainAvailabilityInformationFactory {
                 .serverIp(address != null ? address.getAddress() : null)
                 .serverIpAsn(getAsn(address))
                 .httpSchema(HttpSchema.HTTPS)
+                .httpLocation(rsp.headers().getFirst("Location"))
                 .httpStatus(rsp.httpStatus())
                 .errorClassification(errorClassification)
                 .httpResponseTime(rsp.httpResponseTime()) // Placeholder, actual timing not implemented

code/processes/ping-process/java/nu/marginalia/ping/svc/DomainSecurityInformationFactory.java

@@ -4,7 +4,7 @@ import nu.marginalia.ping.fetcher.response.HttpResponse;
 import nu.marginalia.ping.fetcher.response.HttpsResponse;
 import nu.marginalia.ping.model.DomainSecurityRecord;
 import nu.marginalia.ping.model.HttpSchema;
-import nu.marginalia.ping.ssl.PKIXValidationResult;
+import nu.marginalia.ping.ssl.CertificateValidator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -14,9 +14,7 @@ import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateEncodingException;
 import java.security.cert.X509Certificate;
 import java.time.Instant;
-import java.util.HashSet;
+import java.util.*;
-import java.util.Set;
-import java.util.StringJoiner;
 
 public class DomainSecurityInformationFactory {
     private static final Logger logger = LoggerFactory.getLogger(DomainSecurityInformationFactory.class);
@@ -54,7 +52,7 @@ public class DomainSecurityInformationFactory {
     // HTTPS response
     public DomainSecurityRecord createHttpsSecurityInformation(
             HttpsResponse httpResponse,
-            PKIXValidationResult validationResult,
+            CertificateValidator.ValidationResult validationResult,
             int domainId,
             int nodeId,
             @Nullable Integer asn
@@ -69,8 +67,11 @@ public class DomainSecurityInformationFactory {
         boolean isWildcard = false;
         try {
             if (sslCertificates != null && sslCertificates.length > 0) {
-                for (var sanEntry : sslCertificates[0].getSubjectAlternativeNames()) {
+                Collection<List<?>> sans = sslCertificates[0].getSubjectAlternativeNames();
-
+                if (sans == null) {
+                    sans = Collections.emptyList();
+                }
+                for (var sanEntry : sans) {
 
                     if (sanEntry != null && sanEntry.size() >= 2) {
                         // Check if the SAN entry is a DNS or IP address
@@ -125,6 +126,9 @@ public class DomainSecurityInformationFactory {
                 .sslCertWildcard(isWildcard)
                 .sslCertificateChainLength(sslCertificates.length)
                 .sslCertificateValid(validationResult.isValid())
+                .sslHostValid(validationResult.hostnameValid)
+                .sslChainValid(validationResult.chainValid)
+                .sslDateValid(!validationResult.certificateExpired)
                 .httpVersion(httpResponse.version())
                 .tsLastUpdate(Instant.now())
                 .build();

code/processes/ping-process/java/nu/marginalia/ping/svc/HttpPingService.java

@@ -2,13 +2,13 @@ package nu.marginalia.ping.svc;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import nu.marginalia.coordination.DomainCoordinator;
 import nu.marginalia.ping.fetcher.PingHttpFetcher;
 import nu.marginalia.ping.fetcher.response.*;
 import nu.marginalia.ping.model.*;
 import nu.marginalia.ping.model.comparison.DomainAvailabilityChange;
 import nu.marginalia.ping.model.comparison.SecurityInformationChange;
-import nu.marginalia.ping.ssl.CustomPKIXValidator;
+import nu.marginalia.ping.ssl.CertificateValidator;
-import nu.marginalia.ping.ssl.PKIXValidationResult;
 import nu.marginalia.ping.util.JsonObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,23 +26,23 @@ import java.util.List;
 @Singleton
 public class HttpPingService {
 
+    private final DomainCoordinator domainCoordinator;
     private final PingHttpFetcher pingHttpFetcher;
 
     private final DomainAvailabilityInformationFactory domainAvailabilityInformationFactory;
     private final DomainSecurityInformationFactory domainSecurityInformationFactory;
-
     private static final Logger logger = LoggerFactory.getLogger(HttpPingService.class);
-    CustomPKIXValidator validator;
 
     @Inject
     public HttpPingService(
+            DomainCoordinator domainCoordinator,
             PingHttpFetcher pingHttpFetcher,
             DomainAvailabilityInformationFactory domainAvailabilityInformationFactory,
             DomainSecurityInformationFactory domainSecurityInformationFactory) throws Exception {
+        this.domainCoordinator = domainCoordinator;
         this.pingHttpFetcher = pingHttpFetcher;
         this.domainAvailabilityInformationFactory = domainAvailabilityInformationFactory;
         this.domainSecurityInformationFactory = domainSecurityInformationFactory;
-        this.validator = new CustomPKIXValidator();
     }
 
     private int compareInetAddresses(InetAddress a, InetAddress b) {
@@ -59,7 +59,8 @@ public class HttpPingService {
 
     public List<WritableModel> pingDomain(DomainReference domainReference,
                            @Nullable DomainAvailabilityRecord oldPingStatus,
-                           @Nullable DomainSecurityRecord oldSecurityInformation) throws SQLException {
+                           @Nullable DomainSecurityRecord oldSecurityInformation) throws SQLException, InterruptedException {
+
         // First we figure out if the domain maps to an IP address
 
         List<WritableModel> generatedRecords = new ArrayList<>();
@@ -69,26 +70,31 @@ public class HttpPingService {
 
         if (ipAddress.isEmpty()) {
             result = new UnknownHostError();
-        }
+        } else {
-        else {
+            // lock the domain to prevent concurrent pings
-            String url = "https://" + domainReference.domainName() + "/";
+            try (var _ = domainCoordinator.lockDomain(domainReference.asEdgeDomain())) {
-            String alternateUrl = "http://" + domainReference.domainName() + "/";
+                String url = "https://" + domainReference.domainName() + "/";
+                String alternateUrl = "http://" + domainReference.domainName() + "/";
 
-            result = pingHttpFetcher.fetchUrl(url, Method.HEAD, null, null);
+                result = pingHttpFetcher.fetchUrl(url, Method.HEAD, null, null);
 
-            if (result instanceof HttpsResponse response && shouldTryGET(response.httpStatus())) {
+                if (result instanceof HttpsResponse response && shouldTryGET(response.httpStatus())) {
-                sleep(Duration.ofSeconds(2));
-                result = pingHttpFetcher.fetchUrl(url, Method.GET, null, null);
-            }
-            else if (result instanceof ConnectionError) {
-                var result2 = pingHttpFetcher.fetchUrl(alternateUrl, Method.HEAD, null, null);
-                if (!(result2 instanceof ConnectionError)) {
-                    result = result2;
-                }
-                if (result instanceof HttpResponse response && shouldTryGET(response.httpStatus())) {
                     sleep(Duration.ofSeconds(2));
-                    result = pingHttpFetcher.fetchUrl(alternateUrl, Method.GET, null, null);
+                    result = pingHttpFetcher.fetchUrl(url, Method.GET, null, null);
+                } else if (result instanceof ConnectionError) {
+                    var result2 = pingHttpFetcher.fetchUrl(alternateUrl, Method.HEAD, null, null);
+                    if (!(result2 instanceof ConnectionError)) {
+                        result = result2;
+                    }
+                    if (result instanceof HttpResponse response && shouldTryGET(response.httpStatus())) {
+                        sleep(Duration.ofSeconds(2));
+                        result = pingHttpFetcher.fetchUrl(alternateUrl, Method.GET, null, null);
+                    }
                 }
+
+                // Add a grace sleep before we yield the semaphore, so that another thread doesn't
+                // immediately hammer the same domain after it's released.
+                sleep(Duration.ofSeconds(1));
             }
         }
 
@@ -135,7 +141,7 @@ public class HttpPingService {
                         domainReference.nodeId(),
                         oldPingStatus,
                         ErrorClassification.HTTP_CLIENT_ERROR,
-                        null);
+                        rsp.errorMessage());
                 newSecurityInformation = null;
             }
             case HttpResponse httpResponse -> {
@@ -154,7 +160,11 @@ public class HttpPingService {
                 );
             }
             case HttpsResponse httpsResponse -> {
-                PKIXValidationResult validationResult = validator.validateCertificateChain(domainReference.domainName(), (X509Certificate[]) httpsResponse.sslCertificates());
+                var validationResult = CertificateValidator.validateCertificate(
+                        (X509Certificate[]) httpsResponse.sslCertificates(),
+                        domainReference.domainName(),
+                        true
+                        );
 
                 newPingStatus = domainAvailabilityInformationFactory.createHttpsResponse(
                         domainReference.domainId(),
@@ -186,8 +196,8 @@ public class HttpPingService {
         }
         if (oldSecurityInformation != null && newSecurityInformation != null) {
             compareSecurityInformation(generatedRecords,
-                                       oldSecurityInformation, oldPingStatus,
+                    oldSecurityInformation, oldPingStatus,
-                                       newSecurityInformation, newPingStatus);
+                    newSecurityInformation, newPingStatus);
         }
 
         return generatedRecords;
@@ -286,6 +296,7 @@ public class HttpPingService {
                 change.isCertificatePublicKeyChanged(),
                 change.isCertificateSerialNumberChanged(),
                 change.isCertificateIssuerChanged(),
+                change.schemaChange(),
                 change.oldCertificateTimeToExpiry(),
                 change.isSecurityHeadersChanged(),
                 change.isIpAddressChanged(),

code/processes/ping-process/test/nu/marginalia/ping/AvailabilityJobSchedulerTest.java

@@ -2,6 +2,7 @@ package nu.marginalia.ping;
 
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.coordination.LocalDomainCoordinator;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.ping.fetcher.PingDnsFetcher;
 import nu.marginalia.ping.fetcher.PingHttpFetcher;
@@ -85,11 +86,14 @@ class AvailabilityJobSchedulerTest {
         DomainDnsInformationFactory dnsDomainInformationFactory = new DomainDnsInformationFactory(processConfig, pic);
 
         PingJobScheduler pingJobScheduler = new PingJobScheduler(
-                new HttpPingService(pingHttpFetcher,
+                new HttpPingService(
+                        new LocalDomainCoordinator(),
+                        pingHttpFetcher,
                         new DomainAvailabilityInformationFactory(new GeoIpDictionary(), new BackoffStrategy(pic)),
                         new DomainSecurityInformationFactory()),
                 new DnsPingService(new PingDnsFetcher(List.of("8.8.8.8", "8.8.4.4")),
                         dnsDomainInformationFactory),
+                new LocalDomainCoordinator(),
                 pingDao
         );
 

code/processes/ping-process/test/nu/marginalia/ping/PingDaoTest.java

@@ -243,6 +243,7 @@ class PingDaoTest {
                 .headerServer("Apache/2.4.41 (Ubuntu)")
                 .headerXPoweredBy("PHP/7.4.3")
                 .tsLastUpdate(Instant.now())
+                .sslHostValid(true)
                 .build();
         var svc = new PingDao(dataSource);
         svc.write(foo);
@@ -320,6 +321,7 @@ class PingDaoTest {
                 true,
                 true,
                 false,
+                SchemaChange.NONE,
                 Duration.ofDays(30),
                 false,
                 false,

code/processes/ping-process/test/nu/marginalia/ping/PingHttpServiceTest.java

@@ -2,6 +2,7 @@ package nu.marginalia.ping;
 
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.coordination.LocalDomainCoordinator;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.ping.fetcher.PingHttpFetcher;
 import nu.marginalia.ping.io.HttpClientProvider;
@@ -59,10 +60,12 @@ class PingHttpServiceTest {
 
     }
 
+    @Tag("flaky") // Do not run this test in CI
     @Test
     public void testGetSslInfo() throws Exception {
         var provider = new HttpClientProvider();
         var pingService = new HttpPingService(
+                new LocalDomainCoordinator(),
                 new PingHttpFetcher(provider.get()),
                 new DomainAvailabilityInformationFactory(new GeoIpDictionary(),
                         new BackoffStrategy(PingModule.createPingIntervalsConfiguration())

code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java

@@ -22,6 +22,7 @@ import nu.marginalia.search.model.NavbarModel;
 import nu.marginalia.search.model.ResultsPage;
 import nu.marginalia.search.model.UrlDetails;
 import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData;
+import nu.marginalia.service.server.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,6 +48,8 @@ public class SearchSiteInfoService {
     private final HikariDataSource dataSource;
     private final SearchSiteSubscriptionService searchSiteSubscriptions;
 
+    private final RateLimiter rateLimiter = RateLimiter.custom(60);
+
     @Inject
     public SearchSiteInfoService(SearchOperator searchOperator,
                                  DomainInfoClient domainInfoClient,
@@ -238,6 +241,7 @@ public class SearchSiteInfoService {
         boolean hasScreenshot = screenshotService.hasScreenshot(domainId);
         boolean isSubscribed = searchSiteSubscriptions.isSubscribed(context, domain);
 
+        boolean rateLimited = !rateLimiter.isAllowed();
         if (domainId < 0) {
             domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
             similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
@@ -250,6 +254,12 @@ public class SearchSiteInfoService {
             linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
             feedItemsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
         }
+        else if (rateLimited) {
+            domainInfoFuture = domainInfoClient.domainInformation(domainId);
+            similarSetFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+        }
         else {
             domainInfoFuture = domainInfoClient.domainInformation(domainId);
             similarSetFuture = domainInfoClient.similarDomains(domainId, 25);
@@ -257,7 +267,14 @@ public class SearchSiteInfoService {
             feedItemsFuture = feedsClient.getFeed(domainId);
         }
 
-        List<UrlDetails> sampleResults = searchOperator.doSiteSearch(domainName, domainId,5, 1).results;
+        List<UrlDetails> sampleResults;
+        if (rateLimited) {
+            sampleResults = List.of();
+        }
+        else {
+            sampleResults = searchOperator.doSiteSearch(domainName, domainId, 5, 1).results;
+        }
+
         if (!sampleResults.isEmpty()) {
             url = sampleResults.getFirst().url.withPathAndParam("/", null).toString();
         }
@@ -276,8 +293,9 @@ public class SearchSiteInfoService {
                 sampleResults
         );
 
-        requestMissingScreenshots(result);
+        if (!rateLimited) {
-
+            requestMissingScreenshots(result);
+        }
         return result;
     }
 

code/services-application/search-service/resources/jte/unavailable.jte

@@ -0,0 +1,11 @@
+@param String message
+
+<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8">
+<title>Unavailable</title></head>
+<body>
+<h1>Service Overloaded</h1>
+<p>${message}</p>
+</body>
+</html>

code/services-core/assistant-service/build.gradle

@@ -37,6 +37,7 @@ dependencies {
     implementation project(':code:functions:domain-info')
     implementation project(':code:functions:domain-info:api')
 
+    implementation project(':code:libraries:domain-lock')
     implementation project(':code:libraries:geo-ip')
     implementation project(':code:libraries:language-processing')
     implementation project(':code:libraries:term-frequency-dict')

code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import io.jooby.ExecutionMode;
 import io.jooby.Jooby;
+import nu.marginalia.coordination.DomainCoordinationModule;
 import nu.marginalia.livecapture.LivecaptureModule;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.ServiceId;
@@ -29,6 +30,7 @@ public class AssistantMain extends MainClass {
         Injector injector = Guice.createInjector(
                 new AssistantModule(),
                 new LivecaptureModule(),
+                new DomainCoordinationModule(),
                 new ServiceConfigurationModule(ServiceId.Assistant),
                 new ServiceDiscoveryModule(),
                 new DatabaseModule(false)

settings.gradle

@@ -55,6 +55,7 @@ include 'code:libraries:braille-block-punch-cards'
 include 'code:libraries:language-processing'
 include 'code:libraries:term-frequency-dict'
 include 'code:libraries:test-helpers'
+include 'code:libraries:domain-lock'
 
 include 'code:libraries:message-queue'