(ping) Persist more detailed information about why a cert is invalid

The change also alters the validator to be less judgemental, and accept some invalid chains based on looking like we've simply not got access to a (valid) intermediate cert.

code/common/db/resources/db/migration/V25_01_0_003__ping_domains__dsi_cert_details.sql

@@ -0,0 +1,7 @@
+-- Add additional summary columns to DOMAIN_SECURITY_INFORMATION table
+-- to make it easier to get more information about the SSL certificate's validity
+
+ALTER TABLE DOMAIN_SECURITY_INFORMATION ADD COLUMN SSL_CHAIN_VALID BOOLEAN DEFAULT NULL;
+ALTER TABLE DOMAIN_SECURITY_INFORMATION ADD COLUMN SSL_HOST_VALID BOOLEAN DEFAULT NULL;
+ALTER TABLE DOMAIN_SECURITY_INFORMATION ADD COLUMN SSL_DATE_VALID BOOLEAN DEFAULT NULL;
+OPTIMIZE TABLE DOMAIN_SECURITY_INFORMATION;

code/common/db/resources/db/migration/V25_01_0_004__ping_domains__schema_change.sql

@@ -0,0 +1,5 @@
+-- Add additional summary columns to DOMAIN_SECURITY_EVENTS table
+-- to make it easier to make sense of certificate changes
+
+ALTER TABLE DOMAIN_SECURITY_EVENTS ADD COLUMN CHANGE_SCHEMA ENUM('NONE', 'HTTP_TO_HTTPS', 'HTTPS_TO_HTTP', 'UNKNOWN') NOT NULL DEFAULT 'UNKNOWN';
+OPTIMIZE TABLE DOMAIN_SECURITY_EVENTS;

code/libraries/domain-lock/java/nu/marginalia/coordination/LocalDomainCoordinator.java

@@ -1,5 +1,6 @@
 package nu.marginalia.coordination;
 
+import com.google.inject.Singleton;
 import nu.marginalia.model.EdgeDomain;
 
 import java.time.Duration;
@@ -9,6 +10,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+@Singleton
 public class LocalDomainCoordinator implements DomainCoordinator {
     // The locks are stored in a map, with the domain name as the key.  This map will grow
     // relatively big, but should be manageable since the number of domains is limited to

code/libraries/domain-lock/java/nu/marginalia/coordination/ZookeeperDomainCoordinator.java

@@ -1,6 +1,7 @@
 package nu.marginalia.coordination;
 
 import com.google.inject.Inject;
+import com.google.inject.Singleton;
 import com.google.inject.name.Named;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
@@ -13,11 +14,14 @@ import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
+@Singleton
 public class ZookeeperDomainCoordinator implements DomainCoordinator {
     // The locks are stored in a map, with the domain name as the key.  This map will grow
     // relatively big, but should be manageable since the number of domains is limited to
     // a few hundred thousand typically.
     private final Map<String, InterProcessSemaphoreV2> locks = new ConcurrentHashMap<>();
+    private final Map<String, Integer> waitCounts = new ConcurrentHashMap<>();
+
     private final ServiceRegistryIf serviceRegistry;
     private final int nodeId;
 
@@ -32,27 +36,35 @@ public class ZookeeperDomainCoordinator implements DomainCoordinator {
      * and may be held by another thread.  The caller is responsible for locking and  releasing the lock.
      */
     public DomainLock lockDomain(EdgeDomain domain) throws InterruptedException {
-        var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::createSemapore);
+        final String key = domain.topDomain.toLowerCase();
+        var sem = locks.computeIfAbsent(key, this::createSemapore);
 
+        // Increment or add a wait count for the domain
+        waitCounts.compute(key, (k,value) -> (value == null ? 1 : value + 1));
         try {
-            var lease = sem.acquire();
-
-            return new ZkDomainLock(sem, lease);
+            return new ZkDomainLock(sem, sem.acquire());
         }
         catch (Exception e) {
             throw new RuntimeException("Failed to acquire lock for domain: " + domain.topDomain, e);
         }
+        finally {
+            // Decrement or remove the wait count for the domain
+            waitCounts.compute(key, (k,value) -> (value == null || value <= 1) ? null : value - 1);
+        }
     }
 
+
     public Optional<DomainLock> tryLockDomain(EdgeDomain domain) throws InterruptedException {
         return tryLockDomain(domain, Duration.ofSeconds(1)); // Underlying semaphore doesn't have a tryLock method, so we use a short timeout
     }
 
 
     public Optional<DomainLock> tryLockDomain(EdgeDomain domain, Duration timeout) throws InterruptedException {
+        final String key = domain.topDomain.toLowerCase();
+        var sem = locks.computeIfAbsent(key, this::createSemapore);
 
-        var sem = locks.computeIfAbsent(domain.topDomain.toLowerCase(), this::createSemapore);
-
+        // Increment or add a wait count for the domain
+        waitCounts.compute(key, (k,value) -> (value == null ? 1 : value + 1));
         try {
             var lease = sem.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS); // Acquire with timeout
             if (lease != null) {
@@ -65,6 +77,9 @@ public class ZookeeperDomainCoordinator implements DomainCoordinator {
         catch (Exception e) {
             return Optional.empty(); // If we fail to acquire the lock, we return an empty optional
         }
+        finally {
+            waitCounts.compute(key, (k,value) -> (value == null || value <= 1) ? null : value - 1);
+        }
     }
 
     private InterProcessSemaphoreV2 createSemapore(String topDomain){
@@ -81,7 +96,7 @@ public class ZookeeperDomainCoordinator implements DomainCoordinator {
      * after this method returns true)
      */
     public boolean isLockableHint(EdgeDomain domain) {
-        return true; // Curator does not provide a way to check if a lock is available without acquiring it
+        return !waitCounts.containsKey(domain.topDomain.toLowerCase());
     }
 
     public static class ZkDomainLock implements DomainLock {

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -36,7 +36,6 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
 import org.apache.hc.core5.http.message.MessageSupport;
 import org.apache.hc.core5.http.protocol.HttpContext;
 import org.apache.hc.core5.pool.PoolStats;
-import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.apache.hc.core5.util.TimeValue;
 import org.apache.hc.core5.util.Timeout;
 import org.jsoup.Jsoup;
@@ -49,15 +48,12 @@ import org.slf4j.MarkerFactory;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.net.URISyntaxException;
 import java.net.UnknownHostException;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
-import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
@@ -99,42 +95,12 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
                 .setValidateAfterInactivity(TimeValue.ofSeconds(5))
                 .build();
 
-        // No-op up front validation of server certificates.
-        //
-        // We will validate certificates later, after the connection is established
-        // as we want to store the certificate chain and validation
-        // outcome to the database.
-
-        var trustMeBro = new X509TrustManager() {
-            private X509Certificate[] lastServerCertChain;
-
-            @Override
-            public void checkClientTrusted(X509Certificate[] chain, String authType) {
-            }
-
-            @Override
-            public void checkServerTrusted(X509Certificate[] chain, String authType) {
-                this.lastServerCertChain = chain.clone();
-            }
-
-            @Override
-            public X509Certificate[] getAcceptedIssuers() {
-                return new X509Certificate[0];
-            }
-
-            public X509Certificate[] getLastServerCertChain() {
-                return lastServerCertChain != null ? lastServerCertChain.clone() : null;
-            }
-        };
-
-        SSLContext sslContext = SSLContextBuilder.create().build();
-        sslContext.init(null, new TrustManager[]{trustMeBro}, null);
 
         connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
                 .setMaxConnPerRoute(2)
                 .setMaxConnTotal(5000)
                 .setDefaultConnectionConfig(connectionConfig)
-                .setTlsSocketStrategy(new DefaultClientTlsStrategy(sslContext))
+                .setTlsSocketStrategy(new DefaultClientTlsStrategy(SSLContext.getDefault()))
                 .build();
 
         connectionManager.setDefaultSocketConfig(SocketConfig.custom()

code/processes/ping-process/java/nu/marginalia/ping/PingDao.java

@@ -185,12 +185,12 @@ public class PingDao {
         return null;
     }
 
-    public List<UpdateSchedule.UpdateJob<Long, HistoricalAvailabilityData>> getDomainUpdateSchedule(int nodeId) {
-        List<UpdateSchedule.UpdateJob<Long, HistoricalAvailabilityData>> updateJobs = new ArrayList<>();
+    public List<UpdateSchedule.UpdateJob<DomainReference, HistoricalAvailabilityData>> getDomainUpdateSchedule(int nodeId) {
+        List<UpdateSchedule.UpdateJob<DomainReference, HistoricalAvailabilityData>> updateJobs = new ArrayList<>();
 
         try (var conn = dataSource.getConnection();
              var ps = conn.prepareStatement("""
-                SELECT ID, NEXT_SCHEDULED_UPDATE
+                SELECT ID, DOMAIN_NAME, NEXT_SCHEDULED_UPDATE
                 FROM EC_DOMAIN
                 LEFT JOIN DOMAIN_AVAILABILITY_INFORMATION
                 ON EC_DOMAIN.ID = DOMAIN_AVAILABILITY_INFORMATION.DOMAIN_ID
@@ -200,11 +200,13 @@ public class PingDao {
             ps.setInt(1, nodeId);
             ResultSet rs = ps.executeQuery();
             while (rs.next()) {
-                long domainId = rs.getLong("ID");
+                int domainId = rs.getInt("ID");
+                String domainName = rs.getString("DOMAIN_NAME");
                 var ts = rs.getTimestamp("NEXT_SCHEDULED_UPDATE");
                 Instant nextUpdate = ts == null ? Instant.now() : ts.toInstant();
 
-                updateJobs.add(new UpdateSchedule.UpdateJob<>(domainId, nextUpdate));
+                var ref = new DomainReference(domainId, nodeId, domainName.toLowerCase());
+                updateJobs.add(new UpdateSchedule.UpdateJob<>(ref, nextUpdate));
             }
         } catch (SQLException e) {
             throw new RuntimeException("Failed to retrieve domain update schedule", e);
@@ -241,7 +243,7 @@ public class PingDao {
                 else {
                     var record = new DomainDnsRecord(rs);
                     updateJobs.add(new UpdateSchedule.UpdateJob<>(
-                            new RootDomainReference.ById(dnsRootDomainId),
+                            new RootDomainReference.ByIdAndName(dnsRootDomainId, rootDomainName),
                             Objects.requireNonNullElseGet(record.tsNextScheduledUpdate(), Instant::now))
                     );
                 }

code/processes/ping-process/java/nu/marginalia/ping/PingJobScheduler.java

@@ -2,6 +2,7 @@ package nu.marginalia.ping;
 
 import com.google.inject.Inject;
 import nu.marginalia.coordination.DomainCoordinator;
+import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.ping.model.*;
 import nu.marginalia.ping.svc.DnsPingService;
 import nu.marginalia.ping.svc.HttpPingService;
@@ -31,7 +32,7 @@ public class PingJobScheduler {
 
     private static final UpdateSchedule<RootDomainReference, RootDomainReference> dnsUpdateSchedule
             = new UpdateSchedule<>(250_000);
-    private static final UpdateSchedule<Long, HistoricalAvailabilityData> availabilityUpdateSchedule
+    private static final UpdateSchedule<DomainReference, HistoricalAvailabilityData> availabilityUpdateSchedule
             = new UpdateSchedule<>(250_000);
 
     public volatile Instant dnsLastSync = Instant.now();
@@ -138,7 +139,15 @@ public class PingJobScheduler {
                     continue;
                 }
 
-                long nextId = availabilityUpdateSchedule.next();
+                DomainReference ref = availabilityUpdateSchedule.nextIf(domain -> {
+                    EdgeDomain domainObj = new EdgeDomain(domain.domainName());
+                    if (!domainCoordinator.isLockableHint(domainObj)) {
+                        return false; // Skip locked domains
+                    }
+                    return true; // Process this domain
+                });
+
+                long nextId = ref.domainId();
                 var data = pingDao.getHistoricalAvailabilityData(nextId);
                 if (data == null) {
                     logger.warn("No availability data found for ID: {}", nextId);
@@ -163,7 +172,7 @@ public class PingJobScheduler {
                     for (var object : objects) {
                         var ts = object.nextUpdateTime();
                         if (ts != null) {
-                            availabilityUpdateSchedule.add(nextId, ts);
+                            availabilityUpdateSchedule.add(ref, ts);
                             break;
                         }
                     }
@@ -194,7 +203,7 @@ public class PingJobScheduler {
 
                 try {
                     List<WritableModel> objects = switch(ref) {
-                        case RootDomainReference.ById(long id) -> {
+                        case RootDomainReference.ByIdAndName(long id, String name) -> {
                             var oldRecord = Objects.requireNonNull(pingDao.getDomainDnsRecord(id));
                             yield dnsPingService.pingDomain(oldRecord.rootDomainName(), oldRecord);
                         }

code/processes/ping-process/java/nu/marginalia/ping/UpdateSchedule.java

@@ -2,9 +2,8 @@ package nu.marginalia.ping;
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.PriorityQueue;
+import java.util.*;
+import java.util.function.Predicate;
 
 /** In-memory schedule for updates, allowing jobs to be added and processed in order of their scheduled time.
  * This is not a particularly high-performance implementation, but exists to take contention off the database's
@@ -23,6 +22,9 @@ public class UpdateSchedule<T, T2> {
         notifyAll();
     }
 
+    /** Returns the next job in the queue that is due to be processed.
+     * If no jobs are due, it will block until a job is added or a job becomes due.
+     * */
     public synchronized T next() throws InterruptedException {
         while (true) {
             if (updateQueue.isEmpty()) {
@@ -44,6 +46,56 @@ public class UpdateSchedule<T, T2> {
         }
     }
 
+
+    /** Returns the first job in the queue matching the predicate that is not scheduled into the future,
+     * blocking until a job is added or a job becomes due.
+     */
+    public synchronized T nextIf(Predicate<T> predicate) throws InterruptedException {
+        List<UpdateJob<T, T2>> rejectedJobs = new ArrayList<>();
+
+        try {
+            while (true) {
+                if (updateQueue.isEmpty()) {
+                    wait(); // Wait for a new job to be added
+                    continue;
+                }
+
+                UpdateJob<T, T2> job = updateQueue.peek();
+                Instant now = Instant.now();
+
+                if (job.updateTime.isAfter(now)) {
+                    Duration toWait = Duration.between(now, job.updateTime);
+
+                    // Return the rejected jobs to the queue for other threads to process
+                    updateQueue.addAll(rejectedJobs);
+                    if (!rejectedJobs.isEmpty())
+                        notifyAll();
+                    rejectedJobs.clear();
+
+                    wait(Math.max(1, toWait.toMillis()));
+                } else {
+                    var candidate = updateQueue.poll(); // Remove the job from the queue since it's due
+
+                    assert candidate != null : "Update job should not be null at this point, since we just peeked it in a synchronized block";
+
+                    if (!predicate.test(candidate.key())) {
+                        rejectedJobs.add(candidate);
+                    }
+                    else {
+                        return candidate.key();
+                    }
+                }
+            }
+        }
+        finally {
+            // Return the rejected jobs to the queue for other threads to process
+            updateQueue.addAll(rejectedJobs);
+            if (!rejectedJobs.isEmpty())
+                notifyAll();
+        }
+
+    }
+
     public synchronized void clear() {
         updateQueue.clear();
         notifyAll();

code/processes/ping-process/java/nu/marginalia/ping/fetcher/PingHttpFetcher.java

@@ -4,12 +4,14 @@ import com.google.inject.Inject;
 import nu.marginalia.UserAgent;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.ping.fetcher.response.*;
+import org.apache.hc.client5.http.HttpHostConnectException;
 import org.apache.hc.client5.http.classic.HttpClient;
 import org.apache.hc.client5.http.protocol.HttpClientContext;
 import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
 
+import javax.net.ssl.SSLHandshakeException;
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.time.Duration;
@@ -82,9 +84,12 @@ public class PingHttpFetcher {
             });
         } catch (SocketTimeoutException ex) {
             return new TimeoutResponse(ex.getMessage());
-        } catch (IOException e) {
+        } catch (HttpHostConnectException | SSLHandshakeException e) {
             return new ConnectionError(e.getClass().getSimpleName());
+        } catch (IOException e) {
+            return new ProtocolError(e.getClass().getSimpleName());
         }
+
     }
 
 }

code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java

@@ -18,13 +18,18 @@ import org.apache.hc.core5.http.HttpResponse;
 import org.apache.hc.core5.http.io.SocketConfig;
 import org.apache.hc.core5.http.message.MessageSupport;
 import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.apache.hc.core5.util.TimeValue;
 import org.apache.hc.core5.util.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
 import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 
@@ -37,24 +42,55 @@ public class HttpClientProvider implements Provider<HttpClient> {
     static {
         try {
             client = createClient();
-        } catch (NoSuchAlgorithmException e) {
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
-    private static CloseableHttpClient createClient() throws NoSuchAlgorithmException {
+    private static CloseableHttpClient createClient() throws NoSuchAlgorithmException, KeyManagementException {
         final ConnectionConfig connectionConfig = ConnectionConfig.custom()
                 .setSocketTimeout(15, TimeUnit.SECONDS)
                 .setConnectTimeout(15, TimeUnit.SECONDS)
                 .setValidateAfterInactivity(TimeValue.ofSeconds(5))
                 .build();
 
+        // No-op up front validation of server certificates.
+        //
+        // We will validate certificates later, after the connection is established
+        // as we want to store the certificate chain and validation
+        // outcome to the database.
+
+        var trustMeBro = new X509TrustManager() {
+            private X509Certificate[] lastServerCertChain;
+
+            @Override
+            public void checkClientTrusted(X509Certificate[] chain, String authType) {
+            }
+
+            @Override
+            public void checkServerTrusted(X509Certificate[] chain, String authType) {
+                this.lastServerCertChain = chain.clone();
+            }
+
+            @Override
+            public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+
+            public X509Certificate[] getLastServerCertChain() {
+                return lastServerCertChain != null ? lastServerCertChain.clone() : null;
+            }
+        };
+
+        SSLContext sslContext = SSLContextBuilder.create().build();
+        sslContext.init(null, new TrustManager[]{trustMeBro}, null);
+
         connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
                 .setMaxConnPerRoute(2)
                 .setMaxConnTotal(50)
                 .setDefaultConnectionConfig(connectionConfig)
                 .setTlsSocketStrategy(
-                        new DefaultClientTlsStrategy(SSLContext.getDefault(), NoopHostnameVerifier.INSTANCE))
+                        new DefaultClientTlsStrategy(sslContext, NoopHostnameVerifier.INSTANCE))
                 .build();
 
         connectionManager.setDefaultSocketConfig(SocketConfig.custom()

code/processes/ping-process/java/nu/marginalia/ping/io/RetryStrategy.java

@@ -1,5 +1,6 @@
 package nu.marginalia.ping.io;
 
+import org.apache.hc.client5.http.HttpHostConnectException;
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.core5.http.HttpRequest;
 import org.apache.hc.core5.http.HttpResponse;
@@ -22,6 +23,7 @@ public class RetryStrategy implements HttpRequestRetryStrategy {
             case SocketTimeoutException ste -> false;
             case SSLException ssle -> false;
             case UnknownHostException uhe -> false;
+            case HttpHostConnectException ex -> executionCount <= 2; // Only retry once for connection errors
             default -> executionCount <= 3;
         };
     }

code/processes/ping-process/java/nu/marginalia/ping/model/DomainAvailabilityRecord.java

@@ -1,5 +1,7 @@
 package nu.marginalia.ping.model;
 
+import org.apache.commons.lang3.StringUtils;
+
 import javax.annotation.Nullable;
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -279,7 +281,7 @@ implements WritableModel
         }
 
         public Builder httpLocation(String httpLocation) {
-            this.httpLocation = httpLocation;
+            this.httpLocation = StringUtils.abbreviate(httpLocation, "...",255);
             return this;
         }
 

code/processes/ping-process/java/nu/marginalia/ping/model/DomainSecurityEvent.java

@@ -18,6 +18,7 @@ public record DomainSecurityEvent(
         boolean certificatePublicKeyChanged,
         boolean certificateSerialNumberChanged,
         boolean certificateIssuerChanged,
+        SchemaChange schemaChange,
         Duration oldCertificateTimeToExpiry,
         boolean securityHeadersChanged,
         boolean ipChanged,
@@ -45,8 +46,9 @@ public record DomainSecurityEvent(
                 security_signature_before,
                 security_signature_after,
                 change_certificate_serial_number,
-                change_certificate_issuer
-                ) VALUES    (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                change_certificate_issuer,
+                change_schema
+                ) VALUES    (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
             """))
         {
 
@@ -81,6 +83,7 @@ public record DomainSecurityEvent(
 
             ps.setBoolean(15, certificateSerialNumberChanged());
             ps.setBoolean(16, certificateIssuerChanged());
+            ps.setString(17, schemaChange.name());
 
             ps.executeUpdate();
         }

code/processes/ping-process/java/nu/marginalia/ping/model/DomainSecurityRecord.java

@@ -1,6 +1,7 @@
 package nu.marginalia.ping.model;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
 
 import javax.annotation.Nullable;
 import java.sql.Connection;
@@ -42,7 +43,10 @@ public record DomainSecurityRecord(
         @Nullable String headerXXssProtection,
         @Nullable String headerServer,
         @Nullable String headerXPoweredBy,
-        @Nullable Instant tsLastUpdate
+        @Nullable Instant tsLastUpdate,
+        @Nullable Boolean sslChainValid,
+        @Nullable Boolean sslHostValid,
+        @Nullable Boolean sslDateValid
         )
     implements WritableModel
 {
@@ -102,7 +106,11 @@ public record DomainSecurityRecord(
              rs.getString("DOMAIN_SECURITY_INFORMATION.HEADER_X_XSS_PROTECTION"),
              rs.getString("DOMAIN_SECURITY_INFORMATION.HEADER_SERVER"),
              rs.getString("DOMAIN_SECURITY_INFORMATION.HEADER_X_POWERED_BY"),
-             rs.getObject("DOMAIN_SECURITY_INFORMATION.TS_LAST_UPDATE", Instant.class));
+             rs.getObject("DOMAIN_SECURITY_INFORMATION.TS_LAST_UPDATE", Instant.class),
+             rs.getObject("SSL_CHAIN_VALID", Boolean.class),
+             rs.getObject("SSL_HOST_VALID", Boolean.class),
+             rs.getObject("SSL_DATE_VALID", Boolean.class)
+                );
     }
 
     private static HttpSchema httpSchemaFromString(@Nullable String schema) {
@@ -149,8 +157,11 @@ public record DomainSecurityRecord(
                              header_x_powered_by,
                              ssl_cert_public_key_hash,
                              asn,
-                             ts_last_update)
-                         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                             ts_last_update,
+                             ssl_chain_valid,
+                             ssl_host_valid,
+                             ssl_date_valid)
+                         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                         """))
         {
             ps.setInt(1, domainId());
@@ -294,6 +305,25 @@ public record DomainSecurityRecord(
             } else {
                 ps.setTimestamp(32, java.sql.Timestamp.from(tsLastUpdate()));
             }
+
+            if (sslChainValid() == null) {
+                ps.setNull(33, java.sql.Types.BOOLEAN);
+            } else {
+                ps.setBoolean(33, sslChainValid());
+            }
+
+            if (sslHostValid() == null) {
+                ps.setNull(34, java.sql.Types.BOOLEAN);
+            } else {
+                ps.setBoolean(34, sslHostValid());
+            }
+
+            if (sslDateValid() == null) {
+                ps.setNull(35, java.sql.Types.BOOLEAN);
+            } else {
+                ps.setBoolean(35, sslDateValid());
+            }
+
             ps.executeUpdate();
         }
     }
@@ -332,6 +362,13 @@ public record DomainSecurityRecord(
         private String headerXPoweredBy;
         private Instant tsLastUpdate;
 
+        private Boolean isCertChainValid;
+        private Boolean isCertHostValid;
+        private Boolean isCertDateValid;
+
+
+        private static Instant MAX_UNIX_TIMESTAMP = Instant.ofEpochSecond(Integer.MAX_VALUE);
+
         public Builder() {
             // Default values for boolean fields
             this.sslCertWildcard = false;
@@ -374,12 +411,18 @@ public record DomainSecurityRecord(
             return this;
         }
 
-        public Builder sslCertNotBefore(Instant sslCertNotBefore) {
+        public Builder sslCertNotBefore(@NotNull Instant sslCertNotBefore) {
+            if (sslCertNotBefore.isAfter(MAX_UNIX_TIMESTAMP)) {
+                sslCertNotBefore = MAX_UNIX_TIMESTAMP;
+            }
             this.sslCertNotBefore = sslCertNotBefore;
             return this;
         }
 
-        public Builder sslCertNotAfter(Instant sslCertNotAfter) {
+        public Builder sslCertNotAfter(@NotNull Instant sslCertNotAfter) {
+            if (sslCertNotAfter.isAfter(MAX_UNIX_TIMESTAMP)) {
+                sslCertNotAfter = MAX_UNIX_TIMESTAMP;
+            }
             this.sslCertNotAfter = sslCertNotAfter;
             return this;
         }
@@ -499,6 +542,21 @@ public record DomainSecurityRecord(
             return this;
         }
 
+        public Builder sslChainValid(@Nullable Boolean isCertChainValid) {
+            this.isCertChainValid = isCertChainValid;
+            return this;
+        }
+
+        public Builder sslHostValid(@Nullable Boolean isCertHostValid) {
+            this.isCertHostValid = isCertHostValid;
+            return this;
+        }
+
+        public Builder sslDateValid(@Nullable Boolean isCertDateValid) {
+            this.isCertDateValid = isCertDateValid;
+            return this;
+        }
+
         public DomainSecurityRecord build() {
             return new DomainSecurityRecord(
                     domainId,
@@ -532,7 +590,10 @@ public record DomainSecurityRecord(
                     headerXXssProtection,
                     headerServer,
                     headerXPoweredBy,
-                    tsLastUpdate
+                    tsLastUpdate,
+                    isCertChainValid,
+                    isCertHostValid,
+                    isCertDateValid
             );
         }
     }

code/processes/ping-process/java/nu/marginalia/ping/model/RootDomainReference.java

@@ -1,6 +1,6 @@
 package nu.marginalia.ping.model;
 
 public sealed interface RootDomainReference {
-    record ById(long id) implements RootDomainReference { }
+    record ByIdAndName(long id, String name) implements RootDomainReference { }
     record ByName(String name) implements RootDomainReference { }
 }

code/processes/ping-process/java/nu/marginalia/ping/model/SchemaChange.java

@@ -0,0 +1,12 @@
+package nu.marginalia.ping.model;
+
+public enum SchemaChange {
+    UNKNOWN,
+    NONE,
+    HTTP_TO_HTTPS,
+    HTTPS_TO_HTTP;
+
+    public boolean isSignificant() {
+        return this != NONE && this != UNKNOWN;
+    }
+}

code/processes/ping-process/java/nu/marginalia/ping/model/comparison/SecurityInformationChange.java

@@ -2,6 +2,9 @@ package nu.marginalia.ping.model.comparison;
 
 import nu.marginalia.ping.model.DomainAvailabilityRecord;
 import nu.marginalia.ping.model.DomainSecurityRecord;
+import nu.marginalia.ping.model.HttpSchema;
+import nu.marginalia.ping.model.SchemaChange;
+import org.jetbrains.annotations.NotNull;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -20,7 +23,8 @@ public record SecurityInformationChange(
         Duration oldCertificateTimeToExpiry,
         boolean isSecurityHeadersChanged,
         boolean isIpAddressChanged,
-        boolean isSoftwareHeaderChanged
+        boolean isSoftwareHeaderChanged,
+        SchemaChange schemaChange
 ) {
     public static SecurityInformationChange between(
             DomainSecurityRecord before, DomainAvailabilityRecord availabilityBefore,
@@ -43,9 +47,10 @@ public record SecurityInformationChange(
         );
 
         boolean securityHeadersChanged = before.securityHeadersHash() != after.securityHeadersHash();
-
         boolean softwareChanged = !Objects.equals(before.headerServer(), after.headerServer());
 
+        SchemaChange schemaChange = getSchemaChange(before, after);
+
         // Note we don't include IP address changes in the overall change status,
         // as this is not alone considered a change in security information; we may have
         // multiple IP addresses for a domain, and the IP address may change frequently
@@ -55,7 +60,8 @@ public record SecurityInformationChange(
                 || certificateFingerprintChanged
                 || securityHeadersChanged
                 || certificateProfileChanged
-                || softwareChanged;
+                || softwareChanged
+                || schemaChange.isSignificant();
 
         return new SecurityInformationChange(
                 isChanged,
@@ -69,9 +75,36 @@ public record SecurityInformationChange(
                 oldCertificateTimeToExpiry,
                 securityHeadersChanged,
                 ipChanged,
-                softwareChanged
+                softwareChanged,
+                schemaChange
         );
     }
 
+    private static @NotNull SchemaChange getSchemaChange(DomainSecurityRecord before, DomainSecurityRecord after) {
+        if (before.httpSchema() == null || after.httpSchema() == null) {
+            return SchemaChange.UNKNOWN;
+        }
+
+        boolean beforeIsHttp = before.httpSchema() == HttpSchema.HTTP;
+        boolean afterIsHttp = after.httpSchema() == HttpSchema.HTTP;
+        boolean beforeIsHttps = before.httpSchema() == HttpSchema.HTTPS;
+        boolean afterIsHttps = after.httpSchema() == HttpSchema.HTTPS;
+
+        SchemaChange schemaChange;
+
+        if (beforeIsHttp && afterIsHttp) {
+            schemaChange = SchemaChange.NONE;
+        } else if (beforeIsHttps && afterIsHttps) {
+            schemaChange = SchemaChange.NONE;
+        } else if (beforeIsHttp && afterIsHttps) {
+            schemaChange = SchemaChange.HTTP_TO_HTTPS;
+        } else if (beforeIsHttps && afterIsHttp) {
+            schemaChange = SchemaChange.HTTPS_TO_HTTP;
+        } else {
+            schemaChange = SchemaChange.UNKNOWN;
+        }
+        return schemaChange;
+    }
+
 
 }

code/processes/ping-process/java/nu/marginalia/ping/ssl/AIAExtractor.java

@@ -0,0 +1,59 @@
+package nu.marginalia.ping.ssl;
+
+import org.bouncycastle.asn1.ASN1OctetString;
+import org.bouncycastle.asn1.ASN1Primitive;
+import org.bouncycastle.asn1.x509.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AIAExtractor {
+
+    private static final Logger logger = LoggerFactory.getLogger(AIAExtractor.class);
+
+    public static List<String> getCaIssuerUrls(X509Certificate certificate) {
+        List<String> caIssuerUrls = new ArrayList<>();
+
+        try {
+            // Get the AIA extension value
+            byte[] aiaExtensionValue = certificate.getExtensionValue(Extension.authorityInfoAccess.getId());
+            if (aiaExtensionValue == null) {
+                logger.warn("No AIA extension found");
+                return caIssuerUrls;
+            }
+
+            // Parse the extension - first unwrap the OCTET STRING
+            ASN1OctetString octetString = ASN1OctetString.getInstance(aiaExtensionValue);
+            ASN1Primitive aiaObj = ASN1Primitive.fromByteArray(octetString.getOctets());
+
+            // Parse as AuthorityInformationAccess
+            AuthorityInformationAccess aia = AuthorityInformationAccess.getInstance(aiaObj);
+
+            if (aia != null) {
+                AccessDescription[] accessDescriptions = aia.getAccessDescriptions();
+
+                for (AccessDescription accessDesc : accessDescriptions) {
+                    // Check if this is a CA Issuers access method
+                    if (X509ObjectIdentifiers.id_ad_caIssuers.equals(accessDesc.getAccessMethod())) {
+                        GeneralName accessLocation = accessDesc.getAccessLocation();
+
+                        // Check if it's a URI
+                        if (accessLocation.getTagNo() == GeneralName.uniformResourceIdentifier) {
+                            String url = accessLocation.getName().toString();
+                            caIssuerUrls.add(url);
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            logger.error("Error parsing AIA extension: {}", e.getMessage());
+        }
+
+        return caIssuerUrls;
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/ssl/CertificateFetcher.java

@@ -0,0 +1,273 @@
+package nu.marginalia.ping.ssl;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import nu.marginalia.WmsaHome;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
+import org.bouncycastle.asn1.ASN1OctetString;
+import org.bouncycastle.asn1.ASN1Primitive;
+import org.bouncycastle.asn1.x509.*;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cms.CMSSignedData;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.util.Store;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.CertificateFactory;
+import java.security.cert.TrustAnchor;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public class CertificateFetcher {
+
+    private static final Logger logger = LoggerFactory.getLogger(CertificateFetcher.class);
+
+    private static HttpClient client = HttpClientBuilder.create()
+            .build();
+
+    private static Cache<String, X509Certificate> cache = CacheBuilder
+            .newBuilder()
+            .expireAfterAccess(Duration.ofHours(6))
+            .maximumSize(10_000)
+            .build();
+
+
+    public static List<X509Certificate> fetchMissingIntermediates(X509Certificate leafCert) {
+        List<X509Certificate> intermediates = new ArrayList<>();
+
+        // Get CA Issuer URLs from AIA extension
+        List<String> caIssuerUrls = AIAExtractor.getCaIssuerUrls(leafCert);
+
+        for (String url : caIssuerUrls) {
+            try {
+                // Check cache first
+                X509Certificate cached = cache.getIfPresent(url);
+                if (cached != null) {
+                    intermediates.add(cached);
+                    continue;
+                }
+
+                // Download certificate
+                X509Certificate downloaded = downloadCertificate(url);
+                if (downloaded != null) {
+                    // Verify this certificate can actually sign the leaf
+                    if (canSign(downloaded, leafCert)) {
+                        intermediates.add(downloaded);
+                        cache.put(url, downloaded);
+                        logger.info("Downloaded certificate for url: {}", url);
+                    } else {
+                        logger.warn("Downloaded certificate cannot sign leaf cert from: {}", url);
+                    }
+                }
+
+            } catch (Exception e) {
+                logger.error("Failed to fetch certificate from {}: {}", url, e.getMessage());
+            }
+        }
+
+        return intermediates;
+    }
+    private static X509Certificate downloadCertificate(String urlString) {
+        try {
+            ClassicHttpRequest request =  ClassicRequestBuilder.create("GET")
+                    .addHeader("User-Agent", WmsaHome.getUserAgent() + " (Certificate Fetcher)")
+                    .setUri(urlString)
+                    .build();
+
+            byte[] data = client.execute(request, rsp -> {
+                var entity = rsp.getEntity();
+                if (entity == null) {
+                    logger.warn("GET request returned no content for {}", urlString);
+                    return null;
+                }
+                return entity.getContent().readAllBytes();
+            });
+
+            if (data.length == 0) {
+                logger.warn("Empty response from {}", urlString);
+                return null;
+            }
+
+            // Try different formats based on file extension
+            if (urlString.toLowerCase().endsWith(".p7c") || urlString.toLowerCase().endsWith(".p7b")) {
+                return parsePKCS7(data);
+            } else {
+                return parseX509(data);
+            }
+
+        } catch (Exception e) {
+            logger.warn("Failed to fetch certificate from {}: {}", urlString, e.getMessage());
+            return null;
+        }
+    }
+
+    private static List<X509Certificate> parseMultiplePEM(byte[] data) throws Exception {
+        List<X509Certificate> certificates = new ArrayList<>();
+
+        try (StringReader stringReader = new StringReader(new String(data, StandardCharsets.UTF_8));
+             PEMParser pemParser = new PEMParser(stringReader)) {
+
+            JcaX509CertificateConverter converter = new JcaX509CertificateConverter();
+            Object object;
+
+            while ((object = pemParser.readObject()) != null) {
+                if (object instanceof X509CertificateHolder) {
+                    X509CertificateHolder certHolder = (X509CertificateHolder) object;
+                    certificates.add(converter.getCertificate(certHolder));
+                } else if (object instanceof X509Certificate) {
+                    certificates.add((X509Certificate) object);
+                }
+            }
+        }
+
+        return certificates;
+    }
+    private static X509Certificate parseX509(byte[] data) throws Exception {
+        CertificateFactory cf = CertificateFactory.getInstance("X.509");
+        return (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(data));
+    }
+
+    private static X509Certificate parsePKCS7(byte[] data) throws Exception {
+        try {
+            // Parse PKCS#7/CMS structure
+            CMSSignedData cmsData = new CMSSignedData(data);
+            Store<X509CertificateHolder> certStore = cmsData.getCertificates();
+
+            JcaX509CertificateConverter converter = new JcaX509CertificateConverter();
+
+            // Get the first certificate from the store
+            for (X509CertificateHolder certHolder : certStore.getMatches(null)) {
+                X509Certificate cert = converter.getCertificate(certHolder);
+                return cert;
+            }
+
+            logger.warn("No certificates found in PKCS#7 structure");
+            return null;
+
+        } catch (Exception e) {
+            logger.error("Failed to parse PKCS#7 structure from {}: {}", data.length, e.getMessage());
+            return parseX509(data);
+        }
+    }
+
+    private static boolean canSign(X509Certificate issuerCert, X509Certificate subjectCert) {
+        try {
+            // Check if the issuer DN matches
+            if (!issuerCert.getSubjectDN().equals(subjectCert.getIssuerDN())) {
+                return false;
+            }
+
+            // Try to verify the signature
+            subjectCert.verify(issuerCert.getPublicKey());
+            return true;
+
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    // Recursive fetching for complete chains
+    public static List<X509Certificate> buildCompleteChain(X509Certificate leafCert) {
+        List<X509Certificate> completeChain = new ArrayList<>();
+        completeChain.add(leafCert);
+
+        X509Certificate currentCert = leafCert;
+        int maxDepth = 10; // Prevent infinite loops
+
+        while (maxDepth-- > 0) {
+            // If current cert is self-signed (root), we're done
+            if (currentCert.getSubjectDN().equals(currentCert.getIssuerDN())) {
+                break;
+            }
+
+            // Try to find the issuer
+            List<X509Certificate> intermediates = fetchMissingIntermediates(currentCert);
+            if (intermediates.isEmpty()) {
+                logger.error("Could not find issuer for: {}", currentCert.getSubjectDN());
+                break;
+            }
+
+            // Add the first valid intermediate
+            X509Certificate intermediate = intermediates.get(0);
+            completeChain.add(intermediate);
+            currentCert = intermediate;
+        }
+
+        return completeChain;
+    }
+
+    // Add this to your AIAExtractor class if not already present
+    public static List<String> getOCSPUrls(X509Certificate certificate) {
+        List<String> ocspUrls = new ArrayList<>();
+
+        try {
+            byte[] aiaExtensionValue = certificate.getExtensionValue(Extension.authorityInfoAccess.getId());
+            if (aiaExtensionValue == null) {
+                return ocspUrls;
+            }
+
+            ASN1OctetString octetString = ASN1OctetString.getInstance(aiaExtensionValue);
+            ASN1Primitive aiaObj = ASN1Primitive.fromByteArray(octetString.getOctets());
+            AuthorityInformationAccess aia = AuthorityInformationAccess.getInstance(aiaObj);
+
+            if (aia != null) {
+                AccessDescription[] accessDescriptions = aia.getAccessDescriptions();
+
+                for (AccessDescription accessDesc : accessDescriptions) {
+                    if (X509ObjectIdentifiers.id_ad_ocsp.equals(accessDesc.getAccessMethod())) {
+                        GeneralName accessLocation = accessDesc.getAccessLocation();
+                        if (accessLocation.getTagNo() == GeneralName.uniformResourceIdentifier) {
+                            String url = accessLocation.getName().toString();
+                            ocspUrls.add(url);
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            logger.error("Error parsing AIA extension for OCSP: {}", e.getMessage());
+        }
+
+        return ocspUrls;
+    }
+
+    public static Set<TrustAnchor> getRootCerts(String bundleUrl) throws Exception {
+        ClassicHttpRequest request =  ClassicRequestBuilder.create("GET")
+                .addHeader("User-Agent", WmsaHome.getUserAgent() + " (Certificate Fetcher)")
+                .setUri(bundleUrl)
+                .build();
+
+        byte[] data = client.execute(request, rsp -> {
+            var entity = rsp.getEntity();
+            if (entity == null) {
+                logger.warn("GET request returned no content for {}", bundleUrl);
+                return null;
+            }
+            return entity.getContent().readAllBytes();
+        });
+
+        List<TrustAnchor> anchors = new ArrayList<>();
+        for (var cert : parseMultiplePEM(data)) {
+            try {
+                anchors.add(new TrustAnchor(cert, null));
+            } catch (Exception e) {
+                logger.warn("Failed to create TrustAnchor for certificate: {}", e.getMessage());
+            }
+        }
+
+        logger.info("Loaded {} root certificates from {}", anchors.size(), bundleUrl);
+
+        return Set.copyOf(anchors);
+    }
+}

code/processes/ping-process/java/nu/marginalia/ping/ssl/CertificateValidator.java

@@ -0,0 +1,481 @@
+package nu.marginalia.ping.ssl;
+
+import org.bouncycastle.asn1.ASN1OctetString;
+import org.bouncycastle.asn1.ASN1Primitive;
+import org.bouncycastle.asn1.x509.*;
+
+import javax.security.auth.x500.X500Principal;
+import java.security.cert.TrustAnchor;
+import java.security.cert.X509Certificate;
+import java.util.*;
+
+/** Utility class for validating X.509 certificates.
+ * This class provides methods to validate certificate chains, check expiration,
+ * hostname validity, and revocation status.
+ * <p></p>
+ * This is extremely unsuitable for actual SSL/TLS validation,
+ * and is only to be used in analyzing certificates for fingerprinting
+ * and diagnosing servers!
+ */
+public class CertificateValidator {
+    public static class ValidationResult {
+        public boolean chainValid = false;
+        public boolean certificateExpired = false;
+        public boolean certificateRevoked = false;
+        public boolean selfSigned = false;
+        public boolean hostnameValid = false;
+
+        public boolean isValid() {
+            return !selfSigned && !certificateExpired && !certificateRevoked && hostnameValid;
+        }
+
+        public List<String> errors = new ArrayList<>();
+        public List<String> warnings = new ArrayList<>();
+        public Map<String, Object> details = new HashMap<>();
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append("=== Certificate Validation Result ===\n");
+            sb.append("Chain Valid: ").append(chainValid ? "✓" : "✗").append("\n");
+            sb.append("Not Expired: ").append(!certificateExpired ? "✓" : "✗").append("\n");
+            sb.append("Not Revoked: ").append(!certificateRevoked ? "✓" : "✗").append("\n");
+            sb.append("Hostname Valid: ").append(hostnameValid ? "✓" : "✗").append("\n");
+            sb.append("Self-Signed: ").append(selfSigned ? "✓" : "✗").append("\n");
+
+            if (!errors.isEmpty()) {
+                sb.append("\nErrors:\n");
+                for (String error : errors) {
+                    sb.append("  ✗ ").append(error).append("\n");
+                }
+            }
+
+            if (!warnings.isEmpty()) {
+                sb.append("\nWarnings:\n");
+                for (String warning : warnings) {
+                    sb.append("  ⚠ ").append(warning).append("\n");
+                }
+            }
+
+            if (!details.isEmpty()) {
+                sb.append("\nDetails:\n");
+                for (Map.Entry<String, Object> entry : details.entrySet()) {
+                    sb.append("  ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
+                }
+            }
+
+            return sb.toString();
+        }
+    }
+
+    public static ValidationResult validateCertificate(X509Certificate[] certChain,
+                                                       String hostname) {
+        return validateCertificate(certChain, hostname, false);
+    }
+
+    public static ValidationResult validateCertificate(X509Certificate[] certChain,
+                                                       String hostname,
+                                                       boolean autoTrustFetchedRoots) {
+        ValidationResult result = new ValidationResult();
+
+        if (certChain == null || certChain.length == 0) {
+            result.errors.add("No certificates provided");
+            return result;
+        }
+
+        X509Certificate leafCert = certChain[0];
+
+        // 1. Check certificate expiration
+        result.certificateExpired = checkExpiration(leafCert, result);
+
+        // 2. Check hostname validity
+        result.hostnameValid = checkHostname(leafCert, hostname, result);
+
+        result.selfSigned = certChain.length <= 1;
+
+        // 3. Check certificate chain validity (with AIA fetching)
+        result.chainValid = checkChainValidity(certChain, RootCerts.getTrustAnchors(), result, autoTrustFetchedRoots);
+
+        // 4. Check revocation status
+        result.certificateRevoked = checkRevocation(leafCert, result);
+
+        return result;
+    }
+
+    private static boolean checkExpiration(X509Certificate cert, ValidationResult result) {
+        try {
+            cert.checkValidity();
+            result.details.put("validFrom", cert.getNotBefore());
+            result.details.put("validTo", cert.getNotAfter());
+
+            // Warn if expires soon (30 days)
+            long daysUntilExpiry = (cert.getNotAfter().getTime() - System.currentTimeMillis()) / (1000 * 60 * 60 * 24);
+            if (daysUntilExpiry < 30) {
+                result.warnings.add("Certificate expires in " + daysUntilExpiry + " days");
+            }
+
+            return false; // Not expired
+        } catch (Exception e) {
+            result.errors.add("Certificate expired or not yet valid: " + e.getMessage());
+            return true; // Expired
+        }
+    }
+
+    private static boolean checkHostname(X509Certificate cert, String hostname, ValidationResult result) {
+        if (hostname == null || hostname.isEmpty()) {
+            result.warnings.add("No hostname provided for validation");
+            return false;
+        }
+
+        try {
+            // Check Subject CN
+            String subjectCN = getCommonName(cert.getSubjectX500Principal());
+            if (subjectCN != null && matchesHostname(subjectCN, hostname)) {
+                result.details.put("hostnameMatchedBy", "Subject CN: " + subjectCN);
+                return true;
+            }
+
+            // Check Subject Alternative Names
+            Collection<List<?>> subjectAltNames = cert.getSubjectAlternativeNames();
+            if (subjectAltNames != null) {
+                for (List<?> altName : subjectAltNames) {
+                    if (altName.size() >= 2) {
+                        Integer type = (Integer) altName.get(0);
+                        if (type == 2) { // DNS name
+                            String dnsName = (String) altName.get(1);
+                            if (matchesHostname(dnsName, hostname)) {
+                                result.details.put("hostnameMatchedBy", "SAN DNS: " + dnsName);
+                                return true;
+                            }
+                        }
+                    }
+                }
+            }
+
+            result.errors.add("Hostname '" + hostname + "' does not match certificate");
+            result.details.put("subjectCN", subjectCN);
+            result.details.put("subjectAltNames", subjectAltNames);
+            return false;
+
+        } catch (Exception e) {
+            result.errors.add("Error checking hostname: " + e.getMessage());
+            return false;
+        }
+    }
+
+    private static boolean checkChainValidity(X509Certificate[] originalChain,
+                                              Set<TrustAnchor> trustAnchors,
+                                              ValidationResult result,
+                                              boolean autoTrustFetchedRoots) {
+        try {
+            // First try with the original chain
+            ChainValidationResult originalResult = validateChain(originalChain, trustAnchors);
+
+            if (originalResult.isValid) {
+                result.details.put("chainLength", originalChain.length);
+                result.details.put("chainExtended", false);
+                return true;
+            }
+
+            try {
+                List<X509Certificate> repairedChain = CertificateFetcher.buildCompleteChain(originalChain[0]);
+
+                if (!repairedChain.isEmpty()) {
+
+                    X509Certificate[] extendedArray = repairedChain.toArray(new X509Certificate[0]);
+
+                    // Create a copy of trust anchors for potential modification
+                    Set<TrustAnchor> workingTrustAnchors = new HashSet<>(trustAnchors);
+
+                    // If auto-trust is enabled, add any self-signed certs as trusted roots
+                    if (autoTrustFetchedRoots) {
+                        for (X509Certificate cert : extendedArray) {
+                            if (cert.getSubjectX500Principal().equals(cert.getIssuerX500Principal())) {
+                                // Self-signed certificate - add to trust anchors if not already there
+                                boolean alreadyTrusted = false;
+                                for (TrustAnchor anchor : workingTrustAnchors) {
+                                    if (anchor.getTrustedCert().equals(cert)) {
+                                        alreadyTrusted = true;
+                                        break;
+                                    }
+                                }
+                                if (!alreadyTrusted) {
+                                    workingTrustAnchors.add(new TrustAnchor(cert, null));
+                                    result.warnings.add("Auto-trusted fetched root: " + cert.getSubjectX500Principal().getName());
+                                }
+                            }
+                        }
+                    }
+
+                    ChainValidationResult extendedResult = validateChain(extendedArray, workingTrustAnchors);
+
+                    result.details.put("chainLength", extendedArray.length);
+                    result.details.put("originalChainLength", originalChain.length);
+                    result.details.put("chainExtended", true);
+                    result.details.put("fetchedIntermediates", extendedArray.length);
+                    result.details.put("autoTrustedRoots", autoTrustFetchedRoots);
+
+                    if (extendedResult.isValid) {
+                        result.warnings.add("Extended certificate chain with " + extendedArray.length + " fetched intermediates");
+                        return true;
+                    } else {
+                        result.errors.addAll(extendedResult.issues);
+                        return false;
+                    }
+                } else {
+                    result.warnings.add("Could not fetch missing intermediate certificates");
+                    result.details.put("chainLength", originalChain.length);
+                    result.details.put("chainExtended", false);
+                    result.errors.addAll(originalResult.issues);
+                    return false;
+                }
+
+            } catch (Exception e) {
+                result.warnings.add("Failed to fetch intermediates: " + e.getMessage());
+                result.details.put("chainLength", originalChain.length);
+                result.details.put("chainExtended", false);
+                result.errors.addAll(originalResult.issues);
+                return false;
+            }
+
+        } catch (Exception e) {
+            result.errors.add("Error validating chain: " + e.getMessage());
+            return false;
+        }
+    }
+
+
+    private static void debugCertificateChain(List<X509Certificate> certs, Set<TrustAnchor> trustAnchors) {
+        System.out.println("=== Certificate Chain Analysis ===");
+
+        int length = certs.size();
+        System.out.println("Chain length: " + length);
+
+        int i = 0;
+        for (var x509cert : certs) {
+            System.out.println("\nCertificate " + i++ + ":");
+            System.out.println("  Subject: " + x509cert.getSubjectDN().getName());
+            System.out.println("  Issuer:  " + x509cert.getIssuerDN().getName());
+            System.out.println("  Serial:  " + x509cert.getSerialNumber().toString(16));
+            System.out.println("  Valid:   " + x509cert.getNotBefore() + " to " + x509cert.getNotAfter());
+            System.out.println("  Self-signed: " + x509cert.getSubjectDN().equals(x509cert.getIssuerDN()));
+
+            // Check if we have the issuer in our trust anchors
+            boolean issuerFound = false;
+            for (TrustAnchor anchor : trustAnchors) {
+                if (anchor.getTrustedCert().getSubjectDN().equals(x509cert.getIssuerDN())) {
+                    issuerFound = true;
+                    System.out.println("  Issuer found in trust anchors: " + anchor.getTrustedCert().getSubjectDN().getName());
+                    break;
+                }
+            }
+            if (!issuerFound && i == length) {
+                System.out.println("  *** MISSING ISSUER: " + x509cert.getIssuerDN().getName());
+            }
+        }
+    }
+
+    private static class ChainValidationResult {
+        boolean isValid = false;
+        List<String> issues = new ArrayList<>();
+    }
+
+    private static ChainValidationResult validateChain(X509Certificate[] certChain, Set<TrustAnchor> trustAnchors) {
+        ChainValidationResult result = new ChainValidationResult();
+
+        // Check each certificate in the chain
+        for (int i = 0; i < certChain.length; i++) {
+            X509Certificate cert = certChain[i];
+
+            // Check certificate validity dates
+            try {
+                cert.checkValidity();
+            } catch (Exception e) {
+                result.issues.add("Certificate " + i + " expired: " + cert.getSubjectDN());
+            }
+
+            // Check signature (except for self-signed root)
+            if (i < certChain.length - 1) {
+                X509Certificate issuer = certChain[i + 1];
+                try {
+                    cert.verify(issuer.getPublicKey());
+                } catch (Exception e) {
+                    result.issues.add("Certificate " + i + " signature invalid: " + e.getMessage());
+                }
+
+                // Check issuer/subject relationship
+                if (!cert.getIssuerX500Principal().equals(issuer.getSubjectX500Principal())) {
+                    result.issues.add("Certificate " + i + " issuer does not match certificate " + (i + 1) + " subject");
+                }
+            }
+        }
+
+        // Check if chain ends with a trusted root
+        X509Certificate rootCert = certChain[certChain.length - 1];
+        boolean trustedRootFound = false;
+
+        if (rootCert.getSubjectX500Principal().equals(rootCert.getIssuerX500Principal())) {
+            // Self-signed root - check if it's in trust anchors
+            for (TrustAnchor anchor : trustAnchors) {
+                if (anchor.getTrustedCert().equals(rootCert)) {
+                    trustedRootFound = true;
+                    break;
+                }
+            }
+
+            if (!trustedRootFound) {
+                // Check if we trust the root's subject even if the certificate is different
+                for (TrustAnchor anchor : trustAnchors) {
+                    if (anchor.getTrustedCert().getSubjectX500Principal().equals(rootCert.getSubjectX500Principal())) {
+                        trustedRootFound = true;
+                        // Note: we'll add this as a warning in the main result
+                        break;
+                    }
+                }
+            }
+        } else {
+            // Chain doesn't end with self-signed cert - check if issuer is trusted
+            for (TrustAnchor anchor : trustAnchors) {
+                if (anchor.getTrustedCert().getSubjectX500Principal().equals(rootCert.getIssuerX500Principal())) {
+                    trustedRootFound = true;
+                    break;
+                }
+            }
+        }
+
+        if (!trustedRootFound) {
+            result.issues.add("Chain does not end with a trusted root");
+        }
+
+        result.isValid = result.issues.isEmpty();
+        return result;
+    }
+
+    private static boolean checkRevocation(X509Certificate cert, ValidationResult result) {
+        try {
+            // Try OCSP first
+            if (checkOCSP(cert, result)) {
+                return true; // Revoked
+            }
+
+            // Fallback to CRL
+            if (checkCRL(cert, result)) {
+                return true; // Revoked
+            }
+
+            result.warnings.add("Could not check revocation status");
+            return false; // Assume not revoked if we can't check
+
+        } catch (Exception e) {
+            result.warnings.add("Error checking revocation: " + e.getMessage());
+            return false;
+        }
+    }
+
+    private static boolean checkOCSP(X509Certificate cert, ValidationResult result) {
+        // For now, just extract OCSP URL and note that we found it
+        try {
+            List<String> ocspUrls = CertificateFetcher.getOCSPUrls(cert);
+            if (!ocspUrls.isEmpty()) {
+                result.details.put("ocspUrls", ocspUrls);
+                result.warnings.add("OCSP checking not implemented - found OCSP URLs: " + ocspUrls);
+            }
+            return false;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    private static boolean checkCRL(X509Certificate cert, ValidationResult result) {
+        // Basic CRL URL extraction
+        try {
+            List<String> crlUrls = getCRLUrls(cert);
+            if (!crlUrls.isEmpty()) {
+                result.details.put("crlUrls", crlUrls);
+                result.warnings.add("CRL checking not implemented - found CRL URLs: " + crlUrls);
+            }
+            return false;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    // Helper methods
+    private static String getCommonName(X500Principal principal) {
+        String name = principal.getName();
+        String[] parts = name.split(",");
+        for (String part : parts) {
+            part = part.trim();
+            if (part.startsWith("CN=")) {
+                return part.substring(3);
+            }
+        }
+        return null;
+    }
+
+    private static boolean matchesHostname(String certName, String hostname) {
+        if (certName == null || hostname == null) {
+            return false;
+        }
+
+        // Exact match
+        if (certName.equalsIgnoreCase(hostname)) {
+            return true;
+        }
+
+        // Wildcard match
+        if (certName.startsWith("*.")) {
+            String certDomain = certName.substring(2);
+            String hostDomain = hostname;
+            int firstDot = hostname.indexOf('.');
+            if (firstDot > 0) {
+                hostDomain = hostname.substring(firstDot + 1);
+            }
+            return certDomain.equalsIgnoreCase(hostDomain);
+        }
+
+        return false;
+    }
+
+    private static List<String> getCRLUrls(X509Certificate cert) {
+        // This would need to parse the CRL Distribution Points extension
+        // For now, return empty list
+        return new ArrayList<>();
+    }
+
+    // Add this to your AIAExtractor class if not already present
+    public static List<String> getOCSPUrls(X509Certificate certificate) {
+        List<String> ocspUrls = new ArrayList<>();
+
+        try {
+            byte[] aiaExtensionValue = certificate.getExtensionValue(Extension.authorityInfoAccess.getId());
+            if (aiaExtensionValue == null) {
+                return ocspUrls;
+            }
+
+            ASN1OctetString octetString = ASN1OctetString.getInstance(aiaExtensionValue);
+            ASN1Primitive aiaObj = ASN1Primitive.fromByteArray(octetString.getOctets());
+            AuthorityInformationAccess aia = AuthorityInformationAccess.getInstance(aiaObj);
+
+            if (aia != null) {
+                AccessDescription[] accessDescriptions = aia.getAccessDescriptions();
+
+                for (AccessDescription accessDesc : accessDescriptions) {
+                    if (X509ObjectIdentifiers.id_ad_ocsp.equals(accessDesc.getAccessMethod())) {
+                        GeneralName accessLocation = accessDesc.getAccessLocation();
+                        if (accessLocation.getTagNo() == GeneralName.uniformResourceIdentifier) {
+                            String url = accessLocation.getName().toString();
+                            ocspUrls.add(url);
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            System.err.println("Error parsing AIA extension for OCSP: " + e.getMessage());
+        }
+
+        return ocspUrls;
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/ssl/CustomPKIXValidator.java

@@ -1,491 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-import javax.net.ssl.*;
-import java.io.FileInputStream;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.*;
-import java.util.*;
-
-/**
- * Custom PKIX validator for validating X.509 certificate chains with verbose output
- * for db export (i.e. not just SSLException).
- */
-public class CustomPKIXValidator {
-
-    private final Set<TrustAnchor> trustAnchors;
-    private final boolean revocationEnabled;
-    private final boolean anyPolicyInhibited;
-    private final boolean explicitPolicyRequired;
-    private final boolean policyMappingInhibited;
-    private final Set<String> initialPolicies;
-
-    private static final Set<String> EV_POLICY_OIDS = Set.of(
-            "1.3.6.1.4.1.17326.10.14.2.1.2", // Entrust
-            "1.3.6.1.4.1.17326.10.8.12.1.2",  // Entrust
-            "2.16.840.1.114028.10.1.2",       // Entrust/AffirmTrust
-            "1.3.6.1.4.1.6449.1.2.1.5.1",     // Comodo
-            "1.3.6.1.4.1.8024.0.2.100.1.2",   // QuoVadis
-            "2.16.840.1.114404.1.1.2.4.1",    // GoDaddy
-            "2.16.840.1.114413.1.7.23.3",     // DigiCert
-            "2.16.840.1.114414.1.7.23.3",     // DigiCert
-            "1.3.6.1.4.1.14370.1.6",          // GlobalSign
-            "2.16.756.1.89.1.2.1.1",          // SwissSign
-            "1.3.6.1.4.1.4146.1.1"            // GlobalSign
-    );
-
-    // Constructor with default settings
-    public CustomPKIXValidator() throws Exception {
-        this(true, false, false, false, null);
-    }
-
-    // Constructor with custom settings
-    public CustomPKIXValidator(boolean revocationEnabled,
-                               boolean anyPolicyInhibited,
-                               boolean explicitPolicyRequired,
-                               boolean policyMappingInhibited,
-                               Set<String> initialPolicies) throws Exception {
-        this.trustAnchors = loadDefaultTrustAnchors();
-        this.revocationEnabled = revocationEnabled;
-        this.anyPolicyInhibited = anyPolicyInhibited;
-        this.explicitPolicyRequired = explicitPolicyRequired;
-        this.policyMappingInhibited = policyMappingInhibited;
-        this.initialPolicies = initialPolicies;
-    }
-
-    // Constructor with custom trust anchors
-    public CustomPKIXValidator(Set<TrustAnchor> customTrustAnchors,
-                               boolean revocationEnabled) {
-        this.trustAnchors = new HashSet<>(customTrustAnchors);
-        this.revocationEnabled = revocationEnabled;
-        this.anyPolicyInhibited = false;
-        this.explicitPolicyRequired = false;
-        this.policyMappingInhibited = false;
-        this.initialPolicies = null;
-    }
-
-    /**
-     * Validates certificate chain using PKIX algorithm
-     */
-    public PKIXValidationResult validateCertificateChain(String hostname, X509Certificate[] certChain) {
-        EnumSet<PkixValidationError> errors = EnumSet.noneOf(PkixValidationError.class);
-        try {
-            // 1. Basic input validation
-            if (certChain == null || certChain.length == 0) {
-                return new PKIXValidationResult(false, "Certificate chain is empty", errors,
-                        null, null, null, false);
-            }
-
-            if (hostname == null || hostname.trim().isEmpty()) {
-                return new PKIXValidationResult(false, "Hostname is null or empty", errors,
-                        null, null, null, false);
-            }
-
-            // 2. Create certificate path
-            CertPath certPath = createCertificatePath(certChain);
-            if (certPath == null) {
-                return new PKIXValidationResult(false, "Failed to create certificate path", errors,
-                        null, null, null, false);
-            }
-
-            // 3. Build and validate certificate path using PKIX
-            PKIXCertPathValidatorResult pkixResult = performPKIXValidation(certPath, errors);
-
-            // 4. Validate hostname
-            boolean hostnameValid = validateHostname(hostname, certChain[0], errors);
-
-            // 5. Extract critical extensions information
-            Set<String> criticalExtensions = extractCriticalExtensions(certChain);
-
-            boolean overallValid = (pkixResult != null) && hostnameValid;
-            String errorMessage = null;
-
-            if (pkixResult == null) {
-                errorMessage = "PKIX path validation failed";
-            } else if (!hostnameValid) {
-                errorMessage = "Hostname validation failed";
-            }
-
-            return new PKIXValidationResult(overallValid, errorMessage, errors,
-                    pkixResult, certPath, criticalExtensions, hostnameValid);
-
-        } catch (Exception e) {
-            return new PKIXValidationResult(false, "Validation exception: " + e.getMessage(),
-                    errors, null, null, null, false);
-        }
-    }
-
-    /**
-     * Creates a certificate path from the certificate chain
-     */
-    private CertPath createCertificatePath(X509Certificate[] certChain) throws CertificateException {
-        CertificateFactory cf = CertificateFactory.getInstance("X.509");
-        List<Certificate> certList = Arrays.asList(certChain);
-        return cf.generateCertPath(certList);
-    }
-
-    /**
-     * Performs PKIX validation
-     */
-    private PKIXCertPathValidatorResult performPKIXValidation(CertPath certPath, Set<PkixValidationError> warnings) {
-        try {
-            // Create PKIX parameters
-            PKIXParameters params = new PKIXParameters(trustAnchors);
-
-            // Configure PKIX parameters
-            params.setRevocationEnabled(revocationEnabled);
-            params.setAnyPolicyInhibited(anyPolicyInhibited);
-            params.setExplicitPolicyRequired(explicitPolicyRequired);
-            params.setPolicyMappingInhibited(policyMappingInhibited);
-
-            if (initialPolicies != null && !initialPolicies.isEmpty()) {
-                params.setInitialPolicies(initialPolicies);
-            }
-
-            // Set up certificate stores for intermediate certificates if needed
-            // This helps with path building when intermediate certs are missing
-            List<Certificate> intermediateCerts = extractIntermediateCertificates(certPath);
-            if (!intermediateCerts.isEmpty()) {
-                CertStore certStore = CertStore.getInstance("Collection",
-                        new CollectionCertStoreParameters(intermediateCerts));
-                params.addCertStore(certStore);
-            }
-
-            // Configure revocation checking if enabled
-            if (revocationEnabled) {
-                configureRevocationChecking(params);
-            }
-
-            // Create and run validator
-            CertPathValidator validator = CertPathValidator.getInstance("PKIX");
-            PKIXCertPathValidatorResult result = (PKIXCertPathValidatorResult)
-                    validator.validate(certPath, params);
-
-            return result;
-
-        } catch (CertPathValidatorException e) {
-            warnings.add(PkixValidationError.PATH_VALIDATION_FAILED);
-            return null;
-        } catch (InvalidAlgorithmParameterException e) {
-            warnings.add(PkixValidationError.INVALID_PKIX_PARAMETERS);
-            return null;
-        } catch (Exception e) {
-            warnings.add(PkixValidationError.UNKNOWN);
-            return null;
-        }
-    }
-
-    /**
-     * Extracts intermediate certificates from the path
-     */
-    private List<Certificate> extractIntermediateCertificates(CertPath certPath) {
-        List<Certificate> certs = (List<Certificate>) certPath.getCertificates();
-        if (certs.size() <= 2) {
-            return new ArrayList<>(); // Only leaf and root, no intermediates
-        }
-        // Return all but the first (leaf) and potentially last (root)
-        return new ArrayList<>(certs.subList(1, certs.size()));
-    }
-
-    /**
-     * Configures revocation checking (CRL/OCSP)
-     */
-    private void configureRevocationChecking(PKIXParameters params) throws NoSuchAlgorithmException {
-        // Create PKIX revocation checker
-        PKIXRevocationChecker revocationChecker = (PKIXRevocationChecker)
-                CertPathValidator.getInstance("PKIX").getRevocationChecker();
-
-        // Configure revocation checker options
-        Set<PKIXRevocationChecker.Option> options = EnumSet.of(
-                PKIXRevocationChecker.Option.PREFER_CRLS,
-                PKIXRevocationChecker.Option.SOFT_FAIL  // Don't fail if revocation info unavailable
-        );
-        revocationChecker.setOptions(options);
-
-        params.addCertPathChecker(revocationChecker);
-    }
-
-    /**
-     * Comprehensive hostname validation including SAN and CN
-     */
-    private boolean validateHostname(String hostname, X509Certificate cert, Set<PkixValidationError> warnings) {
-        try {
-            // Use Java's built-in hostname verifier as a starting point
-            HostnameVerifier defaultVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
-
-            // Create a mock SSL session for the hostname verifier
-            MockSSLSession mockSession = new MockSSLSession(cert);
-
-            boolean defaultResult = defaultVerifier.verify(hostname, mockSession);
-
-            if (defaultResult) {
-                return true;
-            }
-
-            // If default fails, do manual validation
-            return performManualHostnameValidation(hostname, cert, warnings);
-
-        } catch (Exception e) {
-            warnings.add(PkixValidationError.UNSPECIFIED_HOST_ERROR);
-            return false;
-        }
-    }
-
-    /**
-     * Manual hostname validation implementation
-     */
-    private boolean performManualHostnameValidation(String hostname, X509Certificate cert, Set<PkixValidationError> warnings) {
-        try {
-            // 1. Check Subject Alternative Names (SAN) - preferred method
-            Collection<List<?>> sanEntries = cert.getSubjectAlternativeNames();
-            if (sanEntries != null) {
-                for (List<?> sanEntry : sanEntries) {
-                    if (sanEntry.size() >= 2) {
-                        Integer type = (Integer) sanEntry.get(0);
-                        if (type == 2) { // DNS name
-                            String dnsName = (String) sanEntry.get(1);
-                            if (matchesHostname(hostname, dnsName)) {
-                                return true;
-                            }
-                        } else if (type == 7) { // IP address
-                            String ipAddress = (String) sanEntry.get(1);
-                            if (hostname.equals(ipAddress)) {
-                                return true;
-                            }
-                        }
-                    }
-                }
-                // If SAN is present but no match found, don't check CN (RFC 6125)
-                warnings.add(PkixValidationError.SAN_MISMATCH);
-                return false;
-            }
-
-            // 2. Fallback to Common Name (CN) in subject if no SAN present
-            String subjectDN = cert.getSubjectDN().getName();
-            String cn = extractCommonName(subjectDN);
-            if (cn != null) {
-                if (matchesHostname(hostname, cn)) {
-                    return true;
-                }
-            }
-
-            warnings.add(PkixValidationError.SAN_MISMATCH);
-            return false;
-
-        } catch (Exception e) {
-            warnings.add(PkixValidationError.UNKNOWN);
-            return false;
-        }
-    }
-
-    /**
-     * Checks if hostname matches certificate name (handles wildcards)
-     */
-    private boolean matchesHostname(String hostname, String certName) {
-        if (hostname == null || certName == null) {
-            return false;
-        }
-
-        hostname = hostname.toLowerCase();
-        certName = certName.toLowerCase();
-
-        // Exact match
-        if (hostname.equals(certName)) {
-            return true;
-        }
-
-        // Wildcard matching (*.example.com)
-        if (certName.startsWith("*.")) {
-            String domain = certName.substring(2);
-
-            // Wildcard must match exactly one level
-            if (hostname.endsWith("." + domain)) {
-                String prefix = hostname.substring(0, hostname.length() - domain.length() - 1);
-                // Ensure wildcard doesn't match multiple levels (no dots in prefix)
-                return !prefix.contains(".");
-            }
-        }
-
-        return false;
-    }
-
-    /**
-     * Extracts Common Name from Subject DN
-     */
-    private String extractCommonName(String subjectDN) {
-        if (subjectDN == null) {
-            return null;
-        }
-
-        // Parse DN components
-        String[] components = subjectDN.split(",");
-        for (String component : components) {
-            component = component.trim();
-            if (component.startsWith("CN=")) {
-                return component.substring(3).trim();
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Extracts critical extensions from all certificates in the chain
-     */
-    private Set<String> extractCriticalExtensions(X509Certificate[] certChain) {
-        Set<String> allCriticalExtensions = new HashSet<>();
-
-        for (X509Certificate cert : certChain) {
-            Set<String> criticalExtensions = cert.getCriticalExtensionOIDs();
-            if (criticalExtensions != null) {
-                allCriticalExtensions.addAll(criticalExtensions);
-            }
-        }
-
-        return allCriticalExtensions;
-    }
-
-
-    /**
-     * Gets the key length from a certificate
-     */
-    private int getKeyLength(X509Certificate cert) {
-        try {
-            java.security.PublicKey publicKey = cert.getPublicKey();
-            if (publicKey instanceof java.security.interfaces.RSAPublicKey) {
-                return ((java.security.interfaces.RSAPublicKey) publicKey).getModulus().bitLength();
-            } else if (publicKey instanceof java.security.interfaces.DSAPublicKey) {
-                return ((java.security.interfaces.DSAPublicKey) publicKey).getParams().getP().bitLength();
-            } else if (publicKey instanceof java.security.interfaces.ECPublicKey) {
-                return ((java.security.interfaces.ECPublicKey) publicKey).getParams().getOrder().bitLength();
-            }
-        } catch (Exception e) {
-            // Ignore
-        }
-        return -1;
-    }
-
-    /**
-     * Checks if signature algorithm is considered weak
-     */
-    private boolean isWeakSignatureAlgorithm(String sigAlg) {
-        if (sigAlg == null) return false;
-
-        sigAlg = sigAlg.toLowerCase();
-        return sigAlg.contains("md5") ||
-                sigAlg.contains("sha1") ||
-                sigAlg.equals("md2withrsa") ||
-                sigAlg.equals("md4withrsa");
-    }
-
-    /**
-     * Checks for deprecated or problematic extensions
-     */
-    private void checkDeprecatedExtensions(X509Certificate cert, int index, List<String> warnings) {
-        // Check for Netscape extensions (deprecated)
-        if (cert.getNonCriticalExtensionOIDs() != null) {
-            for (String oid : cert.getNonCriticalExtensionOIDs()) {
-                if (oid.startsWith("2.16.840.1.113730")) { // Netscape OID space
-                    warnings.add("Certificate " + index + " contains deprecated Netscape extension: " + oid);
-                }
-            }
-        }
-
-        // Additional extension checks can be added here
-    }
-
-    /**
-     * Loads default trust anchors from Java's cacerts keystore
-     */
-    private Set<TrustAnchor> loadDefaultTrustAnchors() throws Exception {
-        Set<TrustAnchor> trustAnchors = new HashSet<>();
-
-        // Try to load from default locations
-        String[] keystorePaths = {
-                System.getProperty("javax.net.ssl.trustStore"),
-                System.getProperty("java.home") + "/lib/security/cacerts",
-                System.getProperty("java.home") + "/jre/lib/security/cacerts"
-        };
-
-        String[] keystorePasswords = {
-                System.getProperty("javax.net.ssl.trustStorePassword"),
-                "changeit",
-                ""
-        };
-
-        for (String keystorePath : keystorePaths) {
-            if (keystorePath != null) {
-                for (String password : keystorePasswords) {
-                    try {
-                        KeyStore trustStore = loadKeyStore(keystorePath, password);
-                        if (trustStore != null) {
-                            trustAnchors.addAll(extractTrustAnchors(trustStore));
-                            if (!trustAnchors.isEmpty()) {
-                                return trustAnchors;
-                            }
-                        }
-                    } catch (Exception e) {
-                        // Try next combination
-                    }
-                }
-            }
-        }
-
-        // Fallback: try to get from default trust manager
-        try {
-            TrustManagerFactory tmf = TrustManagerFactory.getInstance(
-                    TrustManagerFactory.getDefaultAlgorithm());
-            tmf.init((KeyStore) null);
-
-            for (TrustManager tm : tmf.getTrustManagers()) {
-                if (tm instanceof X509TrustManager) {
-                    X509TrustManager x509tm = (X509TrustManager) tm;
-                    for (X509Certificate cert : x509tm.getAcceptedIssuers()) {
-                        trustAnchors.add(new TrustAnchor(cert, null));
-                    }
-                }
-            }
-        } catch (Exception e) {
-            throw new Exception("Failed to load any trust anchors", e);
-        }
-
-        if (trustAnchors.isEmpty()) {
-            throw new Exception("No trust anchors could be loaded");
-        }
-
-        return trustAnchors;
-    }
-
-    /**
-     * Loads a keystore from file
-     */
-    private KeyStore loadKeyStore(String keystorePath, String password) throws Exception {
-        KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType());
-        try (FileInputStream fis = new FileInputStream(keystorePath)) {
-            keystore.load(fis, password != null ? password.toCharArray() : null);
-            return keystore;
-        }
-    }
-
-    /**
-     * Extracts trust anchors from a keystore
-     */
-    private Set<TrustAnchor> extractTrustAnchors(KeyStore trustStore) throws KeyStoreException {
-        Set<TrustAnchor> trustAnchors = new HashSet<>();
-
-        Enumeration<String> aliases = trustStore.aliases();
-        while (aliases.hasMoreElements()) {
-            String alias = aliases.nextElement();
-            if (trustStore.isCertificateEntry(alias)) {
-                Certificate cert = trustStore.getCertificate(alias);
-                if (cert instanceof X509Certificate) {
-                    trustAnchors.add(new TrustAnchor((X509Certificate) cert, null));
-                }
-            }
-        }
-
-        return trustAnchors;
-    }
-
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/MockSSLSession.java

@@ -1,116 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSessionContext;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-
-/**
- * Mock SSL session for hostname verification
- */
-public class MockSSLSession implements SSLSession {
-    private final X509Certificate[] peerCertificates;
-
-    public MockSSLSession(X509Certificate cert) {
-        this.peerCertificates = new X509Certificate[]{cert};
-    }
-
-    @Override
-    public Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException {
-        return peerCertificates;
-    }
-
-    // All other methods return default/empty values as they're not used by hostname verification
-    @Override
-    public byte[] getId() {
-        return new byte[0];
-    }
-
-    @Override
-    public SSLSessionContext getSessionContext() {
-        return null;
-    }
-
-    @Override
-    public long getCreationTime() {
-        return 0;
-    }
-
-    @Override
-    public long getLastAccessedTime() {
-        return 0;
-    }
-
-    @Override
-    public void invalidate() {
-    }
-
-    @Override
-    public boolean isValid() {
-        return true;
-    }
-
-    @Override
-    public void putValue(String name, Object value) {
-    }
-
-    @Override
-    public Object getValue(String name) {
-        return null;
-    }
-
-    @Override
-    public void removeValue(String name) {
-    }
-
-    @Override
-    public String[] getValueNames() {
-        return new String[0];
-    }
-
-    @Override
-    public java.security.Principal getPeerPrincipal() throws SSLPeerUnverifiedException {
-        return null;
-    }
-
-    @Override
-    public java.security.Principal getLocalPrincipal() {
-        return null;
-    }
-
-    @Override
-    public String getCipherSuite() {
-        return "";
-    }
-
-    @Override
-    public String getProtocol() {
-        return "";
-    }
-
-    @Override
-    public String getPeerHost() {
-        return "";
-    }
-
-    @Override
-    public int getPeerPort() {
-        return 0;
-    }
-
-    @Override
-    public int getPacketBufferSize() {
-        return 0;
-    }
-
-    @Override
-    public int getApplicationBufferSize() {
-        return 0;
-    }
-
-    @Override
-    public Certificate[] getLocalCertificates() {
-        return new Certificate[0];
-    }
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/PKIXValidationResult.java

@@ -1,14 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-import java.security.cert.CertPath;
-import java.security.cert.PKIXCertPathValidatorResult;
-import java.util.Set;
-
-public record PKIXValidationResult(boolean isValid, String errorMessage,
-                                   Set<PkixValidationError> errors,
-                                   PKIXCertPathValidatorResult pkixResult,
-                                   CertPath validatedPath,
-                                   Set<String> criticalExtensions,
-                                   boolean hostnameValid)
-{
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/PkixValidationError.java

@@ -1,11 +0,0 @@
-package nu.marginalia.ping.ssl;
-
-public enum PkixValidationError {
-    SAN_MISMATCH,
-    EXPIRED,
-    NOT_YET_VALID,
-    PATH_VALIDATION_FAILED,
-    INVALID_PKIX_PARAMETERS,
-    UNKNOWN,
-    UNSPECIFIED_HOST_ERROR;
-}

code/processes/ping-process/java/nu/marginalia/ping/ssl/RootCerts.java

@@ -0,0 +1,57 @@
+package nu.marginalia.ping.ssl;
+
+import java.security.cert.TrustAnchor;
+import java.time.Duration;
+import java.util.Set;
+
+public class RootCerts {
+    private static final String MOZILLA_CA_BUNDLE_URL = "https://curl.se/ca/cacert.pem";
+
+    volatile static boolean initialized = false;
+    volatile static Set<TrustAnchor> trustAnchors;
+
+    public static Set<TrustAnchor> getTrustAnchors() {
+        if (!initialized) {
+            try {
+                synchronized (RootCerts.class) {
+                    while (!initialized) {
+                        RootCerts.class.wait(100);
+                    }
+                }
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException("RootCerts initialization interrupted", e);
+            }
+        }
+        return trustAnchors;
+    }
+
+    static {
+        Thread.ofPlatform()
+                .name("RootCertsUpdater")
+                .daemon()
+                .unstarted(RootCerts::updateTrustAnchors)
+                .start();
+    }
+
+    private static void updateTrustAnchors() {
+        while (true) {
+            try {
+                trustAnchors = CertificateFetcher.getRootCerts(MOZILLA_CA_BUNDLE_URL);
+                synchronized (RootCerts.class) {
+                    initialized = true;
+                    RootCerts.class.notifyAll(); // Notify any waiting threads
+                }
+                Thread.sleep(Duration.ofHours(24));
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break; // Exit if interrupted
+            } catch (Exception e) {
+                // Log the exception and continue to retry
+                System.err.println("Failed to update trust anchors: " + e.getMessage());
+            }
+        }
+    }
+
+}

code/processes/ping-process/java/nu/marginalia/ping/svc/DomainAvailabilityInformationFactory.java

@@ -9,7 +9,7 @@ import nu.marginalia.ping.fetcher.response.HttpsResponse;
 import nu.marginalia.ping.model.DomainAvailabilityRecord;
 import nu.marginalia.ping.model.ErrorClassification;
 import nu.marginalia.ping.model.HttpSchema;
-import nu.marginalia.ping.ssl.PKIXValidationResult;
+import nu.marginalia.ping.ssl.CertificateValidator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -96,6 +96,7 @@ public class DomainAvailabilityInformationFactory {
                 .serverIp(address != null ? address.getAddress() : null)
                 .serverIpAsn(getAsn(address))
                 .httpSchema(HttpSchema.HTTP)
+                .httpLocation(rsp.headers().getFirst("Location"))
                 .httpStatus(rsp.httpStatus())
                 .errorClassification(errorClassification)
                 .httpResponseTime(rsp.httpResponseTime())
@@ -123,7 +124,7 @@ public class DomainAvailabilityInformationFactory {
                                                         int nodeId,
                                                         @Nullable InetAddress address,
                                                         @Nullable DomainAvailabilityRecord previousRecord,
-                                                        PKIXValidationResult validationResult,
+                                                        CertificateValidator.ValidationResult validationResult,
                                                         HttpsResponse rsp) {
         Instant updateTime;
 
@@ -164,6 +165,7 @@ public class DomainAvailabilityInformationFactory {
                 .serverIp(address != null ? address.getAddress() : null)
                 .serverIpAsn(getAsn(address))
                 .httpSchema(HttpSchema.HTTPS)
+                .httpLocation(rsp.headers().getFirst("Location"))
                 .httpStatus(rsp.httpStatus())
                 .errorClassification(errorClassification)
                 .httpResponseTime(rsp.httpResponseTime()) // Placeholder, actual timing not implemented

code/processes/ping-process/java/nu/marginalia/ping/svc/DomainSecurityInformationFactory.java

@@ -4,7 +4,7 @@ import nu.marginalia.ping.fetcher.response.HttpResponse;
 import nu.marginalia.ping.fetcher.response.HttpsResponse;
 import nu.marginalia.ping.model.DomainSecurityRecord;
 import nu.marginalia.ping.model.HttpSchema;
-import nu.marginalia.ping.ssl.PKIXValidationResult;
+import nu.marginalia.ping.ssl.CertificateValidator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -14,9 +14,7 @@ import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateEncodingException;
 import java.security.cert.X509Certificate;
 import java.time.Instant;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.StringJoiner;
+import java.util.*;
 
 public class DomainSecurityInformationFactory {
     private static final Logger logger = LoggerFactory.getLogger(DomainSecurityInformationFactory.class);
@@ -54,7 +52,7 @@ public class DomainSecurityInformationFactory {
     // HTTPS response
     public DomainSecurityRecord createHttpsSecurityInformation(
             HttpsResponse httpResponse,
-            PKIXValidationResult validationResult,
+            CertificateValidator.ValidationResult validationResult,
             int domainId,
             int nodeId,
             @Nullable Integer asn
@@ -69,8 +67,11 @@ public class DomainSecurityInformationFactory {
         boolean isWildcard = false;
         try {
             if (sslCertificates != null && sslCertificates.length > 0) {
-                for (var sanEntry : sslCertificates[0].getSubjectAlternativeNames()) {
-
+                Collection<List<?>> sans = sslCertificates[0].getSubjectAlternativeNames();
+                if (sans == null) {
+                    sans = Collections.emptyList();
+                }
+                for (var sanEntry : sans) {
 
                     if (sanEntry != null && sanEntry.size() >= 2) {
                         // Check if the SAN entry is a DNS or IP address
@@ -125,6 +126,9 @@ public class DomainSecurityInformationFactory {
                 .sslCertWildcard(isWildcard)
                 .sslCertificateChainLength(sslCertificates.length)
                 .sslCertificateValid(validationResult.isValid())
+                .sslHostValid(validationResult.hostnameValid)
+                .sslChainValid(validationResult.chainValid)
+                .sslDateValid(!validationResult.certificateExpired)
                 .httpVersion(httpResponse.version())
                 .tsLastUpdate(Instant.now())
                 .build();

code/processes/ping-process/java/nu/marginalia/ping/svc/HttpPingService.java

@@ -8,8 +8,7 @@ import nu.marginalia.ping.fetcher.response.*;
 import nu.marginalia.ping.model.*;
 import nu.marginalia.ping.model.comparison.DomainAvailabilityChange;
 import nu.marginalia.ping.model.comparison.SecurityInformationChange;
-import nu.marginalia.ping.ssl.CustomPKIXValidator;
-import nu.marginalia.ping.ssl.PKIXValidationResult;
+import nu.marginalia.ping.ssl.CertificateValidator;
 import nu.marginalia.ping.util.JsonObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,9 +31,7 @@ public class HttpPingService {
 
     private final DomainAvailabilityInformationFactory domainAvailabilityInformationFactory;
     private final DomainSecurityInformationFactory domainSecurityInformationFactory;
-
     private static final Logger logger = LoggerFactory.getLogger(HttpPingService.class);
-    CustomPKIXValidator validator;
 
     @Inject
     public HttpPingService(
@@ -46,7 +43,6 @@ public class HttpPingService {
         this.pingHttpFetcher = pingHttpFetcher;
         this.domainAvailabilityInformationFactory = domainAvailabilityInformationFactory;
         this.domainSecurityInformationFactory = domainSecurityInformationFactory;
-        this.validator = new CustomPKIXValidator();
     }
 
     private int compareInetAddresses(InetAddress a, InetAddress b) {
@@ -145,7 +141,7 @@ public class HttpPingService {
                         domainReference.nodeId(),
                         oldPingStatus,
                         ErrorClassification.HTTP_CLIENT_ERROR,
-                        null);
+                        rsp.errorMessage());
                 newSecurityInformation = null;
             }
             case HttpResponse httpResponse -> {
@@ -164,7 +160,11 @@ public class HttpPingService {
                 );
             }
             case HttpsResponse httpsResponse -> {
-                PKIXValidationResult validationResult = validator.validateCertificateChain(domainReference.domainName(), (X509Certificate[]) httpsResponse.sslCertificates());
+                var validationResult = CertificateValidator.validateCertificate(
+                        (X509Certificate[]) httpsResponse.sslCertificates(),
+                        domainReference.domainName(),
+                        true
+                        );
 
                 newPingStatus = domainAvailabilityInformationFactory.createHttpsResponse(
                         domainReference.domainId(),
@@ -296,6 +296,7 @@ public class HttpPingService {
                 change.isCertificatePublicKeyChanged(),
                 change.isCertificateSerialNumberChanged(),
                 change.isCertificateIssuerChanged(),
+                change.schemaChange(),
                 change.oldCertificateTimeToExpiry(),
                 change.isSecurityHeadersChanged(),
                 change.isIpAddressChanged(),

code/processes/ping-process/test/nu/marginalia/ping/AvailabilityJobSchedulerTest.java

@@ -86,7 +86,9 @@ class AvailabilityJobSchedulerTest {
         DomainDnsInformationFactory dnsDomainInformationFactory = new DomainDnsInformationFactory(processConfig, pic);
 
         PingJobScheduler pingJobScheduler = new PingJobScheduler(
-                new HttpPingService(pingHttpFetcher,
+                new HttpPingService(
+                        new LocalDomainCoordinator(),
+                        pingHttpFetcher,
                         new DomainAvailabilityInformationFactory(new GeoIpDictionary(), new BackoffStrategy(pic)),
                         new DomainSecurityInformationFactory()),
                 new DnsPingService(new PingDnsFetcher(List.of("8.8.8.8", "8.8.4.4")),

code/processes/ping-process/test/nu/marginalia/ping/PingDaoTest.java

@@ -243,6 +243,7 @@ class PingDaoTest {
                 .headerServer("Apache/2.4.41 (Ubuntu)")
                 .headerXPoweredBy("PHP/7.4.3")
                 .tsLastUpdate(Instant.now())
+                .sslHostValid(true)
                 .build();
         var svc = new PingDao(dataSource);
         svc.write(foo);
@@ -320,6 +321,7 @@ class PingDaoTest {
                 true,
                 true,
                 false,
+                SchemaChange.NONE,
                 Duration.ofDays(30),
                 false,
                 false,

code/processes/ping-process/test/nu/marginalia/ping/PingHttpServiceTest.java

@@ -2,6 +2,7 @@ package nu.marginalia.ping;
 
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.coordination.LocalDomainCoordinator;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.ping.fetcher.PingHttpFetcher;
 import nu.marginalia.ping.io.HttpClientProvider;
@@ -59,10 +60,12 @@ class PingHttpServiceTest {
 
     }
 
+    @Tag("flaky") // Do not run this test in CI
     @Test
     public void testGetSslInfo() throws Exception {
         var provider = new HttpClientProvider();
         var pingService = new HttpPingService(
+                new LocalDomainCoordinator(),
                 new PingHttpFetcher(provider.get()),
                 new DomainAvailabilityInformationFactory(new GeoIpDictionary(),
                         new BackoffStrategy(PingModule.createPingIntervalsConfiguration())