(ping) Pull root certificates from cacerts.pem

code/processes/ping-process/java/nu/marginalia/ping/ssl/CertificateFetcher.java

@@ -13,20 +13,25 @@ import org.bouncycastle.asn1.x509.*;
 import org.bouncycastle.cert.X509CertificateHolder;
 import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
 import org.bouncycastle.cms.CMSSignedData;
+import org.bouncycastle.openssl.PEMParser;
 import org.bouncycastle.util.Store;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.ByteArrayInputStream;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
 import java.security.cert.CertificateFactory;
+import java.security.cert.TrustAnchor;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
-public class AIACertificateFetcher {
+public class CertificateFetcher {
 
-    private static final Logger logger = LoggerFactory.getLogger(AIACertificateFetcher.class);
+    private static final Logger logger = LoggerFactory.getLogger(CertificateFetcher.class);
 
     private static HttpClient client = HttpClientBuilder.create()
             .build();
@@ -107,6 +112,27 @@ public class AIACertificateFetcher {
         }
     }
 
+    private static List<X509Certificate> parseMultiplePEM(byte[] data) throws Exception {
+        List<X509Certificate> certificates = new ArrayList<>();
+
+        try (StringReader stringReader = new StringReader(new String(data, StandardCharsets.UTF_8));
+             PEMParser pemParser = new PEMParser(stringReader)) {
+
+            JcaX509CertificateConverter converter = new JcaX509CertificateConverter();
+            Object object;
+
+            while ((object = pemParser.readObject()) != null) {
+                if (object instanceof X509CertificateHolder) {
+                    X509CertificateHolder certHolder = (X509CertificateHolder) object;
+                    certificates.add(converter.getCertificate(certHolder));
+                } else if (object instanceof X509Certificate) {
+                    certificates.add((X509Certificate) object);
+                }
+            }
+        }
+
+        return certificates;
+    }
     private static X509Certificate parseX509(byte[] data) throws Exception {
         CertificateFactory cf = CertificateFactory.getInstance("X.509");
         return (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(data));
@@ -215,4 +241,33 @@ public class AIACertificateFetcher {
 
         return ocspUrls;
     }
+
+    public static Set<TrustAnchor> getRootCerts(String bundleUrl) throws Exception {
+        ClassicHttpRequest request =  ClassicRequestBuilder.create("GET")
+                .addHeader("User-Agent", WmsaHome.getUserAgent() + " (Certificate Fetcher)")
+                .setUri(bundleUrl)
+                .build();
+
+        byte[] data = client.execute(request, rsp -> {
+            var entity = rsp.getEntity();
+            if (entity == null) {
+                logger.warn("GET request returned no content for {}", bundleUrl);
+                return null;
+            }
+            return entity.getContent().readAllBytes();
+        });
+
+        List<TrustAnchor> anchors = new ArrayList<>();
+        for (var cert : parseMultiplePEM(data)) {
+            try {
+                anchors.add(new TrustAnchor(cert, null));
+            } catch (Exception e) {
+                logger.warn("Failed to create TrustAnchor for certificate: {}", e.getMessage());
+            }
+        }
+
+        logger.info("Loaded {} root certificates from {}", anchors.size(), bundleUrl);
+
+        return Set.copyOf(anchors);
+    }
 }

code/processes/ping-process/java/nu/marginalia/ping/ssl/CertificateValidator.java

@@ -4,11 +4,7 @@ import org.bouncycastle.asn1.ASN1OctetString;
 import org.bouncycastle.asn1.ASN1Primitive;
 import org.bouncycastle.asn1.x509.*;
 
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
 import javax.security.auth.x500.X500Principal;
-import java.security.KeyStore;
 import java.security.cert.TrustAnchor;
 import java.security.cert.X509Certificate;
 import java.util.*;
@@ -66,32 +62,6 @@ public class CertificateValidator {
         }
     }
 
-    private static final Set<TrustAnchor> trustAnchors = loadDefaultTrustAnchors();
-
-    private static Set<TrustAnchor> loadDefaultTrustAnchors() {
-
-        try {
-            Set<TrustAnchor> trustAnchors = new HashSet<>();
-
-            TrustManagerFactory tmf = TrustManagerFactory
-                    .getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            tmf.init((KeyStore) null);
-
-            for (TrustManager tm : tmf.getTrustManagers()) {
-                if (tm instanceof X509TrustManager x509tm) {
-                    for (X509Certificate cert : x509tm.getAcceptedIssuers()) {
-                        trustAnchors.add(new TrustAnchor(cert, null));
-                    }
-                }
-            }
-
-            return trustAnchors;
-        }
-        catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
     public static ValidationResult validateCertificate(X509Certificate[] certChain,
                                                        String hostname) {
         return validateCertificate(certChain, hostname, false);
@@ -116,7 +86,7 @@ public class CertificateValidator {
         result.hostnameValid = checkHostname(leafCert, hostname, result);
 
         // 3. Check certificate chain validity (with AIA fetching)
-        result.chainValid = checkChainValidity(certChain, trustAnchors, result, autoTrustFetchedRoots);
+        result.chainValid = checkChainValidity(certChain, RootCerts.getTrustAnchors(), result, autoTrustFetchedRoots);
 
         // 4. Check revocation status
         result.certificateRevoked = checkRevocation(leafCert, result);
@@ -200,7 +170,7 @@ public class CertificateValidator {
             }
 
             try {
-                List<X509Certificate> repairedChain = AIACertificateFetcher.buildCompleteChain(originalChain[0]);
+                List<X509Certificate> repairedChain = CertificateFetcher.buildCompleteChain(originalChain[0]);
 
                 if (!repairedChain.isEmpty()) {
 
@@ -397,7 +367,7 @@ public class CertificateValidator {
     private static boolean checkOCSP(X509Certificate cert, ValidationResult result) {
         // For now, just extract OCSP URL and note that we found it
         try {
-            List<String> ocspUrls = AIACertificateFetcher.getOCSPUrls(cert);
+            List<String> ocspUrls = CertificateFetcher.getOCSPUrls(cert);
             if (!ocspUrls.isEmpty()) {
                 result.details.put("ocspUrls", ocspUrls);
                 result.warnings.add("OCSP checking not implemented - found OCSP URLs: " + ocspUrls);

code/processes/ping-process/java/nu/marginalia/ping/ssl/RootCerts.java

@@ -0,0 +1,57 @@
+package nu.marginalia.ping.ssl;
+
+import java.security.cert.TrustAnchor;
+import java.time.Duration;
+import java.util.Set;
+
+public class RootCerts {
+    private static final String MOZILLA_CA_BUNDLE_URL = "https://curl.se/ca/cacert.pem";
+
+    volatile static boolean initialized = false;
+    volatile static Set<TrustAnchor> trustAnchors;
+
+    public static Set<TrustAnchor> getTrustAnchors() {
+        if (!initialized) {
+            try {
+                synchronized (RootCerts.class) {
+                    while (!initialized) {
+                        RootCerts.class.wait(100);
+                    }
+                }
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException("RootCerts initialization interrupted", e);
+            }
+        }
+        return trustAnchors;
+    }
+
+    static {
+        Thread.ofPlatform()
+                .name("RootCertsUpdater")
+                .daemon()
+                .unstarted(RootCerts::updateTrustAnchors)
+                .start();
+    }
+
+    private static void updateTrustAnchors() {
+        while (true) {
+            try {
+                trustAnchors = CertificateFetcher.getRootCerts(MOZILLA_CA_BUNDLE_URL);
+                synchronized (RootCerts.class) {
+                    initialized = true;
+                    RootCerts.class.notifyAll(); // Notify any waiting threads
+                }
+                Thread.sleep(Duration.ofHours(24));
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break; // Exit if interrupted
+            } catch (Exception e) {
+                // Log the exception and continue to retry
+                System.err.println("Failed to update trust anchors: " + e.getMessage());
+            }
+        }
+    }
+
+}

code/processes/ping-process/test/nu/marginalia/ping/PingHttpServiceTest.java

@@ -60,6 +60,7 @@ class PingHttpServiceTest {
 
     }
 
+    @Tag("flaky") // Do not run this test in CI
     @Test
     public void testGetSslInfo() throws Exception {
         var provider = new HttpClientProvider();
@@ -71,7 +72,7 @@ class PingHttpServiceTest {
                         ),
                 new DomainSecurityInformationFactory());
 
-        var output = pingService.pingDomain(new DomainReference(1, 1, "thecavalierclub.co.uk"), null, null);
+        var output = pingService.pingDomain(new DomainReference(1, 1, "www.marginalia.nu"), null, null);
         for (var model : output) {
             System.out.println(model);
         }