(ping) Fix issue where errors were not correctly labeled in availability

Reduce socket and connection timeouts in HttpClient and adjust thread counts for job consumers

code/processes/ping-process/build.gradle

@@ -47,6 +47,7 @@ dependencies {
     implementation libs.bundles.curator
     implementation libs.bundles.mariadb
     implementation libs.bundles.httpcomponents
+    implementation libs.commons.lang3
 
     implementation 'org.bouncycastle:bcprov-jdk18on:1.80'
     implementation 'org.bouncycastle:bcpkix-jdk18on:1.80'

code/processes/ping-process/java/nu/marginalia/ping/PingDao.java

@@ -202,7 +202,7 @@ public class PingDao {
         var query = """
             SELECT * FROM DOMAIN_DNS_INFORMATION
             WHERE TS_NEXT_DNS_CHECK <= ? AND NODE_AFFINITY = ?
-            ORDER BY DNS_CHECK_PRIORITY ASC, TS_NEXT_DNS_CHECK ASC
+            ORDER BY DNS_CHECK_PRIORITY DESC, TS_NEXT_DNS_CHECK ASC
             LIMIT ?
             """;
         try (var conn = dataSource.getConnection();

code/processes/ping-process/java/nu/marginalia/ping/PingJobScheduler.java

@@ -96,14 +96,14 @@ public class PingJobScheduler {
         running = true;
 
         allThreads.add(Thread.ofPlatform().daemon().name("new-dns").start(this::fetchNewDnsRecords));
-        allThreads.add(Thread.ofPlatform().daemon().name("new-pings").start(this::fetchNewAvailabilityJobs));
-        allThreads.add(Thread.ofPlatform().daemon().name("update-pings").start(this::updateAvailabilityJobs));
+        allThreads.add(Thread.ofPlatform().daemon().name("new-availability").start(this::fetchNewAvailabilityJobs));
+        allThreads.add(Thread.ofPlatform().daemon().name("update-availability").start(this::updateAvailabilityJobs));
         allThreads.add(Thread.ofPlatform().daemon().name("update-dns").start(this::updateDnsJobs));
 
-        for (int i = 0; i < 4; i++) {
-            allThreads.add(Thread.ofPlatform().daemon().name("ping-job-consumer-" + i).start(this::availabilityJobConsumer));
+        for (int i = 0; i < 8; i++) {
+            allThreads.add(Thread.ofPlatform().daemon().name("availability-job-consumer-" + i).start(this::availabilityJobConsumer));
         }
-        for (int i = 0; i < 4; i++) {
+        for (int i = 0; i < 1; i++) {
             allThreads.add(Thread.ofPlatform().daemon().name("dns-job-consumer-" + i).start(this::dnsJobConsumer));
         }
     }
@@ -322,7 +322,7 @@ public class PingJobScheduler {
                     };
 
                     if (processingDomainsAvailability.putIfAbsent(job.reference(), true) == null) {
-                        availabilityJobQueue.add(job);
+                        availabilityJobQueue.put(job);
                     }
 
                 }

code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java

@@ -44,8 +44,8 @@ public class HttpClientProvider implements Provider<HttpClient> {
 
     private static CloseableHttpClient createClient() throws NoSuchAlgorithmException {
         final ConnectionConfig connectionConfig = ConnectionConfig.custom()
-                .setSocketTimeout(30, TimeUnit.SECONDS)
-                .setConnectTimeout(30, TimeUnit.SECONDS)
+                .setSocketTimeout(15, TimeUnit.SECONDS)
+                .setConnectTimeout(15, TimeUnit.SECONDS)
                 .setValidateAfterInactivity(TimeValue.ofSeconds(5))
                 .build();
 

code/processes/ping-process/java/nu/marginalia/ping/model/DomainSecurityRecord.java

@@ -1,5 +1,7 @@
 package nu.marginalia.ping.model;
 
+import org.apache.commons.lang3.StringUtils;
+
 import javax.annotation.Nullable;
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -359,12 +361,12 @@ public record DomainSecurityRecord(
         }
 
         public Builder httpVersion(String httpVersion) {
-            this.httpVersion = httpVersion;
+            this.httpVersion = StringUtils.truncate(httpVersion, 10);
             return this;
         }
 
         public Builder httpCompression(String httpCompression) {
-            this.httpCompression = httpCompression;
+            this.httpCompression = StringUtils.truncate(httpCompression, 50);
             return this;
         }
 
@@ -384,12 +386,12 @@ public record DomainSecurityRecord(
         }
 
         public Builder sslCertIssuer(String sslCertIssuer) {
-            this.sslCertIssuer = sslCertIssuer;
+            this.sslCertIssuer = StringUtils.truncate(sslCertIssuer, 255);
             return this;
         }
 
         public Builder sslCertSubject(String sslCertSubject) {
-            this.sslCertSubject = sslCertSubject;
+            this.sslCertSubject = StringUtils.truncate(sslCertSubject, 255);
             return this;
         }
 
@@ -459,37 +461,37 @@ public record DomainSecurityRecord(
         }
 
         public Builder headerStrictTransportSecurity(String headerStrictTransportSecurity) {
-            this.headerStrictTransportSecurity = headerStrictTransportSecurity;
+            this.headerStrictTransportSecurity = StringUtils.truncate(headerStrictTransportSecurity, 255);
             return this;
         }
 
         public Builder headerReferrerPolicy(String headerReferrerPolicy) {
-            this.headerReferrerPolicy = headerReferrerPolicy;
+            this.headerReferrerPolicy = StringUtils.truncate(headerReferrerPolicy, 50);
             return this;
         }
 
         public Builder headerXFrameOptions(String headerXFrameOptions) {
-            this.headerXFrameOptions = headerXFrameOptions;
+            this.headerXFrameOptions = StringUtils.truncate(headerXFrameOptions, 50);
             return this;
         }
 
         public Builder headerXContentTypeOptions(String headerXContentTypeOptions) {
-            this.headerXContentTypeOptions = headerXContentTypeOptions;
+            this.headerXContentTypeOptions = StringUtils.truncate(headerXContentTypeOptions, 50);
             return this;
         }
 
         public Builder headerXXssProtection(String headerXXssProtection) {
-            this.headerXXssProtection = headerXXssProtection;
+            this.headerXXssProtection = StringUtils.truncate(headerXXssProtection, 50);
             return this;
         }
 
         public Builder headerServer(String headerServer) {
-            this.headerServer = headerServer;
+            this.headerServer = StringUtils.truncate(headerServer, 255);
             return this;
         }
 
         public Builder headerXPoweredBy(String headerXPoweredBy) {
-            this.headerXPoweredBy = headerXPoweredBy;
+            this.headerXPoweredBy = StringUtils.truncate(headerXPoweredBy, 255);
             return this;
         }
 

code/processes/ping-process/java/nu/marginalia/ping/svc/DomainAvailabilityInformationFactory.java

@@ -71,23 +71,40 @@ public class DomainAvailabilityInformationFactory {
                                                        @Nullable DomainAvailabilityRecord previousRecord,
                                                        HttpResponse rsp) {
 
-        Instant lastError = previousRecord != null ? previousRecord.tsLastAvailable() : null;
+        final boolean isAvailable;
+        final Instant now = Instant.now();
+        final Instant lastAvailable;
+        final Instant lastError;
+        final ErrorClassification errorClassification;
+
+        if (rsp.httpStatus() >= 400) {
+            isAvailable = false;
+            lastError = now;
+            lastAvailable = previousRecord != null ? previousRecord.tsLastAvailable() : null;
+            errorClassification = ErrorClassification.HTTP_SERVER_ERROR;
+        } else {
+            isAvailable = true;
+            lastAvailable = now;
+            lastError = previousRecord != null ? previousRecord.tsLastError() : null;
+            errorClassification = ErrorClassification.NONE;
+        }
 
         return DomainAvailabilityRecord.builder()
                 .domainId(domainId)
                 .nodeId(nodeId)
-                .serverAvailable(true)
+                .serverAvailable(isAvailable)
                 .serverIp(address != null ? address.getAddress() : null)
                 .serverIpAsn(getAsn(address))
                 .httpSchema(HttpSchema.HTTP)
                 .httpStatus(rsp.httpStatus())
+                .errorClassification(errorClassification)
                 .httpResponseTime(rsp.httpResponseTime())
                 .httpEtag(rsp.headers().getFirst("ETag"))
                 .httpLastModified(rsp.headers().getFirst("Last-Modified"))
-                .tsLastPing(Instant.now())
-                .tsLastAvailable(Instant.now())
+                .tsLastPing(now)
+                .tsLastAvailable(lastAvailable)
                 .tsLastError(lastError)
-                .nextScheduledUpdate(Instant.now().plus(backoffStrategy.getOkInterval()))
+                .nextScheduledUpdate(now.plus(backoffStrategy.getOkInterval()))
                 .backoffFetchInterval(backoffStrategy.getOkInterval())
                 .build();
 
@@ -117,23 +134,44 @@ public class DomainAvailabilityInformationFactory {
             updateTime = Instant.now().plus(backoffStrategy.getOkInterval());
         }
 
-        Instant lastError = previousRecord != null ? previousRecord.tsLastAvailable() : null;
+        final boolean isAvailable;
+        final Instant now = Instant.now();
+        final Instant lastAvailable;
+        final Instant lastError;
+        final ErrorClassification errorClassification;
+
+        if (!validationResult.isValid()) {
+            isAvailable = false;
+            lastError = now;
+            lastAvailable = previousRecord != null ? previousRecord.tsLastAvailable() : null;
+            errorClassification = ErrorClassification.SSL_ERROR;
+        } else if (rsp.httpStatus() >= 400) {
+            isAvailable = false;
+            lastError = now;
+            lastAvailable = previousRecord != null ? previousRecord.tsLastAvailable() : null;
+            errorClassification = ErrorClassification.HTTP_SERVER_ERROR;
+        } else {
+            isAvailable = true;
+            lastAvailable = Instant.now();
+            lastError = previousRecord != null ? previousRecord.tsLastError() : null;
+            errorClassification = ErrorClassification.NONE;
+        }
 
         return DomainAvailabilityRecord.builder()
                 .domainId(domainId)
                 .nodeId(nodeId)
-                .serverAvailable(validationResult.isValid())
+                .serverAvailable(isAvailable)
                 .serverIp(address != null ? address.getAddress() : null)
                 .serverIpAsn(getAsn(address))
                 .httpSchema(HttpSchema.HTTPS)
                 .httpStatus(rsp.httpStatus())
-                .errorClassification(!validationResult.isValid() ? ErrorClassification.SSL_ERROR : ErrorClassification.NONE)
+                .errorClassification(errorClassification)
                 .httpResponseTime(rsp.httpResponseTime()) // Placeholder, actual timing not implemented
                 .httpEtag(rsp.headers().getFirst("ETag"))
                 .httpLastModified(rsp.headers().getFirst("Last-Modified"))
-                .tsLastPing(Instant.now())
+                .tsLastPing(now)
                 .tsLastError(lastError)
-                .tsLastAvailable(Instant.now())
+                .tsLastAvailable(lastAvailable)
                 .nextScheduledUpdate(updateTime)
                 .backoffFetchInterval(backoffStrategy.getOkInterval())
                 .build();