(crawler) Correct issue leading to parquet files not being correctly preconverted

Path.endsWith("str") != String.endsWith(".str")

code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java

@@ -121,6 +121,7 @@ public class ServiceConfigurationModule extends AbstractModule {
 
         while (nets.hasMoreElements()) {
             NetworkInterface netif = nets.nextElement();
+            logger.info("Considering network interface {}:  Up? {},  Loopback? {}", netif.getDisplayName(), netif.isUp(), netif.isLoopback());
             if (!netif.isUp() || netif.isLoopback()) {
                 continue;
             }
@@ -128,6 +129,7 @@ public class ServiceConfigurationModule extends AbstractModule {
             Enumeration<InetAddress> inetAddresses = netif.getInetAddresses();
             while (inetAddresses.hasMoreElements()) {
                 InetAddress addr = inetAddresses.nextElement();
+                logger.info("Considering address {}: SiteLocal? {}, Loopback? {}", addr.getHostAddress(), addr.isSiteLocalAddress(), addr.isLoopbackAddress());
                 if (addr.isSiteLocalAddress() && !addr.isLoopbackAddress()) {
                     return addr.getHostAddress();
                 }

code/common/service/java/nu/marginalia/service/server/MetricsServer.java

@@ -13,7 +13,7 @@ import java.net.InetSocketAddress;
 
 public class MetricsServer {
 
-    private static Logger logger = LoggerFactory.getLogger(MetricsServer.class);
+    private static final Logger logger = LoggerFactory.getLogger(MetricsServer.class);
 
     @Inject
     public MetricsServer(ServiceConfiguration configuration) {
@@ -30,6 +30,8 @@ public class MetricsServer {
 
             context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
 
+            logger.info("MetricsServer listening on {}:{}", configuration.bindAddress(), configuration.metricsPort());
+
             server.start();
         }
         catch (Exception|NoSuchMethodError ex) {

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -266,11 +266,11 @@ public class CrawlerMain extends ProcessMainClass {
 
                 // Start every task we currently can from the deferral list
                 deferredTasks.removeIf(task -> {
-                    if (pendingCrawlTasks.putIfAbsent(crawlSpec.domain(), task) != null) {
-                        return true; // task has already run, duplicate in crawl specs
-                    }
-
                     if (task.canRun()) {
+                        if (pendingCrawlTasks.putIfAbsent(crawlSpec.domain(), task) != null) {
+                            return true; // task has already run, duplicate in crawl specs
+                        }
+
                         // This blocks the caller when the pool is full
                         pool.submitQuietly(task);
                         return true;
@@ -280,7 +280,7 @@ public class CrawlerMain extends ProcessMainClass {
                 });
             }
 
-            // Schedule any lingering tasks
+            // Schedule any lingering tasks for immediate execution
             for (var task : deferredTasks) {
                 if (pendingCrawlTasks.putIfAbsent(task.domain, task) != null)
                     continue;
@@ -522,7 +522,7 @@ public class CrawlerMain extends ProcessMainClass {
     //
     // This must be synchronized as chewing through parquet files in parallel leads to enormous memory overhead
     private synchronized Path migrateParquetData(Path inputPath, String domain, Path crawlDataRoot) throws IOException {
-        if (!inputPath.endsWith(".parquet")) {
+        if (!inputPath.toString().endsWith(".parquet")) {
             return inputPath;
         }
 

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -60,15 +60,7 @@ public class HttpFetcherImpl implements HttpFetcher {
                 .cookieHandler(cookies)
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .connectTimeout(Duration.ofSeconds(8))
-                .executor(Executors.newCachedThreadPool(
+                .executor(Executors.newCachedThreadPool())
-                        r -> Thread.ofPlatform()
-                                .name("FetcherClient")
-                                .daemon(true)
-                                .uncaughtExceptionHandler((t, ex) -> {
-                                    logger.error("Uncaught Exception in " + t.getName(), ex);
-                                })
-                                .start(r)
-                ))
                 .build();
     }
 

code/processes/crawling-process/model/java/nu/marginalia/io/SerializableCrawlDataStream.java

@@ -42,18 +42,20 @@ public interface SerializableCrawlDataStream extends AutoCloseable {
     {
 
         String fileName = fullPath.getFileName().toString();
-        if (fileName.endsWith(".parquet")) {
+
+        if (fileName.endsWith(".slop.zip")) {
             try {
-                return new ParquetSerializableCrawlDataStream(fullPath);
+                return new SlopSerializableCrawlDataStream(fullPath);
             } catch (Exception ex) {
                 logger.error("Error reading domain data from " + fullPath, ex);
                 return SerializableCrawlDataStream.empty();
             }
         }
 
-        if (fileName.endsWith(".slop.zip")) {
+        else if (fileName.endsWith(".parquet")) {
+            logger.error("Opening deprecated parquet-style crawl data stream", new Exception());
             try {
-                return new SlopSerializableCrawlDataStream(fullPath);
+                return new ParquetSerializableCrawlDataStream(fullPath);
             } catch (Exception ex) {
                 logger.error("Error reading domain data from " + fullPath, ex);
                 return SerializableCrawlDataStream.empty();