(*) Fix boot orchestration for the services

This corrects an annoying bug that had the system crash and burn on first start-up due to a race condition in service initialization, where the services were attempting to access the database before it was properly migrated.

A fix was in principle already in place, but it was running too late and did not prevent attempts to access the as-yet uninitialized database.  Move the first boot check into the MainClass instead of the Service constructor.

The change also adds more appropriate docker dependencies to the services to fix rare errors resolving the hostname of the database.

code/common/service/java/nu/marginalia/service/MainClass.java

@@ -1,6 +1,8 @@
 package nu.marginalia.service;
 
 import io.prometheus.client.hotspot.DefaultExports;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,4 +46,27 @@ public abstract class MainClass {
         DefaultExports.initialize();
     }
 
+    /** Ensure that the services boot in the correct order, so that the control service
+     * has the opportunity to migrate the database before the other services attempt to access it
+     * on first boot.
+     */
+    protected static void orchestrateBoot(ServiceRegistryIf serviceRegistry,
+                                          ServiceConfiguration configuration) {
+
+        if (configuration.serviceId() != ServiceId.Control) {
+            // If this is not the control service, we need to wait for the control service to boot
+            try {
+                serviceRegistry.waitForFirstBoot();
+            }
+            catch (InterruptedException e) {
+                // Fail hard here, there is no meaningful recovery
+                System.err.println("Interrupted while waiting for control service to boot");
+                System.exit(1);
+            }
+        }
+        else {
+            // This is the control service, so we need to declare that we have booted successfully
+            serviceRegistry.declareFirstBoot();
+        }
+    }
 }

code/common/service/java/nu/marginalia/service/server/Service.java

@@ -61,17 +61,6 @@ public class Service {
         var config = params.configuration;
         node = config.node();
 
-        if (config.serviceId() == ServiceId.Control) {
-            // Special case for first boot, since the control service
-            // owns database migrations and so on, we need other processes
-            // to wait for this to be done before they start.  This is
-            // only needed once.
-            params.serviceRegistry.declareFirstBoot();
-        }
-        else {
-            params.serviceRegistry.waitForFirstBoot();
-        }
-
         String inboxName = config.serviceName();
         logger.info("Inbox name: {}", inboxName);
 

code/services-application/api-service/java/nu/marginalia/api/ApiMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -25,8 +26,10 @@ public class ApiMain extends MainClass {
                 new ServiceDiscoveryModule(),
                 new ServiceConfigurationModule(ServiceId.Api));
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(ApiMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-application/dating-service/java/nu/marginalia/dating/DatingMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -32,8 +33,10 @@ public class DatingMain extends MainClass {
                 new DatabaseModule(false)
         );
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(DatingMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -32,8 +33,10 @@ public class ExplorerMain extends MainClass {
                 new DatabaseModule(false)
         );
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(ExplorerMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-application/search-service/java/nu/marginalia/search/SearchMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -33,8 +34,10 @@ public class SearchMain extends MainClass {
                 new DatabaseModule(false)
         );
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(SearchMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -30,8 +31,10 @@ public class AssistantMain extends MainClass {
         );
 
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(AssistantMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-core/control-service/java/nu/marginalia/control/ControlMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -26,8 +27,10 @@ public class ControlMain extends MainClass {
                 new ServiceDiscoveryModule(),
                 new ServiceConfigurationModule(ServiceId.Control));
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(ControlMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-core/executor-service/java/nu/marginalia/executor/ExecutorMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.DatabaseModule;
@@ -30,8 +31,10 @@ public class ExecutorMain extends MainClass  {
                 new ServiceConfigurationModule(ServiceId.Executor)
         );
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(NodeStatusWatcher.class);
         injector.getInstance(ExecutorMain.class);

code/services-core/index-service/java/nu/marginalia/index/IndexMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -30,8 +31,10 @@ public class IndexMain extends MainClass {
                 new ServiceConfigurationModule(ServiceId.Index)
         );
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(NodeStatusWatcher.class);
 

code/services-core/query-service/java/nu/marginalia/query/QueryMain.java

@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.module.ServiceConfiguration;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -29,8 +30,10 @@ public class QueryMain extends MainClass {
                 new ServiceConfigurationModule(ServiceId.Query)
         );
 
-        // Ensure that the service registry is initialized early
-        injector.getInstance(ServiceRegistryIf.class);
+        // Orchestrate the boot order for the services
+        var registry = injector.getInstance(ServiceRegistryIf.class);
+        var configuration = injector.getInstance(ServiceConfiguration.class);
+        orchestrateBoot(registry, configuration);
 
         injector.getInstance(QueryMain.class);
         injector.getInstance(Initialization.class).setReady();

run/install/docker-compose-barebones-1.yml.template

@@ -9,8 +9,10 @@ x-svc: &service
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
 x-p1: &partition-1
   env_file:
     - "${INSTALL_DIR}/env/service.env"
@@ -29,8 +31,10 @@ x-p1: &partition-1
   environment:
     - "WMSA_SERVICE_NODE=1"
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
 services:
   index-service-1:
     <<: *partition-1

run/install/docker-compose-barebones-2.yml.template

@@ -9,8 +9,10 @@ x-svc: &service
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
 x-p1: &partition-1
   env_file:
     - "${INSTALL_DIR}/env/service.env"
@@ -27,8 +29,10 @@ x-p1: &partition-1
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
   environment:
     - "WMSA_SERVICE_NODE=1"
 x-p2: &partition-2
@@ -47,8 +51,10 @@ x-p2: &partition-2
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
   environment:
     - "WMSA_SERVICE_NODE=2"
 services:

run/install/docker-compose-marginalia.yml.template

@@ -9,8 +9,10 @@ x-svc: &service
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
 x-p1: &partition-1
   env_file:
     - "${INSTALL_DIR}/env/service.env"
@@ -27,8 +29,10 @@ x-p1: &partition-1
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
   environment:
     - "WMSA_SERVICE_NODE=1"
 x-p2: &partition-2
@@ -47,8 +51,10 @@ x-p2: &partition-2
   networks:
     - wmsa
   depends_on:
-    - mariadb
-    - zookeeper
+    mariadb:
+      condition: service_healthy
+    zookeeper:
+      condition: service_started
   environment:
     - "WMSA_SERVICE_NODE=2"
 services: