(api) Add query parameter 'dc' for specifying the max number of results per domain

added converter.insertFoundDomains property

code/execution/java/nu/marginalia/actor/proc/ScrapeFeedsActor.java

@@ -47,6 +47,8 @@ public class ScrapeFeedsActor extends RecordActorPrototype {
 
     private final Path feedPath = WmsaHome.getHomePath().resolve("data/scrape-urls.txt");
 
+    private static boolean insertFoundDomains = Boolean.getBoolean("loader.insertFoundDomains");
+
     public record Initial() implements ActorStep {}
     @Resume(behavior = ActorResumeBehavior.RETRY)
     public record Wait(String ts) implements ActorStep {}
@@ -57,6 +59,8 @@ public class ScrapeFeedsActor extends RecordActorPrototype {
     public ActorStep transition(ActorStep self) throws Exception {
         return switch(self) {
             case Initial() -> {
+                if (!insertFoundDomains) yield new Error("Domain insertion prohibited, aborting");
+
                 if (nodeConfigurationService.get(nodeId).profile() != NodeProfile.REALTIME) {
                     yield new Error("Invalid node profile for RSS update");
                 }

code/processes/loading-process/java/nu/marginalia/loading/LoaderMain.java

@@ -40,6 +40,8 @@ public class LoaderMain extends ProcessMainClass {
     private final KeywordLoaderService keywordLoaderService;
     private final DocumentLoaderService documentLoaderService;
 
+    private static boolean insertFoundDomains = Boolean.getBoolean("loader.insertFoundDomains");
+
     public static void main(String... args) {
         try {
             new org.mariadb.jdbc.Driver();
@@ -99,6 +101,10 @@ public class LoaderMain extends ProcessMainClass {
 
         try {
             var results = ForkJoinPool.commonPool()
+                .invokeAll(List.of());
+
+            if ( true == insertFoundDomains ) {
+                results = ForkJoinPool.commonPool()
                         .invokeAll(
                             List.of(
                                 () -> linksService.loadLinks(domainIdRegistry, heartbeat, inputData),
@@ -107,6 +113,17 @@ public class LoaderMain extends ProcessMainClass {
                                 () -> domainService.loadDomainMetadata(domainIdRegistry, heartbeat, inputData)
                             )
                 );
+            }
+            else {
+                results = ForkJoinPool.commonPool()
+                        .invokeAll(
+                            List.of(
+                                () -> keywordLoaderService.loadKeywords(domainIdRegistry, heartbeat, inputData),
+                                () -> documentLoaderService.loadDocuments(domainIdRegistry, heartbeat, inputData),
+                                () -> domainService.loadDomainMetadata(domainIdRegistry, heartbeat, inputData)
+                            )
+                );
+            }
 
             for (var result : results) {
                 if (result.state() == Future.State.FAILED) {

code/processes/loading-process/java/nu/marginalia/loading/domains/DomainLoaderService.java

@@ -25,6 +25,8 @@ import java.util.Set;
 @Singleton
 public class DomainLoaderService {
 
+    private static boolean insertFoundDomains = Boolean.getBoolean("loader.insertFoundDomains");
+
     private final HikariDataSource dataSource;
     private final Logger logger = LoggerFactory.getLogger(DomainLoaderService.class);
     private final int nodeId;
@@ -84,6 +86,9 @@ public class DomainLoaderService {
 
             // Add domains that are linked to from the domains we've just crawled, but with -1 affinity meaning they
             // can be grabbed by any index node
+            if ( true == insertFoundDomains ) {
+                logger.info("Adding found domains");
+
                 try (var inserter = new DomainInserter(conn, -1);
                      var processHeartbeat = heartbeat.createAdHocTaskHeartbeat("INSERT_LINKED_DOMAINS")) {
                     // Add linked domains, but with -1 affinity meaning they can be grabbed by any index node
@@ -101,8 +106,14 @@ public class DomainLoaderService {
                                 }
                             }
                         }
+   
                     }
                 }
+            }
+            else {
+                logger.info("Skipping found domains");
+            }
+
 
             taskHeartbeat.progress(Steps.UPDATE_AFFINITY_AND_IP);
 

code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java

@@ -30,10 +30,11 @@ public class ApiSearchOperator {
 
     public ApiSearchResults query(String query,
                                   int count,
+                                  int domainCount,
                                   int index,
                                   NsfwFilterTier filterTier)
     {
-        var rsp = queryClient.search(createParams(query, count, index, filterTier));
+        var rsp = queryClient.search(createParams(query, count, domainCount, index, filterTier));
 
         return new ApiSearchResults("RESTRICTED", query,
                 rsp.results()
@@ -44,13 +45,13 @@ public class ApiSearchOperator {
                 .collect(Collectors.toList()));
     }
 
-    private QueryParams createParams(String query, int count, int index, NsfwFilterTier filterTirer) {
+    private QueryParams createParams(String query, int count, int domainCount, int index, NsfwFilterTier filterTirer) {
         SearchSetIdentifier searchSet = selectSearchSet(index);
 
         return new QueryParams(
                 query,
                 RpcQueryLimits.newBuilder()
-                        .setResultsByDomain(2)
+                        .setResultsByDomain(Math.clamp(domainCount, 1, 100))
                         .setResultsTotal(Math.min(100, count))
                         .setTimeoutMs(150)
                         .setFetchSize(8192)

code/services-application/api-service/java/nu/marginalia/api/ApiService.java

@@ -119,6 +119,7 @@ public class ApiService extends SparkService {
         }
 
         int count = intParam(request, "count", 20);
+        int domainCount = intParam(request, "dc", 2);
         int index = intParam(request, "index", 3);
         int nsfw = intParam(request, "nsfw", 1);
 
@@ -137,7 +138,7 @@ public class ApiService extends SparkService {
                 .labels(license.key)
                 .time(() ->
                         searchOperator
-                        .query(query, count, index, nsfwFilterTier)
+                        .query(query, count, domainCount, index, nsfwFilterTier)
                         .withLicense(license.getLicense())
                 );
     }

code/services-application/search-service-legacy/java/nu/marginalia/search/command/commands/BangCommand.java

@@ -20,7 +20,7 @@ public class BangCommand implements SearchCommandInterface {
     {
         bangsToPattern.put("!g", "https://www.google.com/search?q=%s");
         bangsToPattern.put("!ddg", "https://duckduckgo.com/?q=%s");
-        bangsToPattern.put("!w", "https://search.marginalia.nu/search?query=%s+site:en.wikipedia.org&profile=wiki");
+        bangsToPattern.put("!w", "https://old-search.marginalia.nu/search?query=%s+site:en.wikipedia.org&profile=wiki");
     }
 
     @Override

code/services-application/search-service/java/nu/marginalia/search/command/commands/BangCommand.java

@@ -20,7 +20,7 @@ public class BangCommand implements SearchCommandInterface {
     {
         bangsToPattern.put("!g", "https://www.google.com/search?q=%s");
         bangsToPattern.put("!ddg", "https://duckduckgo.com/?q=%s");
-        bangsToPattern.put("!w", "https://search.marginalia.nu/search?query=%s+site:en.wikipedia.org&profile=wiki");
+        bangsToPattern.put("!w", "/search?query=%s+site:en.wikipedia.org");
     }
 
     @Override
@@ -34,7 +34,7 @@ public class BangCommand implements SearchCommandInterface {
 
             if (match.isPresent()) {
                 var url = String.format(redirectPattern, URLEncoder.encode(match.get(), StandardCharsets.UTF_8));
-                new MapModelAndView("redirect.jte", Map.of("url", url));
+                return Optional.of(new MapModelAndView("redirect.jte", Map.of("url", url)));
             }
         }
 

code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java

@@ -0,0 +1,19 @@
+package nu.marginalia.search.command.commands;
+
+import nu.marginalia.WebsiteUrl;
+import nu.marginalia.search.command.SearchParameters;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class BangCommandTest {
+
+    @Test
+    void testWikipediaRedirect() {
+        BangCommand bc = new BangCommand();
+
+        assertTrue(bc.process(SearchParameters.defaultsForQuery(new WebsiteUrl("test"), "!w plato", 1)).isPresent());
+        assertFalse(bc.process(SearchParameters.defaultsForQuery(new WebsiteUrl("test"), "plato", 1)).isPresent());
+    }
+}