(search) Dyslexia fix

Improve error handling.  This should be a relatively rare case, but we don't want one bad index partition to blow up the entire query.

code/index/api/java/nu/marginalia/index/api/IndexClient.java

@@ -16,20 +16,19 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-
-import static java.lang.Math.clamp;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
 
 @Singleton
 public class IndexClient {
     private static final Logger logger = LoggerFactory.getLogger(IndexClient.class);
     private final GrpcMultiNodeChannelPool<IndexApiGrpc.IndexApiBlockingStub> channelPool;
     private final DomainBlacklistImpl blacklist;
-    private static final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
+    private static final ExecutorService executor = Executors.newCachedThreadPool();
 
     @Inject
     public IndexClient(GrpcChannelPoolFactory channelPoolFactory, DomainBlacklistImpl blacklist) {
@@ -51,40 +50,37 @@ public class IndexClient {
 
     /** Execute a query on the index partitions and return the combined results. */
     public AggregateQueryResponse executeQueries(RpcIndexQuery indexRequest, Pagination pagination) {
-        List<CompletableFuture<Iterator<RpcDecoratedResultItem>>> futures =
-                channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query)
-                        .async(executor)
-                        .runEach(indexRequest);
 
         final int requestedMaxResults = indexRequest.getQueryLimits().getResultsTotal();
-        final int resultsUpperBound = requestedMaxResults * channelPool.getNumNodes();
 
-        List<RpcDecoratedResultItem> results = new ArrayList<>(resultsUpperBound);
+        AtomicInteger totalNumResults = new AtomicInteger(0);
 
-        for (var future : futures) {
-            try {
-                future.get().forEachRemaining(results::add);
-            }
-            catch (Exception e) {
-                logger.error("Downstream exception", e);
-            }
-        }
+        List<RpcDecoratedResultItem> results =
+                channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query)
+                        .async(executor)
+                        .runEach(indexRequest)
+                        .stream()
+                        .map(future -> future.thenApply(iterator -> {
+                            List<RpcDecoratedResultItem> ret = new ArrayList<>(requestedMaxResults);
+                            iterator.forEachRemaining(ret::add);
+                            totalNumResults.addAndGet(ret.size());
+                            return ret;
+                        }))
+                        .mapMulti((CompletableFuture<List<RpcDecoratedResultItem>> fut, Consumer<List<RpcDecoratedResultItem>> c) ->{
+                            try {
+                                c.accept(fut.join());
+                            } catch (Exception e) {
+                                logger.error("Error while fetching results", e);
+                            }
+                        })
+                        .flatMap(List::stream)
+                        .filter(item -> !isBlacklisted(item))
+                        .sorted(comparator)
+                        .skip(Math.max(0, (pagination.page - 1) * pagination.pageSize))
+                        .limit(pagination.pageSize)
+                        .toList();
 
-        // Sort the results by ranking score and remove blacklisted domains
-        results.sort(comparator);
-        results.removeIf(this::isBlacklisted);
-
-        int numReceivedResults = results.size();
-
-        // pagination is typically 1-indexed, so we need to adjust the start and end indices
-        int indexStart = (pagination.page - 1) * pagination.pageSize;
-        int indexEnd = (pagination.page) * pagination.pageSize;
-
-        results = results.subList(
-                clamp(indexStart, 0, Math.max(0, results.size() - 1)), // from is inclusive, so subtract 1 from size()
-                clamp(indexEnd, 0, results.size()));
-
-        return new AggregateQueryResponse(results, pagination.page(), numReceivedResults);
+        return new AggregateQueryResponse(results, pagination.page(), totalNumResults.get());
     }
 
     private boolean isBlacklisted(RpcDecoratedResultItem item) {

code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java

@@ -84,18 +84,33 @@ public record SearchParameters(WebsiteUrl url,
     }
 
     public String renderUrl() {
-        String path = String.format("/search?query=%s&profile=%s&js=%s&adtech=%s&recent=%s&searchTitle=%s&newfilter=%s&page=%d",
-                URLEncoder.encode(query, StandardCharsets.UTF_8),
-                URLEncoder.encode(profile.filterId, StandardCharsets.UTF_8),
-                URLEncoder.encode(js.value, StandardCharsets.UTF_8),
-                URLEncoder.encode(adtech.value, StandardCharsets.UTF_8),
-                URLEncoder.encode(recent.value, StandardCharsets.UTF_8),
-                URLEncoder.encode(searchTitle.value, StandardCharsets.UTF_8),
-                Boolean.valueOf(newFilter).toString(),
-                page
-                );
 
-        return path;
+        StringBuilder pathBuilder = new StringBuilder("/search?");
+        pathBuilder.append("query=").append(URLEncoder.encode(query, StandardCharsets.UTF_8));
+
+        if (profile != SearchProfile.NO_FILTER) {
+            pathBuilder.append("&profile=").append(URLEncoder.encode(profile.filterId, StandardCharsets.UTF_8));
+        }
+        if (js != SearchJsParameter.DEFAULT) {
+            pathBuilder.append("&js=").append(URLEncoder.encode(js.value, StandardCharsets.UTF_8));
+        }
+        if (adtech != SearchAdtechParameter.DEFAULT) {
+            pathBuilder.append("&adtech=").append(URLEncoder.encode(adtech.value, StandardCharsets.UTF_8));
+        }
+        if (recent != SearchRecentParameter.DEFAULT) {
+            pathBuilder.append("&recent=").append(URLEncoder.encode(recent.value, StandardCharsets.UTF_8));
+        }
+        if (searchTitle != SearchTitleParameter.DEFAULT) {
+            pathBuilder.append("&searchTitle=").append(URLEncoder.encode(searchTitle.value, StandardCharsets.UTF_8));
+        }
+        if (page != 1) {
+            pathBuilder.append("&page=").append(page);
+        }
+        if (newFilter) {
+            pathBuilder.append("&newfilter=").append(Boolean.valueOf(newFilter).toString());
+        }
+
+        return pathBuilder.toString();
     }
 
     public RpcTemporalBias.Bias temporalBias() {

code/services-application/search-service/java/nu/marginalia/search/command/commands/SearchCommand.java

@@ -3,27 +3,22 @@ package nu.marginalia.search.command.commands;
 import com.google.inject.Inject;
 import io.jooby.MapModelAndView;
 import io.jooby.ModelAndView;
-import nu.marginalia.search.JteRenderer;
 import nu.marginalia.search.SearchOperator;
 import nu.marginalia.search.command.SearchCommandInterface;
 import nu.marginalia.search.command.SearchParameters;
 import nu.marginalia.search.model.DecoratedSearchResults;
 import nu.marginalia.search.model.NavbarModel;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.Optional;
 
 public class SearchCommand implements SearchCommandInterface {
     private final SearchOperator searchOperator;
-    private final JteRenderer jteRenderer;
 
 
     @Inject
-    public SearchCommand(SearchOperator searchOperator,
-                         JteRenderer jteRenderer) throws IOException {
+    public SearchCommand(SearchOperator searchOperator){
         this.searchOperator = searchOperator;
-        this.jteRenderer = jteRenderer;
     }
 
     @Override

code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java

@@ -137,7 +137,7 @@ public class SearchSiteInfoService {
             @PathParam String domainName,
             @QueryParam String view,
             @QueryParam Integer page
-    ) throws SQLException {
+    ) throws SQLException, ExecutionException {
 
         if (null == domainName || domainName.isBlank()) {
             return null;

code/services-application/search-service/resources/jte/serp/part/searchform.jte

@@ -36,10 +36,11 @@
 
     </div>
 
+    @if (filters.showRecentOption.isSet()) <input type="hidden" name="js" value="${filters.removeJsOption.value()}"> @endif
+    @if (filters.reduceAdtechOption.isSet()) <input type="hidden" name="adtech" value="${filters.reduceAdtechOption.value()}"> @endif
+    @if (filters.searchTitleOption.isSet()) <input type="hidden" name="searchTitle" value="${filters.searchTitleOption.value()}"> @endif
+    @if (filters.showRecentOption.isSet()) <input type="hidden" name="recent" value="${filters.showRecentOption.value()}"> @endif
 
-    <input type="hidden" name="js" value="${filters.removeJsOption.value()}">
-    <input type="hidden" name="adtech" value="${filters.reduceAdtechOption.value()}">
-    <input type="hidden" name="searchTitle" value="${filters.searchTitleOption.value()}">
     <input type="hidden" name="profile" value="${profile}">
-    <input type="hidden" name="recent" value="${filters.showRecentOption.value()}">
+
 </form>

code/services-application/search-service/resources/jte/serp/start.jte

@@ -36,7 +36,7 @@
         <div class="text-slate-700 dark:text-white text-sm p-4">
             <div class="fas fa-gift mr-1 text-margeblue dark:text-slate-200"></div>
             This is the new design and home of Marginalia Search.
-            You can about what this entails <a href="https://about.marginalia-search.com/article/redesign/"  class="underline text-liteblue dark:text-blue-200">here</a>.
+            You can read about what this entails <a href="https://about.marginalia-search.com/article/redesign/"  class="underline text-liteblue dark:text-blue-200">here</a>.
         <p class="my-4"></p>
         The old version of Marginalia Search remains available at
         <a href="https://old-search.marginalia.nu/" class="underline text-liteblue dark:text-blue-200">https://old-search.marginalia.nu/</a>.