(feeds) Add system property for exporting fetched feeds to a slop table for debugging

This is primarily to make the code a bit easier to reason about, and will reduce the level of indirection and data copying in the search-servi->query-service->index-service communication chain.

code/common/db/java/nu/marginalia/db/DbDomainQueries.java

@@ -8,17 +8,18 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.model.EdgeDomain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.sql.SQLException;
-import java.util.NoSuchElementException;
-import java.util.Optional;
-import java.util.OptionalInt;
+import java.util.*;
 import java.util.concurrent.ExecutionException;
 
 @Singleton
 public class DbDomainQueries {
     private final HikariDataSource dataSource;
 
+    private static final Logger logger = LoggerFactory.getLogger(DbDomainQueries.class);
     private final Cache<EdgeDomain, Integer> domainIdCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
 
     @Inject
@@ -101,4 +102,34 @@ public class DbDomainQueries {
             throw new RuntimeException(ex);
         }
     }
+
+    public List<DomainWithNode> otherSubdomains(EdgeDomain domain, int cnt) {
+        List<DomainWithNode> ret = new ArrayList<>();
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("SELECT DOMAIN_NAME, NODE_AFFINITY FROM EC_DOMAIN WHERE DOMAIN_TOP = ? LIMIT ?")) {
+            stmt.setString(1, domain.topDomain);
+            stmt.setInt(2, cnt);
+
+            var rs = stmt.executeQuery();
+            while (rs.next()) {
+                var sibling = new EdgeDomain(rs.getString(1));
+
+                if (sibling.equals(domain))
+                    continue;
+
+                ret.add(new DomainWithNode(sibling, rs.getInt(2)));
+            }
+        } catch (SQLException e) {
+            logger.error("Failed to get domain neighbors");
+        }
+
+        return ret;
+    }
+
+    public record DomainWithNode (EdgeDomain domain, int nodeAffinity) {
+        public boolean isIndexed() {
+            return nodeAffinity > 0;
+        }
+    }
 }

code/common/model/java/nu/marginalia/util/QueryParams.java

@@ -83,6 +83,11 @@ public class QueryParams {
         if (path.endsWith("StoryView.py")) { // folklore.org is neat
             return param.startsWith("project=") || param.startsWith("story=");
         }
+
+        // www.perseus.tufts.edu:
+        if (param.startsWith("collection=")) return true;
+        if (param.startsWith("doc=")) return true;
+
         return false;
     }
 }

code/functions/live-capture/build.gradle

@@ -29,6 +29,7 @@ dependencies {
     implementation libs.jsoup
     implementation project(':third-party:rssreader')
     implementation libs.opencsv
+    implementation libs.slop
     implementation libs.sqlite
     implementation libs.bundles.slf4j
     implementation libs.commons.lang3

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedFetcherService.java

@@ -96,6 +96,7 @@ public class FeedFetcherService {
             throw new IllegalStateException("Already updating feeds, refusing to start another update");
         }
 
+
         try (FeedDbWriter writer = feedDb.createWriter();
              HttpClient client = HttpClient.newBuilder()
                 .connectTimeout(Duration.ofSeconds(15))
@@ -103,6 +104,7 @@ public class FeedFetcherService {
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .version(HttpClient.Version.HTTP_2)
                 .build();
+             FeedJournal feedJournal = FeedJournal.create();
              var heartbeat = serviceHeartbeat.createServiceAdHocTaskHeartbeat("Update Rss Feeds")
         ) {
             updating = true;
@@ -155,6 +157,8 @@ public class FeedFetcherService {
                             case FetchResult.Success(String value, String etag) -> {
                                 writer.saveEtag(feed.domain(), etag);
                                 writer.saveFeed(parseFeed(value, feed));
+
+                                feedJournal.record(feed.feedUrl(), value);
                             }
                             case FetchResult.NotModified() -> {
                                 writer.saveEtag(feed.domain(), ifNoneMatchTag);

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedJournal.java

@@ -0,0 +1,64 @@
+package nu.marginalia.rss.svc;
+
+import nu.marginalia.WmsaHome;
+import nu.marginalia.slop.SlopTable;
+import nu.marginalia.slop.column.string.StringColumn;
+import nu.marginalia.slop.desc.StorageType;
+import org.apache.commons.io.FileUtils;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Utility for recording fetched feeds to a journal, useful in debugging feed parser issues.
+ */
+public interface FeedJournal extends AutoCloseable {
+    StringColumn urlColumn = new StringColumn("url");
+    StringColumn contentsColumn = new StringColumn("contents", StandardCharsets.UTF_8, StorageType.ZSTD);
+
+    void record(String url, String contents) throws IOException;
+    void close() throws IOException;
+
+
+    static FeedJournal create() throws IOException {
+        if (Boolean.getBoolean("feedFetcher.persistJournal")) {
+            Path journalPath = WmsaHome.getDataPath().resolve("feed-journal");
+            if (Files.isDirectory(journalPath)) {
+                FileUtils.deleteDirectory(journalPath.toFile());
+            }
+            Files.createDirectories(journalPath);
+            return new RecordingFeedJournal(journalPath);
+        }
+        else {
+            return new NoOpFeedJournal();
+        }
+    }
+
+    class NoOpFeedJournal implements FeedJournal {
+        @Override
+        public void record(String url, String contents) {}
+
+        @Override
+        public void close() {}
+    }
+
+    class RecordingFeedJournal extends SlopTable implements FeedJournal {
+
+        private final StringColumn.Writer urlWriter;
+        private final StringColumn.Writer contentsWriter;
+
+        public RecordingFeedJournal(Path path) throws IOException {
+            super(path, SlopTable.getNumPages(path, FeedJournal.urlColumn));
+
+            urlWriter = urlColumn.create(this);
+            contentsWriter = contentsColumn.create(this);
+        }
+
+        public synchronized void record(String url, String contents) throws IOException {
+            urlWriter.put(url);
+            contentsWriter.put(contents);
+        }
+
+    }
+}

code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java

@@ -2,9 +2,6 @@ package nu.marginalia.api.searchquery;
 
 import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
-import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.index.query.limit.SpecificationLimitType;
 
@@ -27,37 +24,19 @@ public class IndexProtobufCodec {
                 .build();
     }
 
-    public static  QueryLimits convertQueryLimits(RpcQueryLimits queryLimits) {
-        return new QueryLimits(
-                queryLimits.getResultsByDomain(),
-                queryLimits.getResultsTotal(),
-                queryLimits.getTimeoutMs(),
-                queryLimits.getFetchSize()
-        );
-    }
-
-    public static RpcQueryLimits convertQueryLimits(QueryLimits queryLimits) {
-        return RpcQueryLimits.newBuilder()
-                .setResultsByDomain(queryLimits.resultsByDomain())
-                .setResultsTotal(queryLimits.resultsTotal())
-                .setTimeoutMs(queryLimits.timeoutMs())
-                .setFetchSize(queryLimits.fetchSize())
-                .build();
-    }
-
     public static SearchQuery convertRpcQuery(RpcQuery query) {
-        List<SearchPhraseConstraint> phraeConstraints = new ArrayList<>();
+        List<SearchPhraseConstraint> phraseConstraints = new ArrayList<>();
 
         for (int j = 0; j < query.getPhrasesCount(); j++) {
             var coh = query.getPhrases(j);
             if (coh.getType() == RpcPhrases.TYPE.OPTIONAL) {
-                phraeConstraints.add(new SearchPhraseConstraint.Optional(List.copyOf(coh.getTermsList())));
+                phraseConstraints.add(new SearchPhraseConstraint.Optional(List.copyOf(coh.getTermsList())));
             }
             else if (coh.getType() == RpcPhrases.TYPE.MANDATORY) {
-                phraeConstraints.add(new SearchPhraseConstraint.Mandatory(List.copyOf(coh.getTermsList())));
+                phraseConstraints.add(new SearchPhraseConstraint.Mandatory(List.copyOf(coh.getTermsList())));
             }
             else if (coh.getType() == RpcPhrases.TYPE.FULL) {
-                phraeConstraints.add(new SearchPhraseConstraint.Full(List.copyOf(coh.getTermsList())));
+                phraseConstraints.add(new SearchPhraseConstraint.Full(List.copyOf(coh.getTermsList())));
             }
             else {
                 throw new IllegalArgumentException("Unknown phrase constraint type: " + coh.getType());
@@ -70,7 +49,7 @@ public class IndexProtobufCodec {
                 query.getExcludeList(),
                 query.getAdviceList(),
                 query.getPriorityList(),
-                phraeConstraints
+                phraseConstraints
         );
     }
 
@@ -103,60 +82,4 @@ public class IndexProtobufCodec {
         return subqueryBuilder.build();
     }
 
-    public static ResultRankingParameters convertRankingParameterss(RpcResultRankingParameters params) {
-        if (params == null)
-            return ResultRankingParameters.sensibleDefaults();
-
-        return new ResultRankingParameters(
-                new Bm25Parameters(params.getBm25K(), params.getBm25B()),
-                params.getShortDocumentThreshold(),
-                params.getShortDocumentPenalty(),
-                params.getDomainRankBonus(),
-                params.getQualityPenalty(),
-                params.getShortSentenceThreshold(),
-                params.getShortSentencePenalty(),
-                params.getBm25Weight(),
-                params.getTcfFirstPositionWeight(),
-                params.getTcfVerbatimWeight(),
-                params.getTcfProximityWeight(),
-                ResultRankingParameters.TemporalBias.valueOf(params.getTemporalBias().getBias().name()),
-                params.getTemporalBiasWeight(),
-                params.getExportDebugData()
-        );
-    }
-
-    public static RpcResultRankingParameters convertRankingParameterss(ResultRankingParameters rankingParams,
-                                                                       RpcTemporalBias temporalBias)
-    {
-        if (rankingParams == null) {
-            rankingParams = ResultRankingParameters.sensibleDefaults();
-        }
-
-        var builder = RpcResultRankingParameters.newBuilder()
-                        .setBm25B(rankingParams.bm25Params.b())
-                        .setBm25K(rankingParams.bm25Params.k())
-                        .setShortDocumentThreshold(rankingParams.shortDocumentThreshold)
-                        .setShortDocumentPenalty(rankingParams.shortDocumentPenalty)
-                        .setDomainRankBonus(rankingParams.domainRankBonus)
-                        .setQualityPenalty(rankingParams.qualityPenalty)
-                        .setShortSentenceThreshold(rankingParams.shortSentenceThreshold)
-                        .setShortSentencePenalty(rankingParams.shortSentencePenalty)
-                        .setBm25Weight(rankingParams.bm25Weight)
-                        .setTcfFirstPositionWeight(rankingParams.tcfFirstPosition)
-                        .setTcfProximityWeight(rankingParams.tcfProximity)
-                        .setTcfVerbatimWeight(rankingParams.tcfVerbatim)
-                        .setTemporalBiasWeight(rankingParams.temporalBiasWeight)
-                        .setExportDebugData(rankingParams.exportDebugData);
-
-        if (temporalBias != null && temporalBias.getBias() != RpcTemporalBias.Bias.NONE) {
-            builder.setTemporalBias(temporalBias);
-        }
-        else {
-            builder.setTemporalBias(RpcTemporalBias.newBuilder()
-                    .setBias(RpcTemporalBias.Bias.valueOf(rankingParams.temporalBias.name())));
-        }
-
-        return builder.build();
-    }
-
 }

code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java

@@ -5,7 +5,7 @@ import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.api.searchquery.model.query.QueryResponse;
 import nu.marginalia.api.searchquery.model.query.SearchSpecification;
 import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.api.searchquery.model.results.SearchResultItem;
 import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore;
 import nu.marginalia.api.searchquery.model.results.debug.DebugFactor;
@@ -37,7 +37,7 @@ public class QueryProtobufCodec {
         builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size));
         builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank));
 
-        builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits));
+        builder.setQueryLimits(query.specs.queryLimits);
 
         // Query strategy may be overridden by the query, but if not, use the one from the request
         if (query.specs.queryStrategy != null && query.specs.queryStrategy != QueryStrategy.AUTO)
@@ -45,9 +45,27 @@ public class QueryProtobufCodec {
         else
             builder.setQueryStrategy(request.getQueryStrategy());
 
-        if (query.specs.rankingParams != null) {
-            builder.setParameters(IndexProtobufCodec.convertRankingParameterss(query.specs.rankingParams, request.getTemporalBias()));
+        if (request.getTemporalBias().getBias() != RpcTemporalBias.Bias.NONE) {
+            if (query.specs.rankingParams != null) {
+                builder.setParameters(
+                        RpcResultRankingParameters.newBuilder(query.specs.rankingParams)
+                                .setTemporalBias(request.getTemporalBias())
+                                .build()
+                );
+            } else {
+                builder.setParameters(
+                        RpcResultRankingParameters.newBuilder(PrototypeRankingParameters.sensibleDefaults())
+                                .setTemporalBias(request.getTemporalBias())
+                                .build()
+                );
+            }
+        } else if (query.specs.rankingParams != null) {
+            builder.setParameters(query.specs.rankingParams);
         }
+        // else {
+        // if we have no ranking params, we don't need to set them, the client check and use the default values
+        // so we don't need to send this huge object over the wire
+        // }
 
         return builder.build();
     }
@@ -65,18 +83,13 @@ public class QueryProtobufCodec {
         builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size));
         builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank));
 
-        builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits));
+        builder.setQueryLimits(query.specs.queryLimits);
 
         // Query strategy may be overridden by the query, but if not, use the one from the request
         builder.setQueryStrategy(query.specs.queryStrategy.name());
 
         if (query.specs.rankingParams != null) {
-            builder.setParameters(IndexProtobufCodec.convertRankingParameterss(
-                    query.specs.rankingParams,
-                    RpcTemporalBias.newBuilder().setBias(
-                                    RpcTemporalBias.Bias.NONE)
-                            .build())
-            );
+            builder.setParameters(query.specs.rankingParams);
         }
 
         return builder.build();
@@ -95,10 +108,10 @@ public class QueryProtobufCodec {
                 IndexProtobufCodec.convertSpecLimit(request.getSize()),
                 IndexProtobufCodec.convertSpecLimit(request.getRank()),
                 request.getDomainIdsList(),
-                IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()),
+                request.getQueryLimits(),
                 request.getSearchSetIdentifier(),
                 QueryStrategy.valueOf(request.getQueryStrategy()),
-                ResultRankingParameters.TemporalBias.valueOf(request.getTemporalBias().getBias().name()),
+                RpcTemporalBias.Bias.valueOf(request.getTemporalBias().getBias().name()),
                 request.getPagination().getPage()
         );
     }
@@ -294,9 +307,9 @@ public class QueryProtobufCodec {
                 IndexProtobufCodec.convertSpecLimit(specs.getYear()),
                 IndexProtobufCodec.convertSpecLimit(specs.getSize()),
                 IndexProtobufCodec.convertSpecLimit(specs.getRank()),
-                IndexProtobufCodec.convertQueryLimits(specs.getQueryLimits()),
+                specs.getQueryLimits(),
                 QueryStrategy.valueOf(specs.getQueryStrategy()),
-                IndexProtobufCodec.convertRankingParameterss(specs.getParameters())
+                specs.hasParameters() ? specs.getParameters() : null
         );
     }
 
@@ -307,7 +320,7 @@ public class QueryProtobufCodec {
                 .addAllTacitExcludes(params.tacitExcludes())
                 .addAllTacitPriority(params.tacitPriority())
                 .setHumanQuery(params.humanQuery())
-                .setQueryLimits(IndexProtobufCodec.convertQueryLimits(params.limits()))
+                .setQueryLimits(params.limits())
                 .setQuality(IndexProtobufCodec.convertSpecLimit(params.quality()))
                 .setYear(IndexProtobufCodec.convertSpecLimit(params.year()))
                 .setSize(IndexProtobufCodec.convertSpecLimit(params.size()))
@@ -319,7 +332,7 @@ public class QueryProtobufCodec {
                         .build())
                 .setPagination(RpcQsQueryPagination.newBuilder()
                         .setPage(params.page())
-                        .setPageSize(Math.min(100, params.limits().resultsTotal()))
+                        .setPageSize(Math.min(100, params.limits().getResultsTotal()))
                         .build());
 
         if (params.nearDomain() != null)

code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java

@@ -1,7 +1,7 @@
 package nu.marginalia.api.searchquery.model.query;
 
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.index.query.limit.QueryLimits;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 
@@ -21,14 +21,14 @@ public record QueryParams(
         SpecificationLimit size,
         SpecificationLimit rank,
         List<Integer> domainIds,
-        QueryLimits limits,
+        RpcQueryLimits limits,
         String identifier,
         QueryStrategy queryStrategy,
-        ResultRankingParameters.TemporalBias temporalBias,
+        RpcTemporalBias.Bias temporalBias,
         int page
 )
 {
-    public QueryParams(String query, QueryLimits limits, String identifier) {
+    public QueryParams(String query, RpcQueryLimits limits, String identifier) {
         this(query, null,
                 List.of(),
                 List.of(),
@@ -42,7 +42,7 @@ public record QueryParams(
                 limits,
                 identifier,
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 1 // page
                 );
     }

code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java

@@ -1,10 +1,11 @@
 package nu.marginalia.api.searchquery.model.query;
 
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.index.query.limit.QueryLimits;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 
+import javax.annotation.Nullable;
 import java.util.List;
 
 public class SearchSpecification {
@@ -24,11 +25,12 @@ public class SearchSpecification {
     public SpecificationLimit size;
     public SpecificationLimit rank;
 
-    public final QueryLimits queryLimits;
+    public final RpcQueryLimits queryLimits;
 
     public final QueryStrategy queryStrategy;
 
-    public final ResultRankingParameters rankingParams;
+    @Nullable
+    public final RpcResultRankingParameters rankingParams;
 
     public SearchSpecification(SearchQuery query,
                                List<Integer> domains,
@@ -38,9 +40,9 @@ public class SearchSpecification {
                                SpecificationLimit year,
                                SpecificationLimit size,
                                SpecificationLimit rank,
-                               QueryLimits queryLimits,
+                               RpcQueryLimits queryLimits,
                                QueryStrategy queryStrategy,
-                               ResultRankingParameters rankingParams)
+                               @Nullable RpcResultRankingParameters rankingParams)
     {
         this.query = query;
         this.domains = domains;
@@ -91,7 +93,7 @@ public class SearchSpecification {
         return this.rank;
     }
 
-    public QueryLimits getQueryLimits() {
+    public RpcQueryLimits getQueryLimits() {
         return this.queryLimits;
     }
 
@@ -99,7 +101,7 @@ public class SearchSpecification {
         return this.queryStrategy;
     }
 
-    public ResultRankingParameters getRankingParams() {
+    public RpcResultRankingParameters getRankingParams() {
         return this.rankingParams;
     }
 
@@ -120,9 +122,9 @@ public class SearchSpecification {
         private boolean size$set;
         private SpecificationLimit rank$value;
         private boolean rank$set;
-        private QueryLimits queryLimits;
+        private RpcQueryLimits queryLimits;
         private QueryStrategy queryStrategy;
-        private ResultRankingParameters rankingParams;
+        private RpcResultRankingParameters rankingParams;
 
         SearchSpecificationBuilder() {
         }
@@ -171,7 +173,7 @@ public class SearchSpecification {
             return this;
         }
 
-        public SearchSpecificationBuilder queryLimits(QueryLimits queryLimits) {
+        public SearchSpecificationBuilder queryLimits(RpcQueryLimits queryLimits) {
             this.queryLimits = queryLimits;
             return this;
         }
@@ -181,7 +183,7 @@ public class SearchSpecification {
             return this;
         }
 
-        public SearchSpecificationBuilder rankingParams(ResultRankingParameters rankingParams) {
+        public SearchSpecificationBuilder rankingParams(RpcResultRankingParameters rankingParams) {
             this.rankingParams = rankingParams;
             return this;
         }

code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/PrototypeRankingParameters.java

@@ -0,0 +1,33 @@
+package nu.marginalia.api.searchquery.model.results;
+
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
+
+public class PrototypeRankingParameters {
+
+    /** These are the default ranking parameters that are used when no parameters are specified. */
+
+    private static final RpcResultRankingParameters _sensibleDefaults = RpcResultRankingParameters.newBuilder()
+                .setBm25B(0.5)
+                .setBm25K(1.2)
+                .setShortDocumentThreshold(2000)
+                .setShortDocumentPenalty(2.)
+                .setDomainRankBonus(1 / 100.)
+                .setQualityPenalty(1 / 15.)
+                .setShortSentenceThreshold(2)
+                .setShortSentencePenalty(5)
+                .setBm25Weight(1.)
+                .setTcfVerbatimWeight(1.)
+                .setTcfProximityWeight(1.)
+                .setTcfFirstPositionWeight(5)
+                .setTemporalBias(RpcTemporalBias.newBuilder().setBias(RpcTemporalBias.Bias.NONE))
+                .setTemporalBiasWeight(5.0)
+                .setExportDebugData(false)
+                .setDisablePenalties(false)
+                .build();
+
+    public static RpcResultRankingParameters sensibleDefaults() {
+        return _sensibleDefaults;
+    }
+
+}

code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java

@@ -1,12 +1,13 @@
 package nu.marginalia.api.searchquery.model.results;
 
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
 import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
 
 import java.util.BitSet;
 
 public class ResultRankingContext {
     private final int docCount;
-    public final ResultRankingParameters params;
+    public final RpcResultRankingParameters params;
 
 
     public final BitSet regularMask;
@@ -21,7 +22,7 @@ public class ResultRankingContext {
     public final CqDataInt priorityCounts;
 
     public ResultRankingContext(int docCount,
-                                ResultRankingParameters params,
+                                RpcResultRankingParameters params,
                                 BitSet ngramsMask,
                                 BitSet regularMask,
                                 CqDataInt fullCounts,

code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java

@@ -1,278 +0,0 @@
-package nu.marginalia.api.searchquery.model.results;
-
-import java.util.Objects;
-
-public class ResultRankingParameters {
-
-    /**
-     * Tuning for BM25 when applied to full document matches
-     */
-    public final Bm25Parameters bm25Params;
-
-    /**
-     * Documents below this length are penalized
-     */
-    public int shortDocumentThreshold;
-
-    public double shortDocumentPenalty;
-
-
-    /**
-     * Scaling factor associated with domain rank (unscaled rank value is 0-255; high is good)
-     */
-    public double domainRankBonus;
-
-    /**
-     * Scaling factor associated with document quality (unscaled rank value is 0-15; high is bad)
-     */
-    public double qualityPenalty;
-
-    /**
-     * Average sentence length values below this threshold are penalized, range [0-4), 2 or 3 is probably what you want
-     */
-    public int shortSentenceThreshold;
-
-    /**
-     * Magnitude of penalty for documents with low average sentence length
-     */
-    public double shortSentencePenalty;
-
-    public double bm25Weight;
-    public double tcfFirstPosition;
-    public double tcfVerbatim;
-    public double tcfProximity;
-
-    public TemporalBias temporalBias;
-    public double temporalBiasWeight;
-
-    public boolean exportDebugData;
-
-    public ResultRankingParameters(Bm25Parameters bm25Params, int shortDocumentThreshold, double shortDocumentPenalty, double domainRankBonus, double qualityPenalty, int shortSentenceThreshold, double shortSentencePenalty, double bm25Weight, double tcfFirstPosition, double tcfVerbatim, double tcfProximity, TemporalBias temporalBias, double temporalBiasWeight, boolean exportDebugData) {
-        this.bm25Params = bm25Params;
-        this.shortDocumentThreshold = shortDocumentThreshold;
-        this.shortDocumentPenalty = shortDocumentPenalty;
-        this.domainRankBonus = domainRankBonus;
-        this.qualityPenalty = qualityPenalty;
-        this.shortSentenceThreshold = shortSentenceThreshold;
-        this.shortSentencePenalty = shortSentencePenalty;
-        this.bm25Weight = bm25Weight;
-        this.tcfFirstPosition = tcfFirstPosition;
-        this.tcfVerbatim = tcfVerbatim;
-        this.tcfProximity = tcfProximity;
-        this.temporalBias = temporalBias;
-        this.temporalBiasWeight = temporalBiasWeight;
-        this.exportDebugData = exportDebugData;
-    }
-
-    public static ResultRankingParameters sensibleDefaults() {
-        return builder()
-                .bm25Params(new Bm25Parameters(1.2, 0.5))
-                .shortDocumentThreshold(2000)
-                .shortDocumentPenalty(2.)
-                .domainRankBonus(1 / 100.)
-                .qualityPenalty(1 / 15.)
-                .shortSentenceThreshold(2)
-                .shortSentencePenalty(5)
-                .bm25Weight(1.)
-                .tcfVerbatim(1.)
-                .tcfProximity(1.)
-                .tcfFirstPosition(5)
-                .temporalBias(TemporalBias.NONE)
-                .temporalBiasWeight(5.0)
-                .exportDebugData(false)
-                .build();
-    }
-
-    public static ResultRankingParametersBuilder builder() {
-        return new ResultRankingParametersBuilder();
-    }
-
-    public Bm25Parameters getBm25Params() {
-        return this.bm25Params;
-    }
-
-    public int getShortDocumentThreshold() {
-        return this.shortDocumentThreshold;
-    }
-
-    public double getShortDocumentPenalty() {
-        return this.shortDocumentPenalty;
-    }
-
-    public double getDomainRankBonus() {
-        return this.domainRankBonus;
-    }
-
-    public double getQualityPenalty() {
-        return this.qualityPenalty;
-    }
-
-    public int getShortSentenceThreshold() {
-        return this.shortSentenceThreshold;
-    }
-
-    public double getShortSentencePenalty() {
-        return this.shortSentencePenalty;
-    }
-
-    public double getBm25Weight() {
-        return this.bm25Weight;
-    }
-
-    public double getTcfFirstPosition() {
-        return this.tcfFirstPosition;
-    }
-
-    public double getTcfVerbatim() {
-        return this.tcfVerbatim;
-    }
-
-    public double getTcfProximity() {
-        return this.tcfProximity;
-    }
-
-    public TemporalBias getTemporalBias() {
-        return this.temporalBias;
-    }
-
-    public double getTemporalBiasWeight() {
-        return this.temporalBiasWeight;
-    }
-
-    public boolean isExportDebugData() {
-        return this.exportDebugData;
-    }
-
-    @Override
-    public final boolean equals(Object o) {
-        if (this == o) return true;
-        if (!(o instanceof ResultRankingParameters that)) return false;
-
-        return shortDocumentThreshold == that.shortDocumentThreshold && Double.compare(shortDocumentPenalty, that.shortDocumentPenalty) == 0 && Double.compare(domainRankBonus, that.domainRankBonus) == 0 && Double.compare(qualityPenalty, that.qualityPenalty) == 0 && shortSentenceThreshold == that.shortSentenceThreshold && Double.compare(shortSentencePenalty, that.shortSentencePenalty) == 0 && Double.compare(bm25Weight, that.bm25Weight) == 0 && Double.compare(tcfFirstPosition, that.tcfFirstPosition) == 0 && Double.compare(tcfVerbatim, that.tcfVerbatim) == 0 && Double.compare(tcfProximity, that.tcfProximity) == 0 && Double.compare(temporalBiasWeight, that.temporalBiasWeight) == 0 && exportDebugData == that.exportDebugData && Objects.equals(bm25Params, that.bm25Params) && temporalBias == that.temporalBias;
-    }
-
-    @Override
-    public int hashCode() {
-        int result = Objects.hashCode(bm25Params);
-        result = 31 * result + shortDocumentThreshold;
-        result = 31 * result + Double.hashCode(shortDocumentPenalty);
-        result = 31 * result + Double.hashCode(domainRankBonus);
-        result = 31 * result + Double.hashCode(qualityPenalty);
-        result = 31 * result + shortSentenceThreshold;
-        result = 31 * result + Double.hashCode(shortSentencePenalty);
-        result = 31 * result + Double.hashCode(bm25Weight);
-        result = 31 * result + Double.hashCode(tcfFirstPosition);
-        result = 31 * result + Double.hashCode(tcfVerbatim);
-        result = 31 * result + Double.hashCode(tcfProximity);
-        result = 31 * result + Objects.hashCode(temporalBias);
-        result = 31 * result + Double.hashCode(temporalBiasWeight);
-        result = 31 * result + Boolean.hashCode(exportDebugData);
-        return result;
-    }
-
-    public String toString() {
-        return "ResultRankingParameters(bm25Params=" + this.getBm25Params() + ", shortDocumentThreshold=" + this.getShortDocumentThreshold() + ", shortDocumentPenalty=" + this.getShortDocumentPenalty() + ", domainRankBonus=" + this.getDomainRankBonus() + ", qualityPenalty=" + this.getQualityPenalty() + ", shortSentenceThreshold=" + this.getShortSentenceThreshold() + ", shortSentencePenalty=" + this.getShortSentencePenalty() + ", bm25Weight=" + this.getBm25Weight() + ", tcfFirstPosition=" + this.getTcfFirstPosition() + ", tcfVerbatim=" + this.getTcfVerbatim() + ", tcfProximity=" + this.getTcfProximity() + ", temporalBias=" + this.getTemporalBias() + ", temporalBiasWeight=" + this.getTemporalBiasWeight() + ", exportDebugData=" + this.isExportDebugData() + ")";
-    }
-
-    public enum TemporalBias {
-        RECENT, OLD, NONE
-    }
-
-    public static class ResultRankingParametersBuilder {
-        private Bm25Parameters bm25Params;
-        private int shortDocumentThreshold;
-        private double shortDocumentPenalty;
-        private double domainRankBonus;
-        private double qualityPenalty;
-        private int shortSentenceThreshold;
-        private double shortSentencePenalty;
-        private double bm25Weight;
-        private double tcfFirstPosition;
-        private double tcfVerbatim;
-        private double tcfProximity;
-        private TemporalBias temporalBias;
-        private double temporalBiasWeight;
-        private boolean exportDebugData;
-
-        ResultRankingParametersBuilder() {
-        }
-
-        public ResultRankingParametersBuilder bm25Params(Bm25Parameters bm25Params) {
-            this.bm25Params = bm25Params;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder shortDocumentThreshold(int shortDocumentThreshold) {
-            this.shortDocumentThreshold = shortDocumentThreshold;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder shortDocumentPenalty(double shortDocumentPenalty) {
-            this.shortDocumentPenalty = shortDocumentPenalty;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder domainRankBonus(double domainRankBonus) {
-            this.domainRankBonus = domainRankBonus;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder qualityPenalty(double qualityPenalty) {
-            this.qualityPenalty = qualityPenalty;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder shortSentenceThreshold(int shortSentenceThreshold) {
-            this.shortSentenceThreshold = shortSentenceThreshold;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder shortSentencePenalty(double shortSentencePenalty) {
-            this.shortSentencePenalty = shortSentencePenalty;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder bm25Weight(double bm25Weight) {
-            this.bm25Weight = bm25Weight;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder tcfFirstPosition(double tcfFirstPosition) {
-            this.tcfFirstPosition = tcfFirstPosition;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder tcfVerbatim(double tcfVerbatim) {
-            this.tcfVerbatim = tcfVerbatim;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder tcfProximity(double tcfProximity) {
-            this.tcfProximity = tcfProximity;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder temporalBias(TemporalBias temporalBias) {
-            this.temporalBias = temporalBias;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder temporalBiasWeight(double temporalBiasWeight) {
-            this.temporalBiasWeight = temporalBiasWeight;
-            return this;
-        }
-
-        public ResultRankingParametersBuilder exportDebugData(boolean exportDebugData) {
-            this.exportDebugData = exportDebugData;
-            return this;
-        }
-
-        public ResultRankingParameters build() {
-            return new ResultRankingParameters(this.bm25Params, this.shortDocumentThreshold, this.shortDocumentPenalty, this.domainRankBonus, this.qualityPenalty, this.shortSentenceThreshold, this.shortSentencePenalty, this.bm25Weight, this.tcfFirstPosition, this.tcfVerbatim, this.tcfProximity, this.temporalBias, this.temporalBiasWeight, this.exportDebugData);
-        }
-
-        public String toString() {
-            return "ResultRankingParameters.ResultRankingParametersBuilder(bm25Params=" + this.bm25Params + ", shortDocumentThreshold=" + this.shortDocumentThreshold + ", shortDocumentPenalty=" + this.shortDocumentPenalty + ", domainRankBonus=" + this.domainRankBonus + ", qualityPenalty=" + this.qualityPenalty + ", shortSentenceThreshold=" + this.shortSentenceThreshold + ", shortSentencePenalty=" + this.shortSentencePenalty + ", bm25Weight=" + this.bm25Weight + ", tcfFirstPosition=" + this.tcfFirstPosition + ", tcfVerbatim=" + this.tcfVerbatim + ", tcfProximity=" + this.tcfProximity + ", temporalBias=" + this.temporalBias + ", temporalBiasWeight=" + this.temporalBiasWeight + ", exportDebugData=" + this.exportDebugData + ")";
-        }
-    }
-}

code/functions/search-query/api/src/main/protobuf/query-api.proto

@@ -162,6 +162,7 @@ message RpcResultRankingParameters {
   double temporalBiasWeight = 17;
 
   bool exportDebugData = 18;
+  bool disablePenalties = 19;
 
 }
 

code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java

@@ -3,8 +3,6 @@ package nu.marginalia.index.client;
 import nu.marginalia.api.searchquery.IndexProtobufCodec;
 import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import org.junit.jupiter.api.Test;
 
@@ -22,18 +20,6 @@ class IndexProtobufCodecTest {
         verifyIsIdentityTransformation(SpecificationLimit.lessThan(1), l -> IndexProtobufCodec.convertSpecLimit(IndexProtobufCodec.convertSpecLimit(l)));
     }
 
-    @Test
-    public void testRankingParameters() {
-        verifyIsIdentityTransformation(ResultRankingParameters.sensibleDefaults(),
-                p -> IndexProtobufCodec.convertRankingParameterss(IndexProtobufCodec.convertRankingParameterss(p, null)));
-    }
-
-    @Test
-    public void testQueryLimits() {
-        verifyIsIdentityTransformation(new QueryLimits(1,2,3,4),
-                l -> IndexProtobufCodec.convertQueryLimits(IndexProtobufCodec.convertQueryLimits(l))
-                );
-    }
     @Test
     public void testSubqery() {
         verifyIsIdentityTransformation(new SearchQuery(

code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryFactory.java

@@ -2,8 +2,9 @@ package nu.marginalia.functions.searchquery;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
 import nu.marginalia.api.searchquery.model.query.*;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
 import nu.marginalia.functions.searchquery.query_parser.QueryExpansion;
 import nu.marginalia.functions.searchquery.query_parser.QueryParser;
 import nu.marginalia.functions.searchquery.query_parser.token.QueryToken;
@@ -36,7 +37,7 @@ public class QueryFactory {
 
 
     public ProcessedQuery createQuery(QueryParams params,
-                                      @Nullable ResultRankingParameters rankingParams) {
+                                      @Nullable RpcResultRankingParameters rankingParams) {
         final var query = params.humanQuery();
 
         if (query.length() > 1000) {
@@ -132,7 +133,9 @@ public class QueryFactory {
         var limits = params.limits();
         // Disable limits on number of results per domain if we're searching with a site:-type term
         if (domain != null) {
-            limits = limits.forSingleDomain();
+            limits = RpcQueryLimits.newBuilder(limits)
+                    .setResultsByDomain(limits.getResultsTotal())
+                    .build();
         }
 
         var expansion = queryExpansion.expandQuery(queryBuilder.searchTermsInclude);

code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java

@@ -9,7 +9,7 @@ import nu.marginalia.api.searchquery.*;
 import nu.marginalia.api.searchquery.model.query.ProcessedQuery;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.index.api.IndexClient;
 import nu.marginalia.service.server.DiscoverableService;
 import org.slf4j.Logger;
@@ -55,7 +55,7 @@ public class QueryGRPCService
                     .time(() -> {
 
                 var params = QueryProtobufCodec.convertRequest(request);
-                var query = queryFactory.createQuery(params, ResultRankingParameters.sensibleDefaults());
+                var query = queryFactory.createQuery(params, PrototypeRankingParameters.sensibleDefaults());
 
                 var indexRequest = QueryProtobufCodec.convertQuery(request, query);
 
@@ -102,7 +102,7 @@ public class QueryGRPCService
             String originalQuery,
             QueryParams params,
             IndexClient.Pagination pagination,
-            ResultRankingParameters rankingParameters) {
+            RpcResultRankingParameters rankingParameters) {
 
         var query = queryFactory.createQuery(params, rankingParameters);
         IndexClient.AggregateQueryResponse response = indexClient.executeQueries(QueryProtobufCodec.convertQuery(originalQuery, query), pagination);

code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java

@@ -233,9 +233,19 @@ public class QueryParser {
             entity.replace(new QueryToken.RankTerm(limit, str));
         } else if (str.startsWith("qs=")) {
             entity.replace(new QueryToken.QsTerm(str.substring(3)));
-        } else if (str.contains(":")) {
+        } else if (str.startsWith("site:")
+                || str.startsWith("format:")
+                || str.startsWith("file:")
+                || str.startsWith("tld:")
+                || str.startsWith("ip:")
+                || str.startsWith("as:")
+                || str.startsWith("asn:")
+                || str.startsWith("generator:")
+        )
+        {
             entity.replace(new QueryToken.AdviceTerm(str, t.displayStr()));
         }
+
     }
 
     private static SpecificationLimit parseSpecificationLimit(String str) {

code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java

@@ -1,12 +1,12 @@
 package nu.marginalia.query.svc;
 
 import nu.marginalia.WmsaHome;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.api.searchquery.model.query.SearchSpecification;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
 import nu.marginalia.functions.searchquery.QueryFactory;
 import nu.marginalia.functions.searchquery.query_parser.QueryExpansion;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.index.query.limit.SpecificationLimitType;
@@ -49,10 +49,15 @@ public class QueryFactoryTest {
                         SpecificationLimit.none(),
                         SpecificationLimit.none(),
                         null,
-                        new QueryLimits(100, 100, 100, 100),
+                        RpcQueryLimits.newBuilder()
+                                .setResultsTotal(100)
+                                .setResultsByDomain(100)
+                                .setTimeoutMs(100)
+                                .setFetchSize(100)
+                                .build(),
                         "NONE",
                         QueryStrategy.AUTO,
-                        ResultRankingParameters.TemporalBias.NONE,
+                        RpcTemporalBias.Bias.NONE,
                         0), null).specs;
     }
 
@@ -208,6 +213,12 @@ public class QueryFactoryTest {
         System.out.println(subquery);
     }
 
+    @Test
+    public void testCplusPlus() {
+        var subquery = parseAndGetSpecs("std::vector::push_back vector");
+        System.out.println(subquery);
+    }
+
     @Test
     public void testQuotedApostrophe() {
         var subquery = parseAndGetSpecs("\"bob's cars\"");

code/index/java/nu/marginalia/index/IndexGrpcService.java

@@ -10,12 +10,12 @@ import it.unimi.dsi.fastutil.longs.LongArrayList;
 import nu.marginalia.api.searchquery.IndexApiGrpc;
 import nu.marginalia.api.searchquery.RpcDecoratedResultItem;
 import nu.marginalia.api.searchquery.RpcIndexQuery;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQuery;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong;
 import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
 import nu.marginalia.api.searchquery.model.query.SearchSpecification;
 import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
 import nu.marginalia.array.page.LongQueryBuffer;
 import nu.marginalia.index.index.StatefulIndex;
 import nu.marginalia.index.model.SearchParameters;
@@ -211,7 +211,7 @@ public class IndexGrpcService
     /** This class is responsible for ranking the results and adding the best results to the
      * resultHeap, which depending on the state of the indexLookup threads may or may not block
      */
-    private ResultRankingContext createRankingContext(ResultRankingParameters rankingParams,
+    private ResultRankingContext createRankingContext(RpcResultRankingParameters rankingParams,
                                                       CompiledQuery<String> compiledQuery,
                                                       CompiledQueryLong compiledQueryIds)
     {

code/index/java/nu/marginalia/index/model/SearchParameters.java

@@ -2,12 +2,13 @@ package nu.marginalia.index.model;
 
 import nu.marginalia.api.searchquery.IndexProtobufCodec;
 import nu.marginalia.api.searchquery.RpcIndexQuery;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQuery;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQueryParser;
-import nu.marginalia.api.searchquery.model.query.SearchSpecification;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.model.query.SearchSpecification;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.index.query.IndexSearchBudget;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.searchset.SearchSet;
@@ -23,7 +24,7 @@ public class SearchParameters {
     public final IndexSearchBudget budget;
     public final SearchQuery query;
     public final QueryParams queryParams;
-    public final ResultRankingParameters rankingParams;
+    public final RpcResultRankingParameters rankingParams;
 
     public final int limitByDomain;
     public final int limitTotal;
@@ -41,11 +42,11 @@ public class SearchParameters {
     public SearchParameters(SearchSpecification specsSet, SearchSet searchSet) {
         var limits = specsSet.queryLimits;
 
-        this.fetchSize = limits.fetchSize();
-        this.budget = new IndexSearchBudget(limits.timeoutMs());
+        this.fetchSize = limits.getFetchSize();
+        this.budget = new IndexSearchBudget(limits.getTimeoutMs());
         this.query = specsSet.query;
-        this.limitByDomain = limits.resultsByDomain();
-        this.limitTotal = limits.resultsTotal();
+        this.limitByDomain = limits.getResultsByDomain();
+        this.limitTotal = limits.getResultsTotal();
 
         queryParams = new QueryParams(
                 specsSet.quality,
@@ -62,17 +63,17 @@ public class SearchParameters {
     }
 
     public SearchParameters(RpcIndexQuery request, SearchSet searchSet) {
-        var limits = IndexProtobufCodec.convertQueryLimits(request.getQueryLimits());
+        var limits = request.getQueryLimits();
 
-        this.fetchSize = limits.fetchSize();
+        this.fetchSize = limits.getFetchSize();
 
         // The time budget is halved because this is the point when we start to
         // wrap up the search and return the results.
-        this.budget = new IndexSearchBudget(limits.timeoutMs() / 2);
+        this.budget = new IndexSearchBudget(limits.getTimeoutMs() / 2);
         this.query = IndexProtobufCodec.convertRpcQuery(request.getQuery());
 
-        this.limitByDomain = limits.resultsByDomain();
-        this.limitTotal = limits.resultsTotal();
+        this.limitByDomain = limits.getResultsByDomain();
+        this.limitTotal = limits.getResultsTotal();
 
         queryParams = new QueryParams(
                 convertSpecLimit(request.getQuality()),
@@ -85,7 +86,7 @@ public class SearchParameters {
         compiledQuery = CompiledQueryParser.parse(this.query.compiledQuery);
         compiledQueryIds = compiledQuery.mapToLong(SearchTermsUtil::getWordId);
 
-        rankingParams = IndexProtobufCodec.convertRankingParameterss(request.getParameters());
+        rankingParams = request.hasParameters() ? request.getParameters() : PrototypeRankingParameters.sensibleDefaults();
     }
 
 

code/index/java/nu/marginalia/index/results/Bm25GraphVisitor.java

@@ -2,7 +2,6 @@ package nu.marginalia.index.results;
 
 import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
 import nu.marginalia.api.searchquery.model.compiled.CqExpression;
-import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
 import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
 
 import java.util.BitSet;
@@ -24,14 +23,14 @@ public class Bm25GraphVisitor implements CqExpression.DoubleVisitor {
 
     private final BitSet mask;
 
-    public Bm25GraphVisitor(Bm25Parameters bm25Parameters,
+    public Bm25GraphVisitor(double k1, double b,
                             float[] counts,
                             int length,
                             ResultRankingContext ctx) {
         this.length = length;
 
-        this.k1 = bm25Parameters.k();
-        this.b = bm25Parameters.b();
+        this.k1 = k1;
+        this.b = b;
 
         this.docCount = ctx.termFreqDocCount();
         this.counts = counts;

code/index/java/nu/marginalia/index/results/IndexResultRankingService.java

@@ -156,7 +156,7 @@ public class IndexResultRankingService {
         // for the selected results, as this would be comically expensive to do for all the results we
         // discard along the way
 
-        if (params.rankingParams.exportDebugData) {
+        if (params.rankingParams.getExportDebugData()) {
             var combinedIdsList = new LongArrayList(resultsList.size());
             for (var item : resultsList) {
                 combinedIdsList.add(item.combinedId);

code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java

@@ -2,10 +2,11 @@ package nu.marginalia.index.results;
 
 import it.unimi.dsi.fastutil.ints.IntIterator;
 import it.unimi.dsi.fastutil.ints.IntList;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQuery;
 import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong;
 import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
 import nu.marginalia.api.searchquery.model.results.SearchResultItem;
 import nu.marginalia.api.searchquery.model.results.debug.DebugRankingFactors;
 import nu.marginalia.index.forward.spans.DocumentSpans;
@@ -116,14 +117,14 @@ public class IndexResultScoreCalculator {
 
         float proximitiyFac = getProximitiyFac(decodedPositions, searchTerms.phraseConstraints, verbatimMatches, unorderedMatches, spans);
 
-        double score_firstPosition = params.tcfFirstPosition * (1.0 / Math.sqrt(unorderedMatches.firstPosition));
-        double score_verbatim = params.tcfVerbatim * verbatimMatches.getScore();
-        double score_proximity = params.tcfProximity * proximitiyFac;
-        double score_bM25 = params.bm25Weight
-                * wordFlagsQuery.root.visit(new Bm25GraphVisitor(params.bm25Params, unorderedMatches.getWeightedCounts(), docSize, rankingContext))
+        double score_firstPosition = params.getTcfFirstPositionWeight() * (1.0 / Math.sqrt(unorderedMatches.firstPosition));
+        double score_verbatim = params.getTcfVerbatimWeight() * verbatimMatches.getScore();
+        double score_proximity = params.getTcfProximityWeight() * proximitiyFac;
+        double score_bM25 = params.getBm25Weight()
+                * wordFlagsQuery.root.visit(new Bm25GraphVisitor(params.getBm25K(), params.getBm25B(), unorderedMatches.getWeightedCounts(), docSize, rankingContext))
                 / (Math.sqrt(unorderedMatches.searchableKeywordCount + 1));
-        double score_bFlags = params.bm25Weight
-                * wordFlagsQuery.root.visit(new TermFlagsGraphVisitor(params.bm25Params, wordFlagsQuery.data, unorderedMatches.getWeightedCounts(), rankingContext))
+        double score_bFlags = params.getBm25Weight()
+                * wordFlagsQuery.root.visit(new TermFlagsGraphVisitor(params.getBm25K(), wordFlagsQuery.data, unorderedMatches.getWeightedCounts(), rankingContext))
                 / (Math.sqrt(unorderedMatches.searchableKeywordCount + 1));
 
         double score = normalize(
@@ -245,9 +246,13 @@ public class IndexResultScoreCalculator {
     private double calculateDocumentBonus(long documentMetadata,
                                           int features,
                                           int length,
-                                          ResultRankingParameters rankingParams,
+                                          RpcResultRankingParameters rankingParams,
                                           @Nullable DebugRankingFactors debugRankingFactors) {
 
+        if (rankingParams.getDisablePenalties()) {
+            return 0.;
+        }
+
         int rank = DocumentMetadata.decodeRank(documentMetadata);
         int asl = DocumentMetadata.decodeAvgSentenceLength(documentMetadata);
         int quality = DocumentMetadata.decodeQuality(documentMetadata);
@@ -256,18 +261,18 @@ public class IndexResultScoreCalculator {
         int topology = DocumentMetadata.decodeTopology(documentMetadata);
         int year = DocumentMetadata.decodeYear(documentMetadata);
 
-        double averageSentenceLengthPenalty = (asl >= rankingParams.shortSentenceThreshold ? 0 : -rankingParams.shortSentencePenalty);
+        double averageSentenceLengthPenalty = (asl >= rankingParams.getShortSentenceThreshold() ? 0 : -rankingParams.getShortSentencePenalty());
 
         final double qualityPenalty = calculateQualityPenalty(size, quality, rankingParams);
-        final double rankingBonus = (255. - rank) * rankingParams.domainRankBonus;
+        final double rankingBonus = (255. - rank) * rankingParams.getDomainRankBonus();
         final double topologyBonus = Math.log(1 + topology);
-        final double documentLengthPenalty = length > rankingParams.shortDocumentThreshold ? 0 : -rankingParams.shortDocumentPenalty;
+        final double documentLengthPenalty = length > rankingParams.getShortDocumentThreshold() ? 0 : -rankingParams.getShortDocumentPenalty();
         final double temporalBias;
 
-        if (rankingParams.temporalBias == ResultRankingParameters.TemporalBias.RECENT) {
-            temporalBias = - Math.abs(year - PubDate.MAX_YEAR) * rankingParams.temporalBiasWeight;
-        } else if (rankingParams.temporalBias == ResultRankingParameters.TemporalBias.OLD) {
-            temporalBias = - Math.abs(year - PubDate.MIN_YEAR) * rankingParams.temporalBiasWeight;
+        if (rankingParams.getTemporalBias().getBias() == RpcTemporalBias.Bias.RECENT) {
+            temporalBias = - Math.abs(year - PubDate.MAX_YEAR) * rankingParams.getTemporalBiasWeight();
+        } else if (rankingParams.getTemporalBias().getBias() == RpcTemporalBias.Bias.OLD) {
+            temporalBias = - Math.abs(year - PubDate.MIN_YEAR) * rankingParams.getTemporalBiasWeight();
         } else {
             temporalBias = 0;
         }
@@ -506,14 +511,14 @@ public class IndexResultScoreCalculator {
     }
 
 
-    private double calculateQualityPenalty(int size, int quality, ResultRankingParameters rankingParams) {
+    private double calculateQualityPenalty(int size, int quality, RpcResultRankingParameters rankingParams) {
         if (size < 400) {
             if (quality < 5)
                 return 0;
-            return -quality * rankingParams.qualityPenalty;
+            return -quality * rankingParams.getQualityPenalty();
         }
         else {
-            return -quality * rankingParams.qualityPenalty * 20;
+            return -quality * rankingParams.getQualityPenalty() * 20;
         }
     }
 

code/index/java/nu/marginalia/index/results/TermFlagsGraphVisitor.java

@@ -3,7 +3,6 @@ package nu.marginalia.index.results;
 import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
 import nu.marginalia.api.searchquery.model.compiled.CqDataLong;
 import nu.marginalia.api.searchquery.model.compiled.CqExpression;
-import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
 import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
 import nu.marginalia.model.idx.WordFlags;
 
@@ -15,15 +14,14 @@ public class TermFlagsGraphVisitor implements CqExpression.DoubleVisitor {
     private final CqDataLong wordMetaData;
     private final CqDataInt frequencies;
     private final float[] counts;
-    private final Bm25Parameters bm25Parameters;
-
+    private final double k1;
     private final int docCount;
 
-    public TermFlagsGraphVisitor(Bm25Parameters bm25Parameters,
+    public TermFlagsGraphVisitor(double k1,
                                  CqDataLong wordMetaData,
                                  float[] counts,
                                  ResultRankingContext ctx) {
-        this.bm25Parameters = bm25Parameters;
+        this.k1 = k1;
         this.counts = counts;
         this.docCount = ctx.termFreqDocCount();
         this.wordMetaData = wordMetaData;
@@ -55,7 +53,7 @@ public class TermFlagsGraphVisitor implements CqExpression.DoubleVisitor {
         int freq = frequencies.get(idx);
 
         // note we override b to zero for priority terms as they are independent of document length
-        return invFreq(docCount, freq) * f(bm25Parameters.k(), 0, count, 0);
+        return invFreq(docCount, freq) * f(k1, 0, count, 0);
     }
 
     private double evaluatePriorityScore(int idx) {

code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java

@@ -1,7 +0,0 @@
-package nu.marginalia.index.query.limit;
-
-public record QueryLimits(int resultsByDomain, int resultsTotal, int timeoutMs, int fetchSize) {
-    public QueryLimits forSingleDomain() {
-        return new QueryLimits(resultsTotal, resultsTotal, timeoutMs, fetchSize);
-    }
-}

code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java

@@ -4,10 +4,11 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import nu.marginalia.IndexLocations;
 import nu.marginalia.api.searchquery.RpcDecoratedResultItem;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
 import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
 import nu.marginalia.api.searchquery.model.query.SearchSpecification;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.index.construction.DocIdRewriter;
 import nu.marginalia.index.construction.full.FullIndexConstructor;
 import nu.marginalia.index.construction.prio.PrioIndexConstructor;
@@ -17,7 +18,6 @@ import nu.marginalia.index.forward.construction.ForwardIndexConverter;
 import nu.marginalia.index.index.StatefulIndex;
 import nu.marginalia.index.journal.IndexJournal;
 import nu.marginalia.index.journal.IndexJournalSlopWriter;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.linkdb.docs.DocumentDbReader;
@@ -115,9 +115,16 @@ public class IndexQueryServiceIntegrationSmokeTest {
 
         var rsp = queryService.justQuery(
                 SearchSpecification.builder()
-                        .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
+                        .queryLimits(
+                                RpcQueryLimits.newBuilder()
+                                        .setResultsByDomain(10)
+                                        .setResultsTotal(10)
+                                        .setTimeoutMs(Integer.MAX_VALUE)
+                                        .setFetchSize(4000)
+                                        .build()
+                        )
                         .queryStrategy(QueryStrategy.SENTENCE)
-                        .rankingParams(ResultRankingParameters.sensibleDefaults())
+                        .rankingParams(PrototypeRankingParameters.sensibleDefaults())
                         .domains(new ArrayList<>())
                         .searchSetIdentifier("NONE")
                         .query(
@@ -171,9 +178,16 @@ public class IndexQueryServiceIntegrationSmokeTest {
 
         var rsp = queryService.justQuery(
                 SearchSpecification.builder()
-                        .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
+                        .queryLimits(
+                                RpcQueryLimits.newBuilder()
+                                        .setResultsByDomain(10)
+                                        .setResultsTotal(10)
+                                        .setTimeoutMs(Integer.MAX_VALUE)
+                                        .setFetchSize(4000)
+                                        .build()
+                        )
                         .queryStrategy(QueryStrategy.SENTENCE)
-                        .rankingParams(ResultRankingParameters.sensibleDefaults())
+                        .rankingParams(PrototypeRankingParameters.sensibleDefaults())
                         .domains(new ArrayList<>())
                         .searchSetIdentifier("NONE")
                         .query(
@@ -225,8 +239,15 @@ public class IndexQueryServiceIntegrationSmokeTest {
 
         var rsp = queryService.justQuery(
                 SearchSpecification.builder()
-                        .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
-                        .rankingParams(ResultRankingParameters.sensibleDefaults())
+                        .queryLimits(
+                                RpcQueryLimits.newBuilder()
+                                        .setResultsByDomain(10)
+                                        .setResultsTotal(10)
+                                        .setTimeoutMs(Integer.MAX_VALUE)
+                                        .setFetchSize(4000)
+                                        .build()
+                        )
+                        .rankingParams(PrototypeRankingParameters.sensibleDefaults())
                         .queryStrategy(QueryStrategy.SENTENCE)
                         .domains(List.of(2))
                         .query(
@@ -282,11 +303,18 @@ public class IndexQueryServiceIntegrationSmokeTest {
 
         var rsp = queryService.justQuery(
                 SearchSpecification.builder()
-                        .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
+                        .queryLimits(
+                                RpcQueryLimits.newBuilder()
+                                        .setResultsByDomain(10)
+                                        .setResultsTotal(10)
+                                        .setTimeoutMs(Integer.MAX_VALUE)
+                                        .setFetchSize(4000)
+                                        .build()
+                        )
                         .year(SpecificationLimit.equals(1998))
                         .queryStrategy(QueryStrategy.SENTENCE)
                         .searchSetIdentifier("NONE")
-                        .rankingParams(ResultRankingParameters.sensibleDefaults())
+                        .rankingParams(PrototypeRankingParameters.sensibleDefaults())
                         .query(
                             SearchQuery.builder()
                                 .compiledQuery("4")

code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java

@@ -4,10 +4,11 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import it.unimi.dsi.fastutil.ints.IntList;
 import nu.marginalia.IndexLocations;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
 import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
 import nu.marginalia.api.searchquery.model.query.SearchSpecification;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.hash.MurmurHash3_128;
 import nu.marginalia.index.construction.DocIdRewriter;
 import nu.marginalia.index.construction.full.FullIndexConstructor;
@@ -18,7 +19,6 @@ import nu.marginalia.index.forward.construction.ForwardIndexConverter;
 import nu.marginalia.index.index.StatefulIndex;
 import nu.marginalia.index.journal.IndexJournal;
 import nu.marginalia.index.journal.IndexJournalSlopWriter;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.linkdb.docs.DocumentDbReader;
@@ -389,13 +389,20 @@ public class IndexQueryServiceIntegrationTest {
     SearchSpecification basicQuery(Function<SearchSpecification.SearchSpecificationBuilder, SearchSpecification.SearchSpecificationBuilder> mutator)
     {
         var builder = SearchSpecification.builder()
-                .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
+                .queryLimits(
+                        RpcQueryLimits.newBuilder()
+                                .setResultsByDomain(10)
+                                .setResultsTotal(10)
+                                .setTimeoutMs(Integer.MAX_VALUE)
+                                .setFetchSize(4000)
+                                .build()
+                )
                 .queryStrategy(QueryStrategy.SENTENCE)
                 .year(SpecificationLimit.none())
                 .quality(SpecificationLimit.none())
                 .size(SpecificationLimit.none())
                 .rank(SpecificationLimit.none())
-                .rankingParams(ResultRankingParameters.sensibleDefaults())
+                .rankingParams(PrototypeRankingParameters.sensibleDefaults())
                 .domains(new ArrayList<>())
                 .searchSetIdentifier("NONE");
 

code/processes/converting-process/ft-keyword-extraction/java/nu/marginalia/keyword/DocumentPositionMapper.java

@@ -152,7 +152,10 @@ public class DocumentPositionMapper {
     }
 
     boolean matchesWordPattern(String s) {
-        // this function is an unrolled version of the regexp [\da-zA-Z]{1,15}([.\-_/:+*][\da-zA-Z]{1,10}){0,4}
+        if (s.length() > 48)
+            return false;
+
+        // this function is an unrolled version of the regexp [\da-zA-Z]{1,15}([.\-_/:+*][\da-zA-Z]{1,10}){0,8}
 
         String wordPartSeparator = ".-_/:+*";
 
@@ -169,7 +172,7 @@ public class DocumentPositionMapper {
         if (i == 0)
             return false;
 
-        for (int j = 0; j < 5; j++) {
+        for (int j = 0; j < 8; j++) {
             if (i == s.length()) return true;
 
             if (wordPartSeparator.indexOf(s.charAt(i)) < 0) {

code/processes/converting-process/ft-keyword-extraction/test/nu/marginalia/keyword/DocumentPositionMapperTest.java

@@ -30,9 +30,11 @@ class DocumentPositionMapperTest {
         Assertions.assertFalse(positionMapper.matchesWordPattern("1234567890abcdef"));
 
         Assertions.assertTrue(positionMapper.matchesWordPattern("test-test-test-test-test"));
-        Assertions.assertFalse(positionMapper.matchesWordPattern("test-test-test-test-test-test"));
+        Assertions.assertFalse(positionMapper.matchesWordPattern("test-test-test-test-test-test-test-test-test"));
         Assertions.assertTrue(positionMapper.matchesWordPattern("192.168.1.100/24"));
         Assertions.assertTrue(positionMapper.matchesWordPattern("std::vector"));
+        Assertions.assertTrue(positionMapper.matchesWordPattern("std::vector::push_back"));
+
         Assertions.assertTrue(positionMapper.matchesWordPattern("c++"));
         Assertions.assertTrue(positionMapper.matchesWordPattern("m*a*s*h"));
         Assertions.assertFalse(positionMapper.matchesWordPattern("Stulpnagelstrasse"));

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/CppreferenceSpecialization.java

@@ -0,0 +1,113 @@
+package nu.marginalia.converting.processor.plugin.specialization;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.converting.processor.logic.TitleExtractor;
+import nu.marginalia.converting.processor.summary.SummaryExtractor;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.util.Strings;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+@Singleton
+public class CppreferenceSpecialization extends WikiSpecialization {
+
+    @Inject
+    public CppreferenceSpecialization(SummaryExtractor summaryExtractor, TitleExtractor titleExtractor) {
+        super(summaryExtractor, titleExtractor);
+    }
+
+    @Override
+    public Document prune(Document original) {
+        var doc = original.clone();
+
+        doc.getElementsByClass("t-nv").remove();
+        doc.getElementsByClass("toc").remove();
+        doc.getElementsByClass("mw-head").remove();
+        doc.getElementsByClass("printfooter").remove();
+        doc.getElementsByClass("cpp-footer-base").remove();
+
+        doc.title(doc.title() + " " + Strings.join(extractExtraTokens(doc.title()), ' '));
+
+        return doc;
+    }
+
+    @Override
+    public String getSummary(Document doc, Set<String> importantWords) {
+
+        Element declTable = doc.getElementsByClass("t-dcl-begin").first();
+        if (declTable != null) {
+            var nextPar =  declTable.nextElementSibling();
+            if (nextPar != null) {
+                return nextPar.text();
+            }
+        }
+
+        return super.getSummary(doc, importantWords);
+    }
+
+
+    public List<String> extractExtraTokens(String title) {
+
+        if (!title.contains("::")) {
+            return List.of();
+        }
+        if (!title.contains("-")) {
+            return List.of();
+        }
+
+        title = StringUtils.split(title, '-')[0];
+
+        String name = title;
+        for (;;) {
+            int lbidx = name.indexOf('<');
+            int rbidx = name.indexOf('>');
+
+            if (lbidx > 0 && rbidx > lbidx) {
+                String className = name.substring(0, lbidx);
+                String methodName = name.substring(rbidx + 1);
+                name = className + methodName;
+            } else {
+                break;
+            }
+        }
+
+
+        List<String> tokens = new ArrayList<>();
+
+        for (var part : name.split("\\s*,\\s*")) {
+            if (part.endsWith(")") && !part.endsWith("()")) {
+                int parenStart = part.indexOf('(');
+                if (parenStart > 0) { // foo(...) -> foo
+                    part = part.substring(0, parenStart);
+                }
+                else if (parenStart == 0) { // (foo) -> foo
+                    part = part.substring(1, part.length() - 1);
+                }
+            }
+
+            part = part.trim();
+            if (part.contains("::")) {
+                tokens.add(part);
+                if (part.startsWith("std::")) {
+                    tokens.add(part.substring(5));
+
+                    int ss = part.indexOf("::", 5);
+                    if (ss > 0) {
+                        tokens.add(part.substring(0, ss));
+                        tokens.add(part.substring(ss+2));
+                    }
+
+                }
+            }
+        }
+
+        return tokens;
+    }
+
+
+}

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java

@@ -24,6 +24,7 @@ public class HtmlProcessorSpecializations {
     private final WikiSpecialization wikiSpecialization;
     private final BlogSpecialization blogSpecialization;
     private final GogStoreSpecialization gogStoreSpecialization;
+    private final CppreferenceSpecialization cppreferenceSpecialization;
     private final DefaultSpecialization defaultSpecialization;
 
     @Inject
@@ -37,6 +38,7 @@ public class HtmlProcessorSpecializations {
                                         WikiSpecialization wikiSpecialization,
                                         BlogSpecialization blogSpecialization,
                                         GogStoreSpecialization gogStoreSpecialization,
+                                        CppreferenceSpecialization cppreferenceSpecialization,
                                         DefaultSpecialization defaultSpecialization) {
         this.domainTypes = domainTypes;
         this.lemmySpecialization = lemmySpecialization;
@@ -48,6 +50,7 @@ public class HtmlProcessorSpecializations {
         this.wikiSpecialization = wikiSpecialization;
         this.blogSpecialization = blogSpecialization;
         this.gogStoreSpecialization = gogStoreSpecialization;
+        this.cppreferenceSpecialization = cppreferenceSpecialization;
         this.defaultSpecialization = defaultSpecialization;
     }
 
@@ -66,6 +69,10 @@ public class HtmlProcessorSpecializations {
             return mariadbKbSpecialization;
         }
 
+        if (url.domain.getTopDomain().equals("cppreference.com")) {
+            return cppreferenceSpecialization;
+        }
+
         if (url.domain.toString().equals("store.steampowered.com")) {
             return steamStoreSpecialization;
         }
@@ -86,6 +93,9 @@ public class HtmlProcessorSpecializations {
         if (generator.keywords().contains("javadoc")) {
             return javadocSpecialization;
         }
+
+        // Must be toward the end, as some specializations are for
+        // wiki-generator content
         if (generator.type() == GeneratorType.WIKI) {
             return wikiSpecialization;
         }
@@ -105,7 +115,7 @@ public class HtmlProcessorSpecializations {
 
         boolean shouldIndex(EdgeUrl url);
         double lengthModifier();
-        void amendWords(Document doc, DocumentKeywordsBuilder words);
 
+        default void amendWords(Document doc, DocumentKeywordsBuilder words) {}
     }
 }

code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecialization.java

@@ -4,7 +4,6 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.converting.processor.logic.TitleExtractor;
 import nu.marginalia.converting.processor.summary.SummaryExtractor;
-import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
 import nu.marginalia.model.EdgeUrl;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -93,6 +92,8 @@ public class WikiSpecialization extends DefaultSpecialization {
         return true;
     }
 
-    public void amendWords(Document doc, DocumentKeywordsBuilder words) {
+    @Override
+    public double lengthModifier() {
+        return 2.5;
     }
 }

code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/CppreferenceSpecializationTest.java

@@ -0,0 +1,27 @@
+package nu.marginalia.converting.processor.plugin.specialization;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+class CppreferenceSpecializationTest {
+    CppreferenceSpecialization specialization = new CppreferenceSpecialization(null, null);
+
+    @Test
+    public void testTitleMagic() {
+
+        List<String> ret;
+
+        ret = specialization.extractExtraTokens("std::multimap<Key, T, Compare, Allocator>::crend - cppreference.com");
+        Assertions.assertTrue(ret.contains("std::multimap::crend"));
+        Assertions.assertTrue(ret.contains("multimap::crend"));
+        Assertions.assertTrue(ret.contains("std::multimap"));
+        Assertions.assertTrue(ret.contains("crend"));
+
+        ret = specialization.extractExtraTokens("std::coroutine_handle<Promise>::operator(), std::coroutine_handle<Promise>::resume - cppreference.com");
+        Assertions.assertTrue(ret.contains("std::coroutine_handle::operator()"));
+        Assertions.assertTrue(ret.contains("std::coroutine_handle::resume"));
+    }
+
+}

code/services-application/api-service/build.gradle

@@ -44,6 +44,7 @@ dependencies {
     implementation libs.bundles.jetty
     implementation libs.opencsv
     implementation libs.trove
+    implementation libs.protobuf
     implementation libs.fastutil
     implementation libs.bundles.gson
     implementation libs.bundles.mariadb

code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java

@@ -6,10 +6,10 @@ import nu.marginalia.api.model.ApiSearchResult;
 import nu.marginalia.api.model.ApiSearchResultQueryDetails;
 import nu.marginalia.api.model.ApiSearchResults;
 import nu.marginalia.api.searchquery.QueryClient;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier;
 import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.idx.WordFlags;
 
 import java.util.ArrayList;
@@ -47,11 +47,12 @@ public class ApiSearchOperator {
 
         return new QueryParams(
                 query,
-                new QueryLimits(
-                        2,
-                        Math.min(100, count),
-                        150,
-                        8192),
+                RpcQueryLimits.newBuilder()
+                        .setResultsByDomain(2)
+                        .setResultsTotal(Math.min(100, count))
+                        .setTimeoutMs(150)
+                        .setFetchSize(8192)
+                        .build(),
                 searchSet.name());
     }
 

code/services-application/search-service-legacy/java/nu/marginalia/search/SearchOperator.java

@@ -5,11 +5,11 @@ import com.google.inject.Singleton;
 import nu.marginalia.WebsiteUrl;
 import nu.marginalia.api.math.MathClient;
 import nu.marginalia.api.searchquery.QueryClient;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
 import nu.marginalia.api.searchquery.model.query.QueryResponse;
 import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
 import nu.marginalia.bbpc.BrailleBlockPunchCards;
 import nu.marginalia.db.DbDomainQueries;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.DomainIndexingState;
@@ -155,15 +155,15 @@ public class SearchOperator {
 
 
     public List<UrlDetails> getResultsFromQuery(QueryResponse queryResponse) {
-        final QueryLimits limits = queryResponse.specs().queryLimits;
-        final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.resultsByDomain());
+        final RpcQueryLimits limits = queryResponse.specs().queryLimits;
+        final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.getResultsByDomain());
 
         // Update the query count (this is what you see on the front page)
         searchVisitorCount.registerQuery();
 
         return queryResponse.results().stream()
                 .filter(deduplicator::shouldRetain)
-                .limit(limits.resultsTotal())
+                .limit(limits.getResultsTotal())
                 .map(SearchOperator::createDetails)
                 .toList();
     }

code/services-application/search-service-legacy/java/nu/marginalia/search/SearchQueryParamFactory.java

@@ -1,10 +1,10 @@
 package nu.marginalia.search;
 
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
 import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.search.command.SearchParameters;
@@ -12,6 +12,21 @@ import nu.marginalia.search.command.SearchParameters;
 import java.util.List;
 
 public class SearchQueryParamFactory {
+    static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder()
+            .setResultsTotal(100)
+            .setResultsByDomain(5)
+            .setTimeoutMs(200)
+            .setFetchSize(8192)
+            .build();
+
+
+    static final RpcQueryLimits shallowLimit = RpcQueryLimits.newBuilder()
+            .setResultsTotal(100)
+            .setResultsByDomain(100)
+            .setTimeoutMs(100)
+            .setFetchSize(512)
+            .build();
+
 
     public QueryParams forRegularSearch(SearchParameters userParams) {
         SearchQuery prototype =  new SearchQuery();
@@ -33,7 +48,7 @@ public class SearchQueryParamFactory {
                 profile.getSizeLimit(),
                 SpecificationLimit.none(),
                 List.of(),
-                new QueryLimits(5, 100, 200, 8192),
+                defaultLimits,
                 profile.searchSetIdentifier.name(),
                 userParams.strategy(),
                 userParams.temporalBias(),
@@ -54,10 +69,15 @@ public class SearchQueryParamFactory {
                 SpecificationLimit.none(),
                 SpecificationLimit.none(),
                 List.of(domainId),
-                new QueryLimits(count, count, 100, 512),
+                RpcQueryLimits.newBuilder()
+                        .setResultsTotal(count)
+                        .setResultsByDomain(count)
+                        .setTimeoutMs(100)
+                        .setFetchSize(512)
+                        .build(),
                 SearchSetIdentifier.NONE.name(),
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 1
         );
     }
@@ -74,10 +94,10 @@ public class SearchQueryParamFactory {
                 SpecificationLimit.none(),
                 SpecificationLimit.none(),
                 List.of(),
-                new QueryLimits(100, 100, 100, 512),
+                shallowLimit,
                 SearchSetIdentifier.NONE.name(),
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 1
         );
     }
@@ -94,10 +114,10 @@ public class SearchQueryParamFactory {
                 SpecificationLimit.none(),
                 SpecificationLimit.none(),
                 List.of(),
-                new QueryLimits(100, 100, 100, 512),
+                shallowLimit,
                 SearchSetIdentifier.NONE.name(),
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 1
         );
     }

code/services-application/search-service-legacy/java/nu/marginalia/search/command/SearchParameters.java

@@ -1,7 +1,7 @@
 package nu.marginalia.search.command;
 
 import nu.marginalia.WebsiteUrl;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.search.model.SearchProfile;
@@ -78,15 +78,15 @@ public record SearchParameters(String query,
         return baseUrl.withPath(path);
     }
 
-    public ResultRankingParameters.TemporalBias temporalBias() {
+    public RpcTemporalBias.Bias temporalBias() {
         if (recent == RECENT) {
-            return ResultRankingParameters.TemporalBias.RECENT;
+            return RpcTemporalBias.Bias.RECENT;
         }
         else if (profile == SearchProfile.VINTAGE) {
-            return ResultRankingParameters.TemporalBias.OLD;
+            return RpcTemporalBias.Bias.OLD;
         }
 
-        return ResultRankingParameters.TemporalBias.NONE;
+        return RpcTemporalBias.Bias.NONE;
     }
 
     public QueryStrategy strategy() {

code/services-application/search-service-legacy/resources/static/search/opensearch.xml

@@ -8,8 +8,8 @@
     <ShortName>Marginalia</ShortName>
     <Description>Search Marginalia</Description>
     <InputEncoding>UTF-8</InputEncoding>
-    <Image width="16" height="16" type="image/x-icon">https://search.marginalia.nu/favicon.ico</Image>
+    <Image width="16" height="16" type="image/x-icon">https://old-search.marginalia.nu/favicon.ico</Image>
     <Url type="text/html" method="get"
-         template="https://search.marginalia.nu/search?query={searchTerms}&amp;ref=opensearch"/>
-    <moz:SearchForm>https://search.marginalia.nu/</moz:SearchForm>
+         template="https://old-search.marginalia.nu/search?query={searchTerms}&amp;ref=opensearch"/>
+    <moz:SearchForm>https://old-search.marginalia.nu/</moz:SearchForm>
 </OpenSearchDescription>

code/services-application/search-service-legacy/resources/templates/search/parts/search-header.hdb

@@ -3,9 +3,9 @@
     <nav>
         <a href="#" class="screenreader-only" onClick="">Skip to content</a>
         <a href="https://www.marginalia.nu/">Marginalia</a>
-        <a href="https://memex.marginalia.nu/projects/edge/about.gmi">About</a>
-        <a href="https://memex.marginalia.nu/projects/edge/supporting.gmi">Donate</a>
-        <a class="extra" href="https://search.marginalia.nu/explore/random">Random</a>
+        <a href="https://about.marginalia-search.com/">About</a>
+        <a href="https://about.marginalia-search.com/article/supporting/">Donate</a>
+        <a class="extra" href="https://old-search.marginalia.nu/explore/random">Random</a>
     </nav>
     <div id="theme">
         <label for="theme-select" class="screenreader-only">Color Theme</label>

code/services-application/search-service-legacy/test/nu/marginalia/search/paperdoll/SearchServicePaperDoll.java

@@ -1,362 +0,0 @@
-package nu.marginalia.search.paperdoll;
-
-import com.google.gson.Gson;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
-import nu.marginalia.api.domains.DomainInfoClient;
-import nu.marginalia.api.domains.model.DomainInformation;
-import nu.marginalia.api.domains.model.SimilarDomain;
-import nu.marginalia.api.searchquery.QueryClient;
-import nu.marginalia.api.searchquery.model.query.QueryResponse;
-import nu.marginalia.api.searchquery.model.query.SearchQuery;
-import nu.marginalia.api.searchquery.model.query.SearchSpecification;
-import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.api.searchquery.model.results.SearchResultItem;
-import nu.marginalia.index.query.limit.QueryLimits;
-import nu.marginalia.index.query.limit.QueryStrategy;
-import nu.marginalia.index.query.limit.SpecificationLimit;
-import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.model.EdgeUrl;
-import nu.marginalia.model.crawl.HtmlFeature;
-import nu.marginalia.model.gson.GsonFactory;
-import nu.marginalia.screenshot.ScreenshotService;
-import nu.marginalia.search.SearchModule;
-import nu.marginalia.search.SearchService;
-import nu.marginalia.service.ServiceId;
-import nu.marginalia.service.discovery.ServiceRegistryIf;
-import nu.marginalia.service.discovery.property.ServiceEndpoint;
-import nu.marginalia.service.module.ServiceConfigurationModule;
-import nu.marginalia.test.TestMigrationLoader;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-import org.testcontainers.containers.MariaDBContainer;
-import org.testcontainers.junit.jupiter.Container;
-import org.testcontainers.junit.jupiter.Testcontainers;
-import spark.Spark;
-
-import java.net.URISyntaxException;
-import java.sql.SQLException;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.Mockito.when;
-
-
-/** This class is a special test class that sets up a search service
- * and registers some search results, without actually starting the rest
- * of the environment. This is used to test the search service in isolation
- * when working on the frontend.
- * <p></p>
- * It's not actually a test, but it's in the test directory because it's
- * using test related classes.
- * <p></p>
- * When using gradle, run ./gradlew paperDoll --info to run this test,
- * the system will wait for you to kill the process to stop the test,
- * and the UI is available at port 9999.
- */
-@Testcontainers
-@Tag("paperdoll")
-public class SearchServicePaperDoll extends AbstractModule {
-
-    @Container
-    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
-            .withDatabaseName("WMSA_prod")
-            .withUsername("wmsa")
-            .withPassword("wmsa")
-            .withNetworkAliases("mariadb");
-
-    private static HikariDataSource dataSource;
-
-    private static List<DecoratedSearchResultItem> results = new ArrayList<>();
-    private static List<SimilarDomain> dummyLinks = new ArrayList<>();
-    private static QueryResponse searchResponse;
-    private static final Gson gson = GsonFactory.get();
-
-    void registerSearchResult(
-            String url,
-            String title,
-            String description,
-            Collection<HtmlFeature> features,
-            double quality,
-            double score,
-            long positions)
-    {
-        try {
-            results.add(new DecoratedSearchResultItem(
-                    new SearchResultItem(url.hashCode(), 2, 3, score, 0),
-                    new EdgeUrl(url),
-                    title,
-                    description,
-                    quality,
-                    "HTML5",
-                    HtmlFeature.encode(features),
-                    null,
-                    url.hashCode(),
-                    400,
-                    positions,
-                    score,
-                    4,
-                    null)
-            );
-        }
-        catch (Exception e) {
-            throw new RuntimeException();
-        }
-    }
-
-    @BeforeAll
-    public static void setup() throws URISyntaxException {
-        if (!Boolean.getBoolean("runPaperDoll")) {
-            return;
-        }
-
-        HikariConfig config = new HikariConfig();
-        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
-        config.setUsername("wmsa");
-        config.setPassword("wmsa");
-
-        dataSource = new HikariDataSource(config);
-
-        TestMigrationLoader.flywayMigration(dataSource);
-
-        System.setProperty("service-name", "search");
-        System.setProperty("search.websiteUrl", "http://localhost:9999/");
-
-        try (var conn = dataSource.getConnection();
-             var newsStmt = conn.prepareStatement("""
-                            INSERT INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE)
-                            VALUES (?, ?, ?, ?)
-                            """);
-             var domainStmt = conn.prepareStatement("""
-                            INSERT INTO EC_DOMAIN(ID, DOMAIN_NAME, DOMAIN_TOP, NODE_AFFINITY)
-                            VALUES (?, ?, ?, ?)
-                            """);
-             var randomStmt = conn.prepareStatement("""
-                            INSERT INTO EC_RANDOM_DOMAINS(DOMAIN_ID, DOMAIN_SET)
-                            VALUES (?, ?) 
-                            """)
-             ) {
-            newsStmt.setString(1, "Lex Luthor elected president");
-            newsStmt.setString(2, "https://www.example.com/foo");
-            newsStmt.setString(3, "Daily Planet");
-            newsStmt.setDate(4, new java.sql.Date(System.currentTimeMillis()));
-            newsStmt.execute();
-
-            newsStmt.setString(1, "Besieged Alesian onlookers confused as Caesar builds a wall around his wall around the city walls");
-            newsStmt.setString(2, "https://www.example2.com/bar");
-            newsStmt.setString(3, "The Gaulish Observer");
-            newsStmt.setDate(4, new java.sql.Date(System.currentTimeMillis()));
-            newsStmt.execute();
-
-            newsStmt.setString(1, "Marginalia acquires Google");
-            newsStmt.setString(2, "https://www.example3.com/baz");
-            newsStmt.setString(3, "The Dependent");
-            newsStmt.setDate(4, new java.sql.Date(System.currentTimeMillis()));
-            newsStmt.execute();
-
-            domainStmt.setInt(1, 1);
-            domainStmt.setString(2, "www.example.com");
-            domainStmt.setString(3, "example.com");
-            domainStmt.setInt(4, 1);
-            domainStmt.execute();
-
-            domainStmt.setInt(1, 2);
-            domainStmt.setString(2, "www.example2.com");
-            domainStmt.setString(3, "example2.com");
-            domainStmt.setInt(4, 2);
-            domainStmt.execute();
-
-            domainStmt.setInt(1, 3);
-            domainStmt.setString(2, "www.example3.com");
-            domainStmt.setString(3, "example3.com");
-            domainStmt.setInt(4, 3);
-            domainStmt.execute();
-
-            randomStmt.setInt(1, 1);
-            randomStmt.setInt(2, 0);
-            randomStmt.execute();
-
-            randomStmt.setInt(1, 2);
-            randomStmt.setInt(2, 0);
-            randomStmt.execute();
-
-            randomStmt.setInt(1, 3);
-            randomStmt.setInt(2, 0);
-            randomStmt.execute();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-
-        searchResponse = new QueryResponse(
-                new SearchSpecification(new SearchQuery(), List.of(), "", "test",
-                        SpecificationLimit.none(),
-                        SpecificationLimit.none(),
-                        SpecificationLimit.none(),
-                        SpecificationLimit.none(),
-                        new QueryLimits(10, 20, 3, 4),
-                        QueryStrategy.AUTO,
-                        ResultRankingParameters.sensibleDefaults()
-                        ),
-                results,
-                List.of(),
-                List.of(),
-                1,
-                1,
-                null
-        );
-    }
-
-    @Test
-    public void run() throws Exception {
-        if (!Boolean.getBoolean("runPaperDoll")) {
-            return;
-        }
-
-        var injector = Guice.createInjector(
-                new ServiceConfigurationModule(ServiceId.Search),
-                new SearchModule(),
-                this);
-
-        injector.getInstance(SearchService.class);
-
-        List<String> suggestions = List.of("foo", "bar", "baz");
-
-        Spark.get("/suggest/", (rq, rsp) -> {
-            rsp.type("application/json");
-            return gson.toJson(suggestions);
-        });
-
-        Spark.get("/screenshot/*", (rq, rsp) -> {
-            rsp.type("image/svg+xml");
-            return """
-                <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-                <svg
-                   xmlns="http://www.w3.org/2000/svg"
-                   width="640px"
-                   height="480px"
-                   viewBox="0 0 640 480"
-                   version="1.1">
-                  <g>
-                    <rect
-                       style="fill:#808080"
-                       id="rect288"
-                       width="595.41992"
-                       height="430.01825"
-                       x="23.034981"
-                       y="27.850344" />
-                    <text
-                       xml:space="preserve"
-                      style="font-size:100px;fill:#909090;font-family:sans-serif;"
-                       x="20"
-                       y="120">Placeholder</text>
-                    <text
-                       xml:space="preserve"
-                       style="font-size:32px;fill:#000000;font-family:monospace;"
-                       x="320" y="240" dominant-baseline="middle" text-anchor="middle">Lorem Ipsum As F</text>
-                  </g>
-                </svg>
-                """;
-        });
-
-        registerSearchResult("https://www.example.com/foo", "Foo", "Lorem ipsum dolor sit amet", Set.of(), 0.5, 0.5, ~0L);
-        registerSearchResult("https://www.example2.com/bar", "Bar", "Some text goes here", Set.of(), 0.5, 0.5, 1L);
-        registerSearchResult("https://www.example3.com/baz", "All HTML Features", "This one's got every feature", EnumSet.allOf(HtmlFeature.class), 0.5, 0.5, 1L);
-
-
-
-
-        dummyLinks.add(new SimilarDomain(
-                new EdgeUrl("https://www.example.com/foo"),
-                1,
-                0.5,
-                0.5,
-                true,
-                true,
-                true,
-                true,
-                SimilarDomain.LinkType.FOWARD
-        ));
-        dummyLinks.add(new SimilarDomain(
-                new EdgeUrl("https://www.example2.com/foo"),
-                2,
-                0.5,
-                1,
-                false,
-                false,
-                true,
-                true,
-                SimilarDomain.LinkType.BACKWARD
-        ));
-        dummyLinks.add(new SimilarDomain(
-                new EdgeUrl("https://www.example3.com/foo"),
-                3,
-                0,
-                0.5,
-                false,
-                false,
-                false,
-                false,
-                SimilarDomain.LinkType.BIDIRECTIONAL
-        ));
-
-
-        for (;;);
-    }
-
-    public void configure() {
-        try {
-            var serviceRegistry = Mockito.mock(ServiceRegistryIf.class);
-            when(serviceRegistry.registerService(any(), any(), any())).thenReturn(new ServiceEndpoint("localhost", 9999));
-
-            bind(ServiceRegistryIf.class).toInstance(serviceRegistry);
-            bind(HikariDataSource.class).toInstance(dataSource);
-
-            var qsMock = Mockito.mock(QueryClient.class);
-            when(qsMock.search(any())).thenReturn(searchResponse);
-            bind(QueryClient.class).toInstance(qsMock);
-
-            var asMock = Mockito.mock(DomainInfoClient.class);
-
-            when(asMock.isAccepting()).thenReturn(true);
-            when(asMock.linkedDomains(anyInt(), anyInt())).thenReturn(CompletableFuture.completedFuture(dummyLinks));
-            when(asMock.similarDomains(anyInt(), anyInt())).thenReturn(CompletableFuture.completedFuture(dummyLinks));
-            when(asMock.domainInformation(anyInt())).thenReturn(CompletableFuture.completedFuture(
-                    new DomainInformation(new EdgeDomain("www.example.com"),
-                            false,
-                            123,
-                            123,
-                            123,
-                            123,
-                            123,
-                            1,
-                            0.5,
-                            false,
-                            false,
-                            false,
-                            "127.0.0.1",
-                            1,
-                            "ACME",
-                            "CA",
-                            "CA",
-                            "Exemplary")
-            ));
-
-            bind(DomainInfoClient.class).toInstance(asMock);
-
-            var sss = Mockito.mock(ScreenshotService.class);
-            when(sss.hasScreenshot(anyInt())).thenReturn(true);
-            bind(ScreenshotService.class).toInstance(sss);
-        }
-        catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-}

code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java

@@ -2,14 +2,13 @@ package nu.marginalia.search;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
-import nu.marginalia.WebsiteUrl;
 import nu.marginalia.api.math.MathClient;
 import nu.marginalia.api.searchquery.QueryClient;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
 import nu.marginalia.api.searchquery.model.query.QueryResponse;
 import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
 import nu.marginalia.bbpc.BrailleBlockPunchCards;
 import nu.marginalia.db.DbDomainQueries;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.DomainIndexingState;
@@ -47,7 +46,6 @@ public class SearchOperator {
     private final DbDomainQueries domainQueries;
     private final QueryClient queryClient;
     private final SearchQueryParamFactory paramFactory;
-    private final WebsiteUrl websiteUrl;
     private final SearchUnitConversionService searchUnitConversionService;
     private final SearchQueryCountService searchVisitorCount;
 
@@ -57,7 +55,6 @@ public class SearchOperator {
                           DbDomainQueries domainQueries,
                           QueryClient queryClient,
                           SearchQueryParamFactory paramFactory,
-                          WebsiteUrl websiteUrl,
                           SearchUnitConversionService searchUnitConversionService,
                           SearchQueryCountService searchVisitorCount
                           )
@@ -67,7 +64,6 @@ public class SearchOperator {
         this.domainQueries = domainQueries;
         this.queryClient = queryClient;
         this.paramFactory = paramFactory;
-        this.websiteUrl = websiteUrl;
         this.searchUnitConversionService = searchUnitConversionService;
         this.searchVisitorCount = searchVisitorCount;
     }
@@ -154,8 +150,8 @@ public class SearchOperator {
 
 
     public SimpleSearchResults getResultsFromQuery(QueryResponse queryResponse) {
-        final QueryLimits limits = queryResponse.specs().queryLimits;
-        final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.resultsByDomain());
+        final RpcQueryLimits limits = queryResponse.specs().queryLimits;
+        final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.getResultsByDomain());
 
         // Update the query count (this is what you see on the front page)
         searchVisitorCount.registerQuery();
@@ -164,7 +160,7 @@ public class SearchOperator {
                 .sorted(this::retentionSortOrder) // Sort in an order that makes us more likely to discard the "bad" duplicates
                 .filter(deduplicator::shouldRetain)
                 .sorted() // Return to the presentation sort order before limiting so we don't throw out good results over schema and "ip-ness"
-                .limit(limits.resultsTotal())
+                .limit(limits.getResultsTotal())
                 .map(SearchOperator::createDetails)
                 .toList();
 

code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java

@@ -1,10 +1,10 @@
 package nu.marginalia.search;
 
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.api.searchquery.model.query.SearchQuery;
 import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.search.command.SearchParameters;
@@ -13,6 +13,22 @@ import java.util.List;
 
 public class SearchQueryParamFactory {
 
+
+    static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder()
+            .setResultsTotal(100)
+            .setResultsByDomain(5)
+            .setTimeoutMs(200)
+            .setFetchSize(8192)
+            .build();
+
+
+    static final RpcQueryLimits shallowLimit = RpcQueryLimits.newBuilder()
+            .setResultsTotal(100)
+            .setResultsByDomain(100)
+            .setTimeoutMs(100)
+            .setFetchSize(512)
+            .build();
+
     public QueryParams forRegularSearch(SearchParameters userParams) {
         SearchQuery prototype =  new SearchQuery();
         var profile = userParams.profile();
@@ -29,11 +45,11 @@ public class SearchQueryParamFactory {
                 prototype.searchTermsPriority,
                 prototype.searchTermsAdvice,
                 profile.getQualityLimit(),
-                profile.getYearLimit(),
+                userParams.yearLimit(),
                 profile.getSizeLimit(),
                 SpecificationLimit.none(),
                 List.of(),
-                new QueryLimits(5, 100, 200, 8192),
+                defaultLimits,
                 profile.searchSetIdentifier.name(),
                 userParams.strategy(),
                 userParams.temporalBias(),
@@ -54,10 +70,15 @@ public class SearchQueryParamFactory {
                 SpecificationLimit.none(),
                 SpecificationLimit.none(),
                 List.of(domainId),
-                new QueryLimits(count, count, 100, 512),
+                RpcQueryLimits.newBuilder()
+                        .setResultsTotal(count)
+                        .setResultsByDomain(count)
+                        .setTimeoutMs(100)
+                        .setFetchSize(512)
+                        .build(),
                 SearchSetIdentifier.NONE.name(),
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 page
         );
     }
@@ -74,10 +95,10 @@ public class SearchQueryParamFactory {
                 SpecificationLimit.none(),
                 SpecificationLimit.none(),
                 List.of(),
-                new QueryLimits(100, 100, 100, 512),
+                shallowLimit,
                 SearchSetIdentifier.NONE.name(),
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 page
         );
     }
@@ -94,10 +115,10 @@ public class SearchQueryParamFactory {
                 SpecificationLimit.none(),
                 SpecificationLimit.none(),
                 List.of(),
-                new QueryLimits(100, 100, 100, 512),
+                shallowLimit,
                 SearchSetIdentifier.NONE.name(),
                 QueryStrategy.AUTO,
-                ResultRankingParameters.TemporalBias.NONE,
+                RpcTemporalBias.Bias.NONE,
                 1
         );
     }

code/services-application/search-service/java/nu/marginalia/search/SearchService.java

@@ -1,15 +1,14 @@
 package nu.marginalia.search;
 
 import com.google.inject.Inject;
+import io.jooby.Context;
 import io.jooby.Jooby;
 import io.prometheus.client.Counter;
 import io.prometheus.client.Histogram;
-import nu.marginalia.WebsiteUrl;
 import nu.marginalia.search.svc.*;
 import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.server.BaseServiceParams;
 import nu.marginalia.service.server.JoobyService;
-import nu.marginalia.service.server.StaticResources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,8 +33,6 @@ public class SearchService extends JoobyService {
 
     @Inject
     public SearchService(BaseServiceParams params,
-                         WebsiteUrl websiteUrl,
-                         StaticResources staticResources,
                          SearchFrontPageService frontPageService,
                          SearchAddToCrawlQueueService addToCrawlQueueService,
                          SearchSiteSubscriptionService siteSubscriptionService,
@@ -62,7 +59,25 @@ public class SearchService extends JoobyService {
     public void startJooby(Jooby jooby) {
         super.startJooby(jooby);
 
+        final String startTimeAttribute = "start-time";
+
         jooby.get("/export-opml", siteSubscriptionService::exportOpml);
+        jooby.before((Context ctx) -> {
+            ctx.setAttribute(startTimeAttribute, System.nanoTime());
+        });
+
+        jooby.after((Context ctx, Object result, Throwable failure) -> {
+            if  (failure != null) {
+                wmsa_search_service_error_count.labels(ctx.getRoute().getPattern(), ctx.getMethod()).inc();
+            }
+            else {
+                Long startTime = ctx.getAttribute(startTimeAttribute);
+                if (startTime != null) {
+                    wmsa_search_service_request_time.labels(ctx.getRoute().getPattern(), ctx.getMethod())
+                            .observe((System.nanoTime() - startTime) / 1e9);
+                }
+            }
+        });
     }
 
 

code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java

@@ -1,7 +1,7 @@
 package nu.marginalia.search.command;
 
 import nu.marginalia.WebsiteUrl;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 import nu.marginalia.model.EdgeDomain;
@@ -98,15 +98,15 @@ public record SearchParameters(WebsiteUrl url,
         return path;
     }
 
-    public ResultRankingParameters.TemporalBias temporalBias() {
+    public RpcTemporalBias.Bias temporalBias() {
         if (recent == RECENT) {
-            return ResultRankingParameters.TemporalBias.RECENT;
+            return RpcTemporalBias.Bias.RECENT;
         }
         else if (profile == SearchProfile.VINTAGE) {
-            return ResultRankingParameters.TemporalBias.OLD;
+            return RpcTemporalBias.Bias.OLD;
         }
 
-        return ResultRankingParameters.TemporalBias.NONE;
+        return RpcTemporalBias.Bias.NONE;
     }
 
     public QueryStrategy strategy() {

code/services-application/search-service/java/nu/marginalia/search/svc/SearchAddToCrawlQueueService.java

@@ -47,18 +47,23 @@ public class SearchAddToCrawlQueueService {
         return new MapModelAndView("redirect.jte", Map.of("url", "/site/"+domainName));
     }
 
-    private void addToCrawlQueue(int id) throws SQLException {
+    /** Mark a domain for crawling by setting node affinity to zero,
+     * unless it is already marked for crawling, then node affinity should
+     * be left unchanged.
+     * */
+    void addToCrawlQueue(int domainId) throws SQLException {
         try (var conn = dataSource.getConnection();
              var stmt = conn.prepareStatement("""
-                     INSERT IGNORE INTO CRAWL_QUEUE(DOMAIN_NAME, SOURCE)
-                     SELECT DOMAIN_NAME, "user" FROM EC_DOMAIN WHERE ID=?
+                     UPDATE EC_DOMAIN 
+                     SET WMSA_prod.EC_DOMAIN.NODE_AFFINITY = 0 
+                     WHERE ID=? AND WMSA_prod.EC_DOMAIN.NODE_AFFINITY < 0
                      """)) {
-            stmt.setInt(1, id);
+            stmt.setInt(1, domainId);
             stmt.executeUpdate();
         }
     }
 
-    private String getDomainName(int id) {
+    String getDomainName(int id) {
         var domain = domainQueries.getDomain(id);
         if (domain.isEmpty())
             throw new IllegalArgumentException();

code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java

@@ -37,7 +37,7 @@ public class SearchQueryService {
             @QueryParam String profile,
             @QueryParam String js,
             @QueryParam String recent,
-            @QueryParam String title,
+            @QueryParam String searchTitle,
             @QueryParam String adtech,
             @QueryParam Integer page
     ) {
@@ -47,7 +47,7 @@ public class SearchQueryService {
                     SearchProfile.getSearchProfile(profile),
                     SearchJsParameter.parse(js),
                     SearchRecentParameter.parse(recent),
-                    SearchTitleParameter.parse(title),
+                    SearchTitleParameter.parse(searchTitle),
                     SearchAdtechParameter.parse(adtech),
                     false,
                     Objects.requireNonNullElse(page,1));

code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java

@@ -197,7 +197,6 @@ public class SearchSiteInfoService {
 
         var domain = new EdgeDomain(domainName);
         final int domainId = domainQueries.tryGetDomainId(domain).orElse(-1);
-        boolean viableAliasDomain = domain.aliasDomain().map(alias -> domainQueries.tryGetDomainId(alias).isPresent()).orElse(false);
 
         final Future<DomainInformation> domainInfoFuture;
         final Future<List<SimilarDomain>> similarSetFuture;
@@ -232,9 +231,10 @@ public class SearchSiteInfoService {
             url = sampleResults.getFirst().url.withPathAndParam("/", null).toString();
         }
 
+
         var result = new SiteInfoWithContext(domainName,
                 isSubscribed,
-                viableAliasDomain ? domain.aliasDomain().map(EdgeDomain::toString) : Optional.empty(),
+                domainQueries.otherSubdomains(domain, 5),
                 domainId,
                 url,
                 hasScreenshot,
@@ -352,7 +352,7 @@ public class SearchSiteInfoService {
 
     public record SiteInfoWithContext(String domain,
                                       boolean isSubscribed,
-                                      Optional<String> aliasDomain,
+                                      List<DbDomainQueries.DomainWithNode> siblingDomains,
                                       int domainId,
                                       String siteUrl,
                                       boolean hasScreenshot,

code/services-application/search-service/readme.md

@@ -2,13 +2,24 @@
 
 This service handles search traffic and is the service
 you're most directly interacting with when visiting
-[search.marginalia.nu](https://search.marginalia.nu). 
+[marginalia-search.com](https://marginalia-search.com). 
 
 It interprets a "human" query and translates it into a
 request that gets passed into to the index service, which finds
 related documents, which this service then ranks and returns
 to the user.
 
+The UI is built using [JTE templates](https://jte.gg/syntax/) and the [Jooby framework](https://jooby.io), primarily using
+its MVC facilities.
+
+When developing, it's possible to set up a mock version of the UI by running
+the gradle command 
+
+```$ ./gradlew paperDoll -i```  
+
+The UI will be available at http://localhost:9999/, and has hot reloading of JTE classes 
+and static resources.
+
 
 ![image](../../../doc/diagram/search-service-map.svg)
 

code/services-application/search-service/resources/jte/siteinfo/view/overview.jte

@@ -1,3 +1,5 @@
+@import nu.marginalia.db.DbDomainQueries
+@import nu.marginalia.model.EdgeDomain
 @import nu.marginalia.search.svc.SearchSiteInfoService
 @import nu.marginalia.search.svc.SearchSiteInfoService.*
 @import nu.marginalia.search.model.UrlDetails
@@ -13,18 +15,18 @@
             <span>${siteInfo.domain()}</span>
             <div class="grow">
             </div>
-            <a rel="nofollow noopener external" href="${siteInfo.siteUrl()}" class="fa-solid fa-arrow-up-right-from-square" ></a>
+            <a href="https://web.archive.org/web/*/${siteInfo.domain()}"
+               class="p-1.5 text-white px-4"
+               title="Wayback Machine">
+                <i class="fas fa-clock-rotate-left text-sm"></i>
+            </a>
+            <a title="Visit ${siteInfo.domain()}" rel="nofollow noopener external" href="${siteInfo.siteUrl()}" class="fa-solid fa-arrow-up-right-from-square" ></a>
         </div>
 
         @if (siteInfo.hasScreenshot())
             <a class="mx-3 " tabindex="-1" rel="nofollow noopener external" href="${siteInfo.siteUrl()}">
                 <img class="border dark:border-gray-600 shadow-inner" src="/screenshot/${siteInfo.domainId()}" alt="Screenshot of ${siteInfo.domain()}">
             </a>
-        @elseif (siteInfo.aliasDomain().isPresent())
-            <div class="mx-3 my-3 text-xs text-slate-800 dark:text-white">
-                The search engine is also aware of links to <a class="underline text-liteblue dark:text-blue-200" href="/site/${siteInfo.aliasDomain().get()}">${siteInfo.aliasDomain().get()}</a>,
-                this may be the canonical address.
-            </div>
         @endif
 
         @if (siteInfo.hasFeed())
@@ -80,6 +82,34 @@
         @endif
 
 
+        @if (!siteInfo.siblingDomains().isEmpty())
+            <div class="mx-3 flex place-items-baseline space-x-2 p-2 bg-gray-100 dark:bg-gray-600 rounded">
+                <i class="fas fa-globe"></i>
+                <span>Related Subdomains</span>
+            </div>
+
+            <table class="min-w-full divide-y divide-gray-200 dark:divide-gray-600 mx-4">
+                <thead>
+                <tr class="bg-gray-50 dark:bg-gray-700">
+                    <th scope="col" class="px-2 py-2 text-left text-xs font-medium text-gray-500 dark:text-gray-100 uppercase tracking-wider">Domain Name</th>
+                </tr>
+                </thead>
+                <tbody class="bg-white dark:bg-gray-800 divide-y divide-gray-200 dark:divide-gray-600 text-xs">
+                @for (DbDomainQueries.DomainWithNode sibling : siteInfo.siblingDomains())
+                    <tr>
+                        <td class="px-3 py-6 md:py-3 whitespace-nowrap">
+                            <a class="text-liteblue dark:text-blue-200" href="/site/${sibling.domain().toString()}">${sibling.domain().toString()}</a>
+
+                            @if (!sibling.isIndexed())
+                                <i class="ml-1 fa-regular fa-question-circle text-gray-400 dark:text-gray-600 text-xs" title="Not indexed"></i>
+                            @endif
+                        </td>
+                    </tr>
+                @endfor
+                </tbody>
+            </table>
+        @endif
+
         @if (siteInfo.domainInformation().isUnknownDomain())
             <div class="mx-3 flex place-items-baseline space-x-2 p-2 bg-gray-100 dark:bg-gray-600 rounded">
                 <i class="fa-regular fa-circle-question"></i>

code/services-application/search-service/resources/static/opensearch.xml

@@ -8,8 +8,8 @@
     <ShortName>Marginalia</ShortName>
     <Description>Search Marginalia</Description>
     <InputEncoding>UTF-8</InputEncoding>
-    <Image width="16" height="16" type="image/x-icon">https://search.marginalia.nu/favicon.ico</Image>
+    <Image width="16" height="16" type="image/x-icon">https://marginalia-search.com/favicon.ico</Image>
     <Url type="text/html" method="get"
-         template="https://search.marginalia.nu/search?query={searchTerms}&amp;ref=opensearch"/>
-    <moz:SearchForm>https://search.marginalia.nu/</moz:SearchForm>
+         template="https://marginalia-search.com/search?query={searchTerms}&amp;ref=opensearch"/>
+    <moz:SearchForm>https://marginalia-search.com/</moz:SearchForm>
 </OpenSearchDescription>

code/services-application/search-service/test/nu/marginalia/search/rendering/MockedSearchResults.java

@@ -6,6 +6,7 @@ import nu.marginalia.api.domains.model.SimilarDomain;
 import nu.marginalia.api.searchquery.model.results.SearchResultItem;
 import nu.marginalia.browse.model.BrowseResult;
 import nu.marginalia.browse.model.BrowseResultSet;
+import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.DomainIndexingState;
@@ -18,7 +19,6 @@ import nu.marginalia.search.svc.SearchSiteInfoService;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 import java.util.concurrent.ThreadLocalRandom;
 
 public class MockedSearchResults {
@@ -133,7 +133,10 @@ public class MockedSearchResults {
         return new SearchSiteInfoService.SiteInfoWithContext(
                 "www.example.com",
                 false,
-                Optional.of("other.example.com"),
+                List.of(
+                        new DbDomainQueries.DomainWithNode(new EdgeDomain("example.com"), 1),
+                        new DbDomainQueries.DomainWithNode(new EdgeDomain("example.com"), 0)
+                        ),
                 14,
                 "https://www.example.com",
                 true,

code/services-application/search-service/test/nu/marginalia/search/svc/SearchAddToCrawlQueueServiceTest.java

@@ -0,0 +1,85 @@
+package nu.marginalia.search.svc;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.db.DbDomainQueries;
+import nu.marginalia.model.EdgeDomain;
+import nu.marginalia.test.TestMigrationLoader;
+import org.junit.jupiter.api.*;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.sql.SQLException;
+
+@Tag("slow")
+@Testcontainers
+class SearchAddToCrawlQueueServiceTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+
+    private DbDomainQueries domainQueries;
+    private SearchAddToCrawlQueueService addToCrawlQueueService;
+
+    @BeforeEach
+    public void setUp() throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.createStatement()) {
+            stmt.executeQuery("DELETE FROM EC_DOMAIN"); // Wipe any old state from other test runs
+
+            stmt.executeQuery("INSERT INTO EC_DOMAIN (DOMAIN_NAME, DOMAIN_TOP, NODE_AFFINITY) VALUES ('known.example.com', 'example.com', -1)");
+            stmt.executeQuery("INSERT INTO EC_DOMAIN (DOMAIN_NAME, DOMAIN_TOP, NODE_AFFINITY) VALUES ('added.example.com', 'example.com', 0)");
+            stmt.executeQuery("INSERT INTO EC_DOMAIN (DOMAIN_NAME, DOMAIN_TOP, NODE_AFFINITY) VALUES ('indexed.example.com', 'example.com', 1)");
+        }
+
+        domainQueries = new DbDomainQueries(dataSource);
+        addToCrawlQueueService = new SearchAddToCrawlQueueService(domainQueries, dataSource);
+    }
+
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+        TestMigrationLoader.flywayMigration(dataSource);
+    }
+
+    private int getNodeAffinity(String domainName) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("SELECT NODE_AFFINITY FROM EC_DOMAIN WHERE DOMAIN_NAME=?"))
+        {
+            stmt.setString(1, domainName);
+            var rsp = stmt.executeQuery();
+            if (rsp.next()) {
+                return rsp.getInt(1);
+            }
+        }
+
+        return -1;
+    }
+
+    @Test
+    void addToCrawlQueue() throws SQLException {
+        int knownId = domainQueries.getDomainId(new EdgeDomain("known.example.com"));
+        int addedId = domainQueries.getDomainId(new EdgeDomain("added.example.com"));
+        int indexedId = domainQueries.getDomainId(new EdgeDomain("indexed.example.com"));
+
+        addToCrawlQueueService.addToCrawlQueue(knownId);
+        addToCrawlQueueService.addToCrawlQueue(addedId);
+        addToCrawlQueueService.addToCrawlQueue(indexedId);
+
+        Assertions.assertEquals(0, getNodeAffinity("known.example.com"));
+        Assertions.assertEquals(0, getNodeAffinity("added.example.com"));
+        Assertions.assertEquals(1, getNodeAffinity("indexed.example.com"));
+    }
+
+}

code/services-core/control-service/build.gradle

@@ -55,6 +55,7 @@ dependencies {
 
     implementation libs.duckdb
     implementation libs.jsoup
+    implementation libs.protobuf
 
     implementation libs.trove
     implementation dependencies.create(libs.spark.get()) {

code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java

@@ -2,11 +2,10 @@ package nu.marginalia.control.app.svc;
 
 import com.google.inject.Inject;
 import nu.marginalia.api.searchquery.QueryClient;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.control.ControlRendererFactory;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.EdgeUrl;
-import nu.marginalia.nodecfg.NodeConfigurationService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -22,18 +21,16 @@ public class SearchToBanService {
     private final ControlRendererFactory rendererFactory;
     private final QueryClient queryClient;
     private final Logger logger = LoggerFactory.getLogger(getClass());
-    private final NodeConfigurationService nodeConfigurationService;
 
     @Inject
     public SearchToBanService(ControlBlacklistService blacklistService,
                               ControlRendererFactory rendererFactory,
-                              QueryClient queryClient, NodeConfigurationService nodeConfigurationService)
+                              QueryClient queryClient)
     {
 
         this.blacklistService = blacklistService;
         this.rendererFactory = rendererFactory;
         this.queryClient = queryClient;
-        this.nodeConfigurationService = nodeConfigurationService;
     }
 
     public void register() throws IOException {
@@ -76,7 +73,14 @@ public class SearchToBanService {
 
     private Object executeQuery(String query) {
         return queryClient.search(new QueryParams(
-                query, new QueryLimits(2, 200, 250, 8192),
+                query,
+                RpcQueryLimits.newBuilder()
+                        .setResultsTotal(100)
+                        .setResultsByDomain(2)
+                        .setTimeoutMs(200)
+                        .setFetchSize(8192)
+                        .build()
+                ,
                 "NONE"
         ));
     }

code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java

@@ -3,12 +3,13 @@ package nu.marginalia.query;
 import com.google.common.base.Strings;
 import com.google.gson.Gson;
 import com.google.inject.Inject;
+import nu.marginalia.api.searchquery.RpcQueryLimits;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
+import nu.marginalia.api.searchquery.RpcTemporalBias;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
-import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.functions.searchquery.QueryGRPCService;
 import nu.marginalia.index.api.IndexClient;
-import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.gson.GsonFactory;
 import nu.marginalia.renderer.MustacheRenderer;
 import nu.marginalia.renderer.RendererFactory;
@@ -53,9 +54,14 @@ public class QueryBasicInterface {
         int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5"));
         String set = requireNonNullElse(request.queryParams("set"), "");
 
-        var params = new QueryParams(queryString, new QueryLimits(
-                domainCount, min(100, count * 10), 250, 8192
-        ), set);
+        var params = new QueryParams(queryString,
+                RpcQueryLimits.newBuilder()
+                        .setResultsByDomain(domainCount)
+                        .setResultsTotal(min(100, count * 10))
+                        .setTimeoutMs(250)
+                        .setFetchSize(8192)
+                        .build()
+        , set);
 
         var pagination = new IndexClient.Pagination(page, count);
 
@@ -63,7 +69,7 @@ public class QueryBasicInterface {
                 queryString,
                 params,
                 pagination,
-                ResultRankingParameters.sensibleDefaults()
+                PrototypeRankingParameters.sensibleDefaults()
         );
 
         var results = detailedDirectResult.result();
@@ -92,7 +98,7 @@ public class QueryBasicInterface {
         String queryString = request.queryParams("q");
         if (queryString == null) {
             // Show the default query form if no query is given
-            return qdebugRenderer.render(Map.of("rankingParams", ResultRankingParameters.sensibleDefaults())
+            return qdebugRenderer.render(Map.of("rankingParams", PrototypeRankingParameters.sensibleDefaults())
             );
         }
 
@@ -101,9 +107,14 @@ public class QueryBasicInterface {
         int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5"));
         String set = requireNonNullElse(request.queryParams("set"), "");
 
-        var queryParams = new QueryParams(queryString, new QueryLimits(
-                domainCount, min(100, count * 10), 250, 8192
-        ), set);
+        var queryParams = new QueryParams(queryString,
+                RpcQueryLimits.newBuilder()
+                    .setResultsByDomain(domainCount)
+                    .setResultsTotal(min(100, count * 10))
+                    .setTimeoutMs(250)
+                    .setFetchSize(8192)
+                    .build(),
+                set);
 
         var pagination = new IndexClient.Pagination(page, count);
 
@@ -126,27 +137,28 @@ public class QueryBasicInterface {
         );
     }
 
-    private ResultRankingParameters debugRankingParamsFromRequest(Request request) {
-        var sensibleDefaults = ResultRankingParameters.sensibleDefaults();
+    private RpcResultRankingParameters debugRankingParamsFromRequest(Request request) {
+        var sensibleDefaults = PrototypeRankingParameters.sensibleDefaults();
 
-        return ResultRankingParameters.builder()
-                .domainRankBonus(doubleFromRequest(request, "domainRankBonus", sensibleDefaults.domainRankBonus))
-                .qualityPenalty(doubleFromRequest(request, "qualityPenalty", sensibleDefaults.qualityPenalty))
-                .shortDocumentThreshold(intFromRequest(request, "shortDocumentThreshold", sensibleDefaults.shortDocumentThreshold))
-                .shortDocumentPenalty(doubleFromRequest(request, "shortDocumentPenalty", sensibleDefaults.shortDocumentPenalty))
-                .tcfFirstPosition(doubleFromRequest(request, "tcfFirstPosition", sensibleDefaults.tcfFirstPosition))
-                .tcfVerbatim(doubleFromRequest(request, "tcfVerbatim", sensibleDefaults.tcfVerbatim))
-                .tcfProximity(doubleFromRequest(request, "tcfProximity", sensibleDefaults.tcfProximity))
-                .bm25Params(new Bm25Parameters(
-                        doubleFromRequest(request, "bm25.k1", sensibleDefaults.bm25Params.k()),
-                        doubleFromRequest(request, "bm25.b", sensibleDefaults.bm25Params.b())
-                ))
-                .temporalBias(ResultRankingParameters.TemporalBias.valueOf(stringFromRequest(request, "temporalBias", sensibleDefaults.temporalBias.toString())))
-                .temporalBiasWeight(doubleFromRequest(request, "temporalBiasWeight", sensibleDefaults.temporalBiasWeight))
-                .shortSentenceThreshold(intFromRequest(request, "shortSentenceThreshold", sensibleDefaults.shortSentenceThreshold))
-                .shortSentencePenalty(doubleFromRequest(request, "shortSentencePenalty", sensibleDefaults.shortSentencePenalty))
-                .bm25Weight(doubleFromRequest(request, "bm25Weight", sensibleDefaults.bm25Weight))
-                .exportDebugData(true)
+        var bias = RpcTemporalBias.Bias.valueOf(stringFromRequest(request, "temporalBias", "NONE"));
+
+        return RpcResultRankingParameters.newBuilder()
+                .setDomainRankBonus(doubleFromRequest(request, "domainRankBonus", sensibleDefaults.getDomainRankBonus()))
+                .setQualityPenalty(doubleFromRequest(request, "qualityPenalty", sensibleDefaults.getQualityPenalty()))
+                .setShortDocumentThreshold(intFromRequest(request, "shortDocumentThreshold", sensibleDefaults.getShortDocumentThreshold()))
+                .setShortDocumentPenalty(doubleFromRequest(request, "shortDocumentPenalty", sensibleDefaults.getShortDocumentPenalty()))
+                .setTcfFirstPositionWeight(doubleFromRequest(request, "tcfFirstPositionWeight", sensibleDefaults.getTcfFirstPositionWeight()))
+                .setTcfVerbatimWeight(doubleFromRequest(request, "tcfVerbatimWeight", sensibleDefaults.getTcfVerbatimWeight()))
+                .setTcfProximityWeight(doubleFromRequest(request, "tcfProximityWeight", sensibleDefaults.getTcfProximityWeight()))
+                .setBm25B(doubleFromRequest(request, "bm25b", sensibleDefaults.getBm25B()))
+                .setBm25K(doubleFromRequest(request, "bm25k", sensibleDefaults.getBm25K()))
+                .setTemporalBias(RpcTemporalBias.newBuilder().setBias(bias).build())
+                .setTemporalBiasWeight(doubleFromRequest(request, "temporalBiasWeight", sensibleDefaults.getTemporalBiasWeight()))
+                .setShortSentenceThreshold(intFromRequest(request, "shortSentenceThreshold", sensibleDefaults.getShortSentenceThreshold()))
+                .setShortSentencePenalty(doubleFromRequest(request, "shortSentencePenalty", sensibleDefaults.getShortSentencePenalty()))
+                .setBm25Weight(doubleFromRequest(request, "bm25Weight", sensibleDefaults.getBm25Weight()))
+                .setDisablePenalties(boolFromRequest(request, "disablePenalties", sensibleDefaults.getDisablePenalties()))
+                .setExportDebugData(true)
                 .build();
     }
 
@@ -154,6 +166,13 @@ public class QueryBasicInterface {
         return Strings.isNullOrEmpty(request.queryParams(param)) ? defaultValue : Double.parseDouble(request.queryParams(param));
     }
 
+    boolean boolFromRequest(Request request, String param, boolean defaultValue) {
+        if (param == null)
+            return defaultValue;
+
+        return Strings.isNullOrEmpty(request.queryParams(param)) ? defaultValue : Boolean.parseBoolean(request.queryParams(param));
+    }
+
     int intFromRequest(Request request, String param, int defaultValue) {
         return Strings.isNullOrEmpty(request.queryParams(param)) ? defaultValue : parseInt(request.queryParams(param));
     }

code/services-core/query-service/resources/templates/qdebug.hdb

@@ -31,20 +31,20 @@
         <div class="col-sm-2"><input type="text" class="form-control" id="shortDocumentPenalty" name="shortDocumentPenalty" value="{{shortDocumentPenalty}}"></div>
     </div>
     <div class="row my-2">
-        <div class="col-sm-2"><label for="tcfFirstPosition">TCF First Position Weight</label></div>
-        <div class="col-sm-2"><input type="text" class="form-control" id="tcfFirstPosition" name="tcfFirstPosition" value="{{tcfFirstPosition}}"></div>
+        <div class="col-sm-2"><label for="tcfFirstPositionWeight">TCF First Position Weight</label></div>
+        <div class="col-sm-2"><input type="text" class="form-control" id="tcfFirstPositionWeight" name="tcfFirstPositionWeight" value="{{tcfFirstPositionWeight}}"></div>
     </div>
     <div class="row my-2">
-        <div class="col-sm-2"><label for="tcfVerbatim">TCF Verbatim</label></div>
-        <div class="col-sm-2"><input type="text" class="form-control" id="tcfVerbatim" name="tcfVerbatim" value="{{tcfVerbatim}}"></div>
-        <div class="col-sm-2"><label for="tcfProximity">TCF Proximity</label></div>
-        <div class="col-sm-2"><input type="text" class="form-control" id="tcfProximity" name="tcfProximity" value="{{tcfProximity}}"></div>
+        <div class="col-sm-2"><label for="tcfVerbatimWeight">TCF Verbatim</label></div>
+        <div class="col-sm-2"><input type="text" class="form-control" id="tcfVerbatimWeight" name="tcfVerbatimWeight" value="{{tcfVerbatimWeight}}"></div>
+        <div class="col-sm-2"><label for="tcfProximityWeight">TCF Proximity</label></div>
+        <div class="col-sm-2"><input type="text" class="form-control" id="tcfProximityWeight" name="tcfProximityWeight" value="{{tcfProximityWeight}}"></div>
     </div>
     <div class="row my-2">
-        <div class="col-sm-2"><label for="bm25.k1">BM25 K1</label></div>
-        <div class="col-sm-2"><input type="text" class="form-control" id="bm25.k1" name="bm25.k1" value="{{bm25Params.k}}"></div>
-        <div class="col-sm-2"><label for="bm25.b">BM25 B</label></div>
-        <div class="col-sm-2"><input type="text" class="form-control" id="bm25.b" name="bm25.b" value="{{bm25Params.b}}"></div>
+        <div class="col-sm-2"><label for="bm25k">BM25 K1</label></div>
+        <div class="col-sm-2"><input type="text" class="form-control" id="bm25k" name="bm25k" value="{{bm25K}}"></div>
+        <div class="col-sm-2"><label for="bm25b">BM25 B</label></div>
+        <div class="col-sm-2"><input type="text" class="form-control" id="bm25b" name="bm25b" value="{{bm25B}}"></div>
     </div>
     <div class="row my-2">
         <div class="col-sm-2"><label for="temporalBias">Temporal Bias</label></div>
@@ -67,6 +67,14 @@
     <div class="row my-2">
         <div class="col-sm-2"><label for="bm25FullWeight">BM25 Weight</label></div>
         <div class="col-sm-2"><input type="text" class="form-control" id="bm25Weight" name="bm25Weight" value="{{bm25Weight}}"></div>
+
+        <div class="col-sm-2"><label for="disablePenalties">Disable Penalties</label></div>
+        <div class="col-sm-2">
+            <select class="form-select" id="disablePenalties" name="disablePenalties">
+                <option value="FALSE" {{#unless disablePenalties}}selected{{/unless}}>FALSE</option>
+                <option value="TRUE" {{#if disablePenalties}}selected{{/if}}>TRUE</option>
+            </select>
+        </div>
     </div>
 
     {{/with}}

code/tools/integration-test/test/nu/marginalia/IntegrationTest.java

@@ -5,7 +5,8 @@ import com.google.inject.Inject;
 import nu.marginalia.api.searchquery.QueryProtobufCodec;
 import nu.marginalia.api.searchquery.RpcQsQuery;
 import nu.marginalia.api.searchquery.RpcQueryLimits;
-import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
+import nu.marginalia.api.searchquery.RpcResultRankingParameters;
+import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
 import nu.marginalia.converting.processor.DomainProcessor;
 import nu.marginalia.converting.writer.ConverterBatchWriter;
 import nu.marginalia.crawl.fetcher.ContentTags;
@@ -211,8 +212,7 @@ public class IntegrationTest {
 
         var params = QueryProtobufCodec.convertRequest(request);
 
-        var p = ResultRankingParameters.sensibleDefaults();
-        p.exportDebugData = true;
+        var p = RpcResultRankingParameters.newBuilder(PrototypeRankingParameters.sensibleDefaults()).setExportDebugData(true).build();
         var query = queryFactory.createQuery(params, p);
 
 

deploy.txt

@@ -1,6 +1,4 @@
 ## This is a token file for automatic deployment
 
-A master HEAD tagged with deploy-core*, deploy-executor*, or deploy-index* will trigger a commit.
-
-2024-12-19-00002: Test deployment of executor
-2024-12-19-00001: Test deployment of executor
+2025-01-08:  Deploy executor.
+2025-01-07:  Deploy executor.

run/readme.md

@@ -11,10 +11,13 @@ platforms, but for lack of suitable hardware, this can not be guaranteed.
 **Docker** - It is a bit of a pain to install, but if you follow
 [this guide](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) you're on the right track for ubuntu-like systems.
 
-**JDK 21** - The code uses Java 21 preview features. 
+**JDK 23** - The code uses Java 23 preview features. 
+
 The civilized way of installing this is to use [SDKMAN](https://sdkman.io/);
 graalce is a good distribution choice but it doesn't matter too much.
 
+**Tailwindcss** - Install NPM and run `npm install -D tailwindcss`
+
 ## Quick Set up
 
 [https://docs.marginalia.nu/](https://docs.marginalia.nu/) has a more comprehensive guide for the install

third-party/rssreader/src/main/java/com/apptasticsoftware/rssreader/AbstractRssReader.java

@@ -701,7 +701,7 @@ public abstract class AbstractRssReader<C extends Channel, I extends Item> {
                     }
                 }
             } catch (XMLStreamException e) {
-                LOGGER.log(Level.WARNING, "Failed to parse XML.", e);
+                LOGGER.log(Level.FINE, "Failed to parse XML.", e);
             }
 
             close();

tools/deployment/deployment.py

@@ -258,6 +258,13 @@ if __name__ == '__main__':
             deploy_tier=2,
             groups={"all", "frontend", "core"}
         ),
+        'search-legacy': ServiceConfig(
+            gradle_target=':code:services-application:search-service-legacy:docker',
+            docker_name='search-service-legacy',
+            instances=None,
+            deploy_tier=3,
+            groups={"all", "frontend", "core"}
+        ),
         'api': ServiceConfig(
             gradle_target=':code:services-application:api-service:docker',
             docker_name='api-service',