(feeds) Add system property for exporting fetched feeds to a slop table for debugging

code/functions/live-capture/build.gradle

@@ -29,6 +29,7 @@ dependencies {
     implementation libs.jsoup
     implementation project(':third-party:rssreader')
     implementation libs.opencsv
+    implementation libs.slop
     implementation libs.sqlite
     implementation libs.bundles.slf4j
     implementation libs.commons.lang3

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedFetcherService.java

@@ -96,6 +96,7 @@ public class FeedFetcherService {
             throw new IllegalStateException("Already updating feeds, refusing to start another update");
         }
 
+
         try (FeedDbWriter writer = feedDb.createWriter();
              HttpClient client = HttpClient.newBuilder()
                 .connectTimeout(Duration.ofSeconds(15))
@@ -103,6 +104,7 @@ public class FeedFetcherService {
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .version(HttpClient.Version.HTTP_2)
                 .build();
+             FeedJournal feedJournal = FeedJournal.create();
              var heartbeat = serviceHeartbeat.createServiceAdHocTaskHeartbeat("Update Rss Feeds")
         ) {
             updating = true;
@@ -155,6 +157,8 @@ public class FeedFetcherService {
                             case FetchResult.Success(String value, String etag) -> {
                                 writer.saveEtag(feed.domain(), etag);
                                 writer.saveFeed(parseFeed(value, feed));
+
+                                feedJournal.record(feed.feedUrl(), value);
                             }
                             case FetchResult.NotModified() -> {
                                 writer.saveEtag(feed.domain(), ifNoneMatchTag);

code/functions/live-capture/java/nu/marginalia/rss/svc/FeedJournal.java

@@ -0,0 +1,64 @@
+package nu.marginalia.rss.svc;
+
+import nu.marginalia.WmsaHome;
+import nu.marginalia.slop.SlopTable;
+import nu.marginalia.slop.column.string.StringColumn;
+import nu.marginalia.slop.desc.StorageType;
+import org.apache.commons.io.FileUtils;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Utility for recording fetched feeds to a journal, useful in debugging feed parser issues.
+ */
+public interface FeedJournal extends AutoCloseable {
+    StringColumn urlColumn = new StringColumn("url");
+    StringColumn contentsColumn = new StringColumn("contents", StandardCharsets.UTF_8, StorageType.ZSTD);
+
+    void record(String url, String contents) throws IOException;
+    void close() throws IOException;
+
+
+    static FeedJournal create() throws IOException {
+        if (Boolean.getBoolean("feedFetcher.persistJournal")) {
+            Path journalPath = WmsaHome.getDataPath().resolve("feed-journal");
+            if (Files.isDirectory(journalPath)) {
+                FileUtils.deleteDirectory(journalPath.toFile());
+            }
+            Files.createDirectories(journalPath);
+            return new RecordingFeedJournal(journalPath);
+        }
+        else {
+            return new NoOpFeedJournal();
+        }
+    }
+
+    class NoOpFeedJournal implements FeedJournal {
+        @Override
+        public void record(String url, String contents) {}
+
+        @Override
+        public void close() {}
+    }
+
+    class RecordingFeedJournal extends SlopTable implements FeedJournal {
+
+        private final StringColumn.Writer urlWriter;
+        private final StringColumn.Writer contentsWriter;
+
+        public RecordingFeedJournal(Path path) throws IOException {
+            super(path, SlopTable.getNumPages(path, FeedJournal.urlColumn));
+
+            urlWriter = urlColumn.create(this);
+            contentsWriter = contentsColumn.create(this);
+        }
+
+        public synchronized void record(String url, String contents) throws IOException {
+            urlWriter.put(url);
+            contentsWriter.put(contents);
+        }
+
+    }
+}

code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java

@@ -15,7 +15,10 @@ import nu.marginalia.api.searchquery.model.results.debug.ResultRankingDetails;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.model.EdgeUrl;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public class QueryProtobufCodec {
 
@@ -42,13 +45,27 @@ public class QueryProtobufCodec {
         else
             builder.setQueryStrategy(request.getQueryStrategy());
 
-        if (query.specs.rankingParams != null && request.getTemporalBias().getBias() != RpcTemporalBias.Bias.NONE) {
-            builder.setParameters(
-                    RpcResultRankingParameters.newBuilder(query.specs.rankingParams)
-                            .setTemporalBias(request.getTemporalBias())
-                            .build()
-            );
+        if (request.getTemporalBias().getBias() != RpcTemporalBias.Bias.NONE) {
+            if (query.specs.rankingParams != null) {
+                builder.setParameters(
+                        RpcResultRankingParameters.newBuilder(query.specs.rankingParams)
+                                .setTemporalBias(request.getTemporalBias())
+                                .build()
+                );
+            } else {
+                builder.setParameters(
+                        RpcResultRankingParameters.newBuilder(PrototypeRankingParameters.sensibleDefaults())
+                                .setTemporalBias(request.getTemporalBias())
+                                .build()
+                );
+            }
+        } else if (query.specs.rankingParams != null) {
+            builder.setParameters(query.specs.rankingParams);
         }
+        // else {
+        // if we have no ranking params, we don't need to set them, the client check and use the default values
+        // so we don't need to send this huge object over the wire
+        // }
 
         return builder.build();
     }
@@ -292,7 +309,7 @@ public class QueryProtobufCodec {
                 IndexProtobufCodec.convertSpecLimit(specs.getRank()),
                 specs.getQueryLimits(),
                 QueryStrategy.valueOf(specs.getQueryStrategy()),
-                Objects.requireNonNullElseGet(specs.getParameters(), PrototypeRankingParameters::sensibleDefaults)
+                specs.hasParameters() ? specs.getParameters() : null
         );
     }
 

code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java

@@ -5,6 +5,7 @@ import nu.marginalia.api.searchquery.RpcResultRankingParameters;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.query.limit.SpecificationLimit;
 
+import javax.annotation.Nullable;
 import java.util.List;
 
 public class SearchSpecification {
@@ -28,6 +29,7 @@ public class SearchSpecification {
 
     public final QueryStrategy queryStrategy;
 
+    @Nullable
     public final RpcResultRankingParameters rankingParams;
 
     public SearchSpecification(SearchQuery query,
@@ -40,7 +42,7 @@ public class SearchSpecification {
                                SpecificationLimit rank,
                                RpcQueryLimits queryLimits,
                                QueryStrategy queryStrategy,
-                               RpcResultRankingParameters rankingParams)
+                               @Nullable RpcResultRankingParameters rankingParams)
     {
         this.query = query;
         this.domains = domains;

code/index/java/nu/marginalia/index/model/SearchParameters.java

@@ -13,8 +13,6 @@ import nu.marginalia.index.query.IndexSearchBudget;
 import nu.marginalia.index.query.limit.QueryStrategy;
 import nu.marginalia.index.searchset.SearchSet;
 
-import java.util.Objects;
-
 import static nu.marginalia.api.searchquery.IndexProtobufCodec.convertSpecLimit;
 
 public class SearchParameters {
@@ -88,7 +86,7 @@ public class SearchParameters {
         compiledQuery = CompiledQueryParser.parse(this.query.compiledQuery);
         compiledQueryIds = compiledQuery.mapToLong(SearchTermsUtil::getWordId);
 
-        rankingParams = Objects.requireNonNullElseGet(request.getParameters(), PrototypeRankingParameters::sensibleDefaults);
+        rankingParams = request.hasParameters() ? request.getParameters() : PrototypeRankingParameters.sensibleDefaults();
     }
 
 

code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java

@@ -37,7 +37,7 @@ public class SearchQueryService {
             @QueryParam String profile,
             @QueryParam String js,
             @QueryParam String recent,
-            @QueryParam String title,
+            @QueryParam String searchTitle,
             @QueryParam String adtech,
             @QueryParam Integer page
     ) {
@@ -47,7 +47,7 @@ public class SearchQueryService {
                     SearchProfile.getSearchProfile(profile),
                     SearchJsParameter.parse(js),
                     SearchRecentParameter.parse(recent),
-                    SearchTitleParameter.parse(title),
+                    SearchTitleParameter.parse(searchTitle),
                     SearchAdtechParameter.parse(adtech),
                     false,
                     Objects.requireNonNullElse(page,1));

deploy.txt

@@ -1,3 +1,4 @@
 ## This is a token file for automatic deployment
 
+2025-01-08:  Deploy executor.
 2025-01-07:  Deploy executor.