(search) Correct exception from misbehaving bots feeding invalid urls

The change also introduces a somewhat convoluted wiremock test to intercept and verify that these headers are in fact sent

code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java

@@ -6,6 +6,7 @@ import nu.marginalia.service.ServiceId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.util.Enumeration;
@@ -115,7 +116,7 @@ public class ServiceConfigurationModule extends AbstractModule {
         }
     }
 
-    public static String getLocalNetworkIP() throws Exception {
+    public static String getLocalNetworkIP() throws IOException {
         Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
 
         while (nets.hasMoreElements()) {

code/execution/java/nu/marginalia/actor/proc/UpdateRssActor.java

@@ -14,6 +14,8 @@ import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.nodecfg.model.NodeProfile;
 import nu.marginalia.service.module.ServiceConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.time.Duration;
 import java.time.LocalDateTime;
@@ -29,6 +31,7 @@ public class UpdateRssActor extends RecordActorPrototype {
 
     private final NodeConfigurationService nodeConfigurationService;
     private final MqPersistence persistence;
+    private static final Logger logger = LoggerFactory.getLogger(UpdateRssActor.class);
 
     @Inject
     public UpdateRssActor(Gson gson,
@@ -101,8 +104,8 @@ public class UpdateRssActor extends RecordActorPrototype {
             case UpdateRefresh(int count, long msgId) -> {
                 MqMessage msg = persistence.waitForMessageTerminalState(msgId, Duration.ofSeconds(10), Duration.ofHours(12));
                 if (msg == null) {
-                    // Retry the update
-                    yield new Error("Failed to update feeds: message not found");
+                    logger.warn("UpdateRefresh is taking a very long time");
+                    yield new UpdateRefresh(count, msgId);
                 } else if (msg.state() != MqMessageState.OK) {
                     // Retry the update
                     yield new Error("Failed to update feeds: " + msg.state());
@@ -119,8 +122,8 @@ public class UpdateRssActor extends RecordActorPrototype {
             case UpdateClean(long msgId) -> {
                 MqMessage msg = persistence.waitForMessageTerminalState(msgId, Duration.ofSeconds(10), Duration.ofHours(12));
                 if (msg == null) {
-                    // Retry the update
-                    yield new Error("Failed to update feeds: message not found");
+                    logger.warn("UpdateClean is taking a very long time");
+                    yield new UpdateClean(msgId);
                 } else if (msg.state() != MqMessageState.OK) {
                     // Retry the update
                     yield new Error("Failed to update feeds: " + msg.state());

code/functions/live-capture/build.gradle

@@ -34,6 +34,7 @@ dependencies {
     implementation libs.bundles.slf4j
     implementation libs.commons.lang3
     implementation libs.commons.io
+    implementation libs.wiremock
 
     implementation libs.prometheus
     implementation libs.guava

code/functions/live-capture/java/nu/marginalia/livecapture/BrowserlessClient.java

@@ -1,6 +1,7 @@
 package nu.marginalia.livecapture;
 
 import com.google.gson.Gson;
+import nu.marginalia.WmsaHome;
 import nu.marginalia.model.gson.GsonFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -12,6 +13,7 @@ import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.time.Duration;
 import java.util.Map;
+import java.util.Optional;
 
 /** Client for local browserless.io API */
 public class BrowserlessClient implements AutoCloseable {
@@ -27,13 +29,16 @@ public class BrowserlessClient implements AutoCloseable {
     private final URI browserlessURI;
     private final Gson gson = GsonFactory.get();
 
+    private final String userAgent = WmsaHome.getUserAgent().uaString();
+
     public BrowserlessClient(URI browserlessURI) {
         this.browserlessURI = browserlessURI;
     }
 
-    public String content(String url, GotoOptions gotoOptions) throws IOException, InterruptedException {
+    public Optional<String> content(String url, GotoOptions gotoOptions) throws IOException, InterruptedException {
         Map<String, Object> requestData = Map.of(
                 "url", url,
+                "userAgent", userAgent,
                 "gotoOptions", gotoOptions
         );
 
@@ -49,10 +54,10 @@ public class BrowserlessClient implements AutoCloseable {
 
         if (rsp.statusCode() >= 300) {
             logger.info("Failed to fetch content for {}, status {}", url, rsp.statusCode());
-            return null;
+            return Optional.empty();
         }
 
-        return rsp.body();
+        return Optional.of(rsp.body());
     }
 
     public byte[] screenshot(String url, GotoOptions gotoOptions, ScreenshotOptions screenshotOptions)
@@ -60,6 +65,7 @@ public class BrowserlessClient implements AutoCloseable {
 
         Map<String, Object> requestData = Map.of(
                 "url", url,
+                "userAgent", userAgent,
                 "options", screenshotOptions,
                 "gotoOptions", gotoOptions
         );
@@ -84,7 +90,7 @@ public class BrowserlessClient implements AutoCloseable {
     }
 
     @Override
-    public void close() throws Exception {
+    public void close() {
         httpClient.shutdownNow();
     }
 

code/functions/live-capture/test/nu/marginalia/livecapture/BrowserlessClientTest.java

@@ -1,5 +1,9 @@
 package nu.marginalia.livecapture;
 
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import nu.marginalia.WmsaHome;
+import nu.marginalia.service.module.ServiceConfigurationModule;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -8,34 +12,86 @@ import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.junit.jupiter.Testcontainers;
 import org.testcontainers.utility.DockerImageName;
 
+import java.io.IOException;
 import java.net.URI;
 import java.util.Map;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.*;
+
+
 @Testcontainers
 @Tag("slow")
 public class BrowserlessClientTest {
     static GenericContainer<?> container = new GenericContainer<>(DockerImageName.parse("browserless/chrome"))
             .withEnv(Map.of("TOKEN", "BROWSERLESS_TOKEN"))
+            .withNetworkMode("bridge")
             .withExposedPorts(3000);
 
+    static WireMockServer wireMockServer =
+            new WireMockServer(WireMockConfiguration.wireMockConfig()
+                    .port(18089));
+
+    static String localIp;
+
+    static URI browserlessURI;
+
     @BeforeAll
-    public static void setup() {
+    public static void setup() throws IOException {
         container.start();
+
+        browserlessURI = URI.create(String.format("http://%s:%d/",
+                container.getHost(),
+                container.getMappedPort(3000))
+        );
+
+        wireMockServer.start();
+        wireMockServer.stubFor(get("/").willReturn(aResponse().withStatus(200).withBody("Ok")));
+
+        localIp = ServiceConfigurationModule.getLocalNetworkIP();
+
+    }
+
+    @Tag("flaky")
+    @Test
+    public void testInspectContentUA__Flaky() throws Exception {
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            client.content("http://" + localIp + ":18089/",
+                    BrowserlessClient.GotoOptions.defaultValues()
+            );
+        }
+
+        wireMockServer.verify(getRequestedFor(urlEqualTo("/")).withHeader("User-Agent", equalTo(WmsaHome.getUserAgent().uaString())));
+    }
+
+    @Tag("flaky")
+    @Test
+    public void testInspectScreenshotUA__Flaky() throws Exception {
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            client.screenshot("http://" + localIp + ":18089/",
+                    BrowserlessClient.GotoOptions.defaultValues(),
+                    BrowserlessClient.ScreenshotOptions.defaultValues()
+            );
+        }
+
+        wireMockServer.verify(getRequestedFor(urlEqualTo("/")).withHeader("User-Agent", equalTo(WmsaHome.getUserAgent().uaString())));
     }
 
     @Test
     public void testContent() throws Exception {
-        try (var client = new BrowserlessClient(URI.create("http://" + container.getHost() + ":" + container.getMappedPort(3000)))) {
-            var content = client.content("https://www.marginalia.nu/", BrowserlessClient.GotoOptions.defaultValues());
-            Assertions.assertNotNull(content, "Content should not be null");
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            var content = client.content("https://www.marginalia.nu/", BrowserlessClient.GotoOptions.defaultValues()).orElseThrow();
+
             Assertions.assertFalse(content.isBlank(), "Content should not be empty");
         }
     }
 
     @Test
     public void testScreenshot() throws Exception {
-        try (var client = new BrowserlessClient(URI.create("http://" + container.getHost() + ":" + container.getMappedPort(3000)))) {
-            var screenshot = client.screenshot("https://www.marginalia.nu/", BrowserlessClient.GotoOptions.defaultValues(), BrowserlessClient.ScreenshotOptions.defaultValues());
+        try (var client = new BrowserlessClient(browserlessURI)) {
+            var screenshot = client.screenshot("https://www.marginalia.nu/",
+                    BrowserlessClient.GotoOptions.defaultValues(),
+                    BrowserlessClient.ScreenshotOptions.defaultValues());
+
             Assertions.assertNotNull(screenshot, "Screenshot should not be null");
         }
     }

code/services-application/search-service/java/nu/marginalia/search/SearchService.java

@@ -3,8 +3,10 @@ package nu.marginalia.search;
 import com.google.inject.Inject;
 import io.jooby.Context;
 import io.jooby.Jooby;
+import io.jooby.StatusCode;
 import io.prometheus.client.Counter;
 import io.prometheus.client.Histogram;
+import nu.marginalia.WebsiteUrl;
 import nu.marginalia.search.svc.*;
 import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.server.BaseServiceParams;
@@ -16,6 +18,7 @@ import java.util.List;
 
 public class SearchService extends JoobyService {
 
+    private final WebsiteUrl websiteUrl;
     private final SearchSiteSubscriptionService siteSubscriptionService;
 
     private static final Logger logger = LoggerFactory.getLogger(SearchService.class);
@@ -33,6 +36,7 @@ public class SearchService extends JoobyService {
 
     @Inject
     public SearchService(BaseServiceParams params,
+                         WebsiteUrl websiteUrl,
                          SearchFrontPageService frontPageService,
                          SearchAddToCrawlQueueService addToCrawlQueueService,
                          SearchSiteSubscriptionService siteSubscriptionService,
@@ -51,6 +55,7 @@ public class SearchService extends JoobyService {
                         new SearchAddToCrawlQueueService_(addToCrawlQueueService),
                         new SearchBrowseService_(searchBrowseService)
                 ));
+        this.websiteUrl = websiteUrl;
 
         this.siteSubscriptionService = siteSubscriptionService;
     }
@@ -62,6 +67,10 @@ public class SearchService extends JoobyService {
         final String startTimeAttribute = "start-time";
 
         jooby.get("/export-opml", siteSubscriptionService::exportOpml);
+
+        jooby.get("/site/https://*", this::handleSiteUrlRedirect);
+        jooby.get("/site/http://*", this::handleSiteUrlRedirect);
+
         jooby.before((Context ctx) -> {
             ctx.setAttribute(startTimeAttribute, System.nanoTime());
         });
@@ -80,5 +89,19 @@ public class SearchService extends JoobyService {
         });
     }
 
+    /** Redirect handler for the case when the user passes
+     * an url like /site/https://example.com/, in this
+     * scenario we want to extract the domain name and redirect
+     * to /site/example.com/
+     */
+    private Context handleSiteUrlRedirect(Context ctx) {
+        var pv = ctx.path("*").value();
+        int trailSlash = pv.indexOf('/');
+        if (trailSlash > 0) {
+            pv = pv.substring(0, trailSlash);
+        }
+        ctx.sendRedirect(StatusCode.TEMPORARY_REDIRECT, websiteUrl.withPath("site/" + pv));
+        return ctx;
+    }
 
 }

code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java

@@ -86,8 +86,10 @@ public record SearchParameters(WebsiteUrl url,
     public String renderUrl() {
 
         StringBuilder pathBuilder = new StringBuilder("/search?");
-        pathBuilder.append("query=").append(URLEncoder.encode(query, StandardCharsets.UTF_8));
 
+        if (query != null) {
+            pathBuilder.append("query=").append(URLEncoder.encode(query, StandardCharsets.UTF_8));
+        }
         if (profile != SearchProfile.NO_FILTER) {
             pathBuilder.append("&profile=").append(URLEncoder.encode(profile.filterId, StandardCharsets.UTF_8));
         }

code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java

@@ -56,7 +56,9 @@ public class SearchQueryService {
         }
         catch (Exception ex) {
             logger.error("Error", ex);
-            return errorPageService.serveError(SearchParameters.defaultsForQuery(websiteUrl, query, page));
+            return errorPageService.serveError(
+                    SearchParameters.defaultsForQuery(websiteUrl, query, Objects.requireNonNullElse(page, 1))
+            );
         }
     }
 

settings.gradle

@@ -239,6 +239,7 @@ dependencyResolutionManagement {
             library('jooby-jte','io.jooby','jooby-jte').version(joobyVersion)
             library('jooby-apt','io.jooby','jooby-apt').version(joobyVersion)
 
+            library('wiremock', 'org.wiremock','wiremock').version('3.11.0')
             library('jte','gg.jte','jte').version('3.1.15')
 
             bundle('jetty', ['jetty-server', 'jetty-util', 'jetty-servlet'])