(search) Clean up the rate limiting

We fail quietly to make life harder for the bot farmers
(search) Add automatic rate limiting to /site
2025-10-06 07:32:38 +02:00 · 2025-06-18 11:20:24 +02:00 · 2025-06-18 11:10:08 +02:00 · 2025-06-18 11:04:36 +02:00 · 2025-06-17 12:26:34 +02:00 · 2025-06-16 22:35:33 +02:00
4 changed files with 24 additions and 2 deletions
--- a/code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java
+++ b/code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java
@@ -112,7 +112,7 @@ public class HttpClientProvider implements Provider<HttpClient> {
        });

        final RequestConfig defaultRequestConfig = RequestConfig.custom()
-                .setCookieSpec(StandardCookieSpec.RELAXED)
+                .setCookieSpec(StandardCookieSpec.IGNORE)
                .setResponseTimeout(10, TimeUnit.SECONDS)
                .setConnectionRequestTimeout(5, TimeUnit.MINUTES)
                .build();
--- a/code/processes/ping-process/java/nu/marginalia/ping/io/RetryStrategy.java
+++ b/code/processes/ping-process/java/nu/marginalia/ping/io/RetryStrategy.java
@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;

 import javax.net.ssl.SSLException;
 import java.io.IOException;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;

@@ -23,7 +24,8 @@ public class RetryStrategy implements HttpRequestRetryStrategy {
            case SocketTimeoutException ste -> false;
            case SSLException ssle -> false;
            case UnknownHostException uhe -> false;
-            case HttpHostConnectException ex -> executionCount <= 2; // Only retry once for connection errors
+            case HttpHostConnectException ex -> executionCount < 2;
+            case SocketException ex -> executionCount < 2;
            default -> executionCount <= 3;
        };
    }
--- a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java
@@ -22,6 +22,7 @@ import nu.marginalia.search.model.NavbarModel;
 import nu.marginalia.search.model.ResultsPage;
 import nu.marginalia.search.model.UrlDetails;
 import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData;
+import nu.marginalia.service.server.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -47,6 +48,8 @@ public class SearchSiteInfoService {
    private final HikariDataSource dataSource;
    private final SearchSiteSubscriptionService searchSiteSubscriptions;

+    private final RateLimiter rateLimiter = RateLimiter.custom(60);
+
    @Inject
    public SearchSiteInfoService(SearchOperator searchOperator,
                                 DomainInfoClient domainInfoClient,
@@ -244,6 +247,12 @@ public class SearchSiteInfoService {
            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
        }
+        else if (!rateLimiter.isAllowed()) {
+            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            similarSetFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+        }
        else if (!domainInfoClient.isAccepting()) {
            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
            similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
--- a/code/services-application/search-service/resources/jte/unavailable.jte
+++ b/code/services-application/search-service/resources/jte/unavailable.jte
@@ -0,0 +1,11 @@
+@param String message
+
+<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8">
+<title>Unavailable</title></head>
+<body>
+<h1>Service Overloaded</h1>
+<p>${message}</p>
+</body>
+</html>
Author	SHA1	Message	Date
Viktor Lofgren	4895a2ac7a	(search) Clean up the rate limiting We fail quietly to make life harder for the bot farmers	2025-06-18 11:20:24 +02:00
Viktor Lofgren	fd32ae9fa7	(search) Add automatic rate limiting to /site Fix typo	2025-06-18 11:10:08 +02:00
Viktor Lofgren	470651ea4c	(search) Add automatic rate limiting to /site	2025-06-18 11:04:36 +02:00
Viktor Lofgren	8d4829e783	(ping) Change cookie specification to ignore cookies	2025-06-17 12:26:34 +02:00
Viktor Lofgren	1290bc15dc	(ping) Reduce retries for SocketException and pals	2025-06-16 22:35:33 +02:00