(search) Clean up the rate limiting

We fail quietly to make life harder for the bot farmers
2025-10-06 07:32:38 +02:00 · 2025-06-18 11:26:30 +02:00 · 2025-06-18 11:22:26 +02:00 · 2025-06-18 11:20:24 +02:00 · 2025-06-18 11:10:08 +02:00 · 2025-06-18 11:04:36 +02:00
3 changed files with 33 additions and 4 deletions
--- a/code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java
+++ b/code/processes/ping-process/java/nu/marginalia/ping/io/HttpClientProvider.java
@@ -112,7 +112,7 @@ public class HttpClientProvider implements Provider<HttpClient> {
        });

        final RequestConfig defaultRequestConfig = RequestConfig.custom()
-                .setCookieSpec(StandardCookieSpec.RELAXED)
+                .setCookieSpec(StandardCookieSpec.IGNORE)
                .setResponseTimeout(10, TimeUnit.SECONDS)
                .setConnectionRequestTimeout(5, TimeUnit.MINUTES)
                .build();
--- a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java
@@ -22,6 +22,7 @@ import nu.marginalia.search.model.NavbarModel;
 import nu.marginalia.search.model.ResultsPage;
 import nu.marginalia.search.model.UrlDetails;
 import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData;
+import nu.marginalia.service.server.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -47,6 +48,8 @@ public class SearchSiteInfoService {
    private final HikariDataSource dataSource;
    private final SearchSiteSubscriptionService searchSiteSubscriptions;

+    private final RateLimiter rateLimiter = RateLimiter.custom(60);
+
    @Inject
    public SearchSiteInfoService(SearchOperator searchOperator,
                                 DomainInfoClient domainInfoClient,
@@ -238,6 +241,7 @@ public class SearchSiteInfoService {
        boolean hasScreenshot = screenshotService.hasScreenshot(domainId);
        boolean isSubscribed = searchSiteSubscriptions.isSubscribed(context, domain);

+        boolean rateLimited = !rateLimiter.isAllowed();
        if (domainId < 0) {
            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
            similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
@@ -250,6 +254,12 @@ public class SearchSiteInfoService {
            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
        }
+        else if (rateLimited) {
+            domainInfoFuture = domainInfoClient.domainInformation(domainId);
+            similarSetFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+        }
        else {
            domainInfoFuture = domainInfoClient.domainInformation(domainId);
            similarSetFuture = domainInfoClient.similarDomains(domainId, 25);
@@ -257,7 +267,14 @@ public class SearchSiteInfoService {
            feedItemsFuture = feedsClient.getFeed(domainId);
        }

-        List<UrlDetails> sampleResults = searchOperator.doSiteSearch(domainName, domainId,5, 1).results;
+        List<UrlDetails> sampleResults;
+        if (rateLimited) {
+            sampleResults = List.of();
+        }
+        else {
+            sampleResults = searchOperator.doSiteSearch(domainName, domainId, 5, 1).results;
+        }
+
        if (!sampleResults.isEmpty()) {
            url = sampleResults.getFirst().url.withPathAndParam("/", null).toString();
        }
@@ -276,8 +293,9 @@ public class SearchSiteInfoService {
                sampleResults
        );

-        requestMissingScreenshots(result);
-
+        if (!rateLimited) {
+            requestMissingScreenshots(result);
+        }
        return result;
    }

--- a/code/services-application/search-service/resources/jte/unavailable.jte
+++ b/code/services-application/search-service/resources/jte/unavailable.jte
@@ -0,0 +1,11 @@
+@param String message
+
+<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8">
+<title>Unavailable</title></head>
+<body>
+<h1>Service Overloaded</h1>
+<p>${message}</p>
+</body>
+</html>
Author	SHA1	Message	Date
Viktor Lofgren	e2c56dc3ca	(search) Clean up the rate limiting We fail quietly to make life harder for the bot farmers	2025-06-18 11:26:30 +02:00
Viktor Lofgren	470b866008	(search) Clean up the rate limiting We fail quietly to make life harder for the bot farmers	2025-06-18 11:22:26 +02:00
Viktor Lofgren	4895a2ac7a	(search) Clean up the rate limiting We fail quietly to make life harder for the bot farmers	2025-06-18 11:20:24 +02:00
Viktor Lofgren	fd32ae9fa7	(search) Add automatic rate limiting to /site Fix typo	2025-06-18 11:10:08 +02:00
Viktor Lofgren	470651ea4c	(search) Add automatic rate limiting to /site	2025-06-18 11:04:36 +02:00
Viktor Lofgren	8d4829e783	(ping) Change cookie specification to ignore cookies	2025-06-17 12:26:34 +02:00