(search) Clean up the rate limiting

We fail quietly to make life harder for the bot farmers
2025-10-06 07:32:38 +02:00 · 2025-06-18 11:22:26 +02:00 · 2025-06-18 11:20:24 +02:00 · 2025-06-18 11:10:08 +02:00 · 2025-06-18 11:04:36 +02:00
2 changed files with 29 additions and 1 deletions
--- a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java
@@ -22,6 +22,7 @@ import nu.marginalia.search.model.NavbarModel;
 import nu.marginalia.search.model.ResultsPage;
 import nu.marginalia.search.model.UrlDetails;
 import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData;
+import nu.marginalia.service.server.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -47,6 +48,8 @@ public class SearchSiteInfoService {
    private final HikariDataSource dataSource;
    private final SearchSiteSubscriptionService searchSiteSubscriptions;

+    private final RateLimiter rateLimiter = RateLimiter.custom(60);
+
    @Inject
    public SearchSiteInfoService(SearchOperator searchOperator,
                                 DomainInfoClient domainInfoClient,
@@ -238,12 +241,19 @@ public class SearchSiteInfoService {
        boolean hasScreenshot = screenshotService.hasScreenshot(domainId);
        boolean isSubscribed = searchSiteSubscriptions.isSubscribed(context, domain);

+        boolean rateLimited = !rateLimiter.isAllowed();
        if (domainId < 0) {
            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
            similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
        }
+        else if (rateLimited) {
+            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            similarSetFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+            feedItemsFuture = CompletableFuture.failedFuture(new Exception("Rate limit exceeded"));
+        }
        else if (!domainInfoClient.isAccepting()) {
            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
            similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
@@ -257,7 +267,14 @@ public class SearchSiteInfoService {
            feedItemsFuture = feedsClient.getFeed(domainId);
        }

-        List<UrlDetails> sampleResults = searchOperator.doSiteSearch(domainName, domainId,5, 1).results;
+        List<UrlDetails> sampleResults;
+        if (rateLimited) {
+            sampleResults = List.of();
+        }
+        else {
+            sampleResults = searchOperator.doSiteSearch(domainName, domainId, 5, 1).results;
+        }
+
        if (!sampleResults.isEmpty()) {
            url = sampleResults.getFirst().url.withPathAndParam("/", null).toString();
        }
--- a/code/services-application/search-service/resources/jte/unavailable.jte
+++ b/code/services-application/search-service/resources/jte/unavailable.jte
@@ -0,0 +1,11 @@
+@param String message
+
+<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8">
+<title>Unavailable</title></head>
+<body>
+<h1>Service Overloaded</h1>
+<p>${message}</p>
+</body>
+</html>
Author	SHA1	Message	Date
Viktor Lofgren	470b866008	(search) Clean up the rate limiting We fail quietly to make life harder for the bot farmers	2025-06-18 11:22:26 +02:00
Viktor Lofgren	4895a2ac7a	(search) Clean up the rate limiting We fail quietly to make life harder for the bot farmers	2025-06-18 11:20:24 +02:00
Viktor Lofgren	fd32ae9fa7	(search) Add automatic rate limiting to /site Fix typo	2025-06-18 11:10:08 +02:00
Viktor Lofgren	470651ea4c	(search) Add automatic rate limiting to /site	2025-06-18 11:04:36 +02:00