(crawler) Fast detection and bail-out for crawler traps

Improve logging and exclude robots.txt from this logic.

ROADMAP.md

@@ -1,4 +1,4 @@
-# Roadmap 2024-2025
+# Roadmap 2025
 
 This is a roadmap with major features planned for Marginalia Search.
 
@@ -30,12 +30,6 @@ Retaining the ability to independently crawl the web is still strongly desirable
 The search engine has a bit of a problem showing spicy content mixed in with the results.  It would be desirable to have a way to filter this out.  It's likely something like a URL blacklist (e.g. [UT1](https://dsi.ut-capitole.fr/blacklists/index_en.php) )
 combined with naive bayesian filter would go a long way, or something more sophisticated...?
 
-## Web Design Overhaul
-
-The design is kinda clunky and hard to maintain, and needlessly outdated-looking.  
-
-In progress: PR [#127](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/127)  -- demo available at https://test.marginalia.nu/
-
 ## Additional Language Support
 
 It would be desirable if the search engine supported more languages than English.  This is partially about
@@ -62,8 +56,31 @@ filter for any API consumer.
 
 I've talked to the stract dev and he does not think it's a good idea to mimic their optics language, which is quite ad-hoc, but instead to work together to find some new common description language for this. 
 
+## Show favicons next to search results
+
+This is expected from search engines.  Basic proof of concept sketch of fetching this data has been done, but the feature is some way from being reality. 
+
+## Specialized crawler for github
+
+One of the search engine's biggest limitations right now is that it does not index github at all.   A specialized crawler that fetches at least the readme.md would go a long way toward providing search capabilities in this domain.
+
 # Completed
 
+## Web Design Overhaul (COMPLETED 2025-01)
+
+The design is kinda clunky and hard to maintain, and needlessly outdated-looking.  
+
+PR [#127](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/127)
+
+## Finalize RSS support (COMPLETED 2024-11)
+
+Marginalia has experimental RSS preview support for a few domains.  This works well and
+it should be extended to all domains.  It would also be interesting to offer search of the
+RSS data itself, or use the RSS set to feed a special live index that updates faster than the
+main dataset. 
+
+Completed with PR [#122](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/122) and PR [#125](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/125)
+
 ## Proper Position Index (COMPLETED 2024-09)
 
 The search engine uses a fixed width bit mask to indicate word positions.  It has the benefit
@@ -76,11 +93,3 @@ list, as is the civilized way of doing this.
 
 Completed with PR [#99](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/99)
 
-## Finalize RSS support (COMPLETED 2024-11)
-
-Marginalia has experimental RSS preview support for a few domains.  This works well and
-it should be extended to all domains.  It would also be interesting to offer search of the
-RSS data itself, or use the RSS set to feed a special live index that updates faster than the
-main dataset. 
-
-Completed with PR [#122](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/122) and PR [#125](https://github.com/MarginaliaSearch/MarginaliaSearch/pull/125)

build.gradle

@@ -47,7 +47,7 @@ ext {
     dockerImageBase='container-registry.oracle.com/graalvm/jdk:23'
     dockerImageTag='latest'
     dockerImageRegistry='marginalia'
-    jibVersion = '3.4.3'
+    jibVersion = '3.4.4'
 
 }
 

code/common/db/java/nu/marginalia/db/DbDomainQueries.java

@@ -20,7 +20,10 @@ public class DbDomainQueries {
     private final HikariDataSource dataSource;
 
     private static final Logger logger = LoggerFactory.getLogger(DbDomainQueries.class);
+
     private final Cache<EdgeDomain, Integer> domainIdCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
+    private final Cache<Integer, EdgeDomain> domainNameCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
+    private final Cache<String, List<DomainWithNode>> siblingsCache = CacheBuilder.newBuilder().maximumSize(10_000).build();
 
     @Inject
     public DbDomainQueries(HikariDataSource dataSource)
@@ -30,16 +33,21 @@ public class DbDomainQueries {
 
 
     public Integer getDomainId(EdgeDomain domain) throws NoSuchElementException {
-        try (var connection = dataSource.getConnection()) {
-
+        try {
             return domainIdCache.get(domain, () -> {
-                try (var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) {
+                try (var connection = dataSource.getConnection();
+                     var stmt = connection.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?")) {
+
                     stmt.setString(1, domain.toString());
                     var rsp = stmt.executeQuery();
                     if (rsp.next()) {
                         return rsp.getInt(1);
                     }
                 }
+                catch (SQLException ex) {
+                    throw new RuntimeException(ex);
+                }
+
                 throw new NoSuchElementException();
             });
         }
@@ -49,9 +57,6 @@ public class DbDomainQueries {
         catch (ExecutionException ex) {
             throw new RuntimeException(ex.getCause());
         }
-        catch (SQLException ex) {
-            throw new RuntimeException(ex);
-        }
     }
 
     public OptionalInt tryGetDomainId(EdgeDomain domain) {
@@ -84,31 +89,38 @@ public class DbDomainQueries {
     }
 
     public Optional<EdgeDomain> getDomain(int id) {
-        try (var connection = dataSource.getConnection()) {
 
+        EdgeDomain existing = domainNameCache.getIfPresent(id);
+        if (existing != null) {
+            return Optional.of(existing);
+        }
+
+        try (var connection = dataSource.getConnection()) {
             try (var stmt = connection.prepareStatement("SELECT DOMAIN_NAME FROM EC_DOMAIN WHERE ID=?")) {
                 stmt.setInt(1, id);
                 var rsp = stmt.executeQuery();
                 if (rsp.next()) {
-                    return Optional.of(new EdgeDomain(rsp.getString(1)));
+                    var val = new EdgeDomain(rsp.getString(1));
+                    domainNameCache.put(id, val);
+                    return Optional.of(val);
                 }
                 return Optional.empty();
             }
         }
-        catch (UncheckedExecutionException ex) {
-            throw new RuntimeException(ex.getCause());
-        }
         catch (SQLException ex) {
             throw new RuntimeException(ex);
         }
     }
 
-    public List<DomainWithNode> otherSubdomains(EdgeDomain domain, int cnt) {
+    public List<DomainWithNode> otherSubdomains(EdgeDomain domain, int cnt) throws ExecutionException {
+        String topDomain = domain.topDomain;
+
+        return siblingsCache.get(topDomain, () -> {
             List<DomainWithNode> ret = new ArrayList<>();
 
             try (var conn = dataSource.getConnection();
                  var stmt = conn.prepareStatement("SELECT DOMAIN_NAME, NODE_AFFINITY FROM EC_DOMAIN WHERE DOMAIN_TOP = ? LIMIT ?")) {
-            stmt.setString(1, domain.topDomain);
+                stmt.setString(1, topDomain);
                 stmt.setInt(2, cnt);
 
                 var rs = stmt.executeQuery();
@@ -123,8 +135,9 @@ public class DbDomainQueries {
             } catch (SQLException e) {
                 logger.error("Failed to get domain neighbors");
             }
-
             return ret;
+        });
+
     }
 
     public record DomainWithNode (EdgeDomain domain, int nodeAffinity) {

code/common/db/java/nu/marginalia/db/DbDomainStatsExportMultitool.java

@@ -1,118 +0,0 @@
-package nu.marginalia.db;
-
-import com.zaxxer.hikari.HikariDataSource;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.OptionalInt;
-
-/** Class used in exporting data.  This is intended to be used for a brief time
- * and then discarded, not kept around as a service.
- */
-public class DbDomainStatsExportMultitool implements AutoCloseable {
-    private final Connection connection;
-    private final int nodeId;
-    private final PreparedStatement knownUrlsQuery;
-    private final PreparedStatement visitedUrlsQuery;
-    private final PreparedStatement goodUrlsQuery;
-    private final PreparedStatement domainNameToId;
-
-    private final PreparedStatement allDomainsQuery;
-    private final PreparedStatement crawlQueueDomains;
-    private final PreparedStatement indexedDomainsQuery;
-
-    public DbDomainStatsExportMultitool(HikariDataSource dataSource, int nodeId) throws SQLException {
-        this.connection = dataSource.getConnection();
-        this.nodeId = nodeId;
-
-        knownUrlsQuery = connection.prepareStatement("""
-                SELECT KNOWN_URLS
-                FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA
-                    ON EC_DOMAIN.ID=DOMAIN_METADATA.ID
-                WHERE DOMAIN_NAME=?
-                """);
-        visitedUrlsQuery = connection.prepareStatement("""
-                SELECT VISITED_URLS
-                FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA
-                    ON EC_DOMAIN.ID=DOMAIN_METADATA.ID
-                WHERE DOMAIN_NAME=?
-                """);
-        goodUrlsQuery = connection.prepareStatement("""
-                SELECT GOOD_URLS
-                FROM EC_DOMAIN INNER JOIN DOMAIN_METADATA
-                    ON EC_DOMAIN.ID=DOMAIN_METADATA.ID
-                WHERE DOMAIN_NAME=?
-                """);
-        domainNameToId = connection.prepareStatement("""
-                SELECT ID
-                FROM EC_DOMAIN
-                WHERE DOMAIN_NAME=?
-                """);
-        allDomainsQuery = connection.prepareStatement("""
-                SELECT DOMAIN_NAME
-                FROM EC_DOMAIN
-                """);
-        crawlQueueDomains = connection.prepareStatement("""
-                SELECT DOMAIN_NAME
-                FROM CRAWL_QUEUE
-                """);
-        indexedDomainsQuery = connection.prepareStatement("""
-                SELECT DOMAIN_NAME
-                FROM EC_DOMAIN
-                WHERE INDEXED > 0
-                """);
-    }
-
-    public OptionalInt getVisitedUrls(String domainName) throws SQLException {
-        return executeNameToIntQuery(domainName, visitedUrlsQuery);
-    }
-
-    public OptionalInt getDomainId(String domainName) throws SQLException {
-        return executeNameToIntQuery(domainName, domainNameToId);
-    }
-
-    public List<String> getCrawlQueueDomains() throws SQLException {
-        return executeListQuery(crawlQueueDomains, 100);
-    }
-    public List<String> getAllIndexedDomains() throws SQLException {
-        return executeListQuery(indexedDomainsQuery, 100_000);
-    }
-
-    private OptionalInt executeNameToIntQuery(String domainName, PreparedStatement statement)
-            throws SQLException {
-        statement.setString(1, domainName);
-        var rs = statement.executeQuery();
-
-        if (rs.next()) {
-            return OptionalInt.of(rs.getInt(1));
-        }
-
-        return OptionalInt.empty();
-    }
-
-    private List<String> executeListQuery(PreparedStatement statement, int sizeHint) throws SQLException {
-        List<String> ret = new ArrayList<>(sizeHint);
-
-        var rs = statement.executeQuery();
-
-        while (rs.next()) {
-            ret.add(rs.getString(1));
-        }
-
-        return ret;
-    }
-
-    @Override
-    public void close() throws SQLException {
-        knownUrlsQuery.close();
-        goodUrlsQuery.close();
-        visitedUrlsQuery.close();
-        allDomainsQuery.close();
-        crawlQueueDomains.close();
-        domainNameToId.close();
-        connection.close();
-    }
-}

code/index/api/java/nu/marginalia/index/api/IndexClient.java

@@ -16,20 +16,19 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-
-import static java.lang.Math.clamp;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
 
 @Singleton
 public class IndexClient {
     private static final Logger logger = LoggerFactory.getLogger(IndexClient.class);
     private final GrpcMultiNodeChannelPool<IndexApiGrpc.IndexApiBlockingStub> channelPool;
     private final DomainBlacklistImpl blacklist;
-    private static final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
+    private static final ExecutorService executor = Executors.newCachedThreadPool();
 
     @Inject
     public IndexClient(GrpcChannelPoolFactory channelPoolFactory, DomainBlacklistImpl blacklist) {
@@ -51,40 +50,37 @@ public class IndexClient {
 
     /** Execute a query on the index partitions and return the combined results. */
     public AggregateQueryResponse executeQueries(RpcIndexQuery indexRequest, Pagination pagination) {
-        List<CompletableFuture<Iterator<RpcDecoratedResultItem>>> futures =
-                channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query)
-                        .async(executor)
-                        .runEach(indexRequest);
 
         final int requestedMaxResults = indexRequest.getQueryLimits().getResultsTotal();
-        final int resultsUpperBound = requestedMaxResults * channelPool.getNumNodes();
 
-        List<RpcDecoratedResultItem> results = new ArrayList<>(resultsUpperBound);
+        AtomicInteger totalNumResults = new AtomicInteger(0);
 
-        for (var future : futures) {
+        List<RpcDecoratedResultItem> results =
+                channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query)
+                        .async(executor)
+                        .runEach(indexRequest)
+                        .stream()
+                        .map(future -> future.thenApply(iterator -> {
+                            List<RpcDecoratedResultItem> ret = new ArrayList<>(requestedMaxResults);
+                            iterator.forEachRemaining(ret::add);
+                            totalNumResults.addAndGet(ret.size());
+                            return ret;
+                        }))
+                        .mapMulti((CompletableFuture<List<RpcDecoratedResultItem>> fut, Consumer<List<RpcDecoratedResultItem>> c) ->{
                             try {
-                future.get().forEachRemaining(results::add);
-            }
-            catch (Exception e) {
-                logger.error("Downstream exception", e);
-            }
+                                c.accept(fut.join());
+                            } catch (Exception e) {
+                                logger.error("Error while fetching results", e);
                             }
+                        })
+                        .flatMap(List::stream)
+                        .filter(item -> !isBlacklisted(item))
+                        .sorted(comparator)
+                        .skip(Math.max(0, (pagination.page - 1) * pagination.pageSize))
+                        .limit(pagination.pageSize)
+                        .toList();
 
-        // Sort the results by ranking score and remove blacklisted domains
-        results.sort(comparator);
-        results.removeIf(this::isBlacklisted);
-
-        int numReceivedResults = results.size();
-
-        // pagination is typically 1-indexed, so we need to adjust the start and end indices
-        int indexStart = (pagination.page - 1) * pagination.pageSize;
-        int indexEnd = (pagination.page) * pagination.pageSize;
-
-        results = results.subList(
-                clamp(indexStart, 0, Math.max(0, results.size() - 1)), // from is inclusive, so subtract 1 from size()
-                clamp(indexEnd, 0, results.size()));
-
-        return new AggregateQueryResponse(results, pagination.page(), numReceivedResults);
+        return new AggregateQueryResponse(results, pagination.page(), totalNumResults.get());
     }
 
     private boolean isBlacklisted(RpcDecoratedResultItem item) {

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcRecorder.java

@@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
 
@@ -89,6 +90,7 @@ public class WarcRecorder implements AutoCloseable {
 
         var call = client.newCall(request);
 
+
         cookieInformation.update(client, request.url());
 
         try (var response = call.execute();
@@ -167,6 +169,25 @@ public class WarcRecorder implements AutoCloseable {
             warcRequest.http(); // force HTTP header to be parsed before body is consumed so that caller can use it
             writer.write(warcRequest);
 
+            if (Duration.between(date, Instant.now()).compareTo(Duration.ofSeconds(9)) > 0
+                    && inputBuffer.size() < 2048
+                    && !request.url().encodedPath().endsWith("robots.txt")) // don't bail on robots.txt
+            {
+                // Fast detection and mitigation of crawler traps that respond with slow
+                // small responses, with a high branching factor
+
+                // Note we bail *after* writing the warc records, this will effectively only
+                // prevent link extraction from the document.
+
+                logger.warn("URL {} took too long to fetch ({}s) and was too small for the effort ({}b)",
+                        requestUri,
+                        Duration.between(date, Instant.now()).getSeconds(),
+                        inputBuffer.size()
+                );
+
+                return new HttpFetchResult.ResultException(new IOException("Likely crawler trap"));
+            }
+
             return new HttpFetchResult.ResultOk(responseUri,
                     response.code(),
                     inputBuffer.headers(),

code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java

@@ -84,18 +84,33 @@ public record SearchParameters(WebsiteUrl url,
     }
 
     public String renderUrl() {
-        String path = String.format("/search?query=%s&profile=%s&js=%s&adtech=%s&recent=%s&searchTitle=%s&newfilter=%s&page=%d",
-                URLEncoder.encode(query, StandardCharsets.UTF_8),
-                URLEncoder.encode(profile.filterId, StandardCharsets.UTF_8),
-                URLEncoder.encode(js.value, StandardCharsets.UTF_8),
-                URLEncoder.encode(adtech.value, StandardCharsets.UTF_8),
-                URLEncoder.encode(recent.value, StandardCharsets.UTF_8),
-                URLEncoder.encode(searchTitle.value, StandardCharsets.UTF_8),
-                Boolean.valueOf(newFilter).toString(),
-                page
-                );
 
-        return path;
+        StringBuilder pathBuilder = new StringBuilder("/search?");
+        pathBuilder.append("query=").append(URLEncoder.encode(query, StandardCharsets.UTF_8));
+
+        if (profile != SearchProfile.NO_FILTER) {
+            pathBuilder.append("&profile=").append(URLEncoder.encode(profile.filterId, StandardCharsets.UTF_8));
+        }
+        if (js != SearchJsParameter.DEFAULT) {
+            pathBuilder.append("&js=").append(URLEncoder.encode(js.value, StandardCharsets.UTF_8));
+        }
+        if (adtech != SearchAdtechParameter.DEFAULT) {
+            pathBuilder.append("&adtech=").append(URLEncoder.encode(adtech.value, StandardCharsets.UTF_8));
+        }
+        if (recent != SearchRecentParameter.DEFAULT) {
+            pathBuilder.append("&recent=").append(URLEncoder.encode(recent.value, StandardCharsets.UTF_8));
+        }
+        if (searchTitle != SearchTitleParameter.DEFAULT) {
+            pathBuilder.append("&searchTitle=").append(URLEncoder.encode(searchTitle.value, StandardCharsets.UTF_8));
+        }
+        if (page != 1) {
+            pathBuilder.append("&page=").append(page);
+        }
+        if (newFilter) {
+            pathBuilder.append("&newfilter=").append(Boolean.valueOf(newFilter).toString());
+        }
+
+        return pathBuilder.toString();
     }
 
     public RpcTemporalBias.Bias temporalBias() {

code/services-application/search-service/java/nu/marginalia/search/command/commands/SearchCommand.java

@@ -3,27 +3,22 @@ package nu.marginalia.search.command.commands;
 import com.google.inject.Inject;
 import io.jooby.MapModelAndView;
 import io.jooby.ModelAndView;
-import nu.marginalia.search.JteRenderer;
 import nu.marginalia.search.SearchOperator;
 import nu.marginalia.search.command.SearchCommandInterface;
 import nu.marginalia.search.command.SearchParameters;
 import nu.marginalia.search.model.DecoratedSearchResults;
 import nu.marginalia.search.model.NavbarModel;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.Optional;
 
 public class SearchCommand implements SearchCommandInterface {
     private final SearchOperator searchOperator;
-    private final JteRenderer jteRenderer;
 
 
     @Inject
-    public SearchCommand(SearchOperator searchOperator,
-                         JteRenderer jteRenderer) throws IOException {
+    public SearchCommand(SearchOperator searchOperator){
         this.searchOperator = searchOperator;
-        this.jteRenderer = jteRenderer;
     }
 
     @Override

code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java

@@ -26,10 +26,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.SQLException;
-import java.time.Duration;
-import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
@@ -69,9 +68,11 @@ public class SearchSiteInfoService {
         this.screenshotService = screenshotService;
         this.dataSource = dataSource;
         this.searchSiteSubscriptions = searchSiteSubscriptions;
+
+        Thread.ofPlatform().name("Recently Added Domains Model Updater").start(this::modelUpdater);
     }
 
-    private volatile SiteOverviewModel model = new SiteOverviewModel(List.of(), Instant.EPOCH);
+    private volatile SiteOverviewModel cachedOverviewModel = new SiteOverviewModel(List.of());
 
     @GET
     @Path("/site")
@@ -81,55 +82,52 @@ public class SearchSiteInfoService {
             return new MapModelAndView("redirect.jte", Map.of("url", "/site/"+domain));
         }
 
-        if (model.age().compareTo(Duration.ofMinutes(15)) > 0) {
-            updateModel();
-        }
-
         return new MapModelAndView("siteinfo/start.jte",
                 Map.of("navbar", NavbarModel.SITEINFO,
-                        "model", model));
-    }
-
-    /**  Update the model if it is older than 15 minutes.
-     *   This query is expensive and should not be run too often,
-     *   and the data doesn't change that often either.
-     *  <p></p>
-     *   This method is synchronized to avoid multiple threads updating the model at the same time.
-     */
-    private synchronized void updateModel() {
-        var currentModel = model;
-        if (currentModel.age().compareTo(Duration.ofMinutes(15)) < 0) {
-            return;
+                        "model", cachedOverviewModel));
     }
 
+    private void modelUpdater() {
+        while (!Thread.interrupted()) {
             List<SiteOverviewModel.DiscoveredDomain> domains = new ArrayList<>();
 
-        try (var conn = dataSource.getConnection();
-             var stmt = conn.prepareStatement("SELECT DOMAIN_NAME, DISCOVER_DATE FROM EC_DOMAIN WHERE NODE_AFFINITY = 0 ORDER BY ID DESC LIMIT 10")) {
+            // This query can be quite expensive, so we can't run it on demand
+            // for every request. Instead, we run it every 15 minutes and cache
+            // the result.
 
+            try (var conn = dataSource.getConnection();
+                 var stmt = conn.prepareStatement("""
+                    SELECT DOMAIN_NAME, DISCOVER_DATE
+                    FROM EC_DOMAIN
+                    WHERE NODE_AFFINITY = 0
+                    ORDER BY ID DESC
+                    LIMIT 10
+                    """))
+            {
                 var rs = stmt.executeQuery();
                 while (rs.next()) {
-                domains.add(new SiteOverviewModel.DiscoveredDomain(rs.getString("DOMAIN_NAME"), rs.getString("DISCOVER_DATE")));
+                    domains.add(new SiteOverviewModel.DiscoveredDomain(
+                            rs.getString("DOMAIN_NAME"),
+                            rs.getString("DISCOVER_DATE"))
+                    );
                 }
-        }
-        catch (SQLException ex) {
-            throw new RuntimeException();
+            } catch (SQLException ex) {
+                logger.warn("Failed to get recently added domains: {}", ex.getMessage());
             }
 
-        model = new SiteOverviewModel(domains);
-    }
-
-    public record SiteOverviewModel(List<DiscoveredDomain> domains, Instant captureTime) {
-
-        public SiteOverviewModel(List<DiscoveredDomain> domains) {
-            this(domains, Instant.now());
+            cachedOverviewModel = new SiteOverviewModel(domains);
+
+            try {
+                TimeUnit.MINUTES.sleep(15);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            }
+        }
     }
 
+    public record SiteOverviewModel(List<DiscoveredDomain> domains) {
         public record DiscoveredDomain(String name, String timestamp) {}
-
-        public Duration age() {
-            return Duration.between(captureTime, Instant.now());
-        }
     }
 
     @GET
@@ -139,7 +137,7 @@ public class SearchSiteInfoService {
             @PathParam String domainName,
             @QueryParam String view,
             @QueryParam Integer page
-    ) throws SQLException {
+    ) throws SQLException, ExecutionException {
 
         if (null == domainName || domainName.isBlank()) {
             return null;
@@ -225,7 +223,7 @@ public class SearchSiteInfoService {
         );
     }
 
-    private SiteInfoWithContext listInfo(Context context, String domainName) {
+    private SiteInfoWithContext listInfo(Context context, String domainName) throws ExecutionException {
 
         var domain = new EdgeDomain(domainName);
         final int domainId = domainQueries.tryGetDomainId(domain).orElse(-1);

code/services-application/search-service/resources/jte/serp/part/searchform.jte

@@ -36,10 +36,11 @@
 
     </div>
 
+    @if (filters.showRecentOption.isSet()) <input type="hidden" name="js" value="${filters.removeJsOption.value()}"> @endif
+    @if (filters.reduceAdtechOption.isSet()) <input type="hidden" name="adtech" value="${filters.reduceAdtechOption.value()}"> @endif
+    @if (filters.searchTitleOption.isSet()) <input type="hidden" name="searchTitle" value="${filters.searchTitleOption.value()}"> @endif
+    @if (filters.showRecentOption.isSet()) <input type="hidden" name="recent" value="${filters.showRecentOption.value()}"> @endif
 
-    <input type="hidden" name="js" value="${filters.removeJsOption.value()}">
-    <input type="hidden" name="adtech" value="${filters.reduceAdtechOption.value()}">
-    <input type="hidden" name="searchTitle" value="${filters.searchTitleOption.value()}">
     <input type="hidden" name="profile" value="${profile}">
-    <input type="hidden" name="recent" value="${filters.showRecentOption.value()}">
+
 </form>

code/services-application/search-service/resources/jte/serp/start.jte

@@ -36,7 +36,7 @@
         <div class="text-slate-700 dark:text-white text-sm p-4">
             <div class="fas fa-gift mr-1 text-margeblue dark:text-slate-200"></div>
             This is the new design and home of Marginalia Search.
-            You can about what this entails <a href="https://about.marginalia-search.com/article/redesign/"  class="underline text-liteblue dark:text-blue-200">here</a>.
+            You can read about what this entails <a href="https://about.marginalia-search.com/article/redesign/"  class="underline text-liteblue dark:text-blue-200">here</a>.
         <p class="my-4"></p>
         The old version of Marginalia Search remains available at
         <a href="https://old-search.marginalia.nu/" class="underline text-liteblue dark:text-blue-200">https://old-search.marginalia.nu/</a>.

code/services-application/search-service/resources/jte/siteinfo/view/overview.jte

@@ -1,5 +1,4 @@
 @import nu.marginalia.db.DbDomainQueries
-@import nu.marginalia.model.EdgeDomain
 @import nu.marginalia.search.svc.SearchSiteInfoService
 @import nu.marginalia.search.svc.SearchSiteInfoService.*
 @import nu.marginalia.search.model.UrlDetails

run/install/barebones-1/docker-compose.yml.template

@@ -72,11 +72,11 @@ services:
     image: "mariadb:lts"
     container_name: "mariadb"
     env_file: "${INSTALL_DIR}/env/mariadb.env"
-    command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
+    command: ['mariadbd', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
     ports:
       - "127.0.0.1:3306:3306/tcp"
     healthcheck:
-      test: mysqladmin ping -h 127.0.0.1 -u ${uval} --password=${pval}
+      test: mariadb-admin ping -h 127.0.0.1 -u ${uval} --password=${pval}
       start_period: 5s
       interval: 5s
       timeout: 5s

run/install/barebones-2/docker-compose.yml.template

@@ -103,11 +103,11 @@ services:
     image: "mariadb:lts"
     container_name: "mariadb"
     env_file: "${INSTALL_DIR}/env/mariadb.env"
-    command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
+    command: ['mariadbd', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
     ports:
       - "127.0.0.1:3306:3306/tcp"
     healthcheck:
-      test: mysqladmin ping -h 127.0.0.1 -u ${uval} --password=${pval}
+      test: mariadb-admin ping -h 127.0.0.1 -u ${uval} --password=${pval}
       start_period: 5s
       interval: 5s
       timeout: 5s

run/install/marginalia-prod-like/docker-compose.yml.template

@@ -129,11 +129,11 @@ services:
     image: "mariadb:lts"
     container_name: "mariadb"
     env_file: "${INSTALL_DIR}/env/mariadb.env"
-    command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
+    command: ['mariadbd', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
     ports:
       - "127.0.0.1:3306:3306/tcp"
     healthcheck:
-      test: mysqladmin ping -h 127.0.0.1 -u ${uval} --password=${pval}
+      test: mariadb-admin ping -h 127.0.0.1 -u ${uval} --password=${pval}
       start_period: 5s
       interval: 5s
       timeout: 5s

run/install/no-docker/docker-compose.yml.template

@@ -3,11 +3,11 @@ services:
     image: "mariadb:lts"
     container_name: "mariadb"
     env_file: "${INSTALL_DIR}/env/mariadb.env"
-    command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
+    command: ['mariadbd', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
     ports:
       - "127.0.0.1:3306:3306/tcp"
     healthcheck:
-      test: mysqladmin ping -h 127.0.0.1 -u ${uval} --password=${pval}
+      test: mariadb-admin ping -h 127.0.0.1 -u ${uval} --password=${pval}
       start_period: 5s
       interval: 5s
       timeout: 5s