(search) Add link promoting the redesign beta

This gives the same upper limit to the live crawler and the big boy crawler, though the live crawler will reject items too large, and the big crawler will truncate at that point.

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/warc/WarcRecorder.java

@@ -34,8 +34,9 @@ import java.util.*;
 public class WarcRecorder implements AutoCloseable {
     /** Maximum time we'll wait on a single request */
     static final int MAX_TIME = 30_000;
-    /** Maximum (decompressed) size we'll fetch */
-    static final int MAX_SIZE = 1024 * 1024 * 10;
+
+    /** Maximum (decompressed) size we'll save */
+    static final int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 10 * 1024 * 1024);
 
     private final WarcWriter writer;
     private final Path warcFile;

code/processes/live-crawling-process/java/nu/marginalia/livecrawler/SimpleLinkScraper.java

@@ -48,6 +48,8 @@ public class SimpleLinkScraper implements AutoCloseable {
     private final Duration readTimeout = Duration.ofSeconds(10);
     private final DomainLocks domainLocks = new DomainLocks();
 
+    private final static int MAX_SIZE = Integer.getInteger("crawler.maxFetchSize", 10 * 1024 * 1024);
+
     public SimpleLinkScraper(LiveCrawlDataSet dataSet,
                              DbDomainQueries domainQueries,
                              DomainBlacklist domainBlacklist) {
@@ -207,7 +209,7 @@ public class SimpleLinkScraper implements AutoCloseable {
                 }
 
                 byte[] body = getResponseData(response);
-                if (body.length > 1024 * 1024) {
+                if (body.length > MAX_SIZE) {
                     return new FetchResult.Error(parsedUrl);
                 }
 

code/services-application/search-service/resources/templates/search/index/index-redesign.hdb

@@ -0,0 +1,14 @@
+<section id="frontpage-tips">
+    <h2>Public Beta Available</h2>
+    <div class="info">
+        <p>
+            A redesigned version of the search engine UI is available for beta testing.
+            Feel free to give it a spin, feedback is welcome!
+            The old one will also be keep being available if you hate it,
+            or have compatibility issues.
+        </p>
+        <p>
+            <a href="https://test.marginalia.nu/">Try it out!</a>
+        </p>
+    </div>
+</section>

code/services-application/search-service/resources/templates/search/index/index.hdb

@@ -24,7 +24,7 @@
 <section id="frontpage">
 {{>search/index/index-news}}
 {{>search/index/index-about}}
-{{>search/index/index-tips}}
+{{>search/index/index-redesign}}
 </section>
 
 {{>search/parts/search-footer}}

tools/deployment/deployment.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from dataclasses import dataclass
 import subprocess, os
 from typing import List, Set, Dict, Optional