(crawler) Lower max conn per route

code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java

@@ -80,11 +80,12 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
     private CloseableHttpClient createClient() throws NoSuchAlgorithmException {
         final ConnectionConfig connectionConfig = ConnectionConfig.custom()
                 .setSocketTimeout(10, TimeUnit.SECONDS)
-                .setConnectTimeout(10, TimeUnit.SECONDS)
+                .setConnectTimeout(30, TimeUnit.SECONDS)
                 .build();
 
         final PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
-                .setMaxConnPerRoute(4)
+                .setMaxConnPerRoute(2)
+                .setMaxConnTotal(5000)
                 .setDefaultConnectionConfig(connectionConfig)
                 .setTlsSocketStrategy(new DefaultClientTlsStrategy(SSLContext.getDefault()))
                 .build();
@@ -613,7 +614,6 @@ public class HttpFetcherImpl implements HttpFetcher, HttpRequestRetryStrategy {
 
     @Override
     public TimeValue getRetryInterval(HttpRequest request, IOException exception, int executionCount, HttpContext context) {
-        logger.info("Error", exception);
         return TimeValue.ofSeconds(1);
     }
 

code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java

@@ -101,7 +101,7 @@ class ContentTypeProberTest {
 
         System.out.println(result);
 
-        assertEquals(result, new HttpFetcher.ContentTypeProbeResult.Ok(htmlEndpoint));
+        assertEquals(result, new HttpFetcher.ContentTypeProbeResult.Redirect(htmlEndpoint));
     }
 
     @Test