(converter) Add truncation att the parser step to prevent the converter from spending too much time on exessively large documents

code/processes/crawling-process/model/java/nu/marginalia/model/crawldata/CrawledDocument.java

@@ -59,9 +59,14 @@ public final class CrawledDocument implements SerializableCrawlData {
     }
 
     public Document parseBody() throws IOException {
+        // Prevent stalls from parsing excessively large documents
+
+        byte[] bytes = documentBodyBytes.length > 200_000
+                ? Arrays.copyOf(documentBodyBytes, 200_000) : documentBodyBytes;
+
         return DocumentBodyToString.getParsedData(
                 ContentType.parse(contentType),
-                documentBodyBytes,
+                bytes,
                 url);
     }
 

code/processes/live-crawling-process/java/nu/marginalia/livecrawler/LiveCrawlDataSet.java

@@ -228,7 +228,7 @@ public class LiveCrawlDataSet implements AutoCloseable {
         }
 
         @Override
-        public boolean hasNext() throws IOException {
+        public boolean hasNext() {
             if (dataStack == null) {
                 query();
             }
@@ -236,7 +236,7 @@ public class LiveCrawlDataSet implements AutoCloseable {
         }
 
         @Override
-        public void close() throws Exception {
+        public void close() {
             dataStack.clear();
         }
     }