(sample) Fix bug where slop files would not be saved despite containing data

We now sometimes get 206 when good.

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java

@@ -74,7 +74,7 @@ public class CrawlerRevisitor {
 
             // If the reference document is empty or the HTTP status is not 200, we'll skip it since it's
             // unlikely to produce anything meaningful for us.
-            if (doc.httpStatus != 200)
+            if (doc.httpStatus != 200 && doc.httpStatus != 206)
                 continue;
             if (!doc.hasBody())
                 continue;

code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java

@@ -58,7 +58,7 @@ public record DocumentWithReference(
         if (null == doc)
             return ContentTags.empty();
 
-        if (doc.documentBodyBytes.length == 0 || doc.httpStatus != 200)
+        if (doc.documentBodyBytes.length == 0 || (doc.httpStatus != 200 && doc.httpStatus != 206))
             return ContentTags.empty();
 
         String lastmod = doc.getLastModified();

code/processes/crawling-process/model/java/nu/marginalia/ContentTypes.java

@@ -1,5 +1,7 @@
 package nu.marginalia;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.util.Set;
 
 public class ContentTypes {
@@ -11,9 +13,9 @@ public class ContentTypes {
             "text/plain");
 
     public static boolean isAccepted(String contentTypeHeader) {
-        String lcHeader = contentTypeHeader.toLowerCase();
+        String lcHeader = StringUtils.substringBefore(contentTypeHeader.toLowerCase(), ';');
         for (var type : acceptedContentTypes) {
-            if (lcHeader.startsWith(type)) {
+            if (lcHeader.equals(type)) {
                 return true;
             }
         }
@@ -21,7 +23,7 @@ public class ContentTypes {
     }
 
     public static boolean isBinary(String contentTypeHeader) {
-        String lcHeader = contentTypeHeader.toLowerCase();
+        String lcHeader = StringUtils.substringBefore(contentTypeHeader.toLowerCase(), ';');
         return lcHeader.startsWith("application/pdf");
     }
 

code/processes/crawling-process/model/java/nu/marginalia/slop/SlopCrawlDataRecord.java

@@ -277,7 +277,8 @@ public record SlopCrawlDataRecord(String domain,
         try (var table = new SlopTable(path)) {
             ShortColumn.Reader statusReader = statusColumn.open(table);
             while (statusReader.hasRemaining()) {
-                if (statusReader.get() == 200) {
+                int status = statusReader.get();
+                if (status == 200 || status == 206) {
                     cnt++;
                 }
             }

code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java

@@ -21,10 +21,7 @@ import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.NoSuchElementException;
+import java.util.*;
 
 public class SampleDataExporter {
     private final FileStorageService storageService;
@@ -127,7 +124,7 @@ public class SampleDataExporter {
              var reader = new SlopCrawlDataRecord.FilteringReader(crawlDataPath) {
                  @Override
                  public boolean filter(String url, int status, String contentType) {
-                     return matchContentTypeHeaderWithMime(contentType, contentTypeFilter)
+                     return Objects.equals(StringUtils.substringBefore(contentType, ';'), contentTypeFilter)
                                 || contentType.startsWith("x-marginalia/"); // metadata records
                  }
              }
@@ -137,7 +134,7 @@ public class SampleDataExporter {
                 var entry = reader.get();
                 writer.write(entry);
 
-                wroteEntry = wroteEntry || contentTypeFilter.equals(entry.contentType());
+                wroteEntry = wroteEntry || Objects.equals(StringUtils.substringBefore(entry.contentType(), ';'), contentTypeFilter);
             }
 
             if (!wroteEntry) {
@@ -154,21 +151,6 @@ public class SampleDataExporter {
         return tempFile;
     }
 
-    private boolean matchContentTypeHeaderWithMime(String contentType, String mime) {
-        if (null == contentType) {
-            return false;
-        }
-
-        /* The content type header may have a charset or other parameters, so we need to
-         * check if the mime type is a prefix of the content type. */
-
-        int semicolonIndex = contentType.indexOf(';');
-        if (semicolonIndex >= 0) {
-            return contentType.substring(0, semicolonIndex).equals(mime);
-        }
-        return contentType.equals(mime);
-    }
-
     private void addFileToTar(TarArchiveOutputStream outputStream, Path file, String fileName) throws IOException {
         var entry = outputStream.createArchiveEntry(file.toFile(), fileName);
         entry.setSize(Files.size(file));