(search) Add debug logging for specific query

Revert "(query) Add debug logging for specific query"
This reverts commit 39ce40bfeb.
2025-10-05 21:22:39 +02:00 · 2025-05-16 23:34:03 +02:00 · 2025-05-16 23:29:06 +02:00 · 2025-05-16 23:23:53 +02:00 · 2025-05-16 23:20:16 +02:00 · 2025-05-16 23:17:01 +02:00
57 changed files with 3296 additions and 183 deletions
--- a/code/common/model/java/nu/marginalia/model/DocumentFormat.java
+++ b/code/common/model/java/nu/marginalia/model/DocumentFormat.java
@@ -0,0 +1,24 @@
+package nu.marginalia.model;
+
+public enum DocumentFormat {
+    PLAIN(0, 1, "text"),
+    PDF(0, 1, "pdf"),
+    UNKNOWN(0, 1, "???"),
+    HTML123(0, 1, "html"),
+    HTML4(-0.1, 1.05, "html"),
+    XHTML(-0.1, 1.05, "html"),
+    HTML5(0.5, 1.1, "html");
+
+    /** Used to tune quality score */
+    public final double offset;
+    /** Used to tune quality score */
+    public final double scale;
+    public final String shortFormat;
+
+    DocumentFormat(double offset, double scale, String shortFormat) {
+        this.offset = offset;
+        this.scale = scale;
+        this.shortFormat = shortFormat;
+    }
+
+}
--- a/code/common/model/java/nu/marginalia/model/crawl/HtmlFeature.java
+++ b/code/common/model/java/nu/marginalia/model/crawl/HtmlFeature.java
@@ -28,6 +28,8 @@ public enum HtmlFeature {

    GA_SPAM("special:gaspam"),

+    PDF("format:pdf"),
+
    /** For fingerprinting and ranking */
    OPENGRAPH("special:opengraph"),
    OPENGRAPH_IMAGE("special:opengraph:image"),
--- a/code/common/model/java/nu/marginalia/model/html/HtmlStandard.java
+++ b/code/common/model/java/nu/marginalia/model/html/HtmlStandard.java
@@ -1,22 +0,0 @@
-package nu.marginalia.model.html;
-
-// This class really doesn't belong anywhere, but will squat here for now
-public enum HtmlStandard {
-    PLAIN(0, 1),
-    UNKNOWN(0, 1),
-    HTML123(0, 1),
-    HTML4(-0.1, 1.05),
-    XHTML(-0.1, 1.05),
-    HTML5(0.5, 1.1);
-
-    /** Used to tune quality score */
-    public final double offset;
-    /** Used to tune quality score */
-    public final double scale;
-
-    HtmlStandard(double offset, double scale) {
-        this.offset = offset;
-        this.scale = scale;
-    }
-
-}
--- a/code/common/model/java/nu/marginalia/model/idx/DocumentFlags.java
+++ b/code/common/model/java/nu/marginalia/model/idx/DocumentFlags.java
@@ -9,7 +9,7 @@ public enum DocumentFlags {
    GeneratorForum,
    GeneratorWiki,
    Sideloaded,
-    Unused7,
+    PdfFile,
    Unused8,
    ;

--- a/code/common/service/resources/log4j2-json.xml
+++ b/code/common/service/resources/log4j2-json.xml
@@ -3,11 +3,18 @@
        <Console name="Console" target="SYSTEM_OUT">
            <PatternLayout pattern="%d{HH:mm:ss,SSS} %style{%-8markerSimpleName}{FG_Cyan} %highlight{%-5level}{FATAL=red, ERROR=red, WARN=yellow} %-24t %-20c{1}  --  %msg%n"/>
            <Filters>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
            </Filters>
        </Console>
+        <Console name="ProcessConsole" target="SYSTEM_OUT">
+            <PatternLayout pattern="%style{P}{FG_Cyan} %msg%n"/>
+            <Filters>
+                <MarkerFilter marker="PROCESS" onMatch="ALLOW" onMismatch="DENY" />
+            </Filters>
+        </Console>
        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
                     ignoreExceptions="false">
            <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true" includeTimeMillis="true"/>
@@ -15,6 +22,7 @@
                <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
            </Filters>
            <SizeBasedTriggeringPolicy size="10MB" />
        </RollingFile>
@@ -31,9 +39,11 @@
    </Appenders>
    <Loggers>
        <Logger name="org.apache.zookeeper" level="WARN" />
-
+        <Logger name="org.apache.pdfbox" level="ERROR" />
+        <Logger name="org.apache.fontbox.ttf" level="ERROR" />
        <Root level="info">
            <AppenderRef ref="Console"/>
+            <AppenderRef ref="ProcessConsole"/>
            <AppenderRef ref="LogToFile"/>
        </Root>
    </Loggers>
--- a/code/common/service/resources/log4j2-prod.xml
+++ b/code/common/service/resources/log4j2-prod.xml
@@ -1,13 +1,51 @@
 <Configuration xmlns="http://logging.apache.org/log4j/2.0/config" >
    <Appenders>
-        <Console name="Console" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{HH:mm:ss,SSS} %style{%-8markerSimpleName}{FG_Cyan} %highlight{%-5level}{FATAL=red, ERROR=red, WARN=yellow} %-24t %-20c{1}  --  %msg%n"/>
+        <Console name="ConsoleInfo" target="SYSTEM_OUT">
+            <PatternLayout pattern="- %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
            <Filters>
+                <LevelMatchFilter level="INFO" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
            </Filters>
        </Console>
+        <Console name="ConsoleWarn" target="SYSTEM_OUT">
+            <PatternLayout pattern="⚠ %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="WARN" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ConsoleError" target="SYSTEM_OUT">
+            <PatternLayout pattern="🔥 %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="ERROR" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ConsoleFatal" target="SYSTEM_OUT">
+            <PatternLayout pattern="💀 %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="FATAL" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="QUERY" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
+                <MarkerFilter marker="CRAWLER" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ProcessConsole" target="SYSTEM_OUT">
+            <PatternLayout pattern="%style{%msg%n}{FG_Cyan}"/>
+            <Filters>
+                <MarkerFilter marker="PROCESS" onMatch="ALLOW" onMismatch="DENY" />
+            </Filters>
+        </Console>
        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:-/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
                     ignoreExceptions="false">
            <PatternLayout>
@@ -34,9 +72,14 @@
    </Appenders>
    <Loggers>
        <Logger name="org.apache.zookeeper" level="WARN" />
-
+        <Logger name="org.apache.pdfbox" level="ERROR" />
+        <Logger name="org.apache.fontbox.ttf" level="ERROR" />
        <Root level="info">
-            <AppenderRef ref="Console"/>
+            <AppenderRef ref="ConsoleInfo"/>
+            <AppenderRef ref="ConsoleWarn"/>
+            <AppenderRef ref="ConsoleError"/>
+            <AppenderRef ref="ConsoleFatal"/>
+            <AppenderRef ref="ProcessConsole"/>
            <AppenderRef ref="LogToFile"/>
        </Root>
    </Loggers>
--- a/code/common/service/resources/log4j2-test.xml
+++ b/code/common/service/resources/log4j2-test.xml
@@ -1,15 +1,50 @@
 <Configuration xmlns="http://logging.apache.org/log4j/2.0/config" >
    <Appenders>
-        <Console name="Console" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{HH:mm:ss,SSS} %style{%-8markerSimpleName}{FG_Cyan} %highlight{%-5level}{FATAL=red, ERROR=red, WARN=yellow} %-24t %-20c{1}  --  %msg%n"/>
+        <Console name="ConsoleInfo" target="SYSTEM_OUT">
+            <PatternLayout pattern="- %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="INFO" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ConsoleWarn" target="SYSTEM_OUT">
+            <PatternLayout pattern="⚠ %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="WARN" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ConsoleError" target="SYSTEM_OUT">
+            <PatternLayout pattern="🔥 %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="ERROR" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ConsoleFatal" target="SYSTEM_OUT">
+            <PatternLayout pattern="💀 %d{HH:mm:ss,SSS} %-20c{1} -- %msg%n"/>
+            <Filters>
+                <LevelMatchFilter level="FATAL" onMatch="ALLOW" onMismatch="DENY"/>
+                <MarkerFilter marker="PROCESS" onMatch="DENY" onMismatch="NEUTRAL" />
+            </Filters>
+        </Console>
+        <Console name="ProcessConsole" target="SYSTEM_OUT">
+            <PatternLayout pattern="%style{%msg%n}{FG_Cyan}"/>
+            <Filters>
+                <MarkerFilter marker="PROCESS" onMatch="ALLOW" onMismatch="DENY" />
+            </Filters>
        </Console>
    </Appenders>
    <Loggers>
        <Logger name="org.apache.zookeeper" level="WARN" />
-
+        <Logger name="org.apache.pdfbox" level="ERROR" />
+        <Logger name="org.apache.fontbox.ttf" level="ERROR" />
        <Root level="info">
-            <AppenderRef ref="Console"/>
-            <AppenderRef ref="LogToFile"/>
+            <AppenderRef ref="ConsoleInfo"/>
+            <AppenderRef ref="ConsoleWarn"/>
+            <AppenderRef ref="ConsoleError"/>
+            <AppenderRef ref="ConsoleFatal"/>
+            <AppenderRef ref="ProcessConsole"/>
        </Root>
    </Loggers>
 </Configuration>
--- a/code/functions/live-capture/java/nu/marginalia/rss/svc/SimpleFeedParser.java
+++ b/code/functions/live-capture/java/nu/marginalia/rss/svc/SimpleFeedParser.java
@@ -79,9 +79,17 @@ public class SimpleFeedParser {
                if (!link.isBlank())
                    break;
                var tag = element.getElementsByTag(attr).first();
+
                if (tag != null) {
-                    link = tag.text();
+                    String linkText = tag.text();
+
+                    if (linkText.isBlank()) {
+                        linkText = tag.attr("href");
+                    }
+
+                    link = linkText;
                }
+
            }

            ret.add(new ItemData(title, description, link, pubDate));
--- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java
+++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java
@@ -1,6 +1,7 @@
 package nu.marginalia.api.searchquery.model.results;

 import nu.marginalia.api.searchquery.model.results.debug.ResultRankingDetails;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import org.jetbrains.annotations.NotNull;

@@ -161,4 +162,14 @@ public class DecoratedSearchResultItem implements Comparable<DecoratedSearchResu
    public String toString() {
        return "DecoratedSearchResultItem(rawIndexResult=" + this.getRawIndexResult() + ", url=" + this.getUrl() + ", title=" + this.getTitle() + ", description=" + this.getDescription() + ", urlQuality=" + this.getUrlQuality() + ", format=" + this.getFormat() + ", features=" + this.getFeatures() + ", pubYear=" + this.getPubYear() + ", dataHash=" + this.getDataHash() + ", wordsTotal=" + this.getWordsTotal() + ", bestPositions=" + this.getBestPositions() + ", rankingScore=" + this.getRankingScore() + ", resultsFromDomain=" + this.getResultsFromDomain() + ", rankingDetails=" + this.getRankingDetails() + ")";
    }
+
+    public String getShortFormat() {
+        try {
+            var df = DocumentFormat.valueOf(format);
+            return df.shortFormat;
+        }
+        catch (IllegalArgumentException e) {
+            return DocumentFormat.UNKNOWN.shortFormat;
+        }
+    }
 }
--- a/code/processes/converting-process/build.gradle
+++ b/code/processes/converting-process/build.gradle
@@ -62,6 +62,7 @@ dependencies {
    implementation libs.jwarc

    implementation libs.jsoup
+    implementation libs.pdfbox

    implementation libs.guava
    implementation dependencies.create(libs.guice.get()) {
--- a/code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java
@@ -1,8 +1,8 @@
 package nu.marginalia.converting.model;

+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.HtmlFeature;
-import nu.marginalia.model.html.HtmlStandard;
 import nu.marginalia.model.idx.DocumentMetadata;

 import javax.annotation.Nullable;
@@ -21,7 +21,7 @@ public class ProcessedDocumentDetails {
    public long hashCode;

    public Set<HtmlFeature> features;
-    public HtmlStandard standard;
+    public DocumentFormat format;

    public List<EdgeUrl> linksInternal;
    public List<EdgeUrl> linksExternal;
@@ -30,6 +30,6 @@ public class ProcessedDocumentDetails {
    public GeneratorType generator;

    public String toString() {
-        return "ProcessedDocumentDetails(title=" + this.title + ", description=" + this.description + ", pubYear=" + this.pubYear + ", length=" + this.length + ", quality=" + this.quality + ", hashCode=" + this.hashCode + ", features=" + this.features + ", standard=" + this.standard + ", linksInternal=" + this.linksInternal + ", linksExternal=" + this.linksExternal + ", metadata=" + this.metadata + ", generator=" + this.generator + ")";
+        return "ProcessedDocumentDetails(title=" + this.title + ", description=" + this.description + ", pubYear=" + this.pubYear + ", length=" + this.length + ", quality=" + this.quality + ", hashCode=" + this.hashCode + ", features=" + this.features + ", standard=" + this.format + ", linksInternal=" + this.linksInternal + ", linksExternal=" + this.linksExternal + ", metadata=" + this.metadata + ", generator=" + this.generator + ")";
    }
 }
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentProcessor.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentProcessor.java
@@ -7,6 +7,7 @@ import nu.marginalia.converting.model.DisqualifiedException;
 import nu.marginalia.converting.model.ProcessedDocument;
 import nu.marginalia.converting.processor.plugin.AbstractDocumentProcessorPlugin;
 import nu.marginalia.converting.processor.plugin.HtmlDocumentProcessorPlugin;
+import nu.marginalia.converting.processor.plugin.PdfDocumentProcessorPlugin;
 import nu.marginalia.converting.processor.plugin.PlainTextDocumentProcessorPlugin;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.model.EdgeDomain;
@@ -33,7 +34,8 @@ public class DocumentProcessor {
    private static final Set<String> acceptedContentTypes = Set.of("application/xhtml+xml",
            "application/xhtml",
            "text/html",
-            "text/plain");
+            "text/plain",
+            "application/pdf");


    private final List<AbstractDocumentProcessorPlugin> processorPlugins = new ArrayList<>();
@@ -42,12 +44,14 @@ public class DocumentProcessor {
    @Inject
    public DocumentProcessor(HtmlDocumentProcessorPlugin htmlDocumentProcessorPlugin,
                             PlainTextDocumentProcessorPlugin plainTextDocumentProcessorPlugin,
+                             PdfDocumentProcessorPlugin pdfDocumentProcessorPlugin,
                             AnchorTextKeywords anchorTextKeywords)
    {
        this.anchorTextKeywords = anchorTextKeywords;

        processorPlugins.add(htmlDocumentProcessorPlugin);
        processorPlugins.add(plainTextDocumentProcessorPlugin);
+        processorPlugins.add(pdfDocumentProcessorPlugin);
    }

    public ProcessedDocument process(CrawledDocument crawledDocument,
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentValuator.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentValuator.java
@@ -2,9 +2,9 @@ package nu.marginalia.converting.processor.logic;

 import crawlercommons.utils.Strings;
 import nu.marginalia.converting.model.DisqualifiedException;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.crawl.HtmlFeature;
 import nu.marginalia.model.crawldata.CrawledDocument;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jetbrains.annotations.NotNull;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -17,7 +17,7 @@ import java.util.Set;
 public class DocumentValuator {

    public double getQuality(CrawledDocument crawledDocument,
-                             HtmlStandard htmlStandard,
+                             DocumentFormat htmlStandard,
                             Document parsedDocument,
                             int textLength) throws DisqualifiedException {

--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java
@@ -1,7 +1,7 @@
 package nu.marginalia.converting.processor.logic;

 import com.google.common.base.Strings;
-import nu.marginalia.model.html.HtmlStandard;
+import nu.marginalia.model.DocumentFormat;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.DocumentType;
 import org.slf4j.Logger;
@@ -12,54 +12,54 @@ public class HtmlStandardExtractor {

    private static final Logger logger = LoggerFactory.getLogger(HtmlStandardExtractor.class);

-    public static HtmlStandard parseDocType(DocumentType docType) {
+    public static DocumentFormat parseDocType(DocumentType docType) {
        if (null == docType) {
-            return HtmlStandard.UNKNOWN;
+            return DocumentFormat.UNKNOWN;
        }

        String publicId = docType.publicId();
        if (Strings.isNullOrEmpty(publicId))
-            return HtmlStandard.HTML5;
+            return DocumentFormat.HTML5;

        publicId = publicId.toUpperCase();
        if (publicId.startsWith("-//SOFTQUAD SOFTWARE//DTD") && publicId.contains("HTML 4")) {
-            return HtmlStandard.HTML4;
+            return DocumentFormat.HTML4;
        }
        if (publicId.startsWith("-//SOFTQUAD SOFTWARE//DTD") && publicId.contains("HTML 3")) {
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        }
        if (publicId.startsWith("-//INTERNET/RFC XXXX//EN"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//NETSCAPE COMM. CORP"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//SQ//DTD HTML 2"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//SOFTQUAD//DTD HTML 2"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//W3O//DTD W3 HTML 2"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//IETF//DTD HTML 2"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//IETF//DTD HTML//EN"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-/W3C//DTD HTML 3"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-/W3C/DTD HTML 3"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//IETF//DTD HTML 3"))
-            return HtmlStandard.HTML123;
+            return DocumentFormat.HTML123;
        if (publicId.startsWith("-//W3C//DTD XHTML"))
-            return HtmlStandard.XHTML;
+            return DocumentFormat.XHTML;
        if (publicId.startsWith("ISO/IEC 15445:2000//DTD"))
-            return HtmlStandard.XHTML;
+            return DocumentFormat.XHTML;
        if (publicId.startsWith("-//W3C//DTD HTML"))
-            return HtmlStandard.HTML4;
+            return DocumentFormat.HTML4;

        logger.debug("Unknown publicID standard {}", publicId);
-        return HtmlStandard.UNKNOWN;
+        return DocumentFormat.UNKNOWN;
    }

-    public static HtmlStandard sniffHtmlStandard(Document parsed) {
+    public static DocumentFormat sniffHtmlStandard(Document parsed) {
        int html4Attributes = 0;
        int html5Attributes = 0;

@@ -73,11 +73,11 @@ public class HtmlStandardExtractor {
            html4Attributes++;
        }
        if (html5Attributes > 0) {
-            return HtmlStandard.HTML5;
+            return DocumentFormat.HTML5;
        }
        if (html4Attributes > 0) {
-            return HtmlStandard.HTML4;
+            return DocumentFormat.HTML4;
        }
-        return HtmlStandard.HTML123;
+        return DocumentFormat.HTML123;
    }
 }
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java
@@ -7,11 +7,11 @@ import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
 import nu.marginalia.language.filter.LanguageFilter;
 import nu.marginalia.language.model.DocumentLanguageData;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.HtmlFeature;
 import nu.marginalia.model.crawl.PubDate;
 import nu.marginalia.model.crawldata.CrawledDocument;
-import nu.marginalia.model.html.HtmlStandard;

 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -73,7 +73,7 @@ public abstract class AbstractDocumentProcessorPlugin {
            return this;
        }

-        public MetaTagsBuilder addFormat(HtmlStandard standard) {
+        public MetaTagsBuilder addFormat(DocumentFormat standard) {

            add("format", standard);

--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
@@ -25,12 +25,12 @@ import nu.marginalia.language.model.DocumentLanguageData;
 import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
 import nu.marginalia.link_parser.FeedExtractor;
 import nu.marginalia.link_parser.LinkParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.HtmlFeature;
 import nu.marginalia.model.crawl.PubDate;
 import nu.marginalia.model.crawldata.CrawledDocument;
-import nu.marginalia.model.html.HtmlStandard;
 import nu.marginalia.model.idx.DocumentFlags;
 import nu.marginalia.model.idx.DocumentMetadata;
 import org.jsoup.nodes.Document;
@@ -137,8 +137,8 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin


        final int length = getLength(doc);
-        final HtmlStandard standard = getHtmlStandard(doc);
-        final double quality = documentValuator.getQuality(crawledDocument, standard, doc, length);
+        final DocumentFormat format = getDocumentFormat(doc);
+        final double quality = documentValuator.getQuality(crawledDocument, format, doc, length);

        if (isDisqualified(documentClass, url, quality, doc.title())) {
            throw new DisqualifiedException(DisqualificationReason.QUALITY);
@@ -152,7 +152,7 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
        var ret = new ProcessedDocumentDetails();

        ret.length = length;
-        ret.standard = standard;
+        ret.format = format;
        ret.title = specialization.getTitle(doc, dld, crawledDocument.url);

        final Set<HtmlFeature> features = featureExtractor.getFeatures(url, doc, documentHeaders, dld);
@@ -161,7 +161,7 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
        ret.quality = documentValuator.adjustQuality(quality, features);
        ret.hashCode = dld.localitySensitiveHashCode();

-        PubDate pubDate = pubDateSniffer.getPubDate(documentHeaders, url, doc, standard, true);
+        PubDate pubDate = pubDateSniffer.getPubDate(documentHeaders, url, doc, format, true);

        EnumSet<DocumentFlags> documentFlags = documentFlags(features, generatorParts.type());

@@ -180,7 +180,7 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
                .addPubDate(pubDate)
                .addUrl(url)
                .addFeatures(features)
-                .addFormat(standard)
+                .addFormat(format)
                .addGenerator(generatorParts.keywords())
                .build();

@@ -316,12 +316,12 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
        return linkTerms;
    }

-    private HtmlStandard getHtmlStandard(Document doc) {
-        HtmlStandard htmlStandard = HtmlStandardExtractor.parseDocType(doc.documentType());
-        if (HtmlStandard.UNKNOWN.equals(htmlStandard)) {
+    private DocumentFormat getDocumentFormat(Document doc) {
+        DocumentFormat format = HtmlStandardExtractor.parseDocType(doc.documentType());
+        if (DocumentFormat.UNKNOWN.equals(format)) {
            return HtmlStandardExtractor.sniffHtmlStandard(doc);
        }
-        return htmlStandard;
+        return format;
    }

    private int getLength(Document doc) {
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPlugin.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPlugin.java
@@ -0,0 +1,286 @@
+package nu.marginalia.converting.processor.plugin;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import nu.marginalia.converting.model.DisqualifiedException;
+import nu.marginalia.converting.model.ProcessedDocumentDetails;
+import nu.marginalia.converting.processor.DocumentClass;
+import nu.marginalia.converting.processor.logic.DocumentLengthLogic;
+import nu.marginalia.converting.processor.plugin.specialization.DefaultSpecialization;
+import nu.marginalia.keyword.DocumentKeywordExtractor;
+import nu.marginalia.keyword.LinkTexts;
+import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
+import nu.marginalia.language.filter.LanguageFilter;
+import nu.marginalia.language.model.DocumentLanguageData;
+import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
+import nu.marginalia.model.DocumentFormat;
+import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.model.crawl.HtmlFeature;
+import nu.marginalia.model.crawl.PubDate;
+import nu.marginalia.model.crawldata.CrawledDocument;
+import nu.marginalia.model.idx.DocumentFlags;
+import nu.marginalia.model.idx.DocumentMetadata;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.text.HeadingAwarePDFTextStripper;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.LocalDate;
+import java.util.*;
+
+
+public class PdfDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin {
+
+    private final int maxTitleLength;
+    private final DocumentKeywordExtractor keywordExtractor;
+    private final ThreadLocalSentenceExtractorProvider sentenceExtractorProvider;
+    private final DocumentLengthLogic documentLengthLogic;
+    private final DefaultSpecialization defaultSpecialization;
+
+    private static final Logger logger = LoggerFactory.getLogger(PdfDocumentProcessorPlugin.class);
+
+    @Inject
+    public PdfDocumentProcessorPlugin(@Named("max-title-length") Integer maxTitleLength,
+                                      LanguageFilter languageFilter,
+                                      ThreadLocalSentenceExtractorProvider sentenceExtractorProvider,
+                                      DocumentKeywordExtractor keywordExtractor,
+                                      DocumentLengthLogic documentLengthLogic,
+                                      DefaultSpecialization defaultSpecialization)
+
+    {
+        super(languageFilter);
+        this.sentenceExtractorProvider = sentenceExtractorProvider;
+        this.documentLengthLogic = documentLengthLogic;
+        this.maxTitleLength = maxTitleLength;
+        this.keywordExtractor = keywordExtractor;
+        this.defaultSpecialization = defaultSpecialization;
+    }
+
+    @Override
+    public boolean isApplicable(CrawledDocument doc) {
+        String contentType = doc.contentType.toLowerCase();
+
+        if (contentType.equals("application/pdf"))
+            return true;
+        if (contentType.startsWith("application/pdf;")) // charset=blabla
+            return true;
+
+        return false;
+    }
+
+    @Override
+    public DetailsWithWords createDetails(CrawledDocument crawledDocument,
+                                          LinkTexts linkTexts,
+                                          DocumentClass documentClass)
+            throws DisqualifiedException, URISyntaxException, IOException {
+
+        String documentBody = crawledDocument.documentBody();
+
+        if (languageFilter.isBlockedUnicodeRange(documentBody)) {
+            throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.LANGUAGE);
+        }
+
+        final EdgeUrl url = new EdgeUrl(crawledDocument.url);
+
+
+        Document doc;
+        try {
+            doc = convertPdfToHtml(crawledDocument.documentBodyBytes);
+        } catch (IOException e) {
+            logger.error("Failed to convert PDF file {} - {}", url, e.getMessage());
+            throw new DisqualifiedException(DisqualifiedException.DisqualificationReason.ERROR);
+        }
+
+        DocumentLanguageData dld = sentenceExtractorProvider.get().extractSentences(doc);
+
+        checkDocumentLanguage(dld);
+
+        documentLengthLogic.validateLength(dld, 1.0);
+
+        var ret = new ProcessedDocumentDetails();
+
+        ret.length = documentBody.length();
+
+        ret.format = DocumentFormat.PDF;
+        ret.title = StringUtils.truncate(defaultSpecialization.getTitle(doc, dld, url.toString()), maxTitleLength);
+
+        ret.quality = -5;
+
+        ret.features = Set.of(HtmlFeature.PDF);
+        ret.description = getDescription(doc);
+        ret.hashCode = dld.localitySensitiveHashCode();
+
+        final PubDate pubDate = new PubDate(LocalDate.ofYearDay(1993, 1));
+
+        EnumSet<DocumentFlags> documentFlags = EnumSet.of(DocumentFlags.PdfFile);
+
+        ret.metadata = new DocumentMetadata(
+                documentLengthLogic.getEncodedAverageLength(dld),
+                pubDate.yearByte(),
+                (int) -ret.quality,
+                documentFlags);
+
+        DocumentKeywordsBuilder words = keywordExtractor.extractKeywords(dld, linkTexts, url);
+
+        var tagWords = new MetaTagsBuilder()
+                .addPubDate(pubDate)
+                .addUrl(url)
+                .addFeatures(ret.features)
+                .addFormat(ret.format)
+                .build();
+
+        words.addAllSyntheticTerms(tagWords);
+
+        if (pubDate.hasYear()) {
+            ret.pubYear = pubDate.year();
+        }
+
+        /* These are assumed to be populated */
+        ret.linksInternal = new ArrayList<>();
+        ret.linksExternal = new ArrayList<>();
+
+        return new DetailsWithWords(ret, words);
+    }
+
+    private String getDescription(Document doc) {
+        int cnt = 0;
+        boolean useNext = false;
+        for (var ptag : doc.getElementsByTag("p")) {
+            String text = ptag.text();
+
+            // Many academic documents have an abstract at the start of the document,
+            // which makes a nice summary.  Though they tend to bleed into the text,
+            // so we check for the word "Abstract" at the start of the paragraph.
+
+            if (text.startsWith("Abstract ")) {
+                return StringUtils.abbreviate(text.substring("Abstract ".length()), "...", 255);
+            }
+            else if (text.equals("Abstract")) {
+                useNext = true;
+            }
+            else if (useNext) {
+                return StringUtils.abbreviate(text, "...", 255);
+            }
+
+            if (++cnt > 15) { // Don't scan the entire document
+                break;
+            }
+        }
+
+        // Fall back to the default specialization
+        return defaultSpecialization.getSummary(doc, Set.of());
+
+    }
+
+    /** Convert the provided PDF bytes into a HTML rendering that can be fed
+     * to the HTML processor.
+     */
+    Document convertPdfToHtml(byte[] pdfBytes) throws IOException {
+        try (var doc = Loader.loadPDF(pdfBytes)) {
+            String docMetaTitle = Objects.requireNonNullElse(doc.getDocumentInformation().getTitle(), "");
+
+            var stripper = new HeadingAwarePDFTextStripper();
+            stripper.setStartPage(1);
+            stripper.setSortByPosition(true);
+            stripper.setWordSeparator(" ");
+
+            // Increase the tolerance for line spacing to deal better with paragraphs.
+            stripper.setDropThreshold(5f);
+
+            stripper.setPageStart("<div>");
+            stripper.setParagraphStart("<p>");
+            stripper.setParagraphEnd("</p>\n");
+            stripper.setPageEnd("</div>\n");
+            stripper.setHeadingStart("<h1>");
+            stripper.setHeadingEnd("</h1>\n");
+            stripper.setLineSeparator("\n");
+
+            String text = stripper.getText(doc);
+
+            StringBuilder htmlBuilder = new StringBuilder(text.length() + 1024);
+            htmlBuilder.append("<html><body>")
+                    .append(text)
+                    .append("</body></html>");
+
+            var parsed = Jsoup.parse(htmlBuilder.toString());
+
+            repairDOM(parsed);
+
+            for (var heading : parsed.getElementsByTag("h1")) {
+                String headingText = heading.text();
+                if (headingText.length() > 2) {
+                    parsed.title(headingText);
+                    break;
+                }
+            }
+
+
+            if (parsed.title().isEmpty()) {
+                // Prefer setting the title to the first paragraph in the
+                // document, as this is almost always correct.  Otherwise,
+                // we fall back on the metadata title, which is almost always
+                // useless
+
+                var firstP = parsed.getElementsByTag("p").first();
+                if (firstP != null) parsed.title(firstP.text());
+                else parsed.title(docMetaTitle);
+            }
+            return parsed;
+        }
+
+
+    }
+
+    /** Repair the DOM to remove some common issues with PDF conversion,
+     * including empty paragraphs, and multiline headers that are split into multiple
+     * conescutive h1 tags.
+     */
+    private void repairDOM(Document parsed) {
+
+        // <p><h1>...</h1></p> -> <h1>...</h1>
+        parsed.getElementsByTag("h1").forEach(h1 -> {
+            var parent = h1.parent();
+            if (parent == null || !"p".equals(parent.tagName())) {
+                return;
+            }
+
+            if (parent.childrenSize() == 1) {
+                parent.replaceWith(h1);
+            }
+        });
+
+        // Remove empty <p> tags
+        parsed.getElementsByTag("p").forEach(p -> {
+            if (p.childrenSize() == 0 && !p.hasText()) {
+                p.remove();
+            }
+        });
+
+        // <h1>...</h1><h1>...</h1> -> <h1>...</h1>
+        parsed.getElementsByTag("h1").forEach(h1 -> {
+            var nextSibling = h1.nextElementSibling();
+            if (nextSibling == null || !"h1".equals(nextSibling.tagName())) {
+                return; // Short-circuit to avoid unnecessary work
+            }
+
+            StringJoiner joiner = new StringJoiner(" ");
+            joiner.add(h1.text());
+
+            for (var sibling : h1.nextElementSiblings()) {
+                if (!"h1".equals(sibling.tagName()))
+                    break;
+                joiner.add(sibling.text());
+                sibling.remove();
+            }
+
+            h1.text(joiner.toString());
+        });
+
+    }
+
+}
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java
@@ -13,10 +13,10 @@ import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
 import nu.marginalia.language.filter.LanguageFilter;
 import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
 import nu.marginalia.model.crawldata.CrawledDocument;
-import nu.marginalia.model.html.HtmlStandard;
 import nu.marginalia.model.idx.DocumentFlags;
 import nu.marginalia.model.idx.DocumentMetadata;
 import org.apache.commons.lang3.StringUtils;
@@ -91,7 +91,7 @@ public class PlainTextDocumentProcessorPlugin extends AbstractDocumentProcessorP

        ret.length = documentBody.length();

-        ret.standard = HtmlStandard.PLAIN;
+        ret.format = DocumentFormat.PLAIN;
        ret.title = StringUtils.truncate(plainTextLogic.getTitle(url, firstFewLines), maxTitleLength);

        ret.quality = -1;
@@ -113,7 +113,7 @@ public class PlainTextDocumentProcessorPlugin extends AbstractDocumentProcessorP
                .addPubDate(pubDate)
                .addUrl(url)
                .addFeatures(ret.features)
-                .addFormat(ret.standard)
+                .addFormat(ret.format)
                .build();

        words.addAllSyntheticTerms(tagWords);
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateFromHtmlStandard.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateFromHtmlStandard.java
@@ -1,12 +1,13 @@
 package nu.marginalia.converting.processor.pubdate;

-import nu.marginalia.model.html.HtmlStandard;
+import nu.marginalia.model.DocumentFormat;

 public class PubDateFromHtmlStandard {
    /** Used to bias pub date heuristics */
-    public static int blindGuess(HtmlStandard standard) {
-        return switch (standard) {
+    public static int blindGuess(DocumentFormat format) {
+        return switch (format) {
            case PLAIN -> 1993;
+            case PDF -> 2010;
            case HTML123 -> 1997;
            case HTML4, XHTML -> 2006;
            case HTML5 -> 2018;
@@ -21,8 +22,8 @@ public class PubDateFromHtmlStandard {
     * Discovering publication year involves a lot of guesswork, this helps
     * keep the guesses relatively sane.
     */
-    public static boolean isGuessPlausible(HtmlStandard standard, int year) {
-        switch (standard) {
+    public static boolean isGuessPlausible(DocumentFormat format, int year) {
+        switch (format) {
            case HTML123:
                return year <= 2000;
            case XHTML:
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateHeuristic.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateHeuristic.java
@@ -1,14 +1,14 @@
 package nu.marginalia.converting.processor.pubdate;

 import nu.marginalia.converting.model.DocumentHeaders;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;

 public interface PubDateHeuristic {

-    Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard);
+    Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard);
 }
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateParser.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateParser.java
@@ -1,7 +1,7 @@
 package nu.marginalia.converting.processor.pubdate;

+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;

 import java.time.DateTimeException;
 import java.time.LocalDate;
@@ -26,7 +26,7 @@ public class PubDateParser {
                .filter(PubDateParser::validateDate);
    }

-    public static Optional<PubDate> attemptParseDate(String date, HtmlStandard standard) {
+    public static Optional<PubDate> attemptParseDate(String date, DocumentFormat standard) {
        return Optional.ofNullable(date)
                .filter(str -> str.length() >= 4 && str.length() < 32)
                .flatMap(str ->
@@ -81,7 +81,7 @@ public class PubDateParser {
    }


-    public static Optional<PubDate> dateFromHighestYearLookingSubstringWithGuess(String maybe, HtmlStandard standard) {
+    public static Optional<PubDate> dateFromHighestYearLookingSubstringWithGuess(String maybe, DocumentFormat standard) {
        int guess = PubDateFromHtmlStandard.blindGuess(standard);

        var matcher = yearPattern.matcher(maybe);
@@ -135,7 +135,7 @@ public class PubDateParser {
        return (max + min) / 2;
    }

-    public static int guessYear(HtmlStandard standard) {
+    public static int guessYear(DocumentFormat standard) {
        // Create some jitter to avoid having documents piling up in the same four years
        // as this would make searching in those years disproportionately useless

--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateSniffer.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/PubDateSniffer.java
@@ -2,9 +2,9 @@ package nu.marginalia.converting.processor.pubdate;

 import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.heuristic.*;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.ArrayList;
@@ -38,7 +38,7 @@ public class PubDateSniffer {
        heuristics.add(new PubDateHeuristicGuessFromHtmlStandard());
    }

-    public PubDate getPubDate(DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard, boolean runExpensive) {
+    public PubDate getPubDate(DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard, boolean runExpensive) {
        final PubDateEffortLevel effortLevel = runExpensive ? PubDateEffortLevel.HIGH : PubDateEffortLevel.LOW;

        for (var heuristic : heuristics) {
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jetbrains.annotations.NotNull;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -19,7 +19,7 @@ import java.util.Optional;
 public class PubDateHeuristicDOMParsingPass1 implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        if (effortLevel == PubDateEffortLevel.LOW)
            return Optional.empty();

@@ -33,9 +33,9 @@ public class PubDateHeuristicDOMParsingPass1 implements PubDateHeuristic {

    private static class DateExtractingNodeVisitorPass implements NodeFilter {
        public PubDate pubDate;
-        private final HtmlStandard htmlStandard;
+        private final DocumentFormat htmlStandard;

-        private DateExtractingNodeVisitorPass(HtmlStandard htmlStandard) {
+        private DateExtractingNodeVisitorPass(DocumentFormat htmlStandard) {
            this.htmlStandard = htmlStandard;
        }

@@ -135,7 +135,7 @@ public class PubDateHeuristicDOMParsingPass1 implements PubDateHeuristic {
        }

        private void parse(String text) {
-            if (htmlStandard == HtmlStandard.UNKNOWN) {
+            if (htmlStandard == DocumentFormat.UNKNOWN) {
                PubDateParser
                        .dateFromHighestYearLookingSubstring(text)
                        .ifPresent(this::setPubDate);
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java
@@ -5,9 +5,9 @@ import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateFromHtmlStandard;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jetbrains.annotations.NotNull;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Node;
@@ -19,7 +19,7 @@ import java.util.Optional;
 public class PubDateHeuristicDOMParsingPass2 implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        if (effortLevel == PubDateEffortLevel.LOW)
            return Optional.empty();

@@ -33,9 +33,9 @@ public class PubDateHeuristicDOMParsingPass2 implements PubDateHeuristic {

    private static class DateExtractingNodeVisitor implements NodeFilter {
        public PubDate pubDate;
-        private final HtmlStandard htmlStandard;
+        private final DocumentFormat htmlStandard;

-        private DateExtractingNodeVisitor(HtmlStandard htmlStandard) {
+        private DateExtractingNodeVisitor(DocumentFormat htmlStandard) {
            this.htmlStandard = htmlStandard;
        }

@@ -73,7 +73,7 @@ public class PubDateHeuristicDOMParsingPass2 implements PubDateHeuristic {
        }

        private void parse(String text) {
-            if (htmlStandard == HtmlStandard.UNKNOWN) {
+            if (htmlStandard == DocumentFormat.UNKNOWN) {
                PubDateParser
                        .dateFromHighestYearLookingSubstring(text)
                        .ifPresent(this::setPubDate);
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,8 +14,8 @@ import java.util.Optional;
 public class PubDateHeuristicGuessFromHtmlStandard implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
-        if (htmlStandard == HtmlStandard.UNKNOWN)
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
+        if (htmlStandard == DocumentFormat.UNKNOWN)
            return Optional.empty();

        return Optional.of(new PubDate(null, PubDateParser.guessYear(htmlStandard)));
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,7 +14,7 @@ import java.util.Optional;
 public class PubDateHeuristicHtml5AnyTimeTag implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        // HTML5, alternative approach
        for (var tag : document.select("time")) {
            var maybeDate = PubDateParser.attemptParseDate(tag.attr("datetime"));
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,7 +14,7 @@ import java.util.Optional;
 public class PubDateHeuristicHtml5ArticleDateTag implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        // HTML5
        for (var tag : document.select("time[pubdate=\"pubdate\"]")) {
            var maybeDate = PubDateParser.attemptParseDate(tag.attr("datetime"));
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,7 +14,7 @@ import java.util.Optional;
 public class PubDateHeuristicHtml5ItempropDateTag implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        for (var tag : document.select("time[itemprop=\"datePublished\"]")) {
            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
            if (maybeDate.isPresent()) {
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicJSONLD.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicJSONLD.java
@@ -8,9 +8,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Collections;
@@ -21,7 +21,7 @@ import java.util.Optional;
 public class PubDateHeuristicJSONLD implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        for (var tag : document.select("script[type=\"application/ld+json\"]")) {
            var maybeDate = parseLdJson(tag.data())
                    .flatMap(PubDateParser::attemptParseDate);
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicLastModified.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicLastModified.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.List;
@@ -15,7 +15,7 @@ import java.util.Optional;
 public class PubDateHeuristicLastModified implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        List<String> lastModified = headers.get("last-modified");
        if (lastModified.isEmpty())
            return Optional.empty();
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicMicrodata.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicMicrodata.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,7 +14,7 @@ import java.util.Optional;
 public class PubDateHeuristicMicrodata implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {

        for (var tag : document.select("meta[itemprop=\"datePublished\"]")) {
            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicOpenGraph.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicOpenGraph.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,7 +14,7 @@ import java.util.Optional;
 public class PubDateHeuristicOpenGraph implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        // OG
        for (var tag : document.select("meta[property=\"article:published_time\"]")) {
            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicRDFaTag.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicRDFaTag.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -14,7 +14,7 @@ import java.util.Optional;
 public class PubDateHeuristicRDFaTag implements PubDateHeuristic {

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        for (var tag : document.select("meta[property=\"datePublished\"]")) {
            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
            if (maybeDate.isPresent()) {
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -21,7 +21,7 @@ public class PubDateHeuristicUrlPatternPass1 implements PubDateHeuristic {
    private static final int MIN_URL_PATTERN_YEAR = 2000;

    @Override
-    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, HtmlStandard htmlStandard) {
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url, Document document, DocumentFormat htmlStandard) {
        final String urlString = url.path;

        var matcher = yearUrlPattern.matcher(urlString);
--- a/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/processor/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java
@@ -4,9 +4,9 @@ import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.PubDateEffortLevel;
 import nu.marginalia.converting.processor.pubdate.PubDateHeuristic;
 import nu.marginalia.converting.processor.pubdate.PubDateParser;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.PubDate;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.nodes.Document;

 import java.util.Optional;
@@ -19,7 +19,7 @@ public class PubDateHeuristicUrlPatternPass2 implements PubDateHeuristic {

    @Override
    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, DocumentHeaders headers, EdgeUrl url,
-                                   Document document, HtmlStandard htmlStandard) {
+                                   Document document, DocumentFormat htmlStandard) {
        final String urlString = url.path;

        var matcher = yearUrlPattern.matcher(urlString);
--- a/code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloaderProcessing.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloaderProcessing.java
@@ -8,12 +8,12 @@ import nu.marginalia.converting.model.ProcessedDocument;
 import nu.marginalia.converting.processor.DocumentClass;
 import nu.marginalia.converting.processor.plugin.HtmlDocumentProcessorPlugin;
 import nu.marginalia.keyword.LinkTexts;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.HtmlFeature;
 import nu.marginalia.model.crawl.PubDate;
 import nu.marginalia.model.crawl.UrlIndexingState;
 import nu.marginalia.model.crawldata.CrawledDocument;
-import nu.marginalia.model.html.HtmlStandard;
 import nu.marginalia.model.idx.DocumentFlags;
 import nu.marginalia.model.idx.DocumentMetadata;
 import nu.marginalia.model.idx.WordFlags;
@@ -83,7 +83,7 @@ public class SideloaderProcessing {
            // that we can't get from the sideloaded data since it's
            // so stripped down

-            ret.details.standard = HtmlStandard.HTML5;
+            ret.details.format = DocumentFormat.HTML5;
            ret.details.pubYear = pubYear;
            ret.details.features.add(HtmlFeature.JS);
            ret.details.features.add(HtmlFeature.TRACKING);
--- a/code/processes/converting-process/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java
@@ -9,13 +9,13 @@ import nu.marginalia.integration.stackexchange.sqlite.StackExchangePostsDb;
 import nu.marginalia.keyword.DocumentKeywordExtractor;
 import nu.marginalia.keyword.LinkTexts;
 import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.crawl.DomainIndexingState;
 import nu.marginalia.model.crawl.HtmlFeature;
 import nu.marginalia.model.crawl.PubDate;
 import nu.marginalia.model.crawl.UrlIndexingState;
-import nu.marginalia.model.html.HtmlStandard;
 import nu.marginalia.model.idx.DocumentFlags;
 import nu.marginalia.model.idx.DocumentMetadata;
 import nu.marginalia.model.idx.WordFlags;
@@ -165,7 +165,7 @@ public class StackexchangeSideloader implements SideloadSource {
            ret.details.description = StringUtils.truncate(doc.body().text(), 255);
            ret.details.length = 128;

-            ret.details.standard = HtmlStandard.HTML5;
+            ret.details.format = DocumentFormat.HTML5;
            ret.details.linksExternal = List.of();
            ret.details.linksInternal = List.of();
            ret.state = UrlIndexingState.OK;
--- a/code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWriter.java
+++ b/code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWriter.java
@@ -124,7 +124,7 @@ public class ConverterBatchWriter implements AutoCloseable, ConverterBatchWriter
                    document.details.title,
                    document.details.description,
                    HtmlFeature.encode(document.details.features),
-                    document.details.standard.name(),
+                    document.details.format.name(),
                    document.details.length,
                    document.details.hashCode,
                    (float) document.details.quality,
--- a/code/processes/converting-process/java/org/apache/pdfbox/text/HeadingAwarePDFTextStripper.java
+++ b/code/processes/converting-process/java/org/apache/pdfbox/text/HeadingAwarePDFTextStripper.java
--- a/code/processes/converting-process/test/nu/marginalia/converting/ConvertingIntegrationTest.java
+++ b/code/processes/converting-process/test/nu/marginalia/converting/ConvertingIntegrationTest.java
@@ -6,6 +6,7 @@ import com.google.inject.Injector;
 import nu.marginalia.converting.model.ProcessedDocument;
 import nu.marginalia.converting.processor.DomainProcessor;
 import nu.marginalia.io.SerializableCrawlDataStream;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.crawl.DomainIndexingState;
 import nu.marginalia.model.crawl.PubDate;
@@ -13,7 +14,6 @@ import nu.marginalia.model.crawl.UrlIndexingState;
 import nu.marginalia.model.crawldata.CrawledDocument;
 import nu.marginalia.model.crawldata.CrawledDomain;
 import nu.marginalia.model.crawldata.SerializableCrawlData;
-import nu.marginalia.model.html.HtmlStandard;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -91,7 +91,7 @@ public class ConvertingIntegrationTest {

            assertTrue(details.title.length() > 4);
            assertTrue(details.description.length() > 4);
-            assertEquals(HtmlStandard.HTML5, details.standard);
+            assertEquals(DocumentFormat.HTML5, details.format);

        }
    }
@@ -125,7 +125,7 @@ public class ConvertingIntegrationTest {
            assertTrue(details.metadata.size() > 0);
            assertTrue(details.title.length() > 4);
            assertTrue(details.description.length() > 4);
-            assertEquals(HtmlStandard.HTML5, details.standard);
+            assertEquals(DocumentFormat.HTML5, details.format);
        }
    }

--- a/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPluginTest.java
+++ b/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPluginTest.java
@@ -0,0 +1,95 @@
+package nu.marginalia.converting.processor.plugin;
+
+import nu.marginalia.WmsaHome;
+import nu.marginalia.converting.processor.DocumentClass;
+import nu.marginalia.converting.processor.logic.DocumentLengthLogic;
+import nu.marginalia.converting.processor.logic.TitleExtractor;
+import nu.marginalia.converting.processor.plugin.specialization.DefaultSpecialization;
+import nu.marginalia.converting.processor.summary.SummaryExtractor;
+import nu.marginalia.converting.processor.summary.heuristic.*;
+import nu.marginalia.keyword.DocumentKeywordExtractor;
+import nu.marginalia.keyword.LinkTexts;
+import nu.marginalia.language.filter.LanguageFilter;
+import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
+import nu.marginalia.model.crawldata.CrawledDocument;
+import nu.marginalia.term_frequency_dict.TermFrequencyDict;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+
+@Tag("flaky")
+class PdfDocumentProcessorPluginTest {
+    static PdfDocumentProcessorPlugin plugin;
+
+    @BeforeAll
+    static void setUpBeforeClass() throws Exception {
+        var lm = WmsaHome.getLanguageModels();
+        plugin = new PdfDocumentProcessorPlugin(255,
+                new LanguageFilter(lm),
+                new ThreadLocalSentenceExtractorProvider(lm),
+                new DocumentKeywordExtractor(new TermFrequencyDict(lm)),
+                new DocumentLengthLogic(100),
+                new DefaultSpecialization(new SummaryExtractor(
+                        255,
+                        new DomFilterHeuristic(255),
+                        new TagDensityHeuristic(255),
+                        new OpenGraphDescriptionHeuristic(),
+                        new MetaDescriptionHeuristic(),
+                        new FallbackHeuristic()
+                ),
+                        new TitleExtractor(255)
+                        ));
+    }
+    public AbstractDocumentProcessorPlugin.DetailsWithWords testPdfFile(byte[] pdfBytes) throws Exception {
+        var doc = new CrawledDocument("test", "https://www.example.com/sample.pdf", "application/pdf", Instant.now().toString(), 200, "OK", "OK", "", pdfBytes, false, null, null);
+        return plugin.createDetails(doc, new LinkTexts(), DocumentClass.NORMAL);
+    }
+
+    public AbstractDocumentProcessorPlugin.DetailsWithWords testPdfFile(Path file) throws Exception {
+        return testPdfFile(Files.readAllBytes(file));
+    }
+
+    private byte[] downloadPDF(String url) throws IOException, URISyntaxException {
+        HttpURLConnection conn = (HttpURLConnection) new URI(url).toURL().openConnection();
+        try {
+            return conn.getInputStream().readAllBytes();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        } finally {
+            conn.disconnect();
+        }
+    }
+
+
+    @Disabled
+    @Test
+    void testingTool() throws Exception {
+        System.out.println(testPdfFile(Path.of("/home/st_work/Work/sample.pdf")).details().title);
+        System.out.println(testPdfFile(Path.of("/home/st_work/Work/sample2.pdf")).details().title);
+        System.out.println(testPdfFile(Path.of("/home/st_work/Work/sample3.pdf")).details().title);
+        System.out.println(testPdfFile(Path.of("/home/st_work/Work/sample4.pdf")).details().title);
+        System.out.println(testPdfFile(Path.of("/home/st_work/Work/sample5.pdf")).details().title);
+        System.out.println(testPdfFile(Path.of("/home/st_work/Work/sample6.pdf")).details().title);
+    }
+
+    @Disabled
+    @Test
+    void testingTool2() throws Exception {
+        System.out.println(plugin.convertPdfToHtml(Files.readAllBytes(Path.of("/home/st_work/Work/sample6.pdf"))));
+    }
+
+    @Test
+    void testMarginaliaSample() throws Exception {
+        var doc = plugin.convertPdfToHtml(downloadPDF("https://www.marginalia.nu/junk/test.pdf"));
+        System.out.println(doc.html());
+    }
+}
--- a/code/processes/converting-process/test/nu/marginalia/converting/processor/pubdate/PubDateSnifferTest.java
+++ b/code/processes/converting-process/test/nu/marginalia/converting/processor/pubdate/PubDateSnifferTest.java
@@ -3,8 +3,8 @@ package nu.marginalia.converting.processor.pubdate;
 import nu.marginalia.WmsaHome;
 import nu.marginalia.converting.model.DocumentHeaders;
 import nu.marginalia.converting.processor.pubdate.heuristic.PubDateHeuristicDOMParsingPass2;
+import nu.marginalia.model.DocumentFormat;
 import nu.marginalia.model.EdgeUrl;
-import nu.marginalia.model.html.HtmlStandard;
 import org.jsoup.Jsoup;
 import org.junit.jupiter.api.Test;

@@ -74,7 +74,7 @@ class PubDateSnifferTest {
                        <time pubdate="pubdate" datetime="2022-08-24">time</time>
                        Wow, sure lor 'em boss
                        </article>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2022-08-24", ret.dateIso8601());
@@ -90,7 +90,7 @@ class PubDateSnifferTest {
                        <time>2022-08-24</time>
                        Wow, sure lor 'em boss
                        </article>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2022-08-24", ret.dateIso8601());
@@ -106,7 +106,7 @@ class PubDateSnifferTest {
                        <time class="published" datetime="July 13, 2006">July 13, 2006</time>
                        Wow, sure lor 'em boss
                        </article>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals(2006, ret.year());
@@ -116,14 +116,14 @@ class PubDateSnifferTest {
    public void testProblemCases() throws IOException, URISyntaxException {
        var ret = dateSniffer.getPubDate(new DocumentHeaders(""),
                new EdgeUrl("https://www.example.com/"),
-                Jsoup.parse(Files.readString(WmsaHome.getHomePath().resolve("test-data/The Switch to Linux Begins .html"))), HtmlStandard.HTML5, true);
+                Jsoup.parse(Files.readString(WmsaHome.getHomePath().resolve("test-data/The Switch to Linux Begins .html"))), DocumentFormat.HTML5, true);

        assertFalse(ret.isEmpty());
        assertEquals(2006, ret.year());

        ret = dateSniffer.getPubDate(new DocumentHeaders(""),
                new EdgeUrl("https://www.example.com/"),
-                Jsoup.parse(Files.readString(WmsaHome.getHomePath().resolve("test-data/Black Hat USA 2010 Understanding and Deploying DNSSEC by Paul Wouters and Patrick Nauber.html"))), HtmlStandard.XHTML, true);
+                Jsoup.parse(Files.readString(WmsaHome.getHomePath().resolve("test-data/Black Hat USA 2010 Understanding and Deploying DNSSEC by Paul Wouters and Patrick Nauber.html"))), DocumentFormat.XHTML, true);

        assertFalse(ret.isEmpty());
        assertEquals(2010, ret.year());
@@ -146,7 +146,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <meta itemprop="datePublished" content="2022-08-24" />
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2022-08-24", ret.dateIso8601());
@@ -160,7 +160,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <meta property="datePublished" content="2022-08-24" />
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2022-08-24", ret.dateIso8601());
@@ -174,7 +174,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <script type="application/ld+json">{"@context":"https:\\/\\/schema.org","@type":"Article","name":"In the Year 2525","url":"https:\\/\\/en.wikipedia.org\\/wiki\\/In_the_Year_2525","sameAs":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","mainEntity":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\\/\\/www.wikimedia.org\\/static\\/images\\/wmf-hor-googpub.png"}},"datePublished":"2004-08-24T14:39:14Z","dateModified":"2022-10-20T11:54:37Z","image":"https:\\/\\/upload.wikimedia.org\\/wikipedia\\/commons\\/4\\/4a\\/In_the_Year_2525_by_Zager_and_Evans_US_vinyl_Side-A_RCA_release.png","headline":"song written and compsoed by Rick Evans, originally recorded by Zager and Evans and released in 1969"}</script><script type="application/ld+json">{"@context":"https:\\/\\/schema.org","@type":"Article","name":"In the Year 2525","url":"https:\\/\\/en.wikipedia.org\\/wiki\\/In_the_Year_2525","sameAs":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","mainEntity":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\\/\\/www.wikimedia.org\\/static\\/images\\/wmf-hor-googpub.png"}},"datePublished":"2004-08-24T14:39:14Z","dateModified":"2022-10-20T11:54:37Z","image":"https:\\/\\/upload.wikimedia.org\\/wikipedia\\/commons\\/4\\/4a\\/In_the_Year_2525_by_Zager_and_Evans_US_vinyl_Side-A_RCA_release.png","headline":"song written and compsoed by Rick Evans, originally recorded by Zager and Evans and released in 1969"}</script>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2004-08-24", ret.dateIso8601());
@@ -188,7 +188,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <script type="application/ld+json" class="aioseop-schema">{"@context":"https://schema.org","@graph":[{"@type":"Organization","@id":"https://socialnomics.net/#organization","url":"https://socialnomics.net/","name":"Socialnomics","sameAs":[]},{"@type":"WebSite","@id":"https://socialnomics.net/#website","url":"https://socialnomics.net/","name":"Socialnomics","publisher":{"@id":"https://socialnomics.net/#organization"}},{"@type":"WebPage","@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/#webpage","url":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/","inLanguage":"en-US","name":"3 Reasons Why You Should Adopt Java-based Technology For Your Business","isPartOf":{"@id":"https://socialnomics.net/#website"},"breadcrumb":{"@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/#breadcrumblist"},"datePublished":"2016-12-27T21:01:36-06:00","dateModified":"2016-12-22T21:02:32-06:00"},{"@type":"Article","@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/#article","isPartOf":{"@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/#webpage"},"author":{"@id":"https://socialnomics.net/author/rahis-saifi/#author"},"headline":"3 Reasons Why You Should Adopt Java-based Technology For Your Business","datePublished":"2016-12-27T21:01:36-06:00","dateModified":"2016-12-22T21:02:32-06:00","commentCount":0,"mainEntityOfPage":{"@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/#webpage"},"publisher":{"@id":"https://socialnomics.net/#organization"},"articleSection":"Business, business, java, Java Developers, programming languages"},{"@type":"Person","@id":"https://socialnomics.net/author/rahis-saifi/#author","name":"Rahis Saifi","sameAs":["https://www.facebook.com/RahisSaifiOfficial","https://www.twitter.com/57rahis"],"image":{"@type":"ImageObject","@id":"https://socialnomics.net/#personlogo","url":"https://secure.gravatar.com/avatar/e67f630f0b8bc87e59e111d5e955961d?s=96&d=mm&r=g","width":96,"height":96,"caption":"Rahis Saifi"}},{"@type":"BreadcrumbList","@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/#breadcrumblist","itemListElement":[{"@type":"ListItem","position":1,"item":{"@type":"WebPage","@id":"https://socialnomics.net/","url":"https://socialnomics.net/","name":"Socialnomics Blog"}},{"@type":"ListItem","position":2,"item":{"@type":"WebPage","@id":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/","url":"https://socialnomics.net/2016/12/27/3-reasons-why-you-should-adopt-java-based-technology-for-your-business/","name":"3 Reasons Why You Should Adopt Java-based Technology For Your Business"}}]}]}</script>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2016-12-27", ret.dateIso8601());
@@ -202,7 +202,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <title>No date in the HTML</title>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertNull(ret.dateIso8601());
@@ -217,7 +217,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <title>No date in the HTML</title>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertEquals("2022-02-03", ret.dateIso8601());
@@ -232,7 +232,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <p>Published 2003, updated 2022</p>
-                        """), HtmlStandard.HTML5, true);
+                        """), DocumentFormat.HTML5, true);

        assertFalse(ret.isEmpty());
        assertNull(ret.dateIso8601());
@@ -258,7 +258,7 @@ class PubDateSnifferTest {
                        <!doctype html>
                        <html>
                        <div style="float: left;">&nbsp;<b>Post subject:</b> Keyboards.</div><div style="float: right;"><span class="postdetails"><b><img src="./styles/subsilver2/imageset/icon_post_target.gif" width="12" height="9" alt="Post" title="Post" /> <a  href="./viewtopic.php?p=34580&amp;sid=cf0c13dedebb4fea1f03fa73e510cd9f#p34580">#1</a></b></span>&nbsp;<b>Posted:</b> Sun Oct 03, 2010 5:37 pm&nbsp;</div>
-                        """), HtmlStandard.UNKNOWN, true);
+                        """), DocumentFormat.UNKNOWN, true);

        assertFalse(ret.isEmpty());
        assertNull(ret.dateIso8601());
--- a/code/processes/crawling-process/model/java/nu/marginalia/ContentTypes.java
+++ b/code/processes/crawling-process/model/java/nu/marginalia/ContentTypes.java
@@ -8,6 +8,8 @@ public class ContentTypes {
    public static final Set<String> acceptedContentTypes = Set.of("application/xhtml+xml",
            "application/xhtml",
            "text/html",
+            "text/markdown",
+            "text/x-markdown",
            "application/pdf",
            "image/x-icon",
            "text/plain");
--- a/code/processes/crawling-process/model/java/nu/marginalia/slop/SlopCrawlDataRecord.java
+++ b/code/processes/crawling-process/model/java/nu/marginalia/slop/SlopCrawlDataRecord.java
@@ -158,11 +158,12 @@ public record SlopCrawlDataRecord(String domain,
                        // and is used to store old responses from previous crawls; in this part of the logic
                        // we treat them the same as a normal response

-                        if (!filterResponse(uaString, response)) {
+                        var filterStatus = filterResponse(uaString, response);
+                        if (filterStatus.isRejected()) {
                            continue;
                        }

-                        slopWriter.write(domain, response);
+                        slopWriter.write(domain, filterStatus, response);
                    } else if (record instanceof WarcXEntityRefused refused) {
                        slopWriter.write(domain, refused);
                    } else if (record instanceof Warcinfo warcinfo) {
@@ -187,25 +188,35 @@ public record SlopCrawlDataRecord(String domain,
        }
    }

-
+    sealed interface ResponseFilterResult {
+        default boolean isRejected() { return false; }
+        record Accept() implements ResponseFilterResult {}
+        record AcceptWithContentType(String contentType) implements ResponseFilterResult {}
+        record AcceptIfPlainText(String contentType) implements ResponseFilterResult {}
+        record Reject() implements ResponseFilterResult {
+            @Override
+            public boolean isRejected() { return true; }
+        }
+    }

    /** Return true if the WarcResponse should be excluded from conversion */
-    private static boolean filterResponse(String uaString, WarcResponse response) throws IOException {
+    private static ResponseFilterResult filterResponse(String uaString, WarcResponse response) throws IOException {

        // We don't want to store robots.txt files, as they are not
        // interesting for the analysis we want to do.  This is important
        // since txt-files in general are interesting, and we don't want to
        // exclude them as a class.

-        if (response.targetURI().getPath().equals("/robots.txt")) {
-            return false;
+        String uriPath = response.targetURI().getPath();
+        if (uriPath.equals("/robots.txt")) {
+            return new ResponseFilterResult.Reject();
        }

        var headers = response.http().headers();
        var robotsTags = headers.all("X-Robots-Tag");

        if (!isXRobotsTagsPermitted(robotsTags, uaString)) {
-            return false;
+            return new ResponseFilterResult.Reject();
        }

        // Strip out responses with content types we aren't interested in
@@ -213,15 +224,29 @@ public record SlopCrawlDataRecord(String domain,
        String contentType = headers.first("Content-Type").orElse("text/plain").toLowerCase();

        if (!ContentTypes.isAccepted(contentType)) {
-            return false;
+            String contentTypeWithoutParams = StringUtils.substringBefore(contentType, ";");
+
+            // Some servers don't understand what a markdown file is
+            if (contentTypeWithoutParams.equals("application/octet-stream")) {
+                if (uriPath.endsWith(".md")) {
+                    // This is a markdown file, which we want to keep
+                    return new ResponseFilterResult.AcceptIfPlainText("text/markdown");
+                }
+                else if (uriPath.endsWith(".pdf")) {
+                    // This is a text file, which we want to keep
+                    return new ResponseFilterResult.AcceptWithContentType("application/pdf");
+                }
+            }
+
+            return new ResponseFilterResult.Reject();
        }

        // If the format is binary, we don't want to translate it if the response is truncated
        if (response.truncated() != WarcTruncationReason.NOT_TRUNCATED && ContentTypes.isBinary(contentType)) {
-            return false;
+            return new ResponseFilterResult.Reject();
        }

-        return true;
+        return new ResponseFilterResult.Accept();
    }

    /**  Check X-Robots-Tag header tag to see if we are allowed to index this page.
@@ -324,7 +349,7 @@ public record SlopCrawlDataRecord(String domain,
            headerColumnWriter.put(record.headers);
        }

-        public void write(String domain, WarcResponse response) throws IOException {
+        public void write(String domain, ResponseFilterResult filterStatus, WarcResponse response) throws IOException {

            HttpFetchResult result = HttpFetchResult.importWarc(response);
            if (!(result instanceof HttpFetchResult.ResultOk fetchOk)) {
@@ -347,6 +372,21 @@ public record SlopCrawlDataRecord(String domain,
                contentType = "";
            }

+            switch (filterStatus) {
+                case ResponseFilterResult.AcceptWithContentType(String ct) -> contentType = ct;
+                case ResponseFilterResult.AcceptIfPlainText(String ct) -> {
+                    try {
+                        // Parse the body as UTF-8
+                        new String(bodyBytes, StandardCharsets.UTF_8);
+                        contentType = ct;
+                    }
+                    catch (RuntimeException ex) { // UTF-8 decoding failed
+                        return;
+                    }
+                }
+                default -> {}
+            }
+
            boolean hasCookies = false;

            String headersStr;
--- a/code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java
+++ b/code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java
@@ -119,12 +119,17 @@ class CrawlerRetreiverTest {


    @Test
-    public void testWarcOutputPDF() throws IOException {
+    public void verifyFileFormatSupport() throws IOException {
+        List<String> urls = List.of(
+                "https://www.marginalia.nu/junk/test.pdf",
+                "https://www.marginalia.nu/junk/test.md"
+        );
+
        var specs = CrawlerMain.CrawlSpecRecord
                .builder()
                .crawlDepth(5)
                .domain("www.marginalia.nu")
-                .urls(List.of("https://www.marginalia.nu/junk/test.pdf"))
+                .urls(urls)
                .build();
        Path tempFile = null;
        Path slopFile = null;
@@ -146,7 +151,11 @@ class CrawlerRetreiverTest {
                    }
                    else if (record instanceof WarcResponse rsp) {
                        responses.add(rsp.target());
-                        System.out.println(rsp.type() + ":" + rsp.target());
+                        try {
+                            System.out.println(rsp.type() + ":" + rsp.target() + ":" + rsp.http().contentType());
+                        } catch (IOException e) {
+                            throw new RuntimeException(e);
+                        }
                    }
                    else {
                        System.out.println(record.type());
@@ -154,7 +163,9 @@ class CrawlerRetreiverTest {
                });
            }

-            assertTrue(requests.contains("https://www.marginalia.nu/junk/test.pdf"));
+            for (var url : urls) {
+                assertTrue(requests.contains(url), "Should have requested " + url);
+            }
            assertEquals(requests, responses);

            // Convert the WARC file to a Slop file
@@ -181,15 +192,18 @@ class CrawlerRetreiverTest {
                throw new RuntimeException(e);
            }

-            // Verify we have a PDF in the Slop file
-            assertNotNull(domain);
-            var pdfDoc = documents.get("https://www.marginalia.nu/junk/test.pdf");
-            assertNotNull(pdfDoc);
-            assertEquals("https://www.marginalia.nu/junk/test.pdf", pdfDoc.url);
-            assertEquals(206, pdfDoc.httpStatus);
-            assertTrue(pdfDoc.documentBodyBytes.length > 100);
-        }
-        finally {
+            for (var url : urls) {
+                // Verify we have the downloaded files in the Slop file
+                assertNotNull(domain);
+                var fetchedDoc = documents.get(url);
+                assertNotNull(fetchedDoc, "Should have a document for " + url);
+                assertEquals(url, fetchedDoc.url);
+                assertTrue(fetchedDoc.httpStatus == 200 || fetchedDoc.httpStatus == 206, "Should be 200 or 206 for " + url);
+                assertTrue(fetchedDoc.documentBodyBytes.length > 32, "Should have a body for " + url);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        } finally {
            if (tempFile != null)
                Files.deleteIfExists(tempFile);
            if (slopFile != null)
--- a/code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java
+++ b/code/processes/export-task-process/java/nu/marginalia/extractor/SampleDataExporter.java
@@ -122,6 +122,8 @@ public class SampleDataExporter {
        }
        Files.createDirectory(tempDir);

+        boolean wroteEntry = false;
+
        try (var writer = new SlopCrawlDataRecord.Writer(tempDir);
             var reader = new SlopCrawlDataRecord.FilteringReader(crawlDataPath) {
                 @Override
@@ -131,14 +133,20 @@ public class SampleDataExporter {
                 }
             }
        ) {
-            boolean wroteEntry = false;
+
            while (reader.hasRemaining()) {
                var entry = reader.get();
                writer.write(entry);

                wroteEntry = wroteEntry || Objects.equals(StringUtils.substringBefore(entry.contentType(), ';'), contentTypeFilter);
            }
+        }
+        catch (Exception ex) {
+            FileUtils.deleteDirectory(tempDir.toFile());
+            throw ex;
+        }

+        try {
            if (!wroteEntry) {
                throw new NoSuchElementException("No relevant entries");
            }
--- a/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java
+++ b/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java
@@ -90,6 +90,7 @@ public class ApiSearchOperator {
                url.getTitle(),
                url.getDescription(),
                sanitizeNaN(url.rankingScore, -100),
+                url.getShortFormat(),
                details
        );
    }
--- a/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResult.java
+++ b/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResult.java
@@ -8,14 +8,16 @@ public class ApiSearchResult {
    public String title;
    public String description;
    public double quality;
+    public String format; // "pdf", "html", "text", etc.

    public List<List<ApiSearchResultQueryDetails>> details = new ArrayList<>();

-    public ApiSearchResult(String url, String title, String description, double quality, List<List<ApiSearchResultQueryDetails>> details) {
+    public ApiSearchResult(String url, String title, String description, double quality, String format, List<List<ApiSearchResultQueryDetails>> details) {
        this.url = url;
        this.title = title;
        this.description = description;
        this.quality = quality;
+        this.format = format;
        this.details = details;
    }

--- a/code/services-application/search-service-legacy/java/nu/marginalia/search/model/UrlDetails.java
+++ b/code/services-application/search-service-legacy/java/nu/marginalia/search/model/UrlDetails.java
@@ -73,6 +73,8 @@ public class UrlDetails implements Comparable<UrlDetails> {
                return "HTML 5";
            case "PLAIN":
                return "Plain Text";
+            case "PDF":
+                return "PDF";
            default:
                return "?";
        }
--- a/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java
@@ -112,6 +112,13 @@ public class SearchOperator {
                .selectStrategy(queryResponse)
                .clusterResults(queryResults, 25);

+        if (queryParams.humanQuery().equals("slackware linux")) {
+            logger.info("Query response: {}", queryResponse.results().subList(0, 5));
+            logger.info("Query results: {}", queryResults.subList(0, 5));
+            logger.info("Clustered results: {}", clusteredResults.subList(0, 5));
+        }
+
+
        // Log the query and results

        logger.info(queryMarker, "Human terms: {}", Strings.join(queryResponse.searchTermsHuman(), ','));
--- a/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java
@@ -17,7 +17,7 @@ public class SearchQueryParamFactory {
    static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder()
            .setResultsTotal(100)
            .setResultsByDomain(5)
-            .setTimeoutMs(200)
+            .setTimeoutMs(250)
            .setFetchSize(8192)
            .build();

--- a/code/services-application/search-service/java/nu/marginalia/search/SearchResultClusterer.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/SearchResultClusterer.java
@@ -23,7 +23,7 @@ public class SearchResultClusterer {
    }

    /** No clustering, just return the results as is */
-    private static List<ClusteredUrlDetails> noOp(List<UrlDetails> results, int total) {
+    public static List<ClusteredUrlDetails> noOp(List<UrlDetails> results, int total) {
        if (results.isEmpty())
            return List.of();

--- a/code/services-application/search-service/java/nu/marginalia/search/SearchService.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/SearchService.java
@@ -85,7 +85,6 @@ public class SearchService extends JoobyService {
        String emptySvg = "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>";
        jooby.get("/site/{domain}/favicon", ctx -> {
            String domain = ctx.path("domain").value();
-            logger.info("Finding icon for domain {}", domain);
            try {
                DbDomainQueries.DomainIdWithNode domainIdWithNode = domainQueries.getDomainIdWithNode(new EdgeDomain(domain));
                var faviconMaybe = faviconClient.getFavicon(domain, domainIdWithNode.nodeAffinity());
--- a/code/services-application/search-service/java/nu/marginalia/search/model/UrlDetails.java
+++ b/code/services-application/search-service/java/nu/marginalia/search/model/UrlDetails.java
@@ -78,6 +78,8 @@ public class UrlDetails implements Comparable<UrlDetails> {
                return "HTML 5";
            case "PLAIN":
                return "Plain Text";
+            case "PDF":
+                return "PDF";
            default:
                return "?";
        }
@@ -92,13 +94,24 @@ public class UrlDetails implements Comparable<UrlDetails> {
    public String displayTitle() {
        StringBuilder sb = new StringBuilder();

+        buildDisplayTitle(sb, title);
+
+        if (sb.isEmpty()) {
+            buildDisplayTitle(sb, url.toDisplayString());
+        }
+
+        return sb.toString();
+    }
+
+    private void buildDisplayTitle(StringBuilder sb, String str) {
+
        int distSinceBreak = 0;

        char c = ' ';
        int prevC = ' ';
-        for (int i = 0; i < title.length(); i++) {
+        for (int i = 0; i < str.length(); i++) {
            prevC = c;
-            c = title.charAt(i);
+            c = str.charAt(i);

            if (Character.isSpaceChar(c)) {
                distSinceBreak = 0;
@@ -135,8 +148,6 @@ public class UrlDetails implements Comparable<UrlDetails> {
                sb.append(c);
            }
        }
-
-        return sb.toString();
    }

    /** Helper that inserts hyphenation hints and escapes
--- a/code/services-application/search-service/resources/jte/serp/part/result.jte
+++ b/code/services-application/search-service/resources/jte/serp/part/result.jte
@@ -21,6 +21,9 @@
                            </h2>

                            <div class="text-sm mt-1">
+                                @if ("PDF".equals(result.first.format))
+                                    <i title="PDF" class="fas fa-file-pdf text-red-500"></i>
+                                @endif
                                <a class="text-liteblue dark:text-blue-200 underline break-all" href="${result.first.url.toString()}"
                                   rel="noopener noreferrer" tabindex="-1">$unsafe{result.first.displayUrl()}</a>
                            </div>
@@ -53,10 +56,13 @@
            <div class="flex mt-2 text-sm flex flex-col space-y-2">
                <p class="text-black dark:text-white ${result.colorScheme.backgroundColor}  p-1 rounded break-words hyphens-auto">Also from ${result.getDomain().toString()}:</p>

-                <ul class="pl-2 mt-2 underline text-liteblue dark:text-blue-200">
+                <ul class="pl-2 mt-2 text-liteblue dark:text-blue-200">
                    @for(UrlDetails item : result.rest)
                        <li class="-indent-4 pl-4 mb-1 break-words hyphens-auto">
-                            <a href="${item.url.toString()}" rel="noopener noreferrer">$unsafe{item.displayTitle()}</a>
+                            @if ("PDF".equals(item.format))
+                                <i title="PDF" class="fas fa-file-pdf text-red-500"></i>
+                            @endif
+                            <a href="${item.url.toString()}" class="underline" rel="noopener noreferrer">$unsafe{item.displayTitle()}</a>
                        </li>
                    @endfor
                </ul>
@@ -74,6 +80,9 @@
            @if (DocumentFlags.PlainText.isPresent(result.getFirst().resultItem.encodedDocMetadata))
                <span class="px-1 bg-blue-100 text-blue-700 dark:border dark:border-blue-600 dark:text-blue-400 dark:bg-black  rounded">Plain text</span>
            @endif
+            @if (DocumentFlags.PdfFile.isPresent(result.getFirst().resultItem.encodedDocMetadata))
+                <span class="px-1 bg-blue-100 text-blue-700 dark:border dark:border-blue-600 dark:text-blue-400 dark:bg-black  rounded">PDF File</span>
+            @endif
            @if (DocumentFlags.GeneratorForum.isPresent(result.getFirst().resultItem.encodedDocMetadata))
                <span class="px-1 bg-blue-100 text-blue-700 dark:border dark:border-blue-600 dark:text-blue-400 dark:bg-black  rounded">Forum</span>
            @endif
--- a/deploy.txt
+++ b/deploy.txt
@@ -7,3 +7,4 @@
 2025-05-04:  Deploy qs, search and api-services.
 2025-05-05:  Deploy executor partition 4.
 2025-05-05:  Deploy control.
+2025-05-08:  Deploy assistant.
--- a/settings.gradle
+++ b/settings.gradle
@@ -236,7 +236,7 @@ dependencyResolutionManagement {
            library('jetty-util','org.eclipse.jetty','jetty-util').version('9.4.54.v20240208')
            library('jetty-servlet','org.eclipse.jetty','jetty-servlet').version('9.4.54.v20240208')

-            library('slop', 'nu.marginalia', 'slop').version('0.0.10-SNAPSHOT')
+            library('slop', 'nu.marginalia', 'slop').version('0.0.11-SNAPSHOT')
            library('jooby-netty','io.jooby','jooby-netty').version(joobyVersion)
            library('jooby-jte','io.jooby','jooby-jte').version(joobyVersion)
            library('jooby-apt','io.jooby','jooby-apt').version(joobyVersion)
@@ -244,6 +244,7 @@ dependencyResolutionManagement {
            library('wiremock', 'org.wiremock','wiremock').version('3.11.0')
            library('jte','gg.jte','jte').version('3.1.15')

+            library('pdfbox', 'org.apache.pdfbox', 'pdfbox').version('3.0.5')
            bundle('jetty', ['jetty-server', 'jetty-util', 'jetty-servlet'])

            bundle('slf4j', ['slf4j.api', 'log4j.api', 'log4j.core', 'log4j.slf4j'])
Author	SHA1	Message	Date
Viktor Lofgren	eb634beec8	(search) Add debug logging for specific query	2025-05-16 23:34:03 +02:00
Viktor Lofgren	269ebd1654	Revert "(query) Add debug logging for specific query" This reverts commit `39ce40bfeb`.	2025-05-16 23:29:06 +02:00
Viktor Lofgren	39ce40bfeb	(query) Add debug logging for specific query	2025-05-16 23:23:53 +02:00
Viktor Lofgren	c187b2e1c1	(search) Re-enable clustering	2025-05-16 23:20:16 +02:00
Viktor Lofgren	42eaa4588b	(search) Disable clustering for a moment	2025-05-16 23:17:01 +02:00
Viktor Lofgren	4f40a5fbeb	(search) Reduce log spam	2025-05-16 23:15:07 +02:00
Viktor Lofgren	3f3d42bc01	(search) Re-enable deduplication	2025-05-16 23:14:54 +02:00
Viktor Lofgren	61c8d53e1b	(search) Disable deduplication for a moment	2025-05-16 23:10:32 +02:00
Viktor Lofgren	a7a3d85be9	(search) Increase search timeout by 50ms	2025-05-16 22:54:12 +02:00
Viktor Lofgren	306232fb54	(pdf) Fix handling of a few corner cases Deal better with documents which change font on blank spaces.	2025-05-13 18:44:28 +02:00
Viktor Lofgren	5aef844f0d	(dependency) Increase slop version to 0.0.11 v0.0.11 uses atomic moves. This ensures we don't encounter a race condition in the backup service with lingering .tmp-files that should have been renamed.	2025-05-12 14:09:16 +02:00
Viktor	d56b5c828a	Merge pull request #198 from MarginaliaSearch/process-pdf-files Add support for processing PDF files. The changeset adds a dependency on pdfbox, and vendors/modifies its PDFTextStripper to extract additional semantics from the documents. Since PDF documents aren't a text based format, but a graphical format which may contain a stream of characters and positions (sometimes overlapping, rotated, out of order) identifying something like a header or a paragraph is a non-trivial task, let alone extracting any text at all. A number of heuristics are used to try to accomplish this task, they aren't perfect, but about as good as you're going to get without going to something like a vision based LLM, which would be ridiculously expensive to apply at an internet search engine scale. The change also adds format information to the JSON API, as well as indicators in the GUI for PDF files.	2025-05-11 16:43:25 +02:00
Viktor Lofgren	ab58a4636f	(pdf) Disable tests that require specific sample data that can't go in the repo	2025-05-11 16:42:23 +02:00
Viktor Lofgren	00be269238	(search) Add PDF indicator in "also from"-segment	2025-05-11 16:35:52 +02:00
Viktor Lofgren	879e6a9424	(pdf) Identify additional headings based on font weight	2025-05-11 16:35:52 +02:00
Viktor Lofgren	fba3455732	(pdf) Clean up code	2025-05-11 16:35:52 +02:00
Viktor Lofgren	14283da7f5	(pdf) Clean up generated DOM Sometimes empty <p>-tags are inserted, which messes with the header joining process. Removes those nodes.	2025-05-11 15:12:09 +02:00
Viktor Lofgren	93df4d1fc0	(pdf) Improve summary extraction for PDFs	2025-05-11 14:33:11 +02:00
Viktor Lofgren	b12a0b998c	(pdf) Use smarter heuristics for paragraph splitting We look at the median line distance, with outliers removed, to figure out when to break lines, as the original approach works poorly with e.g. double line spaced documents.	2025-05-11 14:29:42 +02:00
Viktor Lofgren	3b6f4e321b	(search) Add red PDF indicator to search UI	2025-05-11 13:32:14 +02:00
Viktor Lofgren	8428111771	(pdf) Fix for exception when no text positions are available	2025-05-10 15:12:02 +02:00
Viktor Lofgren	e9fd4415ef	(pdf) Merge consecutive headings. Headings don't follow the same indentation rules as prose and tend to be cut off into multiple "paragraphs" by the text extractor.	2025-05-10 14:38:43 +02:00
Viktor Lofgren	4c95c3dcad	(pdf) Don't look for headings below 75% of the max y-position	2025-05-10 14:38:02 +02:00
Viktor Lofgren	c5281536fb	(api) Add format field to JSON search results API consumers might want to filter out PDF results, etc.	2025-05-10 13:56:22 +02:00
Viktor Lofgren	4431dae7ac	(refac) Rename HtmlStandard -> DocumentFormat The old model made some sense when we only supported HTML and to some extent plain text, but having PDF in an enum called HtmlFormat is a bit of a stretch.	2025-05-10 13:47:26 +02:00
Viktor Lofgren	4df4d0a7a8	(pdf) Increase line spacing tolerance for better paragraph handling	2025-05-10 13:34:04 +02:00
Viktor Lofgren	9f05083b94	(pdf) Add the capability to identify headings This change vendors pdfbox'es PDFTextStripper and modifies it to be able to heuristically identify headings based on their font size, as this is a very useful relevance signal for the search engine, and helps identify the correct title of the article.	2025-05-09 14:04:04 +02:00
Viktor Lofgren	fc92e9b9c0	(feeds) Correct link handling in atom feeds This addresses issue #199	2025-05-09 13:00:07 +02:00
Viktor Lofgren	328fb5d927	(feeds) Correct link handling in atom feeds This addresses issue #199	2025-05-09 12:55:28 +02:00
Viktor Lofgren	36889950e8	(pdf) Migrate to PDFBox 3.0.5 and suppress log spam PDFBox 2.x uses commons logging, which does not route through SLF4j, and thus is a hassle to configure; and is extremely verbose in its default logging settings. Migrating to PDFBox 3.x lets us use slf4j to address the log spam by filtering out the noisy methods.	2025-05-08 18:03:26 +02:00
Viktor Lofgren	c96a94878b	(pdf) Add feature to make pdf-files searchable with format:pdf	2025-05-08 18:03:26 +02:00
Viktor Lofgren	1c57d7d73a	(pdf) Clean up code	2025-05-08 18:03:26 +02:00
Viktor Lofgren	a443d22356	(pdf) Flag the file as a PDF file in the GUI	2025-05-08 18:03:26 +02:00
Viktor Lofgren	aa59d4afa4	(pdf) Somewhat improve title and summary extraction	2025-05-08 18:03:26 +02:00
Viktor Lofgren	df0f18d0e7	(pdf) Read title	2025-05-08 18:03:26 +02:00
Viktor Lofgren	0819d46f97	(pdf) Minimal protytype to get PDFs working	2025-05-08 18:03:26 +02:00
Viktor Lofgren	5e2b63473e	(logging) Change to a terser log format The old log format would often span several screen widths, especially when subprocesses logged. Switching to a terser format that should be much easier to read.	2025-05-08 18:02:22 +02:00
Viktor	f9590703f1	Merge pull request #197 from MarginaliaSearch/crawl-markdown (markdown) Support crawling markdown	2025-05-08 13:35:00 +02:00
Viktor Lofgren	f12fc11337	(markdown) Support crawling markdown	2025-05-08 13:26:22 +02:00
Viktor Lofgren	c309030184	(sample) Ensure we finalize the slop.zip file creation when filtering	2025-05-06 14:52:48 +02:00