eUgEntOptIc44/MarginaliaSearch
1
1
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-10-05 21:22:39 +02:00
Code Issues Activity

(lang+search) Clean up LanguageConfiguration initialization and LangCommand

Browse Source
This commit is contained in:
Viktor Lofgren
2025-09-16 11:47:15 +02:00
parent da5046c3bf
commit ae31bc8498
21 changed files with 111 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
code/functions/search-query/test/nu/marginalia/functions/searchquery/query_parser/QueryParserTest.java
View File

@@ -2,6 +2,7 @@ package nu.marginalia.functions.searchquery.query_parser;
import nu.marginalia.WmsaHome;
import nu.marginalia.functions.searchquery.query_parser.token.QueryToken;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -13,7 +14,7 @@ import java.util.List;
class QueryParserTest {
LanguageConfiguration languageConfiguration = new LanguageConfiguration(WmsaHome.getLanguageModels());
LanguageConfiguration languageConfiguration = new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental());
QueryParserTest() throws IOException, ParserConfigurationException, SAXException {
}

3
code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java
View File

@@ -6,6 +6,7 @@ import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.api.searchquery.model.query.*;
import nu.marginalia.functions.searchquery.QueryFactory;
import nu.marginalia.functions.searchquery.query_parser.QueryExpansion;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.segmentation.NgramLexicon;
import nu.marginalia.term_frequency_dict.TermFrequencyDict;
@@ -31,7 +32,7 @@ public class QueryFactoryTest {
var lm = WmsaHome.getLanguageModels();
queryFactory = new QueryFactory(new QueryExpansion(new TermFrequencyDict(lm), new NgramLexicon(lm)), new LanguageConfiguration(lm));
queryFactory = new QueryFactory(new QueryExpansion(new TermFrequencyDict(lm), new NgramLexicon(lm)), new LanguageConfiguration(lm, new LanguageConfigLocation.Experimental()));
}
public SearchSpecification parseAndGetSpecs(String query) {

3
code/index/index-perftest/java/nu/marginalia/index/perftest/PerfTestMain.java
View File

@@ -23,6 +23,7 @@ import nu.marginalia.index.reverse.PrioReverseIndexReader;
import nu.marginalia.index.reverse.WordLexicon;
import nu.marginalia.index.reverse.query.IndexQuery;
import nu.marginalia.index.searchset.SearchSetAny;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.keywords.KeywordHasher;
import nu.marginalia.linkdb.docs.DocumentDbReader;
@@ -116,7 +117,7 @@ public class PerfTestMain {
new TermFrequencyDict(homeDir.resolve("model/tfreq-new-algo3.bin")),
new NgramLexicon()
),
new LanguageConfiguration(WmsaHome.getLanguageModels())
new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental())
);
}

3
code/libraries/language-processing/java/nu/marginalia/language/LanguageProcessingTool.java
View File

@@ -7,6 +7,7 @@ import io.jooby.ModelAndView;
import nu.marginalia.LanguageModels;
import nu.marginalia.WmsaHome;
import nu.marginalia.keyword.extractors.*;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.DocumentLanguageData;
import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
@@ -33,7 +34,7 @@ public class LanguageProcessingTool extends Jooby {
termFrequencyDict = new TermFrequencyDict(languageModels);
sentenceExtractorProvider = new ThreadLocalSentenceExtractorProvider(
new LanguageConfiguration(languageModels),
new LanguageConfiguration(languageModels, new LanguageConfigLocation.Experimental()),
languageModels
);
Path basePath = Path.of("code/libraries/language-processing/").toAbsolutePath();

43
code/libraries/language-processing/java/nu/marginalia/language/config/LanguageConfigLocation.java Normal file
View File

@@ -0,0 +1,43 @@
package nu.marginalia.language.config;
import nu.marginalia.WmsaHome;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
sealed public interface LanguageConfigLocation {
InputStream findLanguageConfiguration(LanguageConfigLocation languageFile) throws IOException;
final class Auto implements LanguageConfigLocation {
@Override
public InputStream findLanguageConfiguration(LanguageConfigLocation languageFile) throws IOException {
Path filesystemPath = WmsaHome.getLangugeConfig();
if (Files.exists(filesystemPath)) {
return Files.newInputStream(filesystemPath, StandardOpenOption.READ);
}
if (Boolean.getBoolean("language.experimental")) {
return ClassLoader.getSystemResourceAsStream("languages-experimental.xml");
} else {
return ClassLoader.getSystemResourceAsStream("languages-default.xml");
}
}
}
final class Experimental implements LanguageConfigLocation {
@Override
public InputStream findLanguageConfiguration(LanguageConfigLocation languageFile) throws IOException {
return ClassLoader.getSystemResourceAsStream("languages-experimental.xml");
}
}
final class Default implements LanguageConfigLocation {
@Override
public InputStream findLanguageConfiguration(LanguageConfigLocation languageFile) throws IOException {
return ClassLoader.getSystemResourceAsStream("languages-default.xml");
}
}
}

25
code/libraries/language-processing/java/nu/marginalia/language/config/LanguageConfiguration.java
View File

@@ -91,16 +91,20 @@ public class LanguageConfiguration {
@Inject
public LanguageConfiguration() throws IOException, ParserConfigurationException, SAXException {
this(WmsaHome.getLanguageModels());
this(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Auto());
}
public LanguageConfiguration(LanguageModels lm)
public LanguageConfiguration(LanguageConfigLocation languageFile) throws IOException, ParserConfigurationException, SAXException {
this(WmsaHome.getLanguageModels(), languageFile);
}
public LanguageConfiguration(LanguageModels lm, LanguageConfigLocation languageFile)
throws IOException, ParserConfigurationException, SAXException {
fastTextLanguageModel.loadModel(lm.fasttextLanguageModel.toString());
try (var languagesXmlStream = findLanguageConfiguration()) {
try (var languagesXmlStream = languageFile.findLanguageConfiguration(languageFile)) {
if (languagesXmlStream == null)
throw new IllegalStateException("languages.xml resource not found in classpath");
throw new IllegalStateException("languages-default.xml resource not found in classpath");
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
@@ -113,19 +117,6 @@ public class LanguageConfiguration {
logger.info("Loaded language configuration: {}", languages);
}
InputStream findLanguageConfiguration() throws IOException {
Path filesystemPath = WmsaHome.getLangugeConfig();
if (Files.exists(filesystemPath)) {
return Files.newInputStream(filesystemPath, StandardOpenOption.READ);
}
if (Boolean.getBoolean("language.experimental")) {
return ClassLoader.getSystemResourceAsStream("languages-experimental.xml");
}
else {
return ClassLoader.getSystemResourceAsStream("languages-default.xml");
}
}
private void parseLanguages(Document doc) {
NodeList languageNodes = doc.getElementsByTagName("language");

0
code/libraries/language-processing/resources/languages.xml → code/libraries/language-processing/resources/languages-default.xml
View File

3
code/libraries/language-processing/test/nu/marginalia/keyword/DocumentKeywordExtractorTest.java
View File

@@ -2,6 +2,7 @@ package nu.marginalia.keyword;
import nu.marginalia.WmsaHome;
import nu.marginalia.dom.DomPruningFilter;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.UnsupportedLanguageException;
import nu.marginalia.language.sentence.SentenceExtractor;
@@ -31,7 +32,7 @@ class DocumentKeywordExtractorTest {
@BeforeAll
public static void setUpAll() throws IOException, ParserConfigurationException, SAXException {
se = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels()), WmsaHome.getLanguageModels());
se = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental()), WmsaHome.getLanguageModels());
}
@Test

3
code/libraries/language-processing/test/nu/marginalia/keyword/DocumentPositionMapperTest.java
View File

@@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.ints.IntList;
import nu.marginalia.WmsaHome;
import nu.marginalia.keyword.model.DocumentKeywordsBuilder;
import nu.marginalia.keyword.model.DocumentWordSpan;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.DocumentLanguageData;
import nu.marginalia.language.model.DocumentSentence;
@@ -34,7 +35,7 @@ class DocumentPositionMapperTest {
@BeforeAll
public static void setUpAll() throws IOException, ParserConfigurationException, SAXException {
var config = new LanguageConfiguration(WmsaHome.getLanguageModels());
var config = new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental());
se = new SentenceExtractor(config, WmsaHome.getLanguageModels());
english = config.getLanguage("en");
}

3
code/libraries/language-processing/test/nu/marginalia/keyword/SentenceExtractorTest.java
View File

@@ -2,6 +2,7 @@ package nu.marginalia.keyword;
import nu.marginalia.LanguageModels;
import nu.marginalia.WmsaHome;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.LanguageDefinition;
import nu.marginalia.language.model.UnsupportedLanguageException;
@@ -35,7 +36,7 @@ class SentenceExtractorTest {
@BeforeAll
public static void setUpAll() throws IOException, ParserConfigurationException, SAXException {
var config = new LanguageConfiguration(WmsaHome.getLanguageModels());
var config = new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental());
se = new SentenceExtractor(config, WmsaHome.getLanguageModels());
english = config.getLanguage("en");

3
code/libraries/language-processing/test/nu/marginalia/keyword/extractors/ArtifactKeywordsTest.java
View File

@@ -1,5 +1,6 @@
package nu.marginalia.keyword.extractors;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.sentence.SentenceExtractor;
import nu.marginalia.util.TestLanguageModels;
@@ -15,7 +16,7 @@ class ArtifactKeywordsTest {
@Test
public void testExtractArtifacts() throws IOException, ParserConfigurationException, SAXException {
SentenceExtractor se = new SentenceExtractor(new LanguageConfiguration(TestLanguageModels.getLanguageModels()), TestLanguageModels.getLanguageModels());
SentenceExtractor se = new SentenceExtractor(new LanguageConfiguration(TestLanguageModels.getLanguageModels(), new LanguageConfigLocation.Experimental()), TestLanguageModels.getLanguageModels());
var artifacts = new ArtifactKeywords(se.extractSentences("Hello I'm <vlofgren@marginalia.nu>, what's up?", "hello!"));
System.out.println(artifacts.getWords());

5
code/libraries/language-processing/test/nu/marginalia/keyword/extractors/NameLikeKeywordsTest.java
View File

@@ -3,6 +3,7 @@ package nu.marginalia.keyword.extractors;
import com.google.common.collect.Sets;
import nu.marginalia.WmsaHome;
import nu.marginalia.dom.DomPruningFilter;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.LanguageDefinition;
import nu.marginalia.language.model.UnsupportedLanguageException;
@@ -55,8 +56,8 @@ class NameLikeKeywordsTest {
@BeforeAll
public static void setUpAll() throws IOException, ParserConfigurationException, SAXException {
se = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels()), WmsaHome.getLanguageModels());
lc = new LanguageConfiguration(WmsaHome.getLanguageModels());
se = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental()), WmsaHome.getLanguageModels());
lc = new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental());
en = lc.getLanguage("en");
}

3
code/libraries/language-processing/test/nu/marginalia/keyword/extractors/SubjectLikeKeywordsTest.java
View File

@@ -2,6 +2,7 @@ package nu.marginalia.keyword.extractors;
import com.google.common.collect.Sets;
import nu.marginalia.WmsaHome;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.sentence.SentenceExtractor;
import nu.marginalia.term_frequency_dict.TermFrequencyDict;
@@ -49,7 +50,7 @@ class SubjectLikeKeywordsTest {
@BeforeAll
public static void setUpAll() throws IOException, ParserConfigurationException, SAXException {
se = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels()), WmsaHome.getLanguageModels());
se = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental()), WmsaHome.getLanguageModels());
}
@Test

3
code/libraries/language-processing/test/nu/marginalia/keyword/extractors/TitleKeywordsTest.java
View File

@@ -1,6 +1,7 @@
package nu.marginalia.keyword.extractors;
import com.google.common.collect.Sets;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.UnsupportedLanguageException;
import nu.marginalia.language.sentence.SentenceExtractor;
@@ -192,7 +193,7 @@ class TitleKeywordsTest {
@Test
public void extractTitleWords() throws IOException, ParserConfigurationException, SAXException, UnsupportedLanguageException {
var languageConfiguration = new LanguageConfiguration(TestLanguageModels.getLanguageModels());
var languageConfiguration = new LanguageConfiguration(TestLanguageModels.getLanguageModels(), new LanguageConfigLocation.Experimental());
var se = new SentenceExtractor(languageConfiguration, TestLanguageModels.getLanguageModels());
var dld = se.extractSentences(Jsoup.parse(document));

4
code/libraries/language-processing/test/nu/marginalia/language/config/LanguageConfigurationTest.java → code/libraries/language-processing/test/nu/marginalia/language/config/LanguageConfigurationTestFile.java
View File

@@ -13,12 +13,12 @@ import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class LanguageConfigurationTest {
public class LanguageConfigurationTestFile {
private static LanguageConfiguration languageConfiguration;
@BeforeAll
public static void setUpAll() throws IOException, SAXException, ParserConfigurationException {
languageConfiguration = new LanguageConfiguration(TestLanguageModels.getLanguageModels());
languageConfiguration = new LanguageConfiguration(TestLanguageModels.getLanguageModels(), new LanguageConfigLocation.Experimental());
}
@Test

3
code/libraries/language-processing/test/nu/marginalia/language/sentence/SentenceExtractorTest.java
View File

@@ -1,6 +1,7 @@
package nu.marginalia.language.sentence;
import nu.marginalia.WmsaHome;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.UnsupportedLanguageException;
import nu.marginalia.language.sentence.tag.HtmlTag;
@@ -24,7 +25,7 @@ class SentenceExtractorTest {
@BeforeAll
public static void setUp() throws IOException, ParserConfigurationException, SAXException {
languageConfig = new LanguageConfiguration(WmsaHome.getLanguageModels());
languageConfig = new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental());
sentenceExtractor = new SentenceExtractor(languageConfig, WmsaHome.getLanguageModels());
}

3
code/processes/converting-process/test/nu/marginalia/atags/DomainAnchorTagsImplTest.java
View File

@@ -1,6 +1,7 @@
package nu.marginalia.atags;
import nu.marginalia.atags.source.AnchorTagsImpl;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.sentence.SentenceExtractor;
import nu.marginalia.model.EdgeDomain;
@@ -37,7 +38,7 @@ class DomainAnchorTagsImplTest {
System.out.println(tags.forUrl(new EdgeUrl("http://www.chiark.greenend.org.uk/~sgtatham/putty/")));
System.out.println(tags.forUrl(new EdgeUrl("http://www.chiark.greenend.org.uk/~sgtatham/putt")));
var languageConfig = new LanguageConfiguration(TestLanguageModels.getLanguageModels());
var languageConfig = new LanguageConfiguration(TestLanguageModels.getLanguageModels(), new LanguageConfigLocation.Experimental());
var atagsKeywords = new AnchorTextKeywords(new SentenceExtractor(languageConfig, TestLanguageModels.getLanguageModels()), languageConfig);
System.out.println(

5
code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/PdfDocumentProcessorPluginTest.java
View File

@@ -9,6 +9,7 @@ import nu.marginalia.converting.processor.summary.SummaryExtractor;
import nu.marginalia.converting.processor.summary.heuristic.*;
import nu.marginalia.keyword.DocumentKeywordExtractor;
import nu.marginalia.keyword.LinkTexts;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.sentence.ThreadLocalSentenceExtractorProvider;
import nu.marginalia.model.crawldata.CrawledDocument;
@@ -35,8 +36,8 @@ class PdfDocumentProcessorPluginTest {
static void setUpBeforeClass() throws Exception {
var lm = WmsaHome.getLanguageModels();
plugin = new PdfDocumentProcessorPlugin(255,
new LanguageConfiguration(lm),
new ThreadLocalSentenceExtractorProvider(new LanguageConfiguration(lm), lm),
new LanguageConfiguration(lm, new LanguageConfigLocation.Experimental()),
new ThreadLocalSentenceExtractorProvider(new LanguageConfiguration(lm, new LanguageConfigLocation.Experimental()), lm),
new DocumentKeywordExtractor(new TermFrequencyDict(lm)),
new DocumentLengthLogic(100),
new DefaultSpecialization(new SummaryExtractor(

3
code/processes/converting-process/test/nu/marginalia/converting/processor/summary/SummaryExtractorTest.java
View File

@@ -4,6 +4,7 @@ import nu.marginalia.WmsaHome;
import nu.marginalia.converting.processor.summary.heuristic.*;
import nu.marginalia.keyword.DocumentKeywordExtractor;
import nu.marginalia.keyword.LinkTexts;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.language.model.UnsupportedLanguageException;
import nu.marginalia.language.sentence.SentenceExtractor;
@@ -31,7 +32,7 @@ class SummaryExtractorTest {
@BeforeEach
public void setUp() throws IOException, ParserConfigurationException, SAXException {
keywordExtractor = new DocumentKeywordExtractor();
setenceExtractor = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels()), WmsaHome.getLanguageModels());
setenceExtractor = new SentenceExtractor(new LanguageConfiguration(WmsaHome.getLanguageModels(), new LanguageConfigLocation.Experimental()), WmsaHome.getLanguageModels());
summaryExtractor = new SummaryExtractor(255,
new DomFilterHeuristic(255),

32
code/services-application/search-service/java/nu/marginalia/search/command/LangCommand.java
View File

@@ -13,7 +13,7 @@ import java.util.regex.Pattern;
public class LangCommand implements SearchCommandInterface {
private final Pattern queryPatternPredicate = Pattern.compile("(^|\\s)lang:[a-z]{2}(\\s|$)");
private final Pattern queryPatternPredicate = Pattern.compile("(^|\\s)lang:([a-z]{2})(\\s|$)");
private final LanguageConfiguration languageConfiguration;
@Inject
@@ -26,28 +26,24 @@ public class LangCommand implements SearchCommandInterface {
String query = parameters.query();
Matcher matcher = queryPatternPredicate.matcher(query);
if (matcher.find()) {
String lang = query.substring(matcher.start(), matcher.end()).trim().toLowerCase();
if (lang.length() <= 2)
return Optional.empty();
lang = lang.substring(lang.length() - 2, lang.length());
if (lang.equalsIgnoreCase(parameters.languageIsoCode()))
return Optional.empty();
if (!matcher.find())
return Optional.empty();
if (languageConfiguration.getLanguage(lang) == null)
return Optional.empty();
// Extract the language ISO code
String langIsoCode = matcher.group(2).toLowerCase();
StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(matcher.start(), matcher.end(), " ");
if (languageConfiguration.getLanguage(langIsoCode) == null)
return Optional.empty();
String newUrl = parameters.withLanguage(lang).withQuery(newQuery.toString().trim()).renderUrl();
return Optional.of(
new MapModelAndView("redirect.jte", Map.of("url", newUrl))
);
}
String newUrl = parameters
.withLanguage(langIsoCode)
.withQuery(matcher.replaceAll(" ").trim())
.renderUrl();
return Optional.empty();
return Optional.of(
new MapModelAndView("redirect.jte", Map.of("url", newUrl))
);
}
}

13
code/services-application/search-service/test/nu/marginalia/search/command/LangCommandTest.java
View File

@@ -2,14 +2,25 @@ package nu.marginalia.search.command;
import io.jooby.ModelAndView;
import nu.marginalia.WebsiteUrl;
import nu.marginalia.language.config.LanguageConfigLocation;
import nu.marginalia.language.config.LanguageConfiguration;
import nu.marginalia.search.model.SearchParameters;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.util.Optional;
class LangCommandTest {
LangCommand langCommand = new LangCommand();
static LangCommand langCommand;
@BeforeAll
static void setUpAll() throws IOException, ParserConfigurationException, SAXException {
langCommand = new LangCommand(new LanguageConfiguration(new LanguageConfigLocation.Experimental()));
}
@Test
void testProcess() {