eUgEntOptIc44/MarginaliaSearch
1
1
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-10-05 21:22:39 +02:00
Code Issues Activity

(refac) Clean up index code

Browse Source
This commit is contained in:
Viktor Lofgren
2025-09-02 18:52:31 +02:00
parent 87ce4a1b52
commit 6f7530e807
19 changed files with 33 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
code/index/java/nu/marginalia/index/IndexConstructorMain.java
View File

@@ -3,11 +3,11 @@ package nu.marginalia.index;
import com.google.inject.Guice;
import com.google.inject.Inject;
import nu.marginalia.IndexLocations;
import nu.marginalia.index.forward.ForwardIndexFileNames;
import nu.marginalia.index.config.ForwardIndexFileNames;
import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.reverse.ReverseIndexFullFileNames;
import nu.marginalia.index.reverse.ReverseIndexPrioFileNames;
import nu.marginalia.index.config.ReverseIndexFullFileNames;
import nu.marginalia.index.config.ReverseIndexPrioFileNames;
import nu.marginalia.index.reverse.construction.full.FullIndexConstructor;
import nu.marginalia.index.reverse.construction.prio.PrioIndexConstructor;
import nu.marginalia.index.searchset.DomainRankings;

6
code/index/java/nu/marginalia/index/IndexFactory.java
View File

@@ -3,12 +3,12 @@ package nu.marginalia.index;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.IndexLocations;
import nu.marginalia.index.forward.ForwardIndexFileNames;
import nu.marginalia.index.config.ForwardIndexFileNames;
import nu.marginalia.index.config.ReverseIndexFullFileNames;
import nu.marginalia.index.config.ReverseIndexPrioFileNames;
import nu.marginalia.index.forward.ForwardIndexReader;
import nu.marginalia.index.reverse.FullReverseIndexReader;
import nu.marginalia.index.reverse.PrioReverseIndexReader;
import nu.marginalia.index.reverse.ReverseIndexFullFileNames;
import nu.marginalia.index.reverse.ReverseIndexPrioFileNames;
import nu.marginalia.index.reverse.positions.PositionsFileReader;
import nu.marginalia.storage.FileStorageService;

2
code/index/java/nu/marginalia/index/forward/ForwardIndexFileNames.java → code/index/java/nu/marginalia/index/config/ForwardIndexFileNames.java
View File

@@ -1,4 +1,4 @@
package nu.marginalia.index.forward;
package nu.marginalia.index.config;
import java.nio.file.Path;

2
code/index/java/nu/marginalia/index/forward/ForwardIndexParameters.java → code/index/java/nu/marginalia/index/config/ForwardIndexParameters.java
View File

@@ -1,4 +1,4 @@
package nu.marginalia.index.forward;
package nu.marginalia.index.config;
public class ForwardIndexParameters {
public static final int ENTRY_SIZE = 3;

2
code/index/java/nu/marginalia/index/reverse/ReverseIndexFullFileNames.java → code/index/java/nu/marginalia/index/config/ReverseIndexFullFileNames.java
View File

@@ -1,4 +1,4 @@
package nu.marginalia.index.reverse;
package nu.marginalia.index.config;
import java.nio.file.Path;

2
code/index/java/nu/marginalia/index/reverse/ReverseIndexParameters.java → code/index/java/nu/marginalia/index/config/ReverseIndexParameters.java
View File

@@ -1,4 +1,4 @@
package nu.marginalia.index.reverse;
package nu.marginalia.index.config;
import nu.marginalia.btree.model.BTreeBlockSize;
import nu.marginalia.btree.model.BTreeContext;

2
code/index/java/nu/marginalia/index/reverse/ReverseIndexPrioFileNames.java → code/index/java/nu/marginalia/index/config/ReverseIndexPrioFileNames.java
View File

@@ -1,4 +1,4 @@
package nu.marginalia.index.reverse;
package nu.marginalia.index.config;
import java.nio.file.Path;

2
code/index/java/nu/marginalia/index/forward/ForwardIndexReader.java
View File

@@ -18,7 +18,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeoutException;
import static nu.marginalia.index.forward.ForwardIndexParameters.*;
import static nu.marginalia.index.config.ForwardIndexParameters.*;
/** Reads the forward index.
* <p/>

2
code/index/java/nu/marginalia/index/forward/construction/ForwardIndexConverter.java
View File

@@ -3,7 +3,7 @@ package nu.marginalia.index.forward.construction;
import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.LongArrayFactory;
import nu.marginalia.index.forward.ForwardIndexParameters;
import nu.marginalia.index.config.ForwardIndexParameters;
import nu.marginalia.index.forward.spans.IndexSpansWriter;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.searchset.DomainRankings;

2
code/index/java/nu/marginalia/index/forward/spans/SpansCodec.java
View File

@@ -11,6 +11,8 @@ public class SpansCodec {
public static int FOOTER_SIZE = 8;
public enum SpansCodecVersion {
@Deprecated
DEPRECATED_1, // This must not be removed, the ordinal is used to encode the version
PLAIN
}

1
code/index/java/nu/marginalia/index/reverse/FullReverseIndexReader.java
View File

@@ -5,6 +5,7 @@ import nu.marginalia.array.LongArrayFactory;
import nu.marginalia.array.pool.BufferPool;
import nu.marginalia.btree.BTreeReader;
import nu.marginalia.ffi.LinuxSystemCalls;
import nu.marginalia.index.config.ReverseIndexParameters;
import nu.marginalia.index.model.CombinedDocIdList;
import nu.marginalia.index.model.TermMetadataList;
import nu.marginalia.index.reverse.positions.PositionsFileReader;

1
code/index/java/nu/marginalia/index/reverse/PrioIndexEntrySource.java
View File

@@ -61,6 +61,7 @@ public class PrioIndexEntrySource implements EntrySource {
var outputBuffer = buffer.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN);
outputBuffer.clear();
// FYI: The encoding end of this compression algorithm is at PrioDocIdsTransformer
while (outputBuffer.hasRemaining() && readItems++ < numItems) {
int rank;
int domainId;

1
code/index/java/nu/marginalia/index/reverse/PrioReverseIndexReader.java
View File

@@ -4,6 +4,7 @@ import nu.marginalia.array.LongArray;
import nu.marginalia.array.LongArrayFactory;
import nu.marginalia.btree.BTreeReader;
import nu.marginalia.ffi.LinuxSystemCalls;
import nu.marginalia.index.config.ReverseIndexParameters;
import nu.marginalia.index.reverse.query.EmptyEntrySource;
import nu.marginalia.index.reverse.query.EntrySource;
import org.slf4j.Logger;

4
code/index/java/nu/marginalia/index/reverse/construction/full/FullPreindex.java
View File

@@ -3,11 +3,11 @@ package nu.marginalia.index.reverse.construction.full;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.LongArrayFactory;
import nu.marginalia.btree.BTreeWriter;
import nu.marginalia.index.reverse.ReverseIndexParameters;
import nu.marginalia.index.config.ReverseIndexParameters;
import nu.marginalia.index.journal.IndexJournalPage;
import nu.marginalia.index.reverse.construction.CountToOffsetTransformer;
import nu.marginalia.index.reverse.construction.DocIdRewriter;
import nu.marginalia.index.reverse.construction.PositionsFileConstructor;
import nu.marginalia.index.journal.IndexJournalPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

4
code/index/java/nu/marginalia/index/reverse/construction/prio/PrioPreindex.java
View File

@@ -3,10 +3,10 @@ package nu.marginalia.index.reverse.construction.prio;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.LongArrayFactory;
import nu.marginalia.btree.BTreeWriter;
import nu.marginalia.index.reverse.ReverseIndexParameters;
import nu.marginalia.index.config.ReverseIndexParameters;
import nu.marginalia.index.journal.IndexJournalPage;
import nu.marginalia.index.reverse.construction.CountToOffsetTransformer;
import nu.marginalia.index.reverse.construction.DocIdRewriter;
import nu.marginalia.index.journal.IndexJournalPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

4
code/index/test/nu/marginalia/index/CombinedIndexReaderTest.java
View File

@@ -6,11 +6,11 @@ import it.unimi.dsi.fastutil.ints.IntList;
import nu.marginalia.IndexLocations;
import nu.marginalia.array.page.LongQueryBuffer;
import nu.marginalia.hash.MurmurHash3_128;
import nu.marginalia.index.forward.ForwardIndexFileNames;
import nu.marginalia.index.config.ForwardIndexFileNames;
import nu.marginalia.index.config.ReverseIndexFullFileNames;
import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.journal.IndexJournalSlopWriter;
import nu.marginalia.index.reverse.ReverseIndexFullFileNames;
import nu.marginalia.index.reverse.construction.DocIdRewriter;
import nu.marginalia.index.reverse.construction.full.FullIndexConstructor;
import nu.marginalia.index.reverse.query.IndexSearchBudget;

6
code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java
View File

@@ -7,12 +7,12 @@ import nu.marginalia.api.searchquery.RpcDecoratedResultItem;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.*;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.index.forward.ForwardIndexFileNames;
import nu.marginalia.index.config.ForwardIndexFileNames;
import nu.marginalia.index.config.ReverseIndexFullFileNames;
import nu.marginalia.index.config.ReverseIndexPrioFileNames;
import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.journal.IndexJournalSlopWriter;
import nu.marginalia.index.reverse.ReverseIndexFullFileNames;
import nu.marginalia.index.reverse.ReverseIndexPrioFileNames;
import nu.marginalia.index.reverse.construction.DocIdRewriter;
import nu.marginalia.index.reverse.construction.full.FullIndexConstructor;
import nu.marginalia.index.reverse.construction.prio.PrioIndexConstructor;

6
code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java
View File

@@ -8,12 +8,12 @@ import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.*;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.hash.MurmurHash3_128;
import nu.marginalia.index.forward.ForwardIndexFileNames;
import nu.marginalia.index.config.ForwardIndexFileNames;
import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.journal.IndexJournalSlopWriter;
import nu.marginalia.index.reverse.ReverseIndexFullFileNames;
import nu.marginalia.index.reverse.ReverseIndexPrioFileNames;
import nu.marginalia.index.config.ReverseIndexFullFileNames;
import nu.marginalia.index.config.ReverseIndexPrioFileNames;
import nu.marginalia.index.reverse.construction.DocIdRewriter;
import nu.marginalia.index.reverse.construction.full.FullIndexConstructor;
import nu.marginalia.index.reverse.construction.prio.PrioIndexConstructor;

6
code/tools/integration-test/test/nu/marginalia/IntegrationTest.java
View File

@@ -16,13 +16,13 @@ import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
import nu.marginalia.functions.searchquery.QueryFactory;
import nu.marginalia.index.IndexQueryExecution;
import nu.marginalia.index.StatefulIndex;
import nu.marginalia.index.forward.ForwardIndexFileNames;
import nu.marginalia.index.config.ForwardIndexFileNames;
import nu.marginalia.index.config.ReverseIndexFullFileNames;
import nu.marginalia.index.config.ReverseIndexPrioFileNames;
import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.model.SearchContext;
import nu.marginalia.index.results.IndexResultRankingService;
import nu.marginalia.index.reverse.ReverseIndexFullFileNames;
import nu.marginalia.index.reverse.ReverseIndexPrioFileNames;
import nu.marginalia.index.reverse.construction.full.FullIndexConstructor;
import nu.marginalia.index.reverse.construction.prio.PrioIndexConstructor;
import nu.marginalia.index.searchset.DomainRankings;