mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-10-05 21:22:39 +02:00
The primary motivation for this is that in production, the large number of partitioned services has lead to an intermittent exhaustion of available database connections, as each service has a connection pool. The decision to have a separate executor service dates back from when the index service was very slow to start, and the executor didn't always spin off its memory-hungry tasks into separate processes, which meant the executor would sometimes OOM and crash, and it was undesirable to bring the index down with it.
61 lines
1.8 KiB
Java
61 lines
1.8 KiB
Java
package nu.marginalia.svc;
|
|
|
|
import nu.marginalia.storage.FileStorageService;
|
|
import nu.marginalia.storage.model.FileStorage;
|
|
import nu.marginalia.storage.model.FileStorageId;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.mockito.Mockito;
|
|
import spark.Spark;
|
|
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.sql.DriverManager;
|
|
import java.sql.SQLException;
|
|
|
|
import static org.mockito.Mockito.when;
|
|
|
|
class ExecutorFileTransferServiceTest {
|
|
|
|
@Test
|
|
public void test() throws SQLException, InterruptedException {
|
|
// Test requires this file to exist
|
|
if (!Files.exists(Path.of("/tmp/crawl.parquet"))) {
|
|
return;
|
|
}
|
|
|
|
var fileStorage = Mockito.mock(FileStorageService.class);
|
|
|
|
when(fileStorage.getStorage(Mockito.any(FileStorageId.class))).thenReturn(new FileStorage(null,
|
|
null,
|
|
null,
|
|
null,
|
|
"/tmp",
|
|
null,
|
|
null));
|
|
|
|
var svc = new ExecutorFileTransferService(fileStorage);
|
|
|
|
Spark.port(9998);
|
|
Spark.get("/transfer/file/:fid", svc::transferFile);
|
|
Spark.head("/transfer/file/:fid", svc::transferFile);
|
|
|
|
Spark.init();
|
|
|
|
Thread.sleep(1000);
|
|
|
|
|
|
try (var conn = DriverManager.getConnection("jdbc:duckdb:");
|
|
var stmt = conn.createStatement()) {
|
|
var rs = stmt.executeQuery("""
|
|
SELECT COUNT(*) AS cnt, httpStatus
|
|
FROM 'http://localhost:9998/transfer/file/0?path=crawl.parquet'
|
|
GROUP BY httpStatus
|
|
""");
|
|
while (rs.next()) {
|
|
System.out.println(rs.getInt("CNT") + " " + rs.getInt("httpStatus"));
|
|
}
|
|
}
|
|
|
|
Spark.stop();
|
|
}
|
|
} |