1
1
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-10-05 21:22:39 +02:00
Files
MarginaliaSearch/code/execution/test/nu/marginalia/svc/ExecutorFileTransferServiceTest.java
Viktor Lofgren 62b696b1c3 (architecture) Remove the separate executor service and merge it into the index service
The primary motivation for this is that in production, the large number of partitioned services has lead to an intermittent exhaustion of available database connections, as each service has a connection pool.

The decision to have a separate executor service dates back from when the index service was very slow to start, and the executor didn't always spin off its memory-hungry tasks into separate processes, which meant the executor would sometimes OOM and crash, and it was undesirable to bring the index down with it.
2025-07-23 12:57:13 +02:00

61 lines
1.8 KiB
Java

package nu.marginalia.svc;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorage;
import nu.marginalia.storage.model.FileStorageId;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import spark.Spark;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.DriverManager;
import java.sql.SQLException;
import static org.mockito.Mockito.when;
class ExecutorFileTransferServiceTest {
@Test
public void test() throws SQLException, InterruptedException {
// Test requires this file to exist
if (!Files.exists(Path.of("/tmp/crawl.parquet"))) {
return;
}
var fileStorage = Mockito.mock(FileStorageService.class);
when(fileStorage.getStorage(Mockito.any(FileStorageId.class))).thenReturn(new FileStorage(null,
null,
null,
null,
"/tmp",
null,
null));
var svc = new ExecutorFileTransferService(fileStorage);
Spark.port(9998);
Spark.get("/transfer/file/:fid", svc::transferFile);
Spark.head("/transfer/file/:fid", svc::transferFile);
Spark.init();
Thread.sleep(1000);
try (var conn = DriverManager.getConnection("jdbc:duckdb:");
var stmt = conn.createStatement()) {
var rs = stmt.executeQuery("""
SELECT COUNT(*) AS cnt, httpStatus
FROM 'http://localhost:9998/transfer/file/0?path=crawl.parquet'
GROUP BY httpStatus
""");
while (rs.next()) {
System.out.println(rs.getInt("CNT") + " " + rs.getInt("httpStatus"));
}
}
Spark.stop();
}
}