1
1
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-10-06 07:32:38 +02:00

Compare commits

...

2 Commits

Author SHA1 Message Date
Viktor Lofgren
79da622e3b (search) Update front page with new banner about move 2025-01-08 21:38:19 +01:00
Viktor Lofgren
3da8337ba6 (feeds) Add system property for exporting fetched feeds to a slop table for debugging 2025-01-08 20:49:16 +01:00
4 changed files with 74 additions and 5 deletions

View File

@@ -29,6 +29,7 @@ dependencies {
implementation libs.jsoup
implementation project(':third-party:rssreader')
implementation libs.opencsv
implementation libs.slop
implementation libs.sqlite
implementation libs.bundles.slf4j
implementation libs.commons.lang3

View File

@@ -96,6 +96,7 @@ public class FeedFetcherService {
throw new IllegalStateException("Already updating feeds, refusing to start another update");
}
try (FeedDbWriter writer = feedDb.createWriter();
HttpClient client = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(15))
@@ -103,6 +104,7 @@ public class FeedFetcherService {
.followRedirects(HttpClient.Redirect.NORMAL)
.version(HttpClient.Version.HTTP_2)
.build();
FeedJournal feedJournal = FeedJournal.create();
var heartbeat = serviceHeartbeat.createServiceAdHocTaskHeartbeat("Update Rss Feeds")
) {
updating = true;
@@ -155,6 +157,8 @@ public class FeedFetcherService {
case FetchResult.Success(String value, String etag) -> {
writer.saveEtag(feed.domain(), etag);
writer.saveFeed(parseFeed(value, feed));
feedJournal.record(feed.feedUrl(), value);
}
case FetchResult.NotModified() -> {
writer.saveEtag(feed.domain(), ifNoneMatchTag);

View File

@@ -0,0 +1,64 @@
package nu.marginalia.rss.svc;
import nu.marginalia.WmsaHome;
import nu.marginalia.slop.SlopTable;
import nu.marginalia.slop.column.string.StringColumn;
import nu.marginalia.slop.desc.StorageType;
import org.apache.commons.io.FileUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
/** Utility for recording fetched feeds to a journal, useful in debugging feed parser issues.
*/
public interface FeedJournal extends AutoCloseable {
StringColumn urlColumn = new StringColumn("url");
StringColumn contentsColumn = new StringColumn("contents", StandardCharsets.UTF_8, StorageType.ZSTD);
void record(String url, String contents) throws IOException;
void close() throws IOException;
static FeedJournal create() throws IOException {
if (Boolean.getBoolean("feedFetcher.persistJournal")) {
Path journalPath = WmsaHome.getDataPath().resolve("feed-journal");
if (Files.isDirectory(journalPath)) {
FileUtils.deleteDirectory(journalPath.toFile());
}
Files.createDirectories(journalPath);
return new RecordingFeedJournal(journalPath);
}
else {
return new NoOpFeedJournal();
}
}
class NoOpFeedJournal implements FeedJournal {
@Override
public void record(String url, String contents) {}
@Override
public void close() {}
}
class RecordingFeedJournal extends SlopTable implements FeedJournal {
private final StringColumn.Writer urlWriter;
private final StringColumn.Writer contentsWriter;
public RecordingFeedJournal(Path path) throws IOException {
super(path, SlopTable.getNumPages(path, FeedJournal.urlColumn));
urlWriter = urlColumn.create(this);
contentsWriter = contentsColumn.create(this);
}
public synchronized void record(String url, String contents) throws IOException {
urlWriter.put(url);
contentsWriter.put(contents);
}
}
}

View File

@@ -34,12 +34,12 @@
<div class="max-w-7xl mx-auto flex flex-col space-y-4 fill-w">
<div class="border dark:border-gray-600 dark:bg-gray-800 bg-white rounded p-2 m-4 ">
<div class="text-slate-700 dark:text-white text-sm p-4">
<div class="fas fa-wrench mr-1 text-margeblue dark:text-slate-200"></div>
This is the new design and home of Marginalia Search. Migration to the new domain <pre class="inline text-red-800 dark:text-red-100">marginalia-search.com</pre> is currently <em>in progress</em>,
so mind that some things may be a bit broken for a day or two. <a href="https://about.marginalia-search.com/article/redesign/" class="underline text-liteblue dark:text-blue-200">Read more</a>.
<div class="fas fa-gift mr-1 text-margeblue dark:text-slate-200"></div>
This is the new design and home of Marginalia Search.
You can about what this entails <a href="https://about.marginalia-search.com/article/redesign/" class="underline text-liteblue dark:text-blue-200">here</a>.
<p class="my-4"></p>
If you have any issues or feedback regarding this change, please email
<a href="mailto:contact@marginalia-search.com" class="underline text-liteblue dark:text-blue-200">contact@marginalia-search.com</a>.
The old version of Marginalia Search remains available at
<a href="https://old-search.marginalia.nu/" class="underline text-liteblue dark:text-blue-200">https://old-search.marginalia.nu/</a>.
</div>
</div>
<div class="mx-auto flex flex-col sm:flex-row my-4 sm:space-x-2 space-y-2 sm:space-y-0 w-full md:w-auto px-2">