mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-10-06 07:32:38 +02:00
Compare commits
13 Commits
deploy-018
...
deploy-019
Author | SHA1 | Date | |
---|---|---|---|
|
b1814a30f7 | ||
|
f59a9eb025 | ||
|
599534806b | ||
|
7e8253dac7 | ||
|
97a6780ea3 | ||
|
eb634beec8 | ||
|
269ebd1654 | ||
|
39ce40bfeb | ||
|
c187b2e1c1 | ||
|
42eaa4588b | ||
|
4f40a5fbeb | ||
|
3f3d42bc01 | ||
|
61c8d53e1b |
@@ -112,14 +112,6 @@ public class EdgeDomain implements Serializable {
|
|||||||
return topDomain;
|
return topDomain;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getDomainKey() {
|
|
||||||
int cutPoint = topDomain.indexOf('.');
|
|
||||||
if (cutPoint < 0) {
|
|
||||||
return topDomain;
|
|
||||||
}
|
|
||||||
return topDomain.substring(0, cutPoint).toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** If possible, try to provide an alias domain,
|
/** If possible, try to provide an alias domain,
|
||||||
* i.e. a domain name that is very likely to link to this one
|
* i.e. a domain name that is very likely to link to this one
|
||||||
* */
|
* */
|
||||||
|
@@ -8,14 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|||||||
|
|
||||||
class EdgeDomainTest {
|
class EdgeDomainTest {
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSkepdic() throws URISyntaxException {
|
|
||||||
var domain = new EdgeUrl("http://www.skepdic.com/astrology.html");
|
|
||||||
assertEquals("skepdic", domain.getDomain().getDomainKey());
|
|
||||||
var domain2 = new EdgeUrl("http://skepdic.com/astrology.html");
|
|
||||||
assertEquals("skepdic", domain2.getDomain().getDomainKey());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testHkDomain() throws URISyntaxException {
|
public void testHkDomain() throws URISyntaxException {
|
||||||
var domain = new EdgeUrl("http://l7072i3.l7c.net");
|
var domain = new EdgeUrl("http://l7072i3.l7c.net");
|
||||||
|
@@ -61,7 +61,7 @@ public class UrlDeduplicator {
|
|||||||
|
|
||||||
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
||||||
final var domain = details.getUrl().getDomain();
|
final var domain = details.getUrl().getDomain();
|
||||||
final String key = domain.getDomainKey();
|
final String key = domain.toString();
|
||||||
|
|
||||||
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
||||||
}
|
}
|
||||||
|
@@ -23,7 +23,7 @@ public class SearchResultClusterer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** No clustering, just return the results as is */
|
/** No clustering, just return the results as is */
|
||||||
private static List<ClusteredUrlDetails> noOp(List<UrlDetails> results, int total) {
|
public static List<ClusteredUrlDetails> noOp(List<UrlDetails> results, int total) {
|
||||||
if (results.isEmpty())
|
if (results.isEmpty())
|
||||||
return List.of();
|
return List.of();
|
||||||
|
|
||||||
|
@@ -85,7 +85,6 @@ public class SearchService extends JoobyService {
|
|||||||
String emptySvg = "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>";
|
String emptySvg = "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>";
|
||||||
jooby.get("/site/{domain}/favicon", ctx -> {
|
jooby.get("/site/{domain}/favicon", ctx -> {
|
||||||
String domain = ctx.path("domain").value();
|
String domain = ctx.path("domain").value();
|
||||||
logger.info("Finding icon for domain {}", domain);
|
|
||||||
try {
|
try {
|
||||||
DbDomainQueries.DomainIdWithNode domainIdWithNode = domainQueries.getDomainIdWithNode(new EdgeDomain(domain));
|
DbDomainQueries.DomainIdWithNode domainIdWithNode = domainQueries.getDomainIdWithNode(new EdgeDomain(domain));
|
||||||
var faviconMaybe = faviconClient.getFavicon(domain, domainIdWithNode.nodeAffinity());
|
var faviconMaybe = faviconClient.getFavicon(domain, domainIdWithNode.nodeAffinity());
|
||||||
|
@@ -25,6 +25,7 @@ public class UrlDeduplicator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public boolean shouldRemove(DecoratedSearchResultItem details) {
|
public boolean shouldRemove(DecoratedSearchResultItem details) {
|
||||||
|
|
||||||
if (!deduplicateOnSuperficialHash(details))
|
if (!deduplicateOnSuperficialHash(details))
|
||||||
return true;
|
return true;
|
||||||
if (!deduplicateOnLSH(details))
|
if (!deduplicateOnLSH(details))
|
||||||
@@ -61,7 +62,7 @@ public class UrlDeduplicator {
|
|||||||
|
|
||||||
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
||||||
final var domain = details.getUrl().getDomain();
|
final var domain = details.getUrl().getDomain();
|
||||||
final String key = domain.getDomainKey();
|
final String key = domain.toString();
|
||||||
|
|
||||||
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
||||||
}
|
}
|
||||||
|
@@ -8,3 +8,4 @@
|
|||||||
2025-05-05: Deploy executor partition 4.
|
2025-05-05: Deploy executor partition 4.
|
||||||
2025-05-05: Deploy control.
|
2025-05-05: Deploy control.
|
||||||
2025-05-08: Deploy assistant.
|
2025-05-08: Deploy assistant.
|
||||||
|
2025-05-17: Redeploy all.
|
Reference in New Issue
Block a user