mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-10-06 07:32:38 +02:00
Compare commits
8 Commits
deploy-018
...
deploy-019
Author | SHA1 | Date | |
---|---|---|---|
|
b1814a30f7 | ||
|
f59a9eb025 | ||
|
599534806b | ||
|
7e8253dac7 | ||
|
97a6780ea3 | ||
|
eb634beec8 | ||
|
269ebd1654 | ||
|
39ce40bfeb |
@@ -112,14 +112,6 @@ public class EdgeDomain implements Serializable {
|
|||||||
return topDomain;
|
return topDomain;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getDomainKey() {
|
|
||||||
int cutPoint = topDomain.indexOf('.');
|
|
||||||
if (cutPoint < 0) {
|
|
||||||
return topDomain;
|
|
||||||
}
|
|
||||||
return topDomain.substring(0, cutPoint).toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** If possible, try to provide an alias domain,
|
/** If possible, try to provide an alias domain,
|
||||||
* i.e. a domain name that is very likely to link to this one
|
* i.e. a domain name that is very likely to link to this one
|
||||||
* */
|
* */
|
||||||
|
@@ -8,14 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|||||||
|
|
||||||
class EdgeDomainTest {
|
class EdgeDomainTest {
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSkepdic() throws URISyntaxException {
|
|
||||||
var domain = new EdgeUrl("http://www.skepdic.com/astrology.html");
|
|
||||||
assertEquals("skepdic", domain.getDomain().getDomainKey());
|
|
||||||
var domain2 = new EdgeUrl("http://skepdic.com/astrology.html");
|
|
||||||
assertEquals("skepdic", domain2.getDomain().getDomainKey());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testHkDomain() throws URISyntaxException {
|
public void testHkDomain() throws URISyntaxException {
|
||||||
var domain = new EdgeUrl("http://l7072i3.l7c.net");
|
var domain = new EdgeUrl("http://l7072i3.l7c.net");
|
||||||
|
@@ -61,7 +61,7 @@ public class UrlDeduplicator {
|
|||||||
|
|
||||||
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
||||||
final var domain = details.getUrl().getDomain();
|
final var domain = details.getUrl().getDomain();
|
||||||
final String key = domain.getDomainKey();
|
final String key = domain.toString();
|
||||||
|
|
||||||
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
||||||
}
|
}
|
||||||
|
@@ -25,6 +25,7 @@ public class UrlDeduplicator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public boolean shouldRemove(DecoratedSearchResultItem details) {
|
public boolean shouldRemove(DecoratedSearchResultItem details) {
|
||||||
|
|
||||||
if (!deduplicateOnSuperficialHash(details))
|
if (!deduplicateOnSuperficialHash(details))
|
||||||
return true;
|
return true;
|
||||||
if (!deduplicateOnLSH(details))
|
if (!deduplicateOnLSH(details))
|
||||||
@@ -61,7 +62,7 @@ public class UrlDeduplicator {
|
|||||||
|
|
||||||
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
private boolean limitResultsPerDomain(DecoratedSearchResultItem details) {
|
||||||
final var domain = details.getUrl().getDomain();
|
final var domain = details.getUrl().getDomain();
|
||||||
final String key = domain.getDomainKey();
|
final String key = domain.toString();
|
||||||
|
|
||||||
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
return keyCount.adjustOrPutValue(key, 1, 1) <= resultsPerKey;
|
||||||
}
|
}
|
||||||
|
@@ -8,3 +8,4 @@
|
|||||||
2025-05-05: Deploy executor partition 4.
|
2025-05-05: Deploy executor partition 4.
|
||||||
2025-05-05: Deploy control.
|
2025-05-05: Deploy control.
|
||||||
2025-05-08: Deploy assistant.
|
2025-05-08: Deploy assistant.
|
||||||
|
2025-05-17: Redeploy all.
|
Reference in New Issue
Block a user