mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-10-06 17:32:39 +02:00
Compare commits
4 Commits
deploy-000
...
deploy-000
Author | SHA1 | Date | |
---|---|---|---|
|
7622335e84 | ||
|
0da2047eae | ||
|
5ee4321110 | ||
|
9459b9933b |
@@ -38,7 +38,6 @@ import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.BiFunction;
|
||||
@@ -124,31 +123,33 @@ public class FeedFetcherService {
|
||||
|
||||
for (var feed : definitions) {
|
||||
executor.submitQuietly(() -> {
|
||||
try {
|
||||
var oldData = feedDb.getFeed(new EdgeDomain(feed.domain()));
|
||||
|
||||
// If we have existing data, we might skip updating it with a probability that increases with time,
|
||||
// this is to avoid hammering the feeds that are updated very rarely and save some time and resources
|
||||
// on our end
|
||||
|
||||
/* Disable for now:
|
||||
|
||||
if (!oldData.isEmpty()) {
|
||||
Duration duration = feed.durationSinceUpdated();
|
||||
long daysSinceUpdate = duration.toDays();
|
||||
|
||||
|
||||
if (deterministic || (daysSinceUpdate > 2 && ThreadLocalRandom.current()
|
||||
.nextInt(1, 1 + (int) Math.min(10, daysSinceUpdate) / 2) > 1))
|
||||
{
|
||||
.nextInt(1, 1 + (int) Math.min(10, daysSinceUpdate) / 2) > 1)) {
|
||||
// Skip updating this feed, just write the old data back instead
|
||||
writer.saveFeed(oldData);
|
||||
return;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
FetchResult feedData;
|
||||
try (DomainLocks.DomainLock domainLock = domainLocks.lockDomain(new EdgeDomain(feed.domain()))) {
|
||||
feedData = fetchFeedData(feed, client);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
} catch (Exception ex) {
|
||||
feedData = new FetchResult.TransientError();
|
||||
}
|
||||
|
||||
@@ -163,13 +164,17 @@ public class FeedFetcherService {
|
||||
writer.saveFeed(oldData);
|
||||
}
|
||||
}
|
||||
case FetchResult.PermanentError() -> {} // let the definition be forgotten about
|
||||
case FetchResult.PermanentError() -> {
|
||||
} // let the definition be forgotten about
|
||||
}
|
||||
|
||||
}
|
||||
finally {
|
||||
if ((definitionsUpdated.incrementAndGet() % 1_000) == 0) {
|
||||
// Update the progress every 1k feeds, to avoid hammering the database and flooding the logs
|
||||
heartbeat.progress("Updated " + definitionsUpdated + "/" + totalDefinitions + " feeds", definitionsUpdated.get(), totalDefinitions);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
@@ -118,6 +118,7 @@ def deploy_container(container: DockerContainer) -> None:
|
||||
text=True
|
||||
)
|
||||
|
||||
|
||||
# Stream output in real-time
|
||||
while True:
|
||||
output = process.stdout.readline()
|
||||
@@ -156,6 +157,9 @@ def build_and_deploy(plan: DeploymentPlan, service_config: Dict[str, ServiceConf
|
||||
to_deploy.append(container)
|
||||
else:
|
||||
for instance in range(1,config.instances + 1):
|
||||
if config.docker_name in plan.instances_to_hold:
|
||||
continue
|
||||
|
||||
container_name = f"{config.docker_name}-{instance}"
|
||||
if container_name in plan.instances_to_hold:
|
||||
continue
|
||||
@@ -207,22 +211,22 @@ if __name__ == '__main__':
|
||||
instances=2,
|
||||
deploy_tier=1
|
||||
),
|
||||
'api': ServiceConfig(
|
||||
'assistant': ServiceConfig(
|
||||
gradle_target=':code:services-core:assistant-service:docker',
|
||||
docker_name='assistant-service',
|
||||
instances=2,
|
||||
deploy_tier1=2
|
||||
deploy_tier=2
|
||||
),
|
||||
'explorer': ServiceConfig(
|
||||
gradle_target=':code:services-application:explorer-service:docker',
|
||||
docker_name='explorer-service',
|
||||
instances=1,
|
||||
instances=None,
|
||||
deploy_tier=1
|
||||
),
|
||||
'dating': ServiceConfig(
|
||||
gradle_target=':code:services-application:dating-service:docker',
|
||||
docker_name='dating-service',
|
||||
instances=1,
|
||||
instances=None,
|
||||
deploy_tier=1
|
||||
),
|
||||
'index': ServiceConfig(
|
||||
@@ -253,10 +257,8 @@ if __name__ == '__main__':
|
||||
|
||||
try:
|
||||
tags = get_deployment_tag()
|
||||
if tags == None:
|
||||
exit
|
||||
|
||||
print(tags)
|
||||
if tags != None:
|
||||
print("Found deployment tags:", tags)
|
||||
|
||||
plan = parse_deployment_tags(tags, SERVICE_CONFIG)
|
||||
print("\nDeployment Plan:")
|
||||
@@ -266,6 +268,8 @@ if __name__ == '__main__':
|
||||
print("\nExecution Plan:")
|
||||
|
||||
build_and_deploy(plan, SERVICE_CONFIG)
|
||||
else:
|
||||
print("No tags found")
|
||||
|
||||
except ValueError as e:
|
||||
print(f"Error: {e}")
|
||||
|
Reference in New Issue
Block a user