(crawler) Reduce the likelihood of crawler tasks locking on domains before they are ready

Change back to an unbounded queue, tighten sleep times a bit.

code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java

@@ -289,10 +289,13 @@ public class CrawlerMain extends ProcessMainClass {
 
                 if (hasTasks || hasRetryTasks || hasRunningTasks) {
                     retryQueue.drainTo(taskList);
+
+                    // Try to submit any tasks that are in the retry queue (this will block if the pool is full)
                     taskList.removeIf(this::trySubmitDeferredTask);
+
                     // Add a small pause here to avoid busy looping toward the end of the execution cycle when
                     // we might have no new viable tasks to run for hours on end
-                    TimeUnit.MILLISECONDS.sleep(50);
+                    TimeUnit.MILLISECONDS.sleep(5);
                 } else {
                     // We have no tasks to run, and no tasks in the retry queue
                     // but we wait a bit to see if any new tasks come in via the retry queue
@@ -445,7 +448,15 @@ public class CrawlerMain extends ProcessMainClass {
             // We don't have a lock, so we can't run this task
             // we return to avoid blocking the pool for too long
             if (lock.isEmpty()) {
-                retryQueue.add(this);
+                if (retryQueue.remainingCapacity() > 0) {
+                    // Sleep a moment to avoid busy looping via the retry queue
+                    // in the case when few tasks remain and almost all are ineligible for
+                    // immediate restart
+                    Thread.sleep(5);
+                }
+
+                retryQueue.put(this);
+                Thread.currentThread().setName("[idle]");
                 return;
             }
             DomainLocks.DomainLock domainLock = lock.get();