fix: perpetually retry persistent tasks

Author: gajus

README.md

@@ -401,7 +401,7 @@ Retries are configured by passing a `retry` property to the trigger configuratio
 ```ts
 /**
  * @property factor The exponential factor to use. Default is 2.
- * @property maxTimeout The maximum number of milliseconds between two retries. Default is Infinity.
+ * @property maxTimeout The maximum number of milliseconds between two retries. Default is 30,000.
  * @property minTimeout The number of milliseconds before starting the first retry. Default is 1000.
  * @property retries The maximum amount of times to retry the operation. Default is 0. Seting this to 1 means do it once, then retry it once.
  */

src/subscribe.ts

@@ -7,6 +7,8 @@ import {
   type Subscription,
   type Trigger,
 } from './types';
+import { setTimeout } from 'node:timers/promises';
+import { serializeError } from 'serialize-error';
 
 const log = Logger.child({
   namespace: 'subscribe',
@@ -67,11 +69,21 @@ const runTask = async ({
 
   let attempt = -1;
 
-  while (attempt++ < trigger.retry.retries) {
-    const retriesLeft = trigger.retry.retries - attempt;
+  while (true) {
+    attempt++;
 
-    if (retriesLeft < 0) {
-      throw new Error('Expected retries left to be greater than or equal to 0');
+    if (attempt > 0) {
+      const retryFactor = trigger.retry.factor ?? 2;
+      const minTimeout = trigger.retry.minTimeout ?? 1_000;
+      const maxTimeout = trigger.retry.maxTimeout ?? 30_000;
+      const delay = Math.min(
+        attempt * retryFactor * minTimeout,
+        trigger.retry.maxTimeout ?? maxTimeout,
+      );
+
+      log.debug('delaying retry by %dms...', delay);
+
+      await setTimeout(delay);
     }
 
     try {
@@ -92,13 +104,40 @@ const runTask = async ({
         }),
         taskId,
       });
+
+      return;
     } catch (error) {
       if (error.name === 'AbortError') {
         log.warn('%s (%s): task aborted', trigger.name, taskId);
 
         return;
       }
 
+      log.warn(
+        {
+          error: serializeError(error),
+        },
+        'routine produced an error',
+      );
+
+      if (trigger.persistent) {
+        log.warn(
+          '%s (%s): retrying because the trigger is persistent',
+          trigger.name,
+          taskId,
+        );
+
+        continue;
+      }
+
+      const retriesLeft = trigger.retry.retries - attempt;
+
+      if (retriesLeft < 0) {
+        throw new Error(
+          'Expected retries left to be greater than or equal to 0',
+        );
+      }
+
       if (retriesLeft === 0) {
         log.warn(
           '%s (%s): task will not be retried; attempts exhausted',
@@ -229,8 +268,15 @@ export const subscribe = (trigger: Trigger): Subscription => {
         }
       })
       // eslint-disable-next-line promise/prefer-await-to-then
-      .catch(() => {
-        log.warn('%s (%s): task failed', trigger.name, taskId);
+      .catch((error) => {
+        log.warn(
+          {
+            error: serializeError(error),
+          },
+          '%s (%s): task failed',
+          trigger.name,
+          taskId,
+        );
       });
 
     log.debug('%s (%s): started task', trigger.name, taskId);

src/types.ts

@@ -76,7 +76,7 @@ type OnTeardownEventHandler = (event: TeardownEvent) => Promise<void>;
 
 /**
  * @property factor The exponential factor to use. Default is 2.
- * @property maxTimeout The maximum number of milliseconds between two retries. Default is Infinity.
+ * @property maxTimeout The maximum number of milliseconds between two retries. Default is 30,000.
  * @property minTimeout The number of milliseconds before starting the first retry. Default is 1000.
  * @property retries The maximum amount of times to retry the operation. Default is 10. Seting this to 1 means do it once, then retry it once.
  */