feat(http): Support for Retry-After header

lib/util/http/index.ts

@@ -10,11 +10,12 @@ import * as memCache from '../cache/memory';
 import { clone } from '../clone';
 import { hash } from '../hash';
 import { type AsyncResult, Result } from '../result';
-import { resolveBaseUrl } from '../url';
+import { parseUrl, resolveBaseUrl } from '../url';
 import { applyAuthorization, removeAuthorization } from './auth';
 import { hooks } from './hooks';
 import { applyHostRules } from './host-rules';
 import { getQueue } from './queue';
+import { extractRetryAfterHeaderSeconds, wrapWithRetry } from './retry-after';
 import { Throttle, getThrottle } from './throttle';
 import type {
   GotJSONOptions,
@@ -225,7 +226,12 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
         ? () => queue.add<HttpResponse<T>>(throttledTask)
         : throttledTask;
 
-      resPromise = queuedTask();
+      const host = parseUrl(url)?.host ?? /* istanbul ignore next */ url;
+      resPromise = wrapWithRetry(
+        host,
+        queuedTask,
+        extractRetryAfterHeaderSeconds,
+      );
 
       if (memCacheKey) {
         memCache.set(memCacheKey, resPromise);

lib/util/http/retry-after.spec.ts

@@ -0,0 +1,111 @@
+import { RequestError } from 'got';
+import { extractRetryAfterHeaderSeconds, wrapWithRetry } from './retry-after';
+
+describe('util/http/retry-after', () => {
+  describe('wrapWithRetry', () => {
+    beforeEach(() => {
+      jest.useFakeTimers({ advanceTimers: true });
+    });
+
+    afterEach(() => {
+      jest.useRealTimers();
+    });
+
+    it('works', async () => {
+      const task = jest.fn(() => Promise.resolve(42));
+      const res = await wrapWithRetry('http://example.com', task, () => null);
+      expect(res).toBe(42);
+      expect(task).toHaveBeenCalledTimes(1);
+    });
+
+    it('throws', async () => {
+      const task = jest.fn(() => Promise.reject(new Error('foo')));
+
+      await expect(
+        wrapWithRetry('http://example.com', task, () => null),
+      ).rejects.toThrow('foo');
+
+      expect(task).toHaveBeenCalledTimes(1);
+    });
+
+    it('retries', async () => {
+      const task = jest
+        .fn()
+        .mockRejectedValueOnce(new Error('foo'))
+        .mockRejectedValueOnce(new Error('bar'))
+        .mockResolvedValueOnce(42);
+
+      const p = wrapWithRetry('http://example.com', task, () => 1);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      const res = await p;
+      expect(res).toBe(42);
+      expect(task).toHaveBeenCalledTimes(3);
+    });
+
+    it('gives up after max retries', async () => {
+      const task = jest
+        .fn()
+        .mockRejectedValueOnce('foo')
+        .mockRejectedValueOnce('bar')
+        .mockRejectedValueOnce('baz')
+        .mockRejectedValue('qux');
+
+      const p = wrapWithRetry('http://example.com', task, () => 1).catch(
+        (err) => err,
+      );
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await expect(p).resolves.toBe('baz');
+      expect(task).toHaveBeenCalledTimes(3);
+    });
+  });
+
+  describe('extractRetryAfterHeaderSeconds', () => {
+    it('returns null for non-RequestError', () => {
+      expect(extractRetryAfterHeaderSeconds(new Error())).toBeNull();
+    });
+
+    it('returns null for RequestError without response', () => {
+      expect(
+        extractRetryAfterHeaderSeconds(
+          new RequestError('foo', {}, null as never),
+        ),
+      ).toBeNull();
+    });
+
+    it('returns null for status other than 429', () => {
+      const err = new RequestError('foo', {}, null as never);
+      (err as any).response = { statusCode: 302 };
+      expect(extractRetryAfterHeaderSeconds(err)).toBeNull();
+    });
+
+    it('returns null missing "retry-after" header', () => {
+      const err = new RequestError('foo', {}, null as never);
+      (err as any).response = { statusCode: 429, headers: {} };
+      expect(extractRetryAfterHeaderSeconds(err)).toBeNull();
+    });
+
+    it('returns null for non-integer "retry-after" header', () => {
+      const err = new RequestError('foo', {}, null as never);
+      (err as any).response = {
+        statusCode: 429,
+        headers: {
+          'retry-after': 'Wed, 21 Oct 2015 07:28:00 GMT',
+        },
+      };
+      expect(extractRetryAfterHeaderSeconds(err)).toBeNull();
+    });
+
+    it('returns delay in seconds', () => {
+      const err = new RequestError('foo', {}, null as never);
+      (err as any).response = {
+        statusCode: 429,
+        headers: {
+          'retry-after': '42',
+        },
+      };
+      expect(extractRetryAfterHeaderSeconds(err)).toBe(42);
+    });
+  });
+});

lib/util/http/retry-after.ts

@@ -0,0 +1,67 @@
+import { RequestError } from 'got';
+import type { Task } from './types';
+
+const hostBlocks = new Map<string, Promise<void>>();
+
+const maxRetries = 2;
+
+function sleep(seconds: number): Promise<void> {
+  return new Promise((resolve) => global.setTimeout(resolve, seconds * 1000));
+}
+
+export function extractRetryAfterHeaderSeconds(err: unknown): number | null {
+  if (!(err instanceof RequestError)) {
+    return null;
+  }
+
+  if (!err.response) {
+    return null;
+  }
+
+  if (err.response.statusCode !== 429) {
+    return null;
+  }
+
+  const retryAfter = err.response.headers['retry-after'];
+  if (!retryAfter) {
+    return null;
+  }
+
+  const seconds = parseInt(retryAfter, 10);
+  if (Number.isNaN(seconds)) {
+    return null;
+  }
+
+  return seconds;
+}
+
+export async function wrapWithRetry<T>(
+  key: string,
+  task: Task<T>,
+  getDelaySeconds: (err: unknown) => number | null,
+): Promise<T> {
+  let retries = 0;
+  for (;;) {
+    try {
+      const delayPromise = hostBlocks.get(key);
+      if (delayPromise) {
+        await delayPromise;
+        hostBlocks.delete(key);
+      }
+
+      return await task();
+    } catch (err) {
+      if (retries === maxRetries) {
+        throw err;
+      }
+
+      const delaySeconds = getDelaySeconds(err);
+      if (delaySeconds === null) {
+        throw err;
+      }
+
+      hostBlocks.set(key, sleep(delaySeconds));
+      retries += 1;
+    }
+  }
+}

lib/util/http/types.ts

@@ -92,4 +92,5 @@ export interface HttpResponse<T = string> {
   authorization?: boolean;
 }
 
-export type GotTask<T> = () => Promise<HttpResponse<T>>;
+export type Task<T> = () => Promise<T>;
+export type GotTask<T> = Task<HttpResponse<T>>;