feat(http): Support for Retry-After header (renovatebot#25859)

Co-authored-by: Rhys Arkins <[email protected]> Co-authored-by: HonkingGoose <[email protected]> Co-authored-by: Michael Kriese <[email protected]> Co-authored-by: Sebastian Poxhofer <[email protected]>
hiventive · Jan 9, 2024 · 26ff560 · 26ff560
1 parent 2af38bb
commit 26ff560
Show file tree

Hide file tree

Showing 7 changed files with 322 additions and 7 deletions.
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
@@ -1768,6 +1768,26 @@ Example config:
 }
 ```
 
+### maxRetryAfter
+
+A remote host may return a `4xx` response with a `Retry-After` header value, which indicates that Renovate has been rate-limited.
+Renovate may try to contact the host again after waiting a certain time, that's set by the host.
+By default, Renovate tries again after the `Retry-After` header value has passed, up to a maximum of 60 seconds.
+If the `Retry-After` value is more than 60 seconds, Renovate will abort the request instead of waiting.
+
+You can configure a different maximum value in seconds using `maxRetryAfter`:
+
+```json
+{
+  "hostRules": [
+    {
+      "matchHost": "api.github.com",
+      "maxRetryAfter": 25
+    }
+  ]
+}
+```
+
 ### dnsCache
 
 Enable got [dnsCache](https://github.com/sindresorhus/got/blob/v11.5.2/readme.md#dnsCache) support.

diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
@@ -2750,6 +2750,17 @@ const options: RenovateOptions[] = [
     globalOnly: true,
     default: [],
   },
+  {
+    name: 'maxRetryAfter',
+    description:
+      'Maximum retry-after header value to wait for before retrying a failed request.',
+    type: 'integer',
+    default: 60,
+    stage: 'package',
+    parent: 'hostRules',
+    cli: false,
+    env: false,
+  },
 ];
 
 export function getOptions(): RenovateOptions[] {

diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts
@@ -12,6 +12,7 @@ export interface HostRuleSearchResult {
   concurrentRequestLimit?: number;
   maxRequestsPerSecond?: number;
   headers?: Record<string, string | undefined>;
+  maxRetryAfter?: number;
 
   dnsCache?: boolean;
   keepAlive?: boolean;

diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
@@ -15,6 +15,7 @@ import { applyAuthorization, removeAuthorization } from './auth';
 import { hooks } from './hooks';
 import { applyHostRule, findMatchingRule } from './host-rules';
 import { getQueue } from './queue';
+import { getRetryAfter, wrapWithRetry } from './retry-after';
 import { Throttle, getThrottle } from './throttle';
 import type {
   GotJSONOptions,
@@ -130,11 +131,14 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
     protected hostType: string,
     options: HttpOptions = {},
   ) {
-    this.options = merge<GotOptions>(options, { context: { hostType } });
-
-    if (process.env.NODE_ENV === 'test') {
-      this.options.retry = 0;
-    }
+    const retryLimit = process.env.NODE_ENV === 'test' ? 0 : 2;
+    this.options = merge<GotOptions>(options, {
+      context: { hostType },
+      retry: {
+        limit: retryLimit,
+        maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it
+      },
+    });
   }
 
   protected getThrottle(url: string): Throttle | null {
@@ -226,7 +230,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
         ? () => queue.add<HttpResponse<T>>(throttledTask)
         : throttledTask;
 
-      resPromise = queuedTask();
+      const { maxRetryAfter = 60 } = hostRule;
+      resPromise = wrapWithRetry(queuedTask, url, getRetryAfter, maxRetryAfter);
 
       if (memCacheKey) {
         memCache.set(memCacheKey, resPromise);

diff --git a/lib/util/http/retry-after.spec.ts b/lib/util/http/retry-after.spec.ts
@@ -0,0 +1,162 @@
+import { RequestError } from 'got';
+import { getRetryAfter, wrapWithRetry } from './retry-after';
+
+function requestError(
+  response: {
+    statusCode?: number;
+    headers?: Record<string, string | string[]>;
+  } | null = null,
+) {
+  const err = new RequestError('request error', {}, null as never);
+  if (response) {
+    (err as any).response = response;
+  }
+  return err;
+}
+
+describe('util/http/retry-after', () => {
+  beforeEach(() => {
+    jest.useFakeTimers();
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('wrapWithRetry', () => {
+    it('works', async () => {
+      const task = jest.fn(() => Promise.resolve(42));
+      const res = await wrapWithRetry(task, 'foobar', () => null, 60);
+      expect(res).toBe(42);
+      expect(task).toHaveBeenCalledTimes(1);
+    });
+
+    it('throws', async () => {
+      const task = jest.fn(() => Promise.reject(new Error('error')));
+
+      await expect(
+        wrapWithRetry(task, 'http://example.com', () => null, 60),
+      ).rejects.toThrow('error');
+
+      expect(task).toHaveBeenCalledTimes(1);
+    });
+
+    it('retries', async () => {
+      const task = jest
+        .fn()
+        .mockRejectedValueOnce(new Error('error-1'))
+        .mockRejectedValueOnce(new Error('error-2'))
+        .mockResolvedValueOnce(42);
+
+      const p = wrapWithRetry(task, 'http://example.com', () => 1, 60);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      const res = await p;
+      expect(res).toBe(42);
+      expect(task).toHaveBeenCalledTimes(3);
+    });
+
+    it('gives up after max retries', async () => {
+      const task = jest
+        .fn()
+        .mockRejectedValueOnce('error-1')
+        .mockRejectedValueOnce('error-2')
+        .mockRejectedValueOnce('error-3')
+        .mockRejectedValue('error-4');
+
+      const p = wrapWithRetry(task, 'http://example.com', () => 1, 60).catch(
+        (err) => err,
+      );
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await expect(p).resolves.toBe('error-3');
+      expect(task).toHaveBeenCalledTimes(3);
+    });
+
+    it('gives up when delay exceeds maxRetryAfter', async () => {
+      const task = jest.fn().mockRejectedValue('error');
+
+      const p = wrapWithRetry(task, 'http://example.com', () => 61, 60).catch(
+        (err) => err,
+      );
+
+      await expect(p).resolves.toBe('error');
+      expect(task).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('getRetryAfter', () => {
+    it('returns null for non-RequestError', () => {
+      expect(getRetryAfter(new Error())).toBeNull();
+    });
+
+    it('returns null for RequestError without response', () => {
+      expect(getRetryAfter(requestError())).toBeNull();
+    });
+
+    it('returns null for status other than 429', () => {
+      const err = new RequestError('request-error', {}, null as never);
+      (err as any).response = { statusCode: 302 };
+      expect(getRetryAfter(requestError({ statusCode: 302 }))).toBeNull();
+    });
+
+    it('returns null missing "retry-after" header', () => {
+      expect(
+        getRetryAfter(requestError({ statusCode: 429, headers: {} })),
+      ).toBeNull();
+    });
+
+    it('returns null for non-integer "retry-after" header', () => {
+      expect(
+        getRetryAfter(
+          requestError({
+            statusCode: 429,
+            headers: {
+              'retry-after': 'Wed, 21 Oct 2015 07:28:00 GMT',
+            },
+          }),
+        ),
+      ).toBeNull();
+    });
+
+    it('returns delay in seconds from date', () => {
+      jest.setSystemTime(new Date('2020-01-01T00:00:00Z'));
+      expect(
+        getRetryAfter(
+          requestError({
+            statusCode: 429,
+            headers: {
+              'retry-after': 'Wed, 01 Jan 2020 00:00:42 GMT',
+            },
+          }),
+        ),
+      ).toBe(42);
+    });
+
+    it('returns delay in seconds from number', () => {
+      expect(
+        getRetryAfter(
+          requestError({
+            statusCode: 429,
+            headers: {
+              'retry-after': '42',
+            },
+          }),
+        ),
+      ).toBe(42);
+    });
+
+    it('returns null for invalid header value', () => {
+      expect(
+        getRetryAfter(
+          requestError({
+            statusCode: 429,
+            headers: {
+              'retry-after': 'invalid',
+            },
+          }),
+        ),
+      ).toBeNull();
+    });
+  });
+});
diff --git a/lib/util/http/retry-after.ts b/lib/util/http/retry-after.ts
@@ -0,0 +1,115 @@
+import { setTimeout } from 'timers/promises';
+import { RequestError } from 'got';
+import { DateTime } from 'luxon';
+import { logger } from '../../logger';
+import { parseUrl } from '../url';
+import type { Task } from './types';
+
+const hostDelays = new Map<string, Promise<unknown>>();
+
+const maxRetries = 2;
+
+/**
+ * Given a task that returns a promise, retry the task if it fails with a
+ * 429 Too Many Requests or 403 Forbidden response, using the Retry-After
+ * header to determine the delay.
+ *
+ * For response codes other than 429 or 403, or if the Retry-After header
+ * is not present or invalid, the task is not retried, throwing the error.
+ */
+export async function wrapWithRetry<T>(
+  task: Task<T>,
+  url: string,
+  getRetryAfter: (err: unknown) => number | null,
+  maxRetryAfter: number,
+): Promise<T> {
+  const key = parseUrl(url)?.host ?? url;
+
+  let retries = 0;
+  for (;;) {
+    try {
+      await hostDelays.get(key);
+      hostDelays.delete(key);
+
+      return await task();
+    } catch (err) {
+      const delaySeconds = getRetryAfter(err);
+      if (delaySeconds === null) {
+        throw err;
+      }
+
+      if (retries === maxRetries) {
+        logger.debug(
+          `Retry-After: reached maximum retries (${maxRetries}) for ${url}`,
+        );
+        throw err;
+      }
+
+      if (delaySeconds > maxRetryAfter) {
+        logger.debug(
+          `Retry-After: delay ${delaySeconds} seconds exceeds maxRetryAfter ${maxRetryAfter} seconds for ${url}`,
+        );
+        throw err;
+      }
+
+      logger.debug(
+        `Retry-After: will retry ${url} after ${delaySeconds} seconds`,
+      );
+
+      const delay = Promise.all([
+        hostDelays.get(key),
+        setTimeout(1000 * delaySeconds),
+      ]);
+      hostDelays.set(key, delay);
+      retries += 1;
+    }
+  }
+}
+
+export function getRetryAfter(err: unknown): number | null {
+  if (!(err instanceof RequestError)) {
+    return null;
+  }
+
+  if (!err.response) {
+    return null;
+  }
+
+  if (err.response.statusCode < 400 || err.response.statusCode >= 500) {
+    logger.warn(
+      { url: err.response.url },
+      `Retry-After: unexpected status code ${err.response.statusCode}`,
+    );
+    return null;
+  }
+
+  const retryAfter = err.response.headers['retry-after']?.trim();
+  if (!retryAfter) {
+    return null;
+  }
+
+  const date = DateTime.fromHTTP(retryAfter);
+  if (date.isValid) {
+    const seconds = Math.floor(date.diffNow('seconds').seconds);
+    if (seconds < 0) {
+      logger.debug(
+        { url: err.response.url, retryAfter },
+        'Retry-After: date in the past',
+      );
+      return null;
+    }
+
+    return seconds;
+  }
+
+  const seconds = parseInt(retryAfter, 10);
+  if (!Number.isNaN(seconds) && seconds > 0) {
+    return seconds;
+  }
+
+  logger.debug(
+    { url: err.response.url, retryAfter },
+    'Retry-After: unsupported format',
+  );
+  return null;
+}
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
@@ -92,4 +92,5 @@ export interface HttpResponse<T = string> {
   authorization?: boolean;
 }
 
-export type GotTask<T> = () => Promise<HttpResponse<T>>;
+export type Task<T> = () => Promise<T>;
+export type GotTask<T> = Task<HttpResponse<T>>;