feat(config): implement custom header field inside HostRules (renovatebot#26225)

Author: hersentino

docs/usage/configuration-options.md

@@ -1810,6 +1810,30 @@ It uses `QuickLRU` with a `maxSize` of `1000`.
 
 Enable got [http2](https://github.com/sindresorhus/got/blob/v11.5.2/readme.md#http2) support.
 
+### headers
+
+You can provide a `headers` object that includes fields to be forwarded to the HTTP request headers.
+By default, all headers starting with "X-" are allowed.
+
+A bot administrator may configure an override for [`allowedHeaders`](./self-hosted-configuration.md#allowedHeaders) to configure more permitted headers.
+
+`headers` value(s) configured in the bot admin `hostRules` (for example in a `config.js` file) are _not_ validated, so it may contain any header regardless of `allowedHeaders`.
+
+For example:
+
+```json
+{
+  "hostRules": [
+    {
+      "matchHost": "https://domain.com/all-versions",
+      "headers": {
+        "X-custom-header": "secret"
+      }
+    }
+  ]
+}
+```
+
 ### hostType
 
 `hostType` is another way to filter rules and can be either a platform such as `github` and `bitbucket-server`, or it can be a datasource such as `docker` and `rubygems`.

docs/usage/self-hosted-configuration.md

@@ -63,6 +63,44 @@ But before you disable templating completely, try the `allowedPostUpgradeCommand
 
 ## allowScripts
 
+## allowedHeaders
+
+`allowedHeaders` can be useful when a registry uses a authentication system that's not covered by Renovate's default credential handling in `hostRules`.
+By default, all headers starting with "X-" are allowed.
+If needed, you can allow additional headers with the `allowedHeaders` option.
+Any set `allowedHeaders` overrides the default "X-" allowed headers, so you should include them in your config if you wish for them to remain allowed.
+The `allowedHeaders` config option takes an array of minimatch-compatible globs or re2-compatible regex strings.
+
+Examples:
+
+| Example header | Kind of pattern  | Explanation                                 |
+| -------------- | ---------------- | ------------------------------------------- |
+| `/X/`          | Regex            | Any header with `x` anywhere in the name    |
+| `!/X/`         | Regex            | Any header without `X` anywhere in the name |
+| `X-*`          | Global pattern   | Any header starting with `X-`               |
+| `X`            | Exact match glob | Only the header matching exactly `X`        |
+
+```json
+{
+  "hostRules": [
+    {
+      "matchHost": "https://domain.com/all-versions",
+      "headers": {
+        "X-Auth-Token": "secret"
+      }
+    }
+  ]
+}
+```
+
+Or with custom `allowedHeaders`:
+
+```js title="config.js"
+module.exports = {
+  allowedHeaders: ['custom-header'],
+};
+```
+
 ## allowedPostUpgradeCommands
 
 A list of regular expressions that decide which commands in `postUpgradeTasks` are allowed to run.

lib/config/global.ts

@@ -4,6 +4,7 @@ export class GlobalConfig {
   // TODO: once global config work is complete, add a test to make sure this list includes all options with globalOnly=true (#9603)
   private static readonly OPTIONS: (keyof RepoGlobalConfig)[] = [
     'allowCustomCrateRegistries',
+    'allowedHeaders',
     'allowedPostUpgradeCommands',
     'allowPlugins',
     'allowPostUpgradeCommandTemplating',

lib/config/options/index.ts

@@ -5,6 +5,15 @@ import { getVersioningList } from '../../modules/versioning';
 import type { RenovateOptions } from '../types';
 
 const options: RenovateOptions[] = [
+  {
+    name: 'allowedHeaders',
+    description:
+      'List of allowed patterns for header names in repository hostRules config.',
+    type: 'array',
+    default: ['X-*'],
+    subType: 'string',
+    globalOnly: true,
+  },
   {
     name: 'detectGlobalManagerConfig',
     description:
@@ -2394,6 +2403,16 @@ const options: RenovateOptions[] = [
     env: false,
     advancedUse: true,
   },
+  {
+    name: 'headers',
+    description:
+      'Put fields to be forwarded to the HTTP request headers in the headers config option.',
+    type: 'object',
+    parent: 'hostRules',
+    cli: false,
+    env: false,
+    advancedUse: true,
+  },
   {
     name: 'artifactAuth',
     description:

lib/config/types.ts

@@ -127,6 +127,7 @@ export interface RepoGlobalConfig {
   allowPlugins?: boolean;
   allowPostUpgradeCommandTemplating?: boolean;
   allowScripts?: boolean;
+  allowedHeaders?: string[];
   allowedPostUpgradeCommands?: string[];
   binarySource?: 'docker' | 'global' | 'install' | 'hermit';
   cacheHardTtlMinutes?: number;

lib/config/validation.spec.ts

@@ -1,3 +1,4 @@
+import { GlobalConfig } from './global';
 import type { RenovateConfig } from './types';
 import * as configValidation from './validation';
 
@@ -1005,5 +1006,60 @@ describe('config/validation', () => {
         },
       ]);
     });
+
+    it('errors if forbidden header in hostRules', async () => {
+      GlobalConfig.set({ allowedHeaders: ['X-*'] });
+
+      const config = {
+        hostRules: [
+          {
+            matchHost: 'https://domain.com/all-versions',
+            headers: {
+              'X-Auth-Token': 'token',
+              unallowedHeader: 'token',
+            },
+          },
+        ],
+      };
+      const { warnings, errors } = await configValidation.validateConfig(
+        false,
+        config,
+      );
+      expect(warnings).toHaveLength(0);
+      expect(errors).toMatchObject([
+        {
+          message:
+            "hostRules header `unallowedHeader` is not allowed by this bot's `allowedHeaders`.",
+          topic: 'Configuration Error',
+        },
+      ]);
+    });
+
+    it('errors if headers values are not string', async () => {
+      GlobalConfig.set({ allowedHeaders: ['X-*'] });
+
+      const config = {
+        hostRules: [
+          {
+            matchHost: 'https://domain.com/all-versions',
+            headers: {
+              'X-Auth-Token': 10,
+            } as unknown as Record<string, string>,
+          },
+        ],
+      };
+      const { warnings, errors } = await configValidation.validateConfig(
+        false,
+        config,
+      );
+      expect(warnings).toHaveLength(0);
+      expect(errors).toMatchObject([
+        {
+          message:
+            'Invalid hostRules headers value configuration: header must be a string.',
+          topic: 'Configuration Error',
+        },
+      ]);
+    });
   });
 });

lib/config/validation.ts

@@ -7,12 +7,15 @@ import type {
   RegexManagerTemplates,
 } from '../modules/manager/custom/regex/types';
 import type { CustomManager } from '../modules/manager/custom/types';
+import type { HostRule } from '../types/host-rules';
+import { anyMatchRegexOrMinimatch } from '../util/package-rules/match';
 import { configRegexPredicate, isConfigRegex, regEx } from '../util/regex';
 import * as template from '../util/template';
 import {
   hasValidSchedule,
   hasValidTimezone,
 } from '../workers/repository/update/branch/schedule';
+import { GlobalConfig } from './global';
 import { migrateConfig } from './migration';
 import { getOptions } from './options';
 import { resolveConfigPresets } from './presets';
@@ -38,6 +41,7 @@ const topLevelObjects = managerList;
 
 const ignoredNodes = [
   '$schema',
+  'headers',
   'depType',
   'npmToken',
   'packageFile',
@@ -696,6 +700,29 @@ export async function validateConfig(
         }
       }
     }
+
+    if (key === 'hostRules' && is.array(val)) {
+      const allowedHeaders = GlobalConfig.get('allowedHeaders');
+      for (const rule of val as HostRule[]) {
+        if (!rule.headers) {
+          continue;
+        }
+        for (const [header, value] of Object.entries(rule.headers)) {
+          if (!is.string(value)) {
+            errors.push({
+              topic: 'Configuration Error',
+              message: `Invalid hostRules headers value configuration: header must be a string.`,
+            });
+          }
+          if (!anyMatchRegexOrMinimatch(allowedHeaders, header)) {
+            errors.push({
+              topic: 'Configuration Error',
+              message: `hostRules header \`${header}\` is not allowed by this bot's \`allowedHeaders\`.`,
+            });
+          }
+        }
+      }
+    }
   }
 
   function sortAll(a: ValidationMessage, b: ValidationMessage): number {

lib/types/host-rules.ts

@@ -11,6 +11,7 @@ export interface HostRuleSearchResult {
   enableHttp2?: boolean;
   concurrentRequestLimit?: number;
   maxRequestsPerSecond?: number;
+  headers?: Record<string, string>;
   maxRetryAfter?: number;
 
   dnsCache?: boolean;

lib/util/http/host-rules.spec.ts

@@ -1,3 +1,4 @@
+import { GlobalConfig } from '../../config/global';
 import { bootstrap } from '../../proxy';
 import type { HostRule } from '../../types';
 import * as hostRules from '../host-rules';
@@ -542,4 +543,21 @@ describe('util/http/host-rules', () => {
       username: undefined,
     });
   });
+
+  it('should remove forbidden headers from request', () => {
+    GlobalConfig.set({ allowedHeaders: ['X-*'] });
+    const hostRule = {
+      matchHost: 'https://domain.com/all-versions',
+      headers: {
+        'X-Auth-Token': 'token',
+        unallowedHeader: 'token',
+      },
+    };
+
+    expect(applyHostRule(url, {}, hostRule)).toEqual({
+      headers: {
+        'X-Auth-Token': 'token',
+      },
+    });
+  });
 });

lib/util/http/host-rules.ts

@@ -1,4 +1,5 @@
 import is from '@sindresorhus/is';
+import { GlobalConfig } from '../../config/global';
 import {
   BITBUCKET_API_USING_HOST_TYPES,
   GITEA_API_USING_HOST_TYPES,
@@ -9,6 +10,7 @@ import { logger } from '../../logger';
 import { hasProxy } from '../../proxy';
 import type { HostRule } from '../../types';
 import * as hostRules from '../host-rules';
+import { anyMatchRegexOrMinimatch } from '../package-rules/match';
 import { parseUrl } from '../url';
 import { dnsLookup } from './dns';
 import { keepAliveAgents } from './keep-alive';
@@ -162,6 +164,27 @@ export function applyHostRule<GotOptions extends HostRulesGotOptions>(
     options.lookup = dnsLookup;
   }
 
+  if (hostRule.headers) {
+    const allowedHeaders = GlobalConfig.get('allowedHeaders');
+    const filteredHeaders: Record<string, string> = {};
+
+    for (const [header, value] of Object.entries(hostRule.headers)) {
+      if (anyMatchRegexOrMinimatch(allowedHeaders, header)) {
+        filteredHeaders[header] = value;
+      } else {
+        logger.once.error(
+          { allowedHeaders, header },
+          'Disallowed hostRules headers',
+        );
+      }
+    }
+
+    options.headers = {
+      ...filteredHeaders,
+      ...options.headers,
+    };
+  }
+
   if (hostRule.keepAlive) {
     options.agent = keepAliveAgents;
   }