diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index d96d93cf9e207d..c0ecf6eb3ca850 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -706,22 +706,25 @@ You can define custom managers to handle: - Proprietary file formats or conventions - Popular file formats not yet supported as a manager by Renovate -Currently we only have one custom manager. -The `regex` manager which is based on using Regular Expression named capture groups. +Renovate has two custom managers: -You must have a named capture group matching (e.g. `(?.*)`) _or_ configure its corresponding template (e.g. `depNameTemplate`) for these fields: +| Custom manager | Matching engine | +| -------------- | ---------------------------------------------- | +| `regex` | Regular Expression, with named capture groups. | +| `jsonata` | JSONata query. | + +You must capture/extract the following three fields _or_ configure its corresponding template (e.g. `depNameTemplate`) for these fields: - `datasource` - `depName` and / or `packageName` - `currentValue` -Use named capture group matching _or_ set a corresponding template. We recommend you use only _one_ of these methods, or you'll get confused. -We recommend that you also tell Renovate what `versioning` to use. +We also recommend that you also tell Renovate what `versioning` to use. If the `versioning` field is missing, then Renovate defaults to using `semver` versioning. -For more details and examples about it, see our [documentation for the `regex` manager](modules/manager/regex/index.md). +For more details and examples regarding custom managers, see our documentation for the [`regex` manager](modules/manager/regex/index.md) and the [`JSONata` manager](modules/manager/jsonata/index.md). For template fields, use the triple brace `{{{ }}}` notation to avoid Handlebars escaping any special characters. @@ -763,6 +766,10 @@ This will lead to following update where `1.21-alpine` is the newest version of image: my.new.registry/aRepository/andImage:1.21-alpine ``` + +!!! note + Can only be used with the custom regex manager. + ### currentValueTemplate If the `currentValue` for a dependency is not captured with a named group then it can be defined in config using this field. @@ -786,9 +793,24 @@ Example: } ``` +```json title="Parsing a JSON file with a custom manager" +{ + "customManagers": [ + { + "customType": "jsonata", + "fileFormat": "json", + "fileMatch": ["file.json"], + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] + } + ] +} +``` + ### datasourceTemplate -If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field. +If the `datasource` for a dependency is not captured with a named group, then it can be defined in config using this field. It will be compiled using Handlebars and the regex `groups` result. ### depNameTemplate @@ -803,16 +825,43 @@ It will be compiled using Handlebars and the regex `groups` result. ### extractVersionTemplate -If `extractVersion` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. +If `extractVersion` cannot be captured with a named capture group in `matchString`, then it can be defined manually using this field. It will be compiled using Handlebars and the regex `groups` result. +### fileFormat + +`fileFormat` specifies the syntax of the package file that's managed by the custom JSONata manager. +This setting helps the system correctly parse and interpret the configuration file's contents. + +Only the `json` format is supported. + +```json title="Parsing a JSON file with a custom manager" +{ + "customManagers": [ + { + "customType": "jsonata", + "fileFormat": "json", + "fileMatch": [".renovaterc"], + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] + } + ] +} +``` + ### matchStrings -Each `matchStrings` must be a valid regular expression, optionally with named capture groups. +Each `matchStrings` must be one of the following: + +1. A valid regular expression, which may optionally include named capture groups (if using `customType=regex`) +2. Or, a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using `customType=json`) + +Read the [`customType`](#customtype) docs, to learn more. Example: -```json +```json title="matchStrings with a valid regular expression" { "matchStrings": [ "ENV .*?_VERSION=(?.*) # (?.*?)/(?.*?)\\s" @@ -820,6 +869,14 @@ Example: } ``` +```json title="matchStrings with a valid JSONata query" +{ + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] +} +``` + ### matchStringsStrategy `matchStringsStrategy` controls behavior when multiple `matchStrings` values are provided. @@ -829,6 +886,10 @@ Three options are available: - `recursive` - `combination` + +!!! note + `matchStringsStrategy` can only be used in a custom regex manager config! + #### any Each provided `matchString` will be matched individually to the content of the `packageFile`. diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 263cc69f123a74..92b3e29262054f 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2734,18 +2734,26 @@ const options: RenovateOptions[] = [ description: 'Custom manager to use. Valid only within a `customManagers` object.', type: 'string', - allowedValues: ['regex'], + allowedValues: ['jsonata', 'regex'], parents: ['customManagers'], cli: false, env: false, }, { - name: 'matchStrings', + name: 'fileFormat', description: - 'Regex capture rule to use. Valid only within a `customManagers` object.', + 'It specifies the syntax of the package file being managed by the custom JSONata manager.', + type: 'string', + allowedValues: ['json'], + parents: ['customManagers'], + cli: false, + env: false, + }, + { + name: 'matchStrings', + description: 'Queries to use. Valid only within a `customManagers` object.', type: 'array', subType: 'string', - format: 'regex', parents: ['customManagers'], cli: false, env: false, diff --git a/lib/config/validation-helpers/utils.ts b/lib/config/validation-helpers/utils.ts index 5d676bed324800..ac428376027c59 100644 --- a/lib/config/validation-helpers/utils.ts +++ b/lib/config/validation-helpers/utils.ts @@ -1,9 +1,8 @@ import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; import { logger } from '../../logger'; -import type { - RegexManagerConfig, - RegexManagerTemplates, -} from '../../modules/manager/custom/regex/types'; +import type { RegexManagerTemplates } from '../../modules/manager/custom/regex/types'; +import type { CustomManager } from '../../modules/manager/custom/types'; import { regEx } from '../../util/regex'; import type { ValidationMessage } from '../types'; @@ -78,21 +77,20 @@ export function isFalseGlobal( return false; } -function hasField( - customManager: Partial, - field: string, -): boolean { +function hasField(customManager: CustomManager, field: string): boolean { const templateField = `${field}Template` as keyof RegexManagerTemplates; + const fieldStr = + customManager.customType === 'regex' ? `(?<${field}>` : field; return !!( customManager[templateField] ?? customManager.matchStrings?.some((matchString) => - matchString.includes(`(?<${field}>`), + matchString.includes(fieldStr), ) ); } export function validateRegexManagerFields( - customManager: Partial, + customManager: CustomManager, currentPath: string, errors: ValidationMessage[], ): void { @@ -114,7 +112,8 @@ export function validateRegexManagerFields( } else { errors.push({ topic: 'Configuration Error', - message: `Each Custom Manager must contain a non-empty matchStrings array`, + message: + 'Each Custom Manager `matchStrings` array must have at least one item.', }); } @@ -136,3 +135,56 @@ export function validateRegexManagerFields( }); } } + +export function validateJSONataManagerFields( + customManager: CustomManager, + currentPath: string, + errors: ValidationMessage[], +): void { + if (!is.nonEmptyString(customManager.fileFormat)) { + errors.push({ + topic: 'Configuration Error', + message: 'Each JSONata manager must contain a fileFormat field.', + }); + } + + if (is.nonEmptyArray(customManager.matchStrings)) { + for (const matchString of customManager.matchStrings) { + try { + jsonata(matchString); + } catch (err) { + logger.debug( + { err }, + 'customManager.matchStrings JSONata query validation error', + ); + errors.push({ + topic: 'Configuration Error', + message: `Invalid JSONata query for ${currentPath}: \`${matchString}\``, + }); + } + } + } else { + errors.push({ + topic: 'Configuration Error', + message: `Each Custom Manager must contain a non-empty matchStrings array`, + }); + } + + const mandatoryFields = ['currentValue', 'datasource']; + for (const field of mandatoryFields) { + if (!hasField(customManager, field)) { + errors.push({ + topic: 'Configuration Error', + message: `JSONata Managers must contain ${field}Template configuration or ${field} in the query `, + }); + } + } + + const nameFields = ['depName', 'packageName']; + if (!nameFields.some((field) => hasField(customManager, field))) { + errors.push({ + topic: 'Configuration Error', + message: `JSONata Managers must contain depName or packageName in the query or their templates`, + }); + } +} diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts index 927650a07d732d..e66d442cf99cf7 100644 --- a/lib/config/validation.spec.ts +++ b/lib/config/validation.spec.ts @@ -706,7 +706,8 @@ describe('config/validation', () => { currentValueTemplate: 'baz', }, { - customType: 'regex', + customType: 'jsonata', + fileFormat: 'json', fileMatch: ['foo'], depNameTemplate: 'foo', datasourceTemplate: 'bar', @@ -724,7 +725,7 @@ describe('config/validation', () => { expect(errors).toMatchInlineSnapshot(` [ { - "message": "Each Custom Manager must contain a non-empty matchStrings array", + "message": "Each Custom Manager \`matchStrings\` array must have at least one item.", "topic": "Configuration Error", }, { @@ -776,6 +777,60 @@ describe('config/validation', () => { expect(errors).toHaveLength(1); }); + it('error if no fileFormat in custom JSONata manager', async () => { + const config: RenovateConfig = { + customManagers: [ + { + customType: 'jsonata', + fileMatch: ['package.json'], + matchStrings: [ + 'packages.{"depName": name, "currentValue": version, "datasource": "npm"}', + ], + }, + ], + }; + const { warnings, errors } = await configValidation.validateConfig( + 'repo', + config, + true, + ); + expect(warnings).toHaveLength(0); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: 'Each JSONata manager must contain a fileFormat field.', + }, + ]); + }); + + it('validates JSONata query for each matchStrings', async () => { + const config: RenovateConfig = { + customManagers: [ + { + customType: 'jsonata', + fileFormat: 'json', + fileMatch: ['package.json'], + matchStrings: ['packages.{'], + depNameTemplate: 'foo', + datasourceTemplate: 'bar', + currentValueTemplate: 'baz', + }, + ], + }; + const { warnings, errors } = await configValidation.validateConfig( + 'repo', + config, + true, + ); + expect(warnings).toHaveLength(0); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: `Invalid JSONata query for customManagers: \`packages.{\``, + }, + ]); + }); + // testing if we get all errors at once or not (possible), this does not include customType or fileMatch // since they are common to all custom managers it('validates all possible regex manager options', async () => { @@ -811,14 +866,12 @@ describe('config/validation', () => { depTypeTemplate: 'apple', }, { - customType: 'regex', - fileMatch: ['Dockerfile'], - matchStrings: ['ENV (?.*?)\\s'], - packageNameTemplate: 'foo', - datasourceTemplate: 'bar', - registryUrlTemplate: 'foobar', - extractVersionTemplate: '^(?v\\d+\\.\\d+)', - depTypeTemplate: 'apple', + customType: 'jsonata', + fileFormat: 'json', + fileMatch: ['package.json'], + matchStrings: [ + 'packages.{"depName": depName, "currentValue": version, "datasource": "npm"}', + ], }, ], }; @@ -876,6 +929,39 @@ describe('config/validation', () => { expect(errors).toHaveLength(1); }); + it('errors if customManager fields are missing: JSONataManager', async () => { + const config: RenovateConfig = { + customManagers: [ + { + customType: 'jsonata', + fileFormat: 'json', + fileMatch: ['package.json'], + matchStrings: ['packages'], + }, + ], + }; + const { warnings, errors } = await configValidation.validateConfig( + 'repo', + config, + true, + ); + expect(warnings).toHaveLength(0); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: `JSONata Managers must contain currentValueTemplate configuration or currentValue in the query `, + }, + { + topic: 'Configuration Error', + message: `JSONata Managers must contain datasourceTemplate configuration or datasource in the query `, + }, + { + topic: 'Configuration Error', + message: `JSONata Managers must contain depName or packageName in the query or their templates`, + }, + ]); + }); + it('ignore keys', async () => { const config = { $schema: 'renovate.json', diff --git a/lib/config/validation.ts b/lib/config/validation.ts index 551c07ba9f0863..c89f07f4ee8e59 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -37,6 +37,7 @@ import * as regexOrGlobValidator from './validation-helpers/regex-glob-matchers' import { getParentName, isFalseGlobal, + validateJSONataManagerFields, validateNumber, validatePlainObject, validateRegexManagerFields, @@ -486,6 +487,7 @@ export async function validateConfig( const allowedKeys = [ 'customType', 'description', + 'fileFormat', 'fileMatch', 'matchStrings', 'matchStringsStrategy', @@ -527,6 +529,13 @@ export async function validateConfig( errors, ); break; + case 'jsonata': + validateJSONataManagerFields( + customManager, + currentPath, + errors, + ); + break; } } else { errors.push({ diff --git a/lib/modules/manager/custom/api.ts b/lib/modules/manager/custom/api.ts index de5e051ca72ca1..f7dc64aaef3cc6 100644 --- a/lib/modules/manager/custom/api.ts +++ b/lib/modules/manager/custom/api.ts @@ -1,7 +1,9 @@ import type { ManagerApi } from '../types'; +import * as jsonata from './jsonata'; import * as regex from './regex'; const api = new Map(); export default api; api.set('regex', regex); +api.set('jsonata', jsonata); diff --git a/lib/modules/manager/custom/index.spec.ts b/lib/modules/manager/custom/index.spec.ts index 54e922bd1ec517..61a6466dbcd447 100644 --- a/lib/modules/manager/custom/index.spec.ts +++ b/lib/modules/manager/custom/index.spec.ts @@ -10,6 +10,8 @@ describe('modules/manager/custom/index', () => { expect(customManager.isCustomManager('npm')).toBe(false); expect(customManager.isCustomManager('regex')).toBe(true); expect(customManager.isCustomManager('custom.regex')).toBe(false); + expect(customManager.isCustomManager('jsonata')).toBe(true); + expect(customManager.isCustomManager('custom.jsonata')).toBe(false); }); }); }); diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts new file mode 100644 index 00000000000000..9237ba21b62dbf --- /dev/null +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -0,0 +1,278 @@ +import { codeBlock } from 'common-tags'; +import { logger } from '../../../../../test/util'; +import type { JsonataExtractConfig } from './types'; +import { defaultConfig, extractPackageFile } from '.'; + +describe('modules/manager/custom/jsonata/index', () => { + it('has default config', () => { + expect(defaultConfig).toEqual({ + pinDigests: false, + }); + }); + + it('returns null when content does not match specified file format', async () => { + const res = await extractPackageFile('not-json', 'foo-file', { + fileFormat: 'json', + } as JsonataExtractConfig); + expect(res).toBeNull(); + expect(logger.logger.debug).toHaveBeenCalledWith( + expect.anything(), + 'Error while parsing file', + ); + }); + + it('returns null when no content', async () => { + const res = await extractPackageFile('', 'foo-file', { + fileFormat: 'json', + matchStrings: [ + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + ], + } as JsonataExtractConfig); + expect(res).toBeNull(); + }); + + it('extracts data when no templates are used', async () => { + const json = codeBlock` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": "1.2.3", + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "https://registry.npmjs.org", + "dep_type": "dev" + } + ] + }`; + const config = { + fileFormat: 'json', + matchStrings: [ + `packages.{ + "depName": dep_name, + "packageName": package_name, + "currentValue": current_value, + "currentDigest": current_digest, + "datasource": data_source, + "versioning": versioning, + "extractVersion": extract_version, + "registryUrl": registry_url, + "depType": dep_type + }`, + ], + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res).toMatchObject({ + deps: [ + { + depName: 'foo', + packageName: 'fii', + currentValue: '1.2.3', + currentDigest: '1234', + datasource: 'nuget', + versioning: 'maven', + extractVersion: 'custom-extract-version', + registryUrls: ['https://registry.npmjs.org/'], + depType: 'dev', + }, + ], + }); + }); + + it('applies templates', async () => { + const json = codeBlock` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": "1.2.3", + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "https://registry.npmjs.org", + "dep_type": "dev" + }, + { + }] + }`; + const config = { + fileFormat: 'json', + matchStrings: [ + `packages.{ + "depName": dep_name, + "packageName": package_name, + "currentValue": current_value, + "currentDigest": current_digest, + "datasource": data_source, + "versioning": versioning, + "extractVersion": extract_version, + "registryUrl": registry_url, + "depType": dep_type + }`, + ], + depNameTemplate: + '{{#if depName}}{{depName}}{{else}}default-dep-name{{/if}}', + packageNameTemplate: + '{{#if packageName}}{{packageName}}{{else}}default-package-name{{/if}}', + currentValueTemplate: + '{{#if currentValue}}{{currentValue}}{{else}}default-current-value{{/if}}', + currentDigestTemplate: + '{{#if currentDigest}}{{currentDigest}}{{else}}default-current-digest{{/if}}', + datasourceTemplate: + '{{#if datasource}}{{datasource}}{{else}}default-datasource{{/if}}', + versioningTemplate: + '{{#if versioning}}{{versioning}}{{else}}default-versioning{{/if}}', + extractVersionTemplate: + '{{#if extractVersion}}{{extractVersion}}{{else}}default-extract-version{{/if}}', + registryUrlTemplate: + '{{#if registryUrl}}{{registryUrl}}{{else}}https://default.registry.url{{/if}}', + depTypeTemplate: + '{{#if depType}}{{depType}}{{else}}default-dep-type{{/if}}', + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res).toMatchObject({ + deps: [ + { + depName: 'foo', + packageName: 'fii', + currentValue: '1.2.3', + currentDigest: '1234', + datasource: 'nuget', + versioning: 'maven', + extractVersion: 'custom-extract-version', + registryUrls: ['https://registry.npmjs.org/'], + depType: 'dev', + }, + { + depName: 'default-dep-name', + packageName: 'default-package-name', + currentValue: 'default-current-value', + currentDigest: 'default-current-digest', + datasource: 'default-datasource', + versioning: 'default-versioning', + extractVersion: 'default-extract-version', + registryUrls: ['https://default.registry.url/'], + depType: 'default-dep-type', + }, + ], + }); + }); + + it('logs warning if query result does not match schema', async () => { + const json = codeBlock` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": 1, + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "https://registry.npmjs.org", + "dep_type": "dev" + } + ] + }`; + const config = { + fileFormat: 'json', + matchStrings: [ + `packages.{ + "depName": dep_name, + "currentValue": current_value, + "datasource": data_source + }`, + ], + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res).toBeNull(); + expect(logger.logger.warn).toHaveBeenCalledWith( + expect.anything(), + 'Query results failed schema validation', + ); + }); + + it('returns null if no dependencies found', async () => { + const config = { + fileFormat: 'json', + matchStrings: [ + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + ], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(logger.logger.debug).toHaveBeenCalledWith( + { + packageFile: 'unused', + jsonataQuery: + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + }, + 'The jsonata query returned no matches. Possible error, please check your query. Skipping', + ); + expect(res).toBeNull(); + }); + + it('returns null if invalid template', async () => { + const config = { + fileFormat: 'json', + matchStrings: [ + `{"depName": "foo", "currentValue": "1.0.0", "datasource": "npm"}`, + ], + versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).toBeNull(); + expect(logger.logger.debug).toHaveBeenCalledWith( + expect.anything(), + 'Error compiling template for JSONata manager', + ); + }); + + it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { + const config = { + fileFormat: 'json', + matchStrings: [ + `{"depName": "foo", "currentValue": "1.0.0", "datasource": "npm"}`, + ], + registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).not.toBeNull(); + expect(logger.logger.debug).toHaveBeenCalledWith( + { url: 'this-is-not-a-valid-url-foo' }, + 'Invalid JSONata manager registryUrl', + ); + }); + + it('extracts multiple dependencies with multiple matchStrings', async () => { + const config = { + fileFormat: 'json', + matchStrings: [`{"depName": "foo"}`, `{"depName": "bar"}`], + currentValueTemplate: '1.0.0', + datasourceTemplate: 'npm', + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).toMatchObject({ + deps: [ + { + depName: 'foo', + currentValue: '1.0.0', + datasource: 'npm', + }, + { + depName: 'bar', + currentValue: '1.0.0', + datasource: 'npm', + }, + ], + }); + }); +}); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts new file mode 100644 index 00000000000000..cd63c0c0081534 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -0,0 +1,49 @@ +import is from '@sindresorhus/is'; +import type { Category } from '../../../../constants'; +import { logger } from '../../../../logger'; +import { parseJson } from '../../../../util/common'; +import type { PackageFileContent } from '../../types'; +import type { JsonataExtractConfig } from './types'; +import { handleMatching } from './utils'; + +export const categories: Category[] = ['custom']; + +export const defaultConfig = { + pinDigests: false, +}; +export const supportedDatasources = ['*']; +export const displayName = 'JSONata'; + +export async function extractPackageFile( + content: string, + packageFile: string, + config: JsonataExtractConfig, +): Promise { + let json: unknown; + try { + switch (config.fileFormat) { + case 'json': + json = parseJson(content, packageFile); + break; + } + } catch (err) { + logger.debug( + { err, fileName: packageFile, fileFormat: config.fileFormat }, + 'Error while parsing file', + ); + return null; + } + + if (is.nullOrUndefined(json)) { + return null; + } + + const deps = await handleMatching(json, packageFile, config); + if (!deps.length) { + return null; + } + + return { + deps, + }; +} diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md new file mode 100644 index 00000000000000..6e0037cf40d7fc --- /dev/null +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -0,0 +1,203 @@ +With `customManagers` using `JSONata` queries you can configure Renovate so it finds dependencies in JSON files, that are not detected by its other built-in package managers. + +Renovate uses the `jsonata` package to process the `json` file content using the queries. + +For more on the jsonata query language, read the [jsonata query language site](https://docs.jsonata.org/overview.html). + +The JSONata manager is unique in Renovate, because: + +- It can be used with any `datasource` +- It can be configured via [JSONata](https://jsonata.org/) queries +- you can create multiple "JSONata managers" the same repository, using the `customManagers` config + +### Required Fields + +The first two required fields are `fileMatch` and `matchStrings`: + +- `fileMatch` works the same as any manager +- `matchStrings` is a `JSONata` custom manager concept and is used for configuring a jsonata queries + +#### Information that Renovate needs about the dependency + +Before Renovate can look up a dependency and decide about updates, it must have this info about each dependency: + +| Info type | Required | Notes | Docs | +| :--------------------------------------------------- | :------- | :-------------------------------------------------------- | :----------------------------------------------------------------------------- | +| Name of the dependency | Yes | | | +| `datasource` | Yes | Example datasources: npm, Docker, GitHub tags, and so on. | [Supported datasources](../../datasource/index.md#supported-datasources) | +| Version scheme to use. Defaults to `semver-coerced`. | Yes | You may set another version scheme, like `pep440`. | [Supported versioning schemes](../../versioning/index.md#supported-versioning) | + +#### Required fields to be present in the resulting structure returned by the jsonata query + +You must: + +- Capture the `currentValue` of the dependency _or_ use the `currentValueTemplate` template field +- Capture the `depName` or `packageName`. _Or_ use a template field: `depNameTemplate` and `packageNameTemplate` +- Capture the `datasource`, _or_ use the `datasourceTemplate` template field + +#### Optional fields you can include in the resulting structure + +You may use any of these items: + +- `depType`, _or_ use the `depTypeTemplate` template field +- `versioning`, _or_ the use `versioningTemplate` template field. If neither are present, Renovate defaults to `semver-coerced` +- `extractVersion`, _or_ use the `extractVersionTemplate` template field +- `currentDigest` +- `registryUrl`, _or_ use the `registryUrlTemplate` template field. If it's a valid URL, it will be converted to the `registryUrls` field as a single-length array +- `indentation`. Must be empty, _or_ whitespace. Else Renovate restes only `indentation` to an empty string + +### Usage + +When you configure a JSONata manager, use the following syntax: + +```javascript +{ + "customManagers": [ + { + "customType": "jsonata", + "fileFormat": "json", + "fileMatch": [""], + "matchStrings": [''], + ... + } + ] +} +``` + +Overwrite the `` placeholder text with your [JSONata](https://docs.jsonata.org/overview.html) query. +The JSONata query transforms the content to a JSON object, similar to the this: + +```javascript dependencies information extracted usig jsonata query +[ + { + depName: 'some_dep', + currentValue: '1.0.0', + datasource: 'docker', + versioning: 'semver', + }, +]; +``` + +Creating your Renovate JSONata manager config is easier if you understand JSONata queries. +We recommend you follow these steps: + +1. Read the official JSONata query language docs +2. Check our example queries below +3. You're ready to make your own config + +Alternatively you can "try and error" to a working config, by adjusting our examples. + +#### Example queries + +Below are some example queries for the generic JSON manager. +You can also use the [JSONata test website](https://try.jsonata.org) to experiment with queries. + +```json title="Dependencies spread in different nodes, and we want to limit the extraction to a particular node" +{ + "production": [ + { + "version": "1.2.3", + "package": "foo" + } + ], + "development": [ + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +production.{ "depName": package, "currentValue": version } +``` + +```json title="Dependencies spread in different nodes, and we want to extract all of them as if they were in the same node" +{ + "production": [ + { + "version": "1.2.3", + "package": "foo" + } + ], + "development": [ + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +*.{ "depName": package, "currentValue": version } +``` + +```json title="The dependency name is in a JSON node name, and the version is in a child leaf to that node" +{ + "foo": { + "version": "1.2.3" + }, + "bar": { + "version": "4.5.6" + } +} +``` + +Query: + +``` +$each(function($v, $n) { { "depName": $n, "currentValue": $v.version } }) +``` + +```json title="The dependency name and its version are both value nodes of the same parent node" +{ + "packages": [ + { + "version": "1.2.3", + "package": "foo" + }, + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +packages.{ "depName": package, "currentValue": version } +``` + +```json title="The dependency name and version are part of the same string" +{ + "packages": ["foo@1.2.3", "bar@4.5.6"] +} +``` + +Query: + +``` +$map($map(packages, function ($v) { $split($v, "@") }), function ($v) { { "depName": $v[0], "currentVersion": $v[1] } }) +``` + +```json title="JSONata manager config to extract deps from a package.json file in the Renovate repository" +{ + "customType": "jsonata", + "fileMatch": ["package.json"], + "matchStrings": [ + "$each(dependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"dependencies\"}})", + "$each(devDependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"devDependencies\"}})", + "$each(optionalDependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"optionalDependencies\"}})", + "{ \"depName\": \"pnpm\", \"currentValue\": $substring(packageManager, 5), \"depType\": \"packageManager\"}" + ], + "datasourceTemplate": "npm" +} +``` diff --git a/lib/modules/manager/custom/jsonata/schema.ts b/lib/modules/manager/custom/jsonata/schema.ts new file mode 100644 index 00000000000000..8a06911695d91e --- /dev/null +++ b/lib/modules/manager/custom/jsonata/schema.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; + +const DepObjectSchema = z.object({ + currentValue: z.string().optional(), + datasource: z.string().optional(), + depName: z.string().optional(), + packageName: z.string().optional(), + currentDigest: z.string().optional(), + versioning: z.string().optional(), + depType: z.string().optional(), + registryUrl: z.string().optional(), + extractVersion: z.string().optional(), + indentation: z.string().optional(), +}); + +export const QueryResultZodSchema = z.array(DepObjectSchema); diff --git a/lib/modules/manager/custom/jsonata/types.ts b/lib/modules/manager/custom/jsonata/types.ts new file mode 100644 index 00000000000000..70852a81bfc41d --- /dev/null +++ b/lib/modules/manager/custom/jsonata/types.ts @@ -0,0 +1,25 @@ +import type { ExtractConfig } from '../../types'; + +export interface JSONataManagerTemplates { + depNameTemplate?: string; + packageNameTemplate?: string; + datasourceTemplate?: string; + versioningTemplate?: string; + depTypeTemplate?: string; + currentValueTemplate?: string; + currentDigestTemplate?: string; + extractVersionTemplate?: string; + registryUrlTemplate?: string; +} + +export interface JSONataManagerConfig extends JSONataManagerTemplates { + fileFormat: string; + matchStrings: string[]; +} + +export interface JsonataExtractConfig + extends ExtractConfig, + JSONataManagerTemplates { + fileFormat: string; + matchStrings: string[]; +} diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts new file mode 100644 index 00000000000000..c9b396eb91a8ba --- /dev/null +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -0,0 +1,108 @@ +import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; +import { logger } from '../../../../logger'; +import * as template from '../../../../util/template'; +import { parseUrl } from '../../../../util/url'; +import type { PackageDependency } from '../../types'; +import type { ValidMatchFields } from '../utils'; +import { checkIsValidDependency, validMatchFields } from '../utils'; +import { QueryResultZodSchema } from './schema'; +import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; + +export async function handleMatching( + json: unknown, + packageFile: string, + config: JsonataExtractConfig, +): Promise { + let results: Record[] = []; + const { matchStrings: jsonataQueries } = config; + for (const query of jsonataQueries) { + // won't fail as this is verified during config validation + const jsonataExpression = jsonata(query); + // this does not throw error, just returns undefined if no matches + let queryResult = await jsonataExpression.evaluate(json); + + if ( + !queryResult || + is.emptyObject(queryResult) || + is.emptyArray(queryResult) + ) { + logger.debug( + { + jsonataQuery: query, + packageFile, + }, + 'The jsonata query returned no matches. Possible error, please check your query. Skipping', + ); + return []; + } + + queryResult = is.array(queryResult) ? queryResult : [queryResult]; + const parsed = QueryResultZodSchema.safeParse(queryResult); + if (parsed.success) { + results = results.concat(parsed.data); + } else { + logger.warn( + { err: parsed.error, jsonataQuery: query, packageFile, queryResult }, + 'Query results failed schema validation', + ); + } + } + + return results + .map((dep) => createDependency(dep, config)) + .filter(is.truthy) + .filter((dep) => + checkIsValidDependency(dep, packageFile, 'custom.jsonata'), + ); +} + +export function createDependency( + queryResult: Record, + config: JsonataExtractConfig, +): PackageDependency | null { + const dependency: PackageDependency = {}; + + for (const field of validMatchFields) { + const fieldTemplate = `${field}Template` as keyof JSONataManagerTemplates; + const tmpl = config[fieldTemplate]; + if (tmpl) { + try { + const compiled = template.compile(tmpl, queryResult, false); + updateDependency(field, compiled, dependency); + } catch { + logger.debug( + { template: tmpl }, + 'Error compiling template for JSONata manager', + ); + return null; + } + } else if (queryResult[field]) { + updateDependency(field, queryResult[field], dependency); + } + } + return dependency; +} + +function updateDependency( + field: ValidMatchFields, + value: string, + dependency: PackageDependency, +): PackageDependency { + switch (field) { + case 'registryUrl': { + const url = parseUrl(value)?.toString(); + if (!url) { + logger.debug({ url: value }, 'Invalid JSONata manager registryUrl'); + break; + } + dependency.registryUrls = [url]; + break; + } + default: + dependency[field] = value; + break; + } + + return dependency; +} diff --git a/lib/modules/manager/custom/types.ts b/lib/modules/manager/custom/types.ts index de387685f80717..a6aa1ff22ed825 100644 --- a/lib/modules/manager/custom/types.ts +++ b/lib/modules/manager/custom/types.ts @@ -1,10 +1,15 @@ +import type { JSONataManagerConfig } from './jsonata/types'; import type { RegexManagerConfig } from './regex/types'; -export interface CustomExtractConfig extends Partial {} +export interface CustomExtractConfig + extends Partial, + Partial {} -export type CustomManagerName = 'regex'; +export type CustomManagerName = 'jsonata' | 'regex'; -export interface CustomManager extends Partial { +export interface CustomManager + extends Partial, + Partial { customType: CustomManagerName; fileMatch: string[]; }