From e71b81ab010bc4867d1a8058c6c7ab594e7d0fd1 Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Fri, 6 Sep 2024 15:38:30 -0700 Subject: [PATCH 1/3] feat(pypi): support GCloud credentials for Google Artifact Registry --- .../pypi/__snapshots__/index.spec.ts.snap | 43 +++++++++ lib/modules/datasource/pypi/index.spec.ts | 89 +++++++++++++++++++ lib/modules/datasource/pypi/index.ts | 25 +++++- 3 files changed, 155 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap index 1af50f1d6d055b..80a7904e8458e9 100644 --- a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -278,6 +278,49 @@ exports[`modules/datasource/pypi/index getReleases respects constraints 1`] = ` } `; +exports[`modules/datasource/pypi/index supports Google Auth with simple endpoint 1`] = ` +{ + "isPrivate": true, + "registryUrl": "https://someregion-python.pkg.dev/some-project/some-repo/simple", + "releases": [ + { + "version": "0.1.2", + }, + { + "version": "0.1.3", + }, + { + "version": "0.1.4", + }, + { + "version": "0.2.0", + }, + { + "version": "0.2.1", + }, + { + "version": "0.2.2", + }, + { + "version": "0.3.0", + }, + { + "version": "0.4.0", + }, + { + "version": "0.4.1", + }, + { + "version": "0.4.2", + }, + { + "isDeprecated": true, + "version": "0.5.0", + }, + ], +} +`; + exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ instead of https://pypi.org/simple/ 1`] = ` { "registryUrl": "https://pypi.org/simple", diff --git a/lib/modules/datasource/pypi/index.spec.ts b/lib/modules/datasource/pypi/index.spec.ts index 9180829d48e57b..a1c9407d09b9ab 100644 --- a/lib/modules/datasource/pypi/index.spec.ts +++ b/lib/modules/datasource/pypi/index.spec.ts @@ -1,9 +1,14 @@ +import { GoogleAuth as _googleAuth } from 'google-auth-library'; import { getPkgReleases } from '..'; import { Fixtures } from '../../../../test/fixtures'; import * as httpMock from '../../../../test/http-mock'; +import { mocked } from '../../../../test/util'; import * as hostRules from '../../../util/host-rules'; import { PypiDatasource } from '.'; +const googleAuth = mocked(_googleAuth); +jest.mock('google-auth-library'); + const res1 = Fixtures.get('azure-cli-monitor.json'); const htmlResponse = Fixtures.get('versions-html.html'); const mixedCaseResponse = Fixtures.get('versions-html-mixed-case.html'); @@ -125,6 +130,64 @@ describe('modules/datasource/pypi/index', () => { }); }); + it('supports Google Auth', async () => { + httpMock + .scope('https://someregion-python.pkg.dev/some-project/some-repo/') + .get('/azure-cli-monitor/json') + .matchHeader( + 'authorization', + 'Basic b2F1dGgyYWNjZXNzdG9rZW46c29tZS10b2tlbg==', + ) + .reply(200, Fixtures.get('azure-cli-monitor-updated.json')); + const config = { + registryUrls: [ + 'https://someregion-python.pkg.dev/some-project/some-repo', + ], + }; + googleAuth.mockImplementationOnce( + jest.fn().mockImplementationOnce(() => ({ + getAccessToken: jest.fn().mockResolvedValue('some-token'), + })), + ); + const res = await getPkgReleases({ + ...config, + datasource, + packageName: 'azure-cli-monitor', + }); + expect(res?.releases.pop()).toMatchObject({ + version: '0.2.15', + releaseTimestamp: '2019-06-18T13:58:55.000Z', + }); + expect(googleAuth).toHaveBeenCalledTimes(1); + }); + + it('supports Google Auth not being configured', async () => { + httpMock + .scope('https://someregion-python.pkg.dev/some-project/some-repo/') + .get('/azure-cli-monitor/json') + .reply(200, Fixtures.get('azure-cli-monitor-updated.json')); + const config = { + registryUrls: [ + 'https://someregion-python.pkg.dev/some-project/some-repo', + ], + }; + googleAuth.mockImplementation( + jest.fn().mockImplementation(() => ({ + getAccessToken: jest.fn().mockResolvedValue(undefined), + })), + ); + const res = await getPkgReleases({ + ...config, + datasource, + packageName: 'azure-cli-monitor', + }); + expect(res?.releases.pop()).toMatchObject({ + version: '0.2.15', + releaseTimestamp: '2019-06-18T13:58:55.000Z', + }); + expect(googleAuth).toHaveBeenCalledTimes(1); + }); + it('returns non-github home_page', async () => { httpMock .scope(baseUrl) @@ -643,6 +706,32 @@ describe('modules/datasource/pypi/index', () => { }); }); + it('supports Google Auth with simple endpoint', async () => { + httpMock + .scope('https://someregion-python.pkg.dev/some-project/some-repo/simple/') + .get('/dj-database-url/') + .reply(200, htmlResponse); + const config = { + registryUrls: [ + 'https://someregion-python.pkg.dev/some-project/some-repo/simple/', + ], + }; + googleAuth.mockImplementationOnce( + jest.fn().mockImplementationOnce(() => ({ + getAccessToken: jest.fn().mockResolvedValue('some-token'), + })), + ); + expect( + await getPkgReleases({ + datasource, + ...config, + constraints: { python: '2.7' }, + packageName: 'dj-database-url', + }), + ).toMatchSnapshot(); + expect(googleAuth).toHaveBeenCalledTimes(1); + }); + it('uses https://pypi.org/pypi/ instead of https://pypi.org/simple/', async () => { httpMock.scope(baseUrl).get('/azure-cli-monitor/json').reply(200, res1); const config = { diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 546fc0482591e8..b2614866656793 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -4,14 +4,18 @@ import changelogFilenameRegex from 'changelog-filename-regex'; import { logger } from '../../../logger'; import { coerceArray } from '../../../util/array'; import { parse } from '../../../util/html'; +import type { OutgoingHttpHeaders } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; import { ensureTrailingSlash } from '../../../util/url'; import * as pep440 from '../../versioning/pep440'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; +import { getGoogleAuthToken } from '../util'; import { isGitHubRepo, normalizePythonDepName } from './common'; import type { PypiJSON, PypiJSONRelease, Releases } from './types'; +const googleArtifactRegistryDomain = '.pkg.dev'; + export class PypiDatasource extends Datasource { static readonly id = 'pypi'; @@ -82,6 +86,21 @@ export class PypiDatasource extends Datasource { return dependency; } + private async getAuthHeaders( + lookupUrl: string, + ): Promise { + const url = new URL(lookupUrl); + if (url.hostname.endsWith(googleArtifactRegistryDomain)) { + const auth = await getGoogleAuthToken(); + if (auth) { + return { authorization: `Basic ${auth}` }; + } + logger.once.debug({ lookupUrl }, 'Could not get Google access token'); + return {}; + } + return {}; + } + private async getDependency( packageName: string, hostUrl: string, @@ -92,7 +111,8 @@ export class PypiDatasource extends Datasource { ); const dependency: ReleaseResult = { releases: [] }; logger.trace({ lookupUrl }, 'Pypi api got lookup'); - const rep = await this.http.getJson(lookupUrl); + const headers = await this.getAuthHeaders(lookupUrl); + const rep = await this.http.getJson(lookupUrl, { headers }); const dep = rep?.body; if (!dep) { logger.trace({ dependency: packageName }, 'pip package not found'); @@ -237,7 +257,8 @@ export class PypiDatasource extends Datasource { ensureTrailingSlash(normalizePythonDepName(packageName)), ); const dependency: ReleaseResult = { releases: [] }; - const response = await this.http.get(lookupUrl); + const headers = await this.getAuthHeaders(lookupUrl); + const response = await this.http.get(lookupUrl, { headers }); const dep = response?.body; if (!dep) { logger.trace({ dependency: packageName }, 'pip package not found'); From 653d7ce9c64d402629689927bbdd251ab5cae605 Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Sat, 7 Sep 2024 11:36:00 -0700 Subject: [PATCH 2/3] Wrap new URL() in a try-catch --- lib/modules/datasource/pypi/index.spec.ts | 12 ++++++++++++ lib/modules/datasource/pypi/index.ts | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/modules/datasource/pypi/index.spec.ts b/lib/modules/datasource/pypi/index.spec.ts index a1c9407d09b9ab..2277797208af88 100644 --- a/lib/modules/datasource/pypi/index.spec.ts +++ b/lib/modules/datasource/pypi/index.spec.ts @@ -732,6 +732,18 @@ describe('modules/datasource/pypi/index', () => { expect(googleAuth).toHaveBeenCalledTimes(1); }); + it('ignores an invalid URL when checking for auth headers', async () => { + const config = { + registryUrls: ['not-a-url/simple/'], + }; + const res = await getPkgReleases({ + ...config, + datasource, + packageName: 'azure-cli-monitor', + }); + expect(res?.releases.pop()).toBeNil(); + }); + it('uses https://pypi.org/pypi/ instead of https://pypi.org/simple/', async () => { httpMock.scope(baseUrl).get('/azure-cli-monitor/json').reply(200, res1); const config = { diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index b2614866656793..79a0ead1137450 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -89,7 +89,13 @@ export class PypiDatasource extends Datasource { private async getAuthHeaders( lookupUrl: string, ): Promise { - const url = new URL(lookupUrl); + let url: URL; + try { + url = new URL(lookupUrl); + } catch (err) { + logger.once.debug({ lookupUrl, err }, 'Failed to parse URL'); + return {}; + } if (url.hostname.endsWith(googleArtifactRegistryDomain)) { const auth = await getGoogleAuthToken(); if (auth) { From 701e7804ac359fcd0408b8f73f95bc312bc2b45f Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Fri, 13 Sep 2024 08:00:13 -0700 Subject: [PATCH 3/3] Address review comments --- .../pypi/__snapshots__/index.spec.ts.snap | 43 ----------- lib/modules/datasource/pypi/index.spec.ts | 75 ++++++++++++++++--- lib/modules/datasource/pypi/index.ts | 14 ++-- 3 files changed, 70 insertions(+), 62 deletions(-) diff --git a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap index 80a7904e8458e9..1af50f1d6d055b 100644 --- a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -278,49 +278,6 @@ exports[`modules/datasource/pypi/index getReleases respects constraints 1`] = ` } `; -exports[`modules/datasource/pypi/index supports Google Auth with simple endpoint 1`] = ` -{ - "isPrivate": true, - "registryUrl": "https://someregion-python.pkg.dev/some-project/some-repo/simple", - "releases": [ - { - "version": "0.1.2", - }, - { - "version": "0.1.3", - }, - { - "version": "0.1.4", - }, - { - "version": "0.2.0", - }, - { - "version": "0.2.1", - }, - { - "version": "0.2.2", - }, - { - "version": "0.3.0", - }, - { - "version": "0.4.0", - }, - { - "version": "0.4.1", - }, - { - "version": "0.4.2", - }, - { - "isDeprecated": true, - "version": "0.5.0", - }, - ], -} -`; - exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ instead of https://pypi.org/simple/ 1`] = ` { "registryUrl": "https://pypi.org/simple", diff --git a/lib/modules/datasource/pypi/index.spec.ts b/lib/modules/datasource/pypi/index.spec.ts index 2277797208af88..678101cfa03c30 100644 --- a/lib/modules/datasource/pypi/index.spec.ts +++ b/lib/modules/datasource/pypi/index.spec.ts @@ -14,6 +14,62 @@ const htmlResponse = Fixtures.get('versions-html.html'); const mixedCaseResponse = Fixtures.get('versions-html-mixed-case.html'); const withPeriodsResponse = Fixtures.get('versions-html-with-periods.html'); +const azureCliMonitorReleases = [ + { releaseTimestamp: '2017-04-03T16:55:14.000Z', version: '0.0.1' }, + { releaseTimestamp: '2017-04-17T20:32:30.000Z', version: '0.0.2' }, + { releaseTimestamp: '2017-04-28T21:18:54.000Z', version: '0.0.3' }, + { releaseTimestamp: '2017-05-09T21:36:51.000Z', version: '0.0.4' }, + { releaseTimestamp: '2017-05-30T23:13:49.000Z', version: '0.0.5' }, + { releaseTimestamp: '2017-06-13T22:21:05.000Z', version: '0.0.6' }, + { releaseTimestamp: '2017-06-21T22:12:36.000Z', version: '0.0.7' }, + { releaseTimestamp: '2017-07-07T16:22:26.000Z', version: '0.0.8' }, + { releaseTimestamp: '2017-08-28T20:14:33.000Z', version: '0.0.9' }, + { releaseTimestamp: '2017-09-22T23:47:59.000Z', version: '0.0.10' }, + { releaseTimestamp: '2017-10-24T02:14:07.000Z', version: '0.0.11' }, + { releaseTimestamp: '2017-11-14T18:31:57.000Z', version: '0.0.12' }, + { releaseTimestamp: '2017-12-05T18:57:54.000Z', version: '0.0.13' }, + { releaseTimestamp: '2018-01-05T21:26:03.000Z', version: '0.0.14' }, + { releaseTimestamp: '2018-01-17T18:36:39.000Z', version: '0.1.0' }, + { releaseTimestamp: '2018-01-31T18:05:22.000Z', version: '0.1.1' }, + { releaseTimestamp: '2018-02-13T18:17:52.000Z', version: '0.1.2' }, + { releaseTimestamp: '2018-03-13T17:08:20.000Z', version: '0.1.3' }, + { releaseTimestamp: '2018-03-27T17:55:25.000Z', version: '0.1.4' }, + { releaseTimestamp: '2018-04-10T17:25:47.000Z', version: '0.1.5' }, + { releaseTimestamp: '2018-05-07T17:59:09.000Z', version: '0.1.6' }, + { releaseTimestamp: '2018-05-22T17:25:23.000Z', version: '0.1.7' }, + { releaseTimestamp: '2018-07-03T16:18:06.000Z', version: '0.1.8' }, + { releaseTimestamp: '2018-07-18T16:20:01.000Z', version: '0.2.0' }, + { releaseTimestamp: '2018-07-31T15:32:28.000Z', version: '0.2.1' }, + { releaseTimestamp: '2018-08-14T14:55:32.000Z', version: '0.2.2' }, + { releaseTimestamp: '2018-08-28T15:35:01.000Z', version: '0.2.3' }, + { releaseTimestamp: '2018-10-09T18:09:08.000Z', version: '0.2.4' }, + { releaseTimestamp: '2018-10-23T16:54:38.000Z', version: '0.2.5' }, + { releaseTimestamp: '2018-11-06T16:34:51.000Z', version: '0.2.6' }, + { releaseTimestamp: '2018-11-20T20:16:03.000Z', version: '0.2.7' }, + { releaseTimestamp: '2019-01-15T21:08:09.000Z', version: '0.2.8' }, + { releaseTimestamp: '2019-01-30T01:51:15.000Z', version: '0.2.9' }, + { releaseTimestamp: '2019-02-12T18:09:43.000Z', version: '0.2.10' }, + { releaseTimestamp: '2019-03-26T17:57:43.000Z', version: '0.2.11' }, + { releaseTimestamp: '2019-04-09T17:01:09.000Z', version: '0.2.12' }, + { releaseTimestamp: '2019-04-23T17:00:58.000Z', version: '0.2.13' }, + { releaseTimestamp: '2019-05-21T18:43:17.000Z', version: '0.2.14' }, + { releaseTimestamp: '2019-06-18T13:58:55.000Z', version: '0.2.15' }, +]; + +const djDatabaseUrlSimpleReleases = [ + { version: '0.1.2' }, + { version: '0.1.3' }, + { version: '0.1.4' }, + { version: '0.2.0' }, + { version: '0.2.1' }, + { version: '0.2.2' }, + { version: '0.3.0' }, + { version: '0.4.0' }, + { version: '0.4.1' }, + { version: '0.4.2' }, + { isDeprecated: true, version: '0.5.0' }, +]; + const baseUrl = 'https://pypi.org/pypi'; const datasource = PypiDatasource.id; @@ -154,10 +210,7 @@ describe('modules/datasource/pypi/index', () => { datasource, packageName: 'azure-cli-monitor', }); - expect(res?.releases.pop()).toMatchObject({ - version: '0.2.15', - releaseTimestamp: '2019-06-18T13:58:55.000Z', - }); + expect(res).toMatchObject({ releases: azureCliMonitorReleases }); expect(googleAuth).toHaveBeenCalledTimes(1); }); @@ -181,10 +234,7 @@ describe('modules/datasource/pypi/index', () => { datasource, packageName: 'azure-cli-monitor', }); - expect(res?.releases.pop()).toMatchObject({ - version: '0.2.15', - releaseTimestamp: '2019-06-18T13:58:55.000Z', - }); + expect(res).toMatchObject({ releases: azureCliMonitorReleases }); expect(googleAuth).toHaveBeenCalledTimes(1); }); @@ -728,7 +778,12 @@ describe('modules/datasource/pypi/index', () => { constraints: { python: '2.7' }, packageName: 'dj-database-url', }), - ).toMatchSnapshot(); + ).toMatchObject({ + isPrivate: true, + registryUrl: + 'https://someregion-python.pkg.dev/some-project/some-repo/simple', + releases: djDatabaseUrlSimpleReleases, + }); expect(googleAuth).toHaveBeenCalledTimes(1); }); @@ -741,7 +796,7 @@ describe('modules/datasource/pypi/index', () => { datasource, packageName: 'azure-cli-monitor', }); - expect(res?.releases.pop()).toBeNil(); + expect(res).toBeNil(); }); it('uses https://pypi.org/pypi/ instead of https://pypi.org/simple/', async () => { diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 79a0ead1137450..dc8a5437dc259e 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -6,7 +6,7 @@ import { coerceArray } from '../../../util/array'; import { parse } from '../../../util/html'; import type { OutgoingHttpHeaders } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; -import { ensureTrailingSlash } from '../../../util/url'; +import { ensureTrailingSlash, parseUrl } from '../../../util/url'; import * as pep440 from '../../versioning/pep440'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; @@ -14,8 +14,6 @@ import { getGoogleAuthToken } from '../util'; import { isGitHubRepo, normalizePythonDepName } from './common'; import type { PypiJSON, PypiJSONRelease, Releases } from './types'; -const googleArtifactRegistryDomain = '.pkg.dev'; - export class PypiDatasource extends Datasource { static readonly id = 'pypi'; @@ -89,14 +87,12 @@ export class PypiDatasource extends Datasource { private async getAuthHeaders( lookupUrl: string, ): Promise { - let url: URL; - try { - url = new URL(lookupUrl); - } catch (err) { - logger.once.debug({ lookupUrl, err }, 'Failed to parse URL'); + const parsedUrl = parseUrl(lookupUrl); + if (!parsedUrl) { + logger.once.debug({ lookupUrl }, 'Failed to parse URL'); return {}; } - if (url.hostname.endsWith(googleArtifactRegistryDomain)) { + if (parsedUrl.hostname.endsWith('.pkg.dev')) { const auth = await getGoogleAuthToken(); if (auth) { return { authorization: `Basic ${auth}` };