From 5a07d49bda2b4c8ba971e8ad84f613b2a4b87b99 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:41:17 +0100 Subject: [PATCH 01/10] fix(pypi): lookup simple api first --- .../pypi/__snapshots__/index.spec.ts.snap | 144 +++++++++++- lib/modules/datasource/pypi/index.spec.ts | 206 ++++++++++++------ lib/modules/datasource/pypi/index.ts | 171 ++++++++------- 3 files changed, 380 insertions(+), 141 deletions(-) diff --git a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap index 42dc4dcea02587..d03640f84e093f 100644 --- a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -1,6 +1,48 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`modules/datasource/pypi/index getReleases fall back from json and process data from simple endpoint 1`] = ` +exports[`modules/datasource/pypi/index getReleases process data from simple api with pypijson unavailable 1`] = ` +{ + "registryUrl": "https://custom.pypi.net/foo", + "releases": [ + { + "version": "0.1.2", + }, + { + "version": "0.1.3", + }, + { + "version": "0.1.4", + }, + { + "version": "0.2.0", + }, + { + "version": "0.2.1", + }, + { + "version": "0.2.2", + }, + { + "version": "0.3.0", + }, + { + "version": "0.4.0", + }, + { + "version": "0.4.1", + }, + { + "version": "0.4.2", + }, + { + "isDeprecated": true, + "version": "0.5.0", + }, + ], +} +`; + +exports[`modules/datasource/pypi/index getReleases process data from simple api with pypijson unavailable 2`] = ` { "registryUrl": "https://custom.pypi.net/foo", "releases": [ @@ -281,7 +323,7 @@ exports[`modules/datasource/pypi/index getReleases respects constraints 1`] = ` } `; -exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ instead of https://pypi.org/simple/ 1`] = ` +exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ and https://pypi.org/simple/ (no find) 1`] = ` { "registryUrl": "https://pypi.org/simple", "releases": [ @@ -378,3 +420,101 @@ exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ instead of ht "sourceUrl": "https://github.com/Azure/azure-cli", } `; + +exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ and https://pypi.org/simple/ (no find) 2`] = ` +{ + "registryUrl": "https://pypi.org/pypi", + "releases": [ + { + "releaseTimestamp": "2017-04-03T16:55:14.000Z", + "version": "0.0.1", + }, + { + "releaseTimestamp": "2017-04-17T20:32:30.000Z", + "version": "0.0.2", + }, + { + "releaseTimestamp": "2017-04-28T21:18:54.000Z", + "version": "0.0.3", + }, + { + "releaseTimestamp": "2017-05-09T21:36:51.000Z", + "version": "0.0.4", + }, + { + "releaseTimestamp": "2017-05-30T23:13:49.000Z", + "version": "0.0.5", + }, + { + "releaseTimestamp": "2017-06-13T22:21:05.000Z", + "version": "0.0.6", + }, + { + "releaseTimestamp": "2017-06-21T22:12:36.000Z", + "version": "0.0.7", + }, + { + "releaseTimestamp": "2017-07-07T16:22:26.000Z", + "version": "0.0.8", + }, + { + "releaseTimestamp": "2017-08-28T20:14:33.000Z", + "version": "0.0.9", + }, + { + "releaseTimestamp": "2017-09-22T23:47:59.000Z", + "version": "0.0.10", + }, + { + "releaseTimestamp": "2017-10-24T02:14:07.000Z", + "version": "0.0.11", + }, + { + "releaseTimestamp": "2017-11-14T18:31:57.000Z", + "version": "0.0.12", + }, + { + "releaseTimestamp": "2017-12-05T18:57:54.000Z", + "version": "0.0.13", + }, + { + "releaseTimestamp": "2018-01-05T21:26:03.000Z", + "version": "0.0.14", + }, + { + "releaseTimestamp": "2018-01-17T18:36:39.000Z", + "version": "0.1.0", + }, + { + "releaseTimestamp": "2018-01-31T18:05:22.000Z", + "version": "0.1.1", + }, + { + "releaseTimestamp": "2018-02-13T18:17:52.000Z", + "version": "0.1.2", + }, + { + "releaseTimestamp": "2018-03-13T17:08:20.000Z", + "version": "0.1.3", + }, + { + "releaseTimestamp": "2018-03-27T17:55:25.000Z", + "version": "0.1.4", + }, + { + "releaseTimestamp": "2018-04-10T17:25:47.000Z", + "version": "0.1.5", + }, + { + "isDeprecated": true, + "releaseTimestamp": "2018-05-07T17:59:09.000Z", + "version": "0.1.6", + }, + { + "releaseTimestamp": "2018-05-22T17:25:23.000Z", + "version": "0.1.7", + }, + ], + "sourceUrl": "https://github.com/Azure/azure-cli", +} +`; diff --git a/lib/modules/datasource/pypi/index.spec.ts b/lib/modules/datasource/pypi/index.spec.ts index 4172900b268d96..81fe34b366d9fa 100644 --- a/lib/modules/datasource/pypi/index.spec.ts +++ b/lib/modules/datasource/pypi/index.spec.ts @@ -16,7 +16,8 @@ const mixedCaseResponse = Fixtures.get('versions-html-mixed-case.html'); const withPeriodsResponse = Fixtures.get('versions-html-with-periods.html'); const hyphensResponse = Fixtures.get('versions-html-hyphens.html'); -const baseUrl = 'https://pypi.org/pypi'; +const baseJsonUrl = 'https://pypi.org/pypi'; +const baseSimpleUrl = 'https://pypi.org/simple'; const datasource = PypiDatasource.id; describe('modules/datasource/pypi/index', () => { @@ -33,7 +34,8 @@ describe('modules/datasource/pypi/index', () => { }); it('returns null for empty result', async () => { - httpMock.scope(baseUrl).get('/something/json').reply(200); + httpMock.scope(baseJsonUrl).get('/something/json').reply(200); + httpMock.scope(baseSimpleUrl).get('/something/').reply(404); expect( await getPkgReleases({ datasource, @@ -43,8 +45,8 @@ describe('modules/datasource/pypi/index', () => { }); it('returns null for 404', async () => { - httpMock.scope(baseUrl).get('/something/json').reply(404); - httpMock.scope(baseUrl).get('/something/').reply(404); + httpMock.scope(baseJsonUrl).get('/something/json').reply(404); + httpMock.scope(baseSimpleUrl).get('/something/').reply(404); expect( await getPkgReleases({ datasource, @@ -54,7 +56,14 @@ describe('modules/datasource/pypi/index', () => { }); it('processes real data', async () => { - httpMock.scope(baseUrl).get('/azure-cli-monitor/json').reply(200, res1); + httpMock + .scope(baseJsonUrl) + .get('/azure-cli-monitor/json') + .reply(200, res1); + httpMock + .scope(baseSimpleUrl) + .get('/azure-cli-monitor/') + .reply(200, htmlResponse); expect( await getPkgReleases({ datasource, @@ -68,6 +77,10 @@ describe('modules/datasource/pypi/index', () => { .scope('https://custom.pypi.net/foo') .get('/azure-cli-monitor/json') .reply(200, res1); + httpMock + .scope('https://custom.pypi.net/foo') + .get('/azure-cli-monitor/') + .reply(404); const config = { registryUrls: ['https://custom.pypi.net/foo'], }; @@ -90,6 +103,10 @@ describe('modules/datasource/pypi/index', () => { .scope('https://customprivate.pypi.net/foo') .get('/azure-cli-monitor/json') .reply(200, res1); + httpMock + .scope('https://customprivate.pypi.net/foo') + .get('/azure-cli-monitor/') + .reply(404); const config = { registryUrls: ['https://customprivate.pypi.net/foo'], }; @@ -104,16 +121,24 @@ describe('modules/datasource/pypi/index', () => { it('supports multiple custom datasource urls', async () => { httpMock .scope('https://custom.pypi.net/foo') - .get('/azure-cli-monitor/json') + .get('/azure-cli-monitor/') .replyWithError('error'); httpMock .scope('https://second-index/foo') .get('/azure-cli-monitor/json') .reply(200, res1); + httpMock + .scope('https://second-index/foo') + .get('/azure-cli-monitor/') + .reply(404); httpMock .scope('https://third-index/foo') .get('/azure-cli-monitor/json') .reply(200, res2); + httpMock + .scope('https://third-index/foo') + .get('/azure-cli-monitor/') + .reply(404); const config = { registryUrls: [ 'https://custom.pypi.net/foo', @@ -134,7 +159,7 @@ describe('modules/datasource/pypi/index', () => { it('returns non-github home_page', async () => { httpMock - .scope(baseUrl) + .scope(baseJsonUrl) .get('/something/json') .reply(200, { ...JSON.parse(res1), @@ -143,6 +168,7 @@ describe('modules/datasource/pypi/index', () => { home_page: 'https://microsoft.com', }, }); + httpMock.scope(baseSimpleUrl).get('/something/').reply(404); expect( ( await getPkgReleases({ @@ -166,9 +192,10 @@ describe('modules/datasource/pypi/index', () => { }, }; httpMock - .scope(baseUrl) + .scope(baseJsonUrl) .get('/flexget/json') .reply(200, { ...JSON.parse(res1), info }); + httpMock.scope(baseSimpleUrl).get('/flexget/').reply(404); const result = await getPkgReleases({ datasource, packageName: 'flexget', @@ -186,9 +213,10 @@ describe('modules/datasource/pypi/index', () => { }, }; httpMock - .scope(baseUrl) + .scope(baseJsonUrl) .get('/flexget/json') .reply(200, { ...JSON.parse(res1), info }); + httpMock.scope(baseSimpleUrl).get('/flexget/').reply(404); const result = await getPkgReleases({ datasource, packageName: 'flexget', @@ -198,13 +226,17 @@ describe('modules/datasource/pypi/index', () => { it('normalizes the package name according to PEP 503', async () => { const expectedHttpCall = httpMock - .scope(baseUrl) + .scope(baseJsonUrl) .get('/not-normalized-package/json') .reply(200, htmlResponse); + httpMock + .scope(baseSimpleUrl) + .get('/not-normalized-package/') + .reply(200, htmlResponse); await getPkgReleases({ datasource, - registryUrls: [baseUrl], + registryUrls: [baseJsonUrl], packageName: 'not_normalized.Package', }); @@ -213,17 +245,17 @@ describe('modules/datasource/pypi/index', () => { it('normalizes the package name according to PEP 503 when falling back to simple endpoint', async () => { httpMock - .scope(baseUrl) + .scope(baseJsonUrl) .get('/not-normalized-package/json') .reply(404, ''); const expectedFallbackHttpCall = httpMock - .scope(baseUrl) + .scope(baseSimpleUrl) .get('/not-normalized-package/') .reply(200, htmlResponse); await getPkgReleases({ datasource, - registryUrls: [baseUrl], + registryUrls: [baseJsonUrl], packageName: 'not_normalized.Package', }); @@ -236,6 +268,10 @@ describe('modules/datasource/pypi/index', () => { .scope(simpleRegistryUrl) .get('/not-normalized-package/') .reply(200, htmlResponse); + httpMock + .scope(simpleRegistryUrl) + .get('/not-normalized-package/json') + .reply(404); await getPkgReleases({ datasource, @@ -248,7 +284,7 @@ describe('modules/datasource/pypi/index', () => { it('respects constraints', async () => { httpMock - .scope(baseUrl) + .scope(baseJsonUrl) .get('/doit/json') .reply(200, { info: { @@ -265,6 +301,7 @@ describe('modules/datasource/pypi/index', () => { '0.4.1': [], }, }); + httpMock.scope(baseSimpleUrl).get('/doit/').reply(404); expect( await getPkgReleases({ datasource, @@ -276,10 +313,12 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from simple endpoint', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/dj-database-url/') .reply(200, htmlResponse); + httpMock.scope(simpleRegistryUrl).get('/dj-database-url/json').reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -294,10 +333,13 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from +simple endpoint', async () => { + const simpleRegistryUrl = 'https://some.registry.org/+simple/'; httpMock - .scope('https://some.registry.org/+simple/') + .scope(simpleRegistryUrl) .get('/dj-database-url/') .reply(200, htmlResponse); + httpMock.scope(simpleRegistryUrl).get('/dj-database-url/json').reply(404); + const config = { registryUrls: ['https://some.registry.org/+simple/'], }; @@ -312,14 +354,16 @@ describe('modules/datasource/pypi/index', () => { }); it('sets private simple if authorization provided', async () => { + const simpleRegistryUrl = 'https://some.private.registry.org/+simple/'; hostRules.add({ matchHost: 'some.private.registry.org', token: '123test', }); httpMock - .scope('https://some.private.registry.org/+simple/') + .scope(simpleRegistryUrl) .get('/dj-database-url/') .reply(200, htmlResponse); + httpMock.scope(simpleRegistryUrl).get('/dj-database-url/json').reply(404); const config = { registryUrls: ['https://some.private.registry.org/+simple/'], }; @@ -333,10 +377,15 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from simple endpoint with hyphens', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/package-with-hyphens/') .reply(200, hyphensResponse); + httpMock + .scope(simpleRegistryUrl) + .get('/package-with-hyphens/json') + .reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -353,10 +402,12 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from simple endpoint with hyphens replaced with underscores', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/image-collector/') .reply(200, mixedHyphensResponse); + httpMock.scope(simpleRegistryUrl).get('/image-collector/json').reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -371,10 +422,15 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from simple endpoint with mixed-case characters', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/packagewithmixedcase/') .reply(200, mixedCaseResponse); + httpMock + .scope(simpleRegistryUrl) + .get('/packagewithmixedcase/json') + .reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -391,10 +447,15 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from simple endpoint with mixed-case characters when using lower case dependency name', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/packagewithmixedcase/') .reply(200, mixedCaseResponse); + httpMock + .scope(simpleRegistryUrl) + .get('/packagewithmixedcase/json') + .reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -411,10 +472,15 @@ describe('modules/datasource/pypi/index', () => { }); it('process data from simple endpoint with periods', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/package-with-periods/') .reply(200, withPeriodsResponse); + httpMock + .scope(simpleRegistryUrl) + .get('/package-with-periods/json') + .reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -431,10 +497,9 @@ describe('modules/datasource/pypi/index', () => { }); it('returns null for empty response', async () => { - httpMock - .scope('https://some.registry.org/simple/') - .get('/dj-database-url/') - .reply(200); + const simpleRegistryUrl = 'https://some.registry.org/simple/'; + httpMock.scope(simpleRegistryUrl).get('/dj-database-url/').reply(200); + httpMock.scope(simpleRegistryUrl).get('/dj-database-url/json').reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -467,10 +532,12 @@ describe('modules/datasource/pypi/index', () => { }); it('returns null for response with no versions', async () => { + const simpleRegistryUrl = 'https://some.registry.org/simple/'; httpMock - .scope('https://some.registry.org/simple/') + .scope(simpleRegistryUrl) .get('/dj-database-url/') .reply(200, badResponse); + httpMock.scope(simpleRegistryUrl).get('/dj-database-url/json').reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -484,31 +551,38 @@ describe('modules/datasource/pypi/index', () => { ).toBeNull(); }); - it('fall back from json and process data from simple endpoint', async () => { - httpMock - .scope('https://custom.pypi.net/foo') - .get('/dj-database-url/json') - .reply(404); - httpMock - .scope('https://custom.pypi.net/foo') - .get('/dj-database-url/') - .reply(200, htmlResponse); - const config = { - registryUrls: ['https://custom.pypi.net/foo'], - }; - const result = await getPkgReleases({ - datasource, - ...config, - packageName: 'dj-database-url', - }); - expect(result).toMatchSnapshot(); - }); + it.each([404, 403])( + 'process data from simple api with pypijson unavailable', + async (code: number) => { + httpMock + .scope('https://custom.pypi.net/foo') + .get('/dj-database-url/json') + .reply(code); + httpMock + .scope('https://custom.pypi.net/foo') + .get('/dj-database-url/') + .reply(200, htmlResponse); + const config = { + registryUrls: ['https://custom.pypi.net/foo'], + }; + const result = await getPkgReleases({ + datasource, + ...config, + packageName: 'dj-database-url', + }); + expect(result).toMatchSnapshot(); + }, + ); it('parses data-requires-python and respects constraints from simple endpoint', async () => { httpMock .scope('https://some.registry.org/simple/') .get('/dj-database-url/') .reply(200, dataRequiresPythonResponse); + httpMock + .scope('https://some.registry.org/simple/') + .get('/dj-database-url/json') + .reply(404); const config = { registryUrls: ['https://some.registry.org/simple/'], }; @@ -524,18 +598,28 @@ describe('modules/datasource/pypi/index', () => { }); }); - it('uses https://pypi.org/pypi/ instead of https://pypi.org/simple/', async () => { - httpMock.scope(baseUrl).get('/azure-cli-monitor/json').reply(200, res1); - const config = { - registryUrls: ['https://pypi.org/simple/'], - }; - expect( - await getPkgReleases({ - datasource, - ...config, - constraints: { python: '2.7' }, - packageName: 'azure-cli-monitor', - }), - ).toMatchSnapshot(); - }); + it.each([baseSimpleUrl, baseJsonUrl])( + 'uses https://pypi.org/pypi/ and https://pypi.org/simple/ (no find)', + async (registry: string) => { + httpMock + .scope(baseJsonUrl) + .get('/azure-cli-monitor/json') + .reply(200, res1); + httpMock + .scope(baseSimpleUrl) + .get('/azure-cli-monitor/') + .reply(200, htmlResponse); + const config = { + registryUrls: [registry], + }; + expect( + await getPkgReleases({ + datasource, + ...config, + constraints: { python: '2.7' }, + packageName: 'azure-cli-monitor', + }), + ).toMatchSnapshot(); + }, + ); }); diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 11e4c2dd5fb805..3b7c3b8c649f81 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -34,45 +34,51 @@ export class PypiDatasource extends Datasource { packageName, registryUrl, }: GetReleasesConfig): Promise { - let dependency: ReleaseResult | null = null; + const dependency: ReleaseResult = { releases: [] }; // TODO: null check (#22198) - const hostUrl = ensureTrailingSlash( - registryUrl!.replace('https://pypi.org/simple', 'https://pypi.org/pypi'), - ); + let hostUrl = ensureTrailingSlash(registryUrl!); const normalizedLookupName = PypiDatasource.normalizeName(packageName); - // not all simple indexes use this identifier, but most do - if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) { + // convert pypi json api url to simple + hostUrl = hostUrl.replace( + 'https://pypi.org/pypi', + 'https://pypi.org/simple', + ); + + const simpleFound = await this.addResultsViaSimple( + normalizedLookupName, + hostUrl, + dependency, + ).catch((err) => { + if (err.statusCode !== 404) { + throw err; + } logger.trace( - { packageName, hostUrl }, - 'Looking up pypi simple dependency', + 'Simple api not found. Looking up pypijson api as fallback.', ); - dependency = await this.getSimpleDependency( - normalizedLookupName, - hostUrl, - ); - } else { - logger.trace({ packageName, hostUrl }, 'Looking up pypi api dependency'); - try { - // we need to resolve early here so we can catch any 404s and fallback to a simple lookup - dependency = await this.getDependency(normalizedLookupName, hostUrl); - } catch (err) { - if (err.statusCode !== 404) { - throw err; - } - - // error contacting json-style api -- attempt to fallback to a simple-style api - logger.trace( - { packageName, hostUrl }, - 'Looking up pypi simple dependency via fallback', - ); - dependency = await this.getSimpleDependency( - normalizedLookupName, - hostUrl, - ); + return false; + }); + // convert pypi simple api url to json + hostUrl = hostUrl.replace( + 'https://pypi.org/simple', + 'https://pypi.org/pypi', + ); + logger.trace('Querying json api for metadata'); + const jsonFound = await this.addResultsViaPyPiJson( + normalizedLookupName, + hostUrl, + dependency, + ).catch((err) => { + if (!simpleFound) { + throw err; } + logger.trace('Json api lookup failed but got simple results.'); + return false; + }); + if (simpleFound || jsonFound) { + return dependency; } - return dependency; + return null; } private static normalizeName(input: string): string { @@ -83,21 +89,21 @@ export class PypiDatasource extends Datasource { return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-'); } - private async getDependency( + private async addResultsViaPyPiJson( packageName: string, hostUrl: string, - ): Promise { + dependency: ReleaseResult, + ): Promise { const lookupUrl = url.resolve( hostUrl, `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json`, ); - const dependency: ReleaseResult = { releases: [] }; logger.trace({ lookupUrl }, 'Pypi api got lookup'); const rep = await this.http.getJson(lookupUrl); const dep = rep?.body; if (!dep) { logger.trace({ dependency: packageName }, 'pip package not found'); - return null; + return false; } if (rep.authorization) { dependency.isPrivate = true; @@ -148,26 +154,30 @@ export class PypiDatasource extends Datasource { if (dep.releases) { const versions = Object.keys(dep.releases); - dependency.releases = versions.map((version) => { - const releases = coerceArray(dep.releases?.[version]); - const { upload_time: releaseTimestamp } = releases[0] || {}; - const isDeprecated = releases.some(({ yanked }) => yanked); - const result: Release = { - version, - releaseTimestamp, - }; - if (isDeprecated) { - result.isDeprecated = isDeprecated; - } - // There may be multiple releases with different requires_python, so we return all in an array - result.constraints = { - // TODO: string[] isn't allowed here - python: releases.map(({ requires_python }) => requires_python) as any, - }; - return result; - }); + dependency.releases = dependency.releases.concat( + versions.map((version) => { + const releases = coerceArray(dep.releases?.[version]); + const { upload_time: releaseTimestamp } = releases[0] || {}; + const isDeprecated = releases.some(({ yanked }) => yanked); + const result: Release = { + version, + releaseTimestamp, + }; + if (isDeprecated) { + result.isDeprecated = isDeprecated; + } + // There may be multiple releases with different requires_python, so we return all in an array + result.constraints = { + // TODO: string[] isn't allowed here + python: releases.map( + ({ requires_python }) => requires_python, + ) as any, + }; + return result; + }), + ); } - return dependency; + return true; } private static extractVersionFromLinkText( @@ -221,22 +231,25 @@ export class PypiDatasource extends Datasource { ); } - private async getSimpleDependency( + private async addResultsViaSimple( packageName: string, hostUrl: string, - ): Promise { + dependency: ReleaseResult, + ): Promise { const lookupUrl = url.resolve( - hostUrl, + hostUrl.replace('https://pypi.org/pypi', 'https://pypi.org/simple'), ensureTrailingSlash( PypiDatasource.normalizeNameForUrlLookup(packageName), ), ); - const dependency: ReleaseResult = { releases: [] }; const response = await this.http.get(lookupUrl); const dep = response?.body; if (!dep) { - logger.trace({ dependency: packageName }, 'pip package not found'); - return null; + logger.trace( + { dependency: packageName }, + 'pip package not found via simple api', + ); + return false; } if (response.authorization) { dependency.isPrivate = true; @@ -264,22 +277,24 @@ export class PypiDatasource extends Datasource { } } const versions = Object.keys(releases); - dependency.releases = versions.map((version) => { - const versionReleases = coerceArray(releases[version]); - const isDeprecated = versionReleases.some(({ yanked }) => yanked); - const result: Release = { version }; - if (isDeprecated) { - result.isDeprecated = isDeprecated; - } - // There may be multiple releases with different requires_python, so we return all in an array - result.constraints = { - // TODO: string[] isn't allowed here - python: versionReleases.map( - ({ requires_python }) => requires_python, - ) as any, - }; - return result; - }); - return dependency; + dependency.releases = dependency.releases.concat( + versions.map((version) => { + const versionReleases = coerceArray(releases[version]); + const isDeprecated = versionReleases.some(({ yanked }) => yanked); + const result: Release = { version }; + if (isDeprecated) { + result.isDeprecated = isDeprecated; + } + // There may be multiple releases with different requires_python, so we return all in an array + result.constraints = { + // TODO: string[] isn't allowed here + python: versionReleases.map( + ({ requires_python }) => requires_python, + ) as any, + }; + return result; + }), + ); + return true; } } From db6e529e7ee1b9ae6d5b10cccb2dfe68bb5d0984 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Fri, 5 Jan 2024 17:13:11 +0100 Subject: [PATCH 02/10] use const hostUrls --- lib/modules/datasource/pypi/index.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 3b7c3b8c649f81..3633dec4be661b 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -36,18 +36,17 @@ export class PypiDatasource extends Datasource { }: GetReleasesConfig): Promise { const dependency: ReleaseResult = { releases: [] }; // TODO: null check (#22198) - let hostUrl = ensureTrailingSlash(registryUrl!); + const hostUrl = ensureTrailingSlash(registryUrl!); const normalizedLookupName = PypiDatasource.normalizeName(packageName); - // convert pypi json api url to simple - hostUrl = hostUrl.replace( + const simpleHostUrl = hostUrl.replace( 'https://pypi.org/pypi', 'https://pypi.org/simple', ); const simpleFound = await this.addResultsViaSimple( normalizedLookupName, - hostUrl, + simpleHostUrl, dependency, ).catch((err) => { if (err.statusCode !== 404) { @@ -58,15 +57,14 @@ export class PypiDatasource extends Datasource { ); return false; }); - // convert pypi simple api url to json - hostUrl = hostUrl.replace( + logger.trace('Querying json api for metadata'); + const pypiJsonHostUrl = hostUrl.replace( 'https://pypi.org/simple', 'https://pypi.org/pypi', ); - logger.trace('Querying json api for metadata'); const jsonFound = await this.addResultsViaPyPiJson( normalizedLookupName, - hostUrl, + pypiJsonHostUrl, dependency, ).catch((err) => { if (!simpleFound) { @@ -237,7 +235,7 @@ export class PypiDatasource extends Datasource { dependency: ReleaseResult, ): Promise { const lookupUrl = url.resolve( - hostUrl.replace('https://pypi.org/pypi', 'https://pypi.org/simple'), + hostUrl, ensureTrailingSlash( PypiDatasource.normalizeNameForUrlLookup(packageName), ), From 2cadbb7adcf1a077f342c4963d8d4e3038112f30 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Fri, 5 Jan 2024 17:19:44 +0100 Subject: [PATCH 03/10] log with variables --- lib/modules/datasource/pypi/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 3633dec4be661b..9b763e489e61bd 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -53,11 +53,12 @@ export class PypiDatasource extends Datasource { throw err; } logger.trace( + { packageName, hostUrl }, 'Simple api not found. Looking up pypijson api as fallback.', ); return false; }); - logger.trace('Querying json api for metadata'); + logger.trace({ packageName, hostUrl }, 'Querying json api for metadata'); const pypiJsonHostUrl = hostUrl.replace( 'https://pypi.org/simple', 'https://pypi.org/pypi', @@ -70,7 +71,10 @@ export class PypiDatasource extends Datasource { if (!simpleFound) { throw err; } - logger.trace('Json api lookup failed but got simple results.'); + logger.trace( + { packageName, hostUrl }, + 'Json api lookup failed but got simple results.', + ); return false; }); if (simpleFound || jsonFound) { From 1e7fab7b62ce68c591c403e8d5e35a7d7c7751b4 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Fri, 5 Jan 2024 20:11:19 +0100 Subject: [PATCH 04/10] extract dependency merge --- lib/modules/datasource/artifactory/index.ts | 2 +- lib/modules/datasource/pypi/index.ts | 115 +++++++++----------- 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/lib/modules/datasource/artifactory/index.ts b/lib/modules/datasource/artifactory/index.ts index 86fe6918b04b44..eb7d4813d110ec 100644 --- a/lib/modules/datasource/artifactory/index.ts +++ b/lib/modules/datasource/artifactory/index.ts @@ -69,7 +69,7 @@ export class ArtifactoryDatasource extends Datasource { : node.innerHTML; const published = ArtifactoryDatasource.parseReleaseTimestamp( - node.nextSibling!.text, // TODO: can be null (#22198) + node.nextSibling.text, // TODO: can be null (#22198) ); const thisRelease: Release = { diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 9b763e489e61bd..bf48ebf414292d 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -34,7 +34,6 @@ export class PypiDatasource extends Datasource { packageName, registryUrl, }: GetReleasesConfig): Promise { - const dependency: ReleaseResult = { releases: [] }; // TODO: null check (#22198) const hostUrl = ensureTrailingSlash(registryUrl!); const normalizedLookupName = PypiDatasource.normalizeName(packageName); @@ -43,11 +42,9 @@ export class PypiDatasource extends Datasource { 'https://pypi.org/pypi', 'https://pypi.org/simple', ); - - const simpleFound = await this.addResultsViaSimple( + const simpleDependencies = await this.getResultsViaSimple( normalizedLookupName, simpleHostUrl, - dependency, ).catch((err) => { if (err.statusCode !== 404) { throw err; @@ -56,31 +53,31 @@ export class PypiDatasource extends Datasource { { packageName, hostUrl }, 'Simple api not found. Looking up pypijson api as fallback.', ); - return false; + return null; }); logger.trace({ packageName, hostUrl }, 'Querying json api for metadata'); const pypiJsonHostUrl = hostUrl.replace( 'https://pypi.org/simple', 'https://pypi.org/pypi', ); - const jsonFound = await this.addResultsViaPyPiJson( + const pypiJsonDependencies = await this.getResultsViaPyPiJson( normalizedLookupName, pypiJsonHostUrl, - dependency, ).catch((err) => { - if (!simpleFound) { + if (simpleDependencies === null) { throw err; } logger.trace( { packageName, hostUrl }, 'Json api lookup failed but got simple results.', ); - return false; + return null; }); - if (simpleFound || jsonFound) { - return dependency; + if (simpleDependencies === null && pypiJsonDependencies === null) { + return null; } - return null; + // merge results + return Object.assign({}, simpleDependencies, pypiJsonDependencies); } private static normalizeName(input: string): string { @@ -91,11 +88,11 @@ export class PypiDatasource extends Datasource { return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-'); } - private async addResultsViaPyPiJson( + private async getResultsViaPyPiJson( packageName: string, hostUrl: string, - dependency: ReleaseResult, - ): Promise { + ): Promise { + const dependency: ReleaseResult = { releases: [] }; const lookupUrl = url.resolve( hostUrl, `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json`, @@ -105,7 +102,7 @@ export class PypiDatasource extends Datasource { const dep = rep?.body; if (!dep) { logger.trace({ dependency: packageName }, 'pip package not found'); - return false; + return null; } if (rep.authorization) { dependency.isPrivate = true; @@ -156,30 +153,26 @@ export class PypiDatasource extends Datasource { if (dep.releases) { const versions = Object.keys(dep.releases); - dependency.releases = dependency.releases.concat( - versions.map((version) => { - const releases = coerceArray(dep.releases?.[version]); - const { upload_time: releaseTimestamp } = releases[0] || {}; - const isDeprecated = releases.some(({ yanked }) => yanked); - const result: Release = { - version, - releaseTimestamp, - }; - if (isDeprecated) { - result.isDeprecated = isDeprecated; - } - // There may be multiple releases with different requires_python, so we return all in an array - result.constraints = { - // TODO: string[] isn't allowed here - python: releases.map( - ({ requires_python }) => requires_python, - ) as any, - }; - return result; - }), - ); + dependency.releases = versions.map((version) => { + const releases = coerceArray(dep.releases?.[version]); + const { upload_time: releaseTimestamp } = releases[0] || {}; + const isDeprecated = releases.some(({ yanked }) => yanked); + const result: Release = { + version, + releaseTimestamp, + }; + if (isDeprecated) { + result.isDeprecated = isDeprecated; + } + // There may be multiple releases with different requires_python, so we return all in an array + result.constraints = { + // TODO: string[] isn't allowed here + python: releases.map(({ requires_python }) => requires_python) as any, + }; + return result; + }); } - return true; + return dependency; } private static extractVersionFromLinkText( @@ -233,11 +226,11 @@ export class PypiDatasource extends Datasource { ); } - private async addResultsViaSimple( + private async getResultsViaSimple( packageName: string, hostUrl: string, - dependency: ReleaseResult, - ): Promise { + ): Promise { + const dependency: ReleaseResult = { releases: [] }; const lookupUrl = url.resolve( hostUrl, ensureTrailingSlash( @@ -251,7 +244,7 @@ export class PypiDatasource extends Datasource { { dependency: packageName }, 'pip package not found via simple api', ); - return false; + return null; } if (response.authorization) { dependency.isPrivate = true; @@ -279,24 +272,22 @@ export class PypiDatasource extends Datasource { } } const versions = Object.keys(releases); - dependency.releases = dependency.releases.concat( - versions.map((version) => { - const versionReleases = coerceArray(releases[version]); - const isDeprecated = versionReleases.some(({ yanked }) => yanked); - const result: Release = { version }; - if (isDeprecated) { - result.isDeprecated = isDeprecated; - } - // There may be multiple releases with different requires_python, so we return all in an array - result.constraints = { - // TODO: string[] isn't allowed here - python: versionReleases.map( - ({ requires_python }) => requires_python, - ) as any, - }; - return result; - }), - ); - return true; + dependency.releases = versions.map((version) => { + const versionReleases = coerceArray(releases[version]); + const isDeprecated = versionReleases.some(({ yanked }) => yanked); + const result: Release = { version }; + if (isDeprecated) { + result.isDeprecated = isDeprecated; + } + // There may be multiple releases with different requires_python, so we return all in an array + result.constraints = { + // TODO: string[] isn't allowed here + python: versionReleases.map( + ({ requires_python }) => requires_python, + ) as any, + }; + return result; + }); + return dependency; } } From dda6ddef5d76e85ac1aaf16c167a04440754f290 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Fri, 5 Jan 2024 20:20:33 +0100 Subject: [PATCH 05/10] revert wrong lint fix --- lib/modules/datasource/artifactory/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/artifactory/index.ts b/lib/modules/datasource/artifactory/index.ts index eb7d4813d110ec..86fe6918b04b44 100644 --- a/lib/modules/datasource/artifactory/index.ts +++ b/lib/modules/datasource/artifactory/index.ts @@ -69,7 +69,7 @@ export class ArtifactoryDatasource extends Datasource { : node.innerHTML; const published = ArtifactoryDatasource.parseReleaseTimestamp( - node.nextSibling.text, // TODO: can be null (#22198) + node.nextSibling!.text, // TODO: can be null (#22198) ); const thisRelease: Release = { From aa68d1bb0e840f9a51672ef7fcc1a246c1068fd9 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Fri, 5 Jan 2024 20:23:40 +0100 Subject: [PATCH 06/10] cleanup diff --- lib/modules/datasource/pypi/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index bf48ebf414292d..24df7bcffbd9f0 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -92,11 +92,11 @@ export class PypiDatasource extends Datasource { packageName: string, hostUrl: string, ): Promise { - const dependency: ReleaseResult = { releases: [] }; const lookupUrl = url.resolve( hostUrl, `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json`, ); + const dependency: ReleaseResult = { releases: [] }; logger.trace({ lookupUrl }, 'Pypi api got lookup'); const rep = await this.http.getJson(lookupUrl); const dep = rep?.body; @@ -230,13 +230,13 @@ export class PypiDatasource extends Datasource { packageName: string, hostUrl: string, ): Promise { - const dependency: ReleaseResult = { releases: [] }; const lookupUrl = url.resolve( hostUrl, ensureTrailingSlash( PypiDatasource.normalizeNameForUrlLookup(packageName), ), ); + const dependency: ReleaseResult = { releases: [] }; const response = await this.http.get(lookupUrl); const dep = response?.body; if (!dep) { From ad026680843e640184af31c1e1b7763b4db0f67c Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Sat, 6 Jan 2024 16:17:15 +0100 Subject: [PATCH 07/10] reorder and use constant --- lib/modules/datasource/pypi/index.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 24df7bcffbd9f0..2903d90638891f 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -11,6 +11,9 @@ import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import { isGitHubRepo } from './common'; import type { PypiJSON, PypiJSONRelease, Releases } from './types'; +const jsonPyPiUrl = 'https://pypi.org/pypi'; +const simplePyPiUrl = 'https://pypi.org/simple'; + export class PypiDatasource extends Datasource { static readonly id = 'pypi'; @@ -36,12 +39,10 @@ export class PypiDatasource extends Datasource { }: GetReleasesConfig): Promise { // TODO: null check (#22198) const hostUrl = ensureTrailingSlash(registryUrl!); + const simpleHostUrl = hostUrl.replace(jsonPyPiUrl, simplePyPiUrl); + const pypiJsonHostUrl = hostUrl.replace(simplePyPiUrl, jsonPyPiUrl); const normalizedLookupName = PypiDatasource.normalizeName(packageName); - const simpleHostUrl = hostUrl.replace( - 'https://pypi.org/pypi', - 'https://pypi.org/simple', - ); const simpleDependencies = await this.getResultsViaSimple( normalizedLookupName, simpleHostUrl, @@ -56,10 +57,6 @@ export class PypiDatasource extends Datasource { return null; }); logger.trace({ packageName, hostUrl }, 'Querying json api for metadata'); - const pypiJsonHostUrl = hostUrl.replace( - 'https://pypi.org/simple', - 'https://pypi.org/pypi', - ); const pypiJsonDependencies = await this.getResultsViaPyPiJson( normalizedLookupName, pypiJsonHostUrl, From 42dd5f309ef9742e4eb959b2caadcc6532692501 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Sat, 6 Jan 2024 16:18:02 +0100 Subject: [PATCH 08/10] Apply suggestions from code review Co-authored-by: Sebastian Poxhofer --- lib/modules/datasource/pypi/index.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 2903d90638891f..4ba8a8408f04d0 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -51,12 +51,12 @@ export class PypiDatasource extends Datasource { throw err; } logger.trace( - { packageName, hostUrl }, + { packageName, hostUrl: simpleHostUrl }, 'Simple api not found. Looking up pypijson api as fallback.', ); return null; }); - logger.trace({ packageName, hostUrl }, 'Querying json api for metadata'); + logger.trace({ packageName, hostUrl: pypiJsonHostUrl }, 'Querying json api for metadata'); const pypiJsonDependencies = await this.getResultsViaPyPiJson( normalizedLookupName, pypiJsonHostUrl, @@ -74,7 +74,10 @@ export class PypiDatasource extends Datasource { return null; } // merge results - return Object.assign({}, simpleDependencies, pypiJsonDependencies); + return { + ...simpleDependencies, + ...pypiJsonDependencies + }; } private static normalizeName(input: string): string { From 75b196635dce62c5b9220d8fe4f75d4e35ea139b Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Sat, 6 Jan 2024 17:50:04 +0100 Subject: [PATCH 09/10] fix formatting and linter --- lib/modules/datasource/pypi/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 4ba8a8408f04d0..319ef000bec6ed 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -56,7 +56,10 @@ export class PypiDatasource extends Datasource { ); return null; }); - logger.trace({ packageName, hostUrl: pypiJsonHostUrl }, 'Querying json api for metadata'); + logger.trace( + { packageName, hostUrl: pypiJsonHostUrl }, + 'Querying json api for metadata', + ); const pypiJsonDependencies = await this.getResultsViaPyPiJson( normalizedLookupName, pypiJsonHostUrl, @@ -75,8 +78,9 @@ export class PypiDatasource extends Datasource { } // merge results return { + releases: [], ...simpleDependencies, - ...pypiJsonDependencies + ...pypiJsonDependencies, }; } From a6532a3f3038f6487e970e159ce1cacc62195fc8 Mon Sep 17 00:00:00 2001 From: Jonas Dittrich <58814480+Kakadus@users.noreply.github.com> Date: Sun, 7 Jan 2024 09:37:59 +0100 Subject: [PATCH 10/10] inline snapshots for new tests --- .../pypi/__snapshots__/index.spec.ts.snap | 280 ------------------ lib/modules/datasource/pypi/index.spec.ts | 136 ++++++++- 2 files changed, 134 insertions(+), 282 deletions(-) diff --git a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap index d03640f84e093f..c23fac1c07756b 100644 --- a/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/modules/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -1,89 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`modules/datasource/pypi/index getReleases process data from simple api with pypijson unavailable 1`] = ` -{ - "registryUrl": "https://custom.pypi.net/foo", - "releases": [ - { - "version": "0.1.2", - }, - { - "version": "0.1.3", - }, - { - "version": "0.1.4", - }, - { - "version": "0.2.0", - }, - { - "version": "0.2.1", - }, - { - "version": "0.2.2", - }, - { - "version": "0.3.0", - }, - { - "version": "0.4.0", - }, - { - "version": "0.4.1", - }, - { - "version": "0.4.2", - }, - { - "isDeprecated": true, - "version": "0.5.0", - }, - ], -} -`; - -exports[`modules/datasource/pypi/index getReleases process data from simple api with pypijson unavailable 2`] = ` -{ - "registryUrl": "https://custom.pypi.net/foo", - "releases": [ - { - "version": "0.1.2", - }, - { - "version": "0.1.3", - }, - { - "version": "0.1.4", - }, - { - "version": "0.2.0", - }, - { - "version": "0.2.1", - }, - { - "version": "0.2.2", - }, - { - "version": "0.3.0", - }, - { - "version": "0.4.0", - }, - { - "version": "0.4.1", - }, - { - "version": "0.4.2", - }, - { - "isDeprecated": true, - "version": "0.5.0", - }, - ], -} -`; - exports[`modules/datasource/pypi/index getReleases parses data-requires-python and respects constraints from simple endpoint 1`] = ` { "registryUrl": "https://some.registry.org/simple", @@ -322,199 +238,3 @@ exports[`modules/datasource/pypi/index getReleases respects constraints 1`] = ` ], } `; - -exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ and https://pypi.org/simple/ (no find) 1`] = ` -{ - "registryUrl": "https://pypi.org/simple", - "releases": [ - { - "releaseTimestamp": "2017-04-03T16:55:14.000Z", - "version": "0.0.1", - }, - { - "releaseTimestamp": "2017-04-17T20:32:30.000Z", - "version": "0.0.2", - }, - { - "releaseTimestamp": "2017-04-28T21:18:54.000Z", - "version": "0.0.3", - }, - { - "releaseTimestamp": "2017-05-09T21:36:51.000Z", - "version": "0.0.4", - }, - { - "releaseTimestamp": "2017-05-30T23:13:49.000Z", - "version": "0.0.5", - }, - { - "releaseTimestamp": "2017-06-13T22:21:05.000Z", - "version": "0.0.6", - }, - { - "releaseTimestamp": "2017-06-21T22:12:36.000Z", - "version": "0.0.7", - }, - { - "releaseTimestamp": "2017-07-07T16:22:26.000Z", - "version": "0.0.8", - }, - { - "releaseTimestamp": "2017-08-28T20:14:33.000Z", - "version": "0.0.9", - }, - { - "releaseTimestamp": "2017-09-22T23:47:59.000Z", - "version": "0.0.10", - }, - { - "releaseTimestamp": "2017-10-24T02:14:07.000Z", - "version": "0.0.11", - }, - { - "releaseTimestamp": "2017-11-14T18:31:57.000Z", - "version": "0.0.12", - }, - { - "releaseTimestamp": "2017-12-05T18:57:54.000Z", - "version": "0.0.13", - }, - { - "releaseTimestamp": "2018-01-05T21:26:03.000Z", - "version": "0.0.14", - }, - { - "releaseTimestamp": "2018-01-17T18:36:39.000Z", - "version": "0.1.0", - }, - { - "releaseTimestamp": "2018-01-31T18:05:22.000Z", - "version": "0.1.1", - }, - { - "releaseTimestamp": "2018-02-13T18:17:52.000Z", - "version": "0.1.2", - }, - { - "releaseTimestamp": "2018-03-13T17:08:20.000Z", - "version": "0.1.3", - }, - { - "releaseTimestamp": "2018-03-27T17:55:25.000Z", - "version": "0.1.4", - }, - { - "releaseTimestamp": "2018-04-10T17:25:47.000Z", - "version": "0.1.5", - }, - { - "isDeprecated": true, - "releaseTimestamp": "2018-05-07T17:59:09.000Z", - "version": "0.1.6", - }, - { - "releaseTimestamp": "2018-05-22T17:25:23.000Z", - "version": "0.1.7", - }, - ], - "sourceUrl": "https://github.com/Azure/azure-cli", -} -`; - -exports[`modules/datasource/pypi/index uses https://pypi.org/pypi/ and https://pypi.org/simple/ (no find) 2`] = ` -{ - "registryUrl": "https://pypi.org/pypi", - "releases": [ - { - "releaseTimestamp": "2017-04-03T16:55:14.000Z", - "version": "0.0.1", - }, - { - "releaseTimestamp": "2017-04-17T20:32:30.000Z", - "version": "0.0.2", - }, - { - "releaseTimestamp": "2017-04-28T21:18:54.000Z", - "version": "0.0.3", - }, - { - "releaseTimestamp": "2017-05-09T21:36:51.000Z", - "version": "0.0.4", - }, - { - "releaseTimestamp": "2017-05-30T23:13:49.000Z", - "version": "0.0.5", - }, - { - "releaseTimestamp": "2017-06-13T22:21:05.000Z", - "version": "0.0.6", - }, - { - "releaseTimestamp": "2017-06-21T22:12:36.000Z", - "version": "0.0.7", - }, - { - "releaseTimestamp": "2017-07-07T16:22:26.000Z", - "version": "0.0.8", - }, - { - "releaseTimestamp": "2017-08-28T20:14:33.000Z", - "version": "0.0.9", - }, - { - "releaseTimestamp": "2017-09-22T23:47:59.000Z", - "version": "0.0.10", - }, - { - "releaseTimestamp": "2017-10-24T02:14:07.000Z", - "version": "0.0.11", - }, - { - "releaseTimestamp": "2017-11-14T18:31:57.000Z", - "version": "0.0.12", - }, - { - "releaseTimestamp": "2017-12-05T18:57:54.000Z", - "version": "0.0.13", - }, - { - "releaseTimestamp": "2018-01-05T21:26:03.000Z", - "version": "0.0.14", - }, - { - "releaseTimestamp": "2018-01-17T18:36:39.000Z", - "version": "0.1.0", - }, - { - "releaseTimestamp": "2018-01-31T18:05:22.000Z", - "version": "0.1.1", - }, - { - "releaseTimestamp": "2018-02-13T18:17:52.000Z", - "version": "0.1.2", - }, - { - "releaseTimestamp": "2018-03-13T17:08:20.000Z", - "version": "0.1.3", - }, - { - "releaseTimestamp": "2018-03-27T17:55:25.000Z", - "version": "0.1.4", - }, - { - "releaseTimestamp": "2018-04-10T17:25:47.000Z", - "version": "0.1.5", - }, - { - "isDeprecated": true, - "releaseTimestamp": "2018-05-07T17:59:09.000Z", - "version": "0.1.6", - }, - { - "releaseTimestamp": "2018-05-22T17:25:23.000Z", - "version": "0.1.7", - }, - ], - "sourceUrl": "https://github.com/Azure/azure-cli", -} -`; diff --git a/lib/modules/datasource/pypi/index.spec.ts b/lib/modules/datasource/pypi/index.spec.ts index 81fe34b366d9fa..9ea1e6cd25892f 100644 --- a/lib/modules/datasource/pypi/index.spec.ts +++ b/lib/modules/datasource/pypi/index.spec.ts @@ -570,7 +570,45 @@ describe('modules/datasource/pypi/index', () => { ...config, packageName: 'dj-database-url', }); - expect(result).toMatchSnapshot(); + expect(result).toEqual({ + registryUrl: 'https://custom.pypi.net/foo', + releases: [ + { + version: '0.1.2', + }, + { + version: '0.1.3', + }, + { + version: '0.1.4', + }, + { + version: '0.2.0', + }, + { + version: '0.2.1', + }, + { + version: '0.2.2', + }, + { + version: '0.3.0', + }, + { + version: '0.4.0', + }, + { + version: '0.4.1', + }, + { + version: '0.4.2', + }, + { + isDeprecated: true, + version: '0.5.0', + }, + ], + }); }, ); @@ -619,7 +657,101 @@ describe('modules/datasource/pypi/index', () => { constraints: { python: '2.7' }, packageName: 'azure-cli-monitor', }), - ).toMatchSnapshot(); + ).toEqual({ + registryUrl: registry, + releases: [ + { + releaseTimestamp: '2017-04-03T16:55:14.000Z', + version: '0.0.1', + }, + { + releaseTimestamp: '2017-04-17T20:32:30.000Z', + version: '0.0.2', + }, + { + releaseTimestamp: '2017-04-28T21:18:54.000Z', + version: '0.0.3', + }, + { + releaseTimestamp: '2017-05-09T21:36:51.000Z', + version: '0.0.4', + }, + { + releaseTimestamp: '2017-05-30T23:13:49.000Z', + version: '0.0.5', + }, + { + releaseTimestamp: '2017-06-13T22:21:05.000Z', + version: '0.0.6', + }, + { + releaseTimestamp: '2017-06-21T22:12:36.000Z', + version: '0.0.7', + }, + { + releaseTimestamp: '2017-07-07T16:22:26.000Z', + version: '0.0.8', + }, + { + releaseTimestamp: '2017-08-28T20:14:33.000Z', + version: '0.0.9', + }, + { + releaseTimestamp: '2017-09-22T23:47:59.000Z', + version: '0.0.10', + }, + { + releaseTimestamp: '2017-10-24T02:14:07.000Z', + version: '0.0.11', + }, + { + releaseTimestamp: '2017-11-14T18:31:57.000Z', + version: '0.0.12', + }, + { + releaseTimestamp: '2017-12-05T18:57:54.000Z', + version: '0.0.13', + }, + { + releaseTimestamp: '2018-01-05T21:26:03.000Z', + version: '0.0.14', + }, + { + releaseTimestamp: '2018-01-17T18:36:39.000Z', + version: '0.1.0', + }, + { + releaseTimestamp: '2018-01-31T18:05:22.000Z', + version: '0.1.1', + }, + { + releaseTimestamp: '2018-02-13T18:17:52.000Z', + version: '0.1.2', + }, + { + releaseTimestamp: '2018-03-13T17:08:20.000Z', + version: '0.1.3', + }, + { + releaseTimestamp: '2018-03-27T17:55:25.000Z', + version: '0.1.4', + }, + { + releaseTimestamp: '2018-04-10T17:25:47.000Z', + version: '0.1.5', + }, + { + isDeprecated: true, + releaseTimestamp: '2018-05-07T17:59:09.000Z', + version: '0.1.6', + }, + { + releaseTimestamp: '2018-05-22T17:25:23.000Z', + version: '0.1.7', + }, + ], + sourceUrl: 'https://github.com/Azure/azure-cli', + }); }, ); });