diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts index e59f0b3709e68e..ff31d16c158362 100644 --- a/lib/modules/datasource/api.ts +++ b/lib/modules/datasource/api.ts @@ -51,6 +51,7 @@ import { PackagistDatasource } from './packagist'; import { PodDatasource } from './pod'; import { PuppetForgeDatasource } from './puppet-forge'; import { PypiDatasource } from './pypi'; +import { PythonVersionDatasource } from './python-version'; import { RepologyDatasource } from './repology'; import { RubyVersionDatasource } from './ruby-version'; import { RubyGemsDatasource } from './rubygems'; @@ -120,6 +121,7 @@ api.set(PackagistDatasource.id, new PackagistDatasource()); api.set(PodDatasource.id, new PodDatasource()); api.set(PuppetForgeDatasource.id, new PuppetForgeDatasource()); api.set(PypiDatasource.id, new PypiDatasource()); +api.set(PythonVersionDatasource.id, new PythonVersionDatasource()); api.set(RepologyDatasource.id, new RepologyDatasource()); api.set(RubyVersionDatasource.id, new RubyVersionDatasource()); api.set(RubyGemsDatasource.id, new RubyGemsDatasource()); diff --git a/lib/modules/datasource/python-version/__fixtures__/eol.json b/lib/modules/datasource/python-version/__fixtures__/eol.json new file mode 100644 index 00000000000000..4f9d6d19e384d9 --- /dev/null +++ b/lib/modules/datasource/python-version/__fixtures__/eol.json @@ -0,0 +1,4 @@ +[ + {"cycle":"3.12","releaseDate":"2023-10-02","support":"2025-04-02","eol":"2028-10-31","latest":"3.12.2","latestReleaseDate":"2024-02-06","lts":false}, + {"cycle":"3.7","releaseDate":"2018-06-26","support":"2020-06-27","eol":"2023-06-27","latest":"3.7.17","latestReleaseDate":"2023-06-05","lts":false} +] diff --git a/lib/modules/datasource/python-version/__fixtures__/release.json b/lib/modules/datasource/python-version/__fixtures__/release.json new file mode 100644 index 00000000000000..af122e745d4e87 --- /dev/null +++ b/lib/modules/datasource/python-version/__fixtures__/release.json @@ -0,0 +1,7 @@ +[ + {"name": "Python 3.12.0", "slug": "python-3120", "version": 3, "is_published": true, "is_latest": false, "release_date": "2023-10-02T12:50:09Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.12.0/whatsnew/changelog.html#python-3-12-0", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/832/"}, + {"name": "Python 3.12.0a1", "slug": "python-3120a1", "version": 3, "is_published": true, "is_latest": false, "release_date": "2022-10-25T02:16:12Z", "pre_release": true, "release_page": null, "release_notes_url": "", "show_on_download_page": false, "resource_uri": "https://www.python.org/api/v2/downloads/release/767/"}, + {"name": "Python 3.12.2", "slug": "python-3122", "version": 3, "is_published": true, "is_latest": true, "release_date": "2024-02-06T21:40:35Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.12.2/whatsnew/changelog.html#python-3-12-2", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/871/"}, + {"name": "Python 3.7.8", "slug": "python-378", "version": 3, "is_published": true, "is_latest": false, "release_date": "2020-06-27T12:55:01Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.7.8/whatsnew/changelog.html#changelog", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/442/"}, + {"name": "Python 3.7.9", "slug": "python-379", "version": 3, "is_published": true, "is_latest": false, "release_date": "2020-08-17T22:00:00Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.7.9/whatsnew/changelog.html#changelog", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/482/"} +] diff --git a/lib/modules/datasource/python-version/common.ts b/lib/modules/datasource/python-version/common.ts new file mode 100644 index 00000000000000..304693714f1d7d --- /dev/null +++ b/lib/modules/datasource/python-version/common.ts @@ -0,0 +1,5 @@ +export const defaultRegistryUrl = + 'https://www.python.org/api/v2/downloads/release'; +export const githubBaseUrl = 'https://api.github.com/'; + +export const datasource = 'python-version'; diff --git a/lib/modules/datasource/python-version/index.spec.ts b/lib/modules/datasource/python-version/index.spec.ts new file mode 100644 index 00000000000000..4333bfe0cec344 --- /dev/null +++ b/lib/modules/datasource/python-version/index.spec.ts @@ -0,0 +1,148 @@ +import { satisfies } from '@renovatebot/pep440'; +import { getPkgReleases } from '..'; +import { Fixtures } from '../../../../test/fixtures'; +import * as httpMock from '../../../../test/http-mock'; +import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages'; +import * as githubGraphql from '../../../util/github/graphql'; +import { registryUrl as eolRegistryUrl } from '../endoflife-date/common'; +import { datasource, defaultRegistryUrl } from './common'; +import { PythonVersionDatasource } from '.'; + +describe('modules/datasource/python-version/index', () => { + describe('dependent datasources', () => { + it('returns Python EOL data', async () => { + const datasource = new PythonVersionDatasource(); + httpMock + .scope(eolRegistryUrl) + .get('/python.json') + .reply(200, Fixtures.get('eol.json')); + const res = await datasource.getEolReleases(); + expect( + res?.releases.find((release) => release.version === '3.7.17') + ?.isDeprecated, + ).toBeTrue(); + }); + }); + + describe('getReleases', () => { + beforeEach(() => { + httpMock + .scope('https://endoflife.date') + .get('/api/python.json') + .reply(200, Fixtures.get('eol.json')); + + jest.spyOn(githubGraphql, 'queryReleases').mockResolvedValueOnce([ + { + id: 1, + url: 'https://example.com', + name: 'containerbase/python-prebuild', + description: 'some description', + version: '3.12.1', + releaseTimestamp: '2020-03-09T13:00:00Z', + }, + { + id: 2, + url: 'https://example.com', + name: 'containerbase/python-prebuild', + description: 'some description', + version: '3.12.0', + releaseTimestamp: '2020-03-09T13:00:00Z', + }, + { + id: 3, + url: 'https://example.com', + name: 'containerbase/python-prebuild', + description: 'some description', + version: '3.7.8', + releaseTimestamp: '2020-03-09T13:00:00Z', + }, + ]); + }); + + it('throws for 500', async () => { + httpMock.scope(defaultRegistryUrl).get('').reply(500); + await expect( + getPkgReleases({ + datasource, + packageName: 'python', + }), + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + }); + + it('returns null for error', async () => { + httpMock.scope(defaultRegistryUrl).get('').replyWithError('error'); + expect( + await getPkgReleases({ + datasource, + packageName: 'python', + }), + ).toBeNull(); + }); + + it('returns null for empty 200 OK', async () => { + httpMock.scope(defaultRegistryUrl).get('').reply(200, []); + expect( + await getPkgReleases({ + datasource, + packageName: 'python', + }), + ).toBeNull(); + }); + + describe('processes real data', () => { + beforeEach(() => { + httpMock + .scope(defaultRegistryUrl) + .get('') + .reply(200, Fixtures.get('release.json')); + }); + + it('returns the correct data', async () => { + const res = await getPkgReleases({ + datasource, + packageName: 'python', + }); + expect(res?.releases[0]).toEqual({ + isDeprecated: true, + isStable: true, + releaseTimestamp: '2020-06-27T12:55:01.000Z', + version: '3.7.8', + }); + }); + + it('only returns stable versions', async () => { + const res = await getPkgReleases({ + datasource, + packageName: 'python', + }); + expect(res?.releases).toHaveLength(2); + for (const release of res?.releases ?? []) { + expect(release.isStable).toBeTrue(); + } + }); + + it('only returns versions that are prebuilt', async () => { + const res = await getPkgReleases({ + datasource, + packageName: 'python', + }); + expect( + res?.releases.filter((release) => + satisfies(release.version, '>3.12.1'), + ), + ).toHaveLength(0); + }); + + it('returns isDeprecated status for Python 3 minor releases', async () => { + const res = await getPkgReleases({ + datasource, + packageName: 'python', + }); + expect(res?.releases).toHaveLength(2); + for (const release of res?.releases ?? []) { + expect(release.isDeprecated).toBeBoolean(); + } + }); + }); + }); +}); diff --git a/lib/modules/datasource/python-version/index.ts b/lib/modules/datasource/python-version/index.ts new file mode 100644 index 00000000000000..682c6ceb928747 --- /dev/null +++ b/lib/modules/datasource/python-version/index.ts @@ -0,0 +1,92 @@ +import { cache } from '../../../util/cache/package/decorator'; +import { id as versioning } from '../../versioning/python'; +import { Datasource } from '../datasource'; +import { EndoflifeDatePackagesource } from '../endoflife-date'; +import { registryUrl as eolRegistryUrl } from '../endoflife-date/common'; +import { GithubReleasesDatasource } from '../github-releases'; +import type { GetReleasesConfig, ReleaseResult } from '../types'; +import { datasource, defaultRegistryUrl, githubBaseUrl } from './common'; +import { PythonRelease } from './schema'; + +export class PythonVersionDatasource extends Datasource { + static readonly id = datasource; + pythonPrebuildDatasource: GithubReleasesDatasource; + pythonEolDatasource: EndoflifeDatePackagesource; + + constructor() { + super(datasource); + this.pythonPrebuildDatasource = new GithubReleasesDatasource(); + this.pythonEolDatasource = new EndoflifeDatePackagesource(); + } + + override readonly customRegistrySupport = false; + + override readonly defaultRegistryUrls = [defaultRegistryUrl]; + + override readonly defaultVersioning = versioning; + + override readonly caching = true; + + async getPrebuildReleases(): Promise { + return await this.pythonPrebuildDatasource.getReleases({ + registryUrl: githubBaseUrl, + packageName: 'containerbase/python-prebuild', + }); + } + + async getEolReleases(): Promise { + return await this.pythonEolDatasource.getReleases({ + registryUrl: eolRegistryUrl, + packageName: 'python', + }); + } + + @cache({ + namespace: `datasource-${datasource}`, + key: ({ registryUrl }: GetReleasesConfig) => `${registryUrl}`, + }) + async getReleases({ + registryUrl, + }: GetReleasesConfig): Promise { + // istanbul ignore if + if (!registryUrl) { + return null; + } + const pythonPrebuildReleases = await this.getPrebuildReleases(); + const pythonPrebuildVersions = new Set( + pythonPrebuildReleases?.releases.map((release) => release.version), + ); + const pythonEolReleases = await this.getEolReleases(); + const pythonEolVersions = new Map( + pythonEolReleases?.releases + .filter((release) => release.isDeprecated !== undefined) + .map((release) => [ + release.version.split('.').slice(0, 2).join('.'), + release.isDeprecated, + ]), + ); + const result: ReleaseResult = { + homepage: 'https://python.org', + sourceUrl: 'https://github.com/python/cpython', + registryUrl, + releases: [], + }; + try { + const response = await this.http.getJson(registryUrl, PythonRelease); + result.releases.push( + ...response.body + .filter((release) => release.isStable) + .filter((release) => pythonPrebuildVersions.has(release.version)), + ); + } catch (err) { + this.handleGenericErrors(err); + } + for (const release of result.releases) { + release.isDeprecated = pythonEolVersions.get( + release.version.split('.').slice(0, 2).join('.'), + ); + } + + return result.releases.length ? result : null; + } +} diff --git a/lib/modules/datasource/python-version/readme.md b/lib/modules/datasource/python-version/readme.md new file mode 100644 index 00000000000000..f904048f539810 --- /dev/null +++ b/lib/modules/datasource/python-version/readme.md @@ -0,0 +1,36 @@ +This datasource returns Python releases from the [python.org API](https://www.python.org/api/v2/downloads/release/). + +It also fetches deprecated versions from the [Endoflife Date datasource](../endoflife-date/index.md). + +Because Renovate depends on [`containerbase/python-prebuild`](https://github.com/containerbase/python-prebuild/releases) it will also fetch releases from the GitHub API. + +## Example custom manager + +Below is a [custom regex manager](../../manager/regex/index.md) to update the Python versions in a Dockerfile. +Python versions sometimes drop the dot that separate the major and minor number: so `3.11` becomes `311`. +The example below handles this case. + +```dockerfile +ARG PYTHON_VERSION=311 +FROM image-python${PYTHON_VERSION}-builder:1.0.0 +``` + +```json +{ + "customManagers": [ + { + "customType": "regex", + "fileMatch": ["^Dockerfile$"], + "matchStringsStrategy": "any", + "matchStrings": [ + "ARG PYTHON_VERSION=\"?(?3(?\\d+))\"?\\s" + ], + "autoReplaceStringTemplate": "ARG PYTHON_VERSION={{{replace '\\.' '' newValue}}}\n", + "currentValueTemplate": "3.{{{minor}}}", + "datasourceTemplate": "python-version", + "versioningTemplate": "python", + "depNameTemplate": "python" + } + ] +} +``` diff --git a/lib/modules/datasource/python-version/schema.ts b/lib/modules/datasource/python-version/schema.ts new file mode 100644 index 00000000000000..2f804ef8a42f4d --- /dev/null +++ b/lib/modules/datasource/python-version/schema.ts @@ -0,0 +1,31 @@ +import { z } from 'zod'; +import type { Release } from '../types'; + +export const PythonRelease = z + .object({ + /** e.g: "Python 3.9.0b1" */ + name: z.string(), + /** e.g: "python-390b1" */ + slug: z.string(), + /** Major version e.g: 3 */ + version: z.number(), + /** is latest major version, true for Python 2.7.18 and latest Python 3 */ + is_latest: z.boolean(), + is_published: z.boolean(), + release_date: z.string(), + pre_release: z.boolean(), + release_page: z.string().nullable(), + show_on_download_page: z.boolean(), + /** Changelog e.g: "https://docs.python.org/…html#python-3-9-0-beta-1" */ + release_notes_url: z.string(), + /** Download URL e.g: "https://www.python.org/api/v2/downloads/release/436/" */ + resource_uri: z.string(), + }) + .transform( + ({ name, release_date: releaseTimestamp, pre_release }): Release => { + const version = name?.replace('Python', '').trim(); + const isStable = pre_release === false; + return { version, releaseTimestamp, isStable }; + }, + ) + .array(); diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts index 6d600df7e14e70..e301414c99be6e 100644 --- a/lib/util/cache/package/types.ts +++ b/lib/util/cache/package/types.ts @@ -90,6 +90,7 @@ export type PackageCacheNamespace = | 'datasource-packagist-public-files' | 'datasource-packagist' | 'datasource-pod' + | 'datasource-python-version' | 'datasource-releases' | 'datasource-repology-list' | 'datasource-ruby-version'