From 9c2c0297e6042c37494b318d27a90b4a94e39f61 Mon Sep 17 00:00:00 2001 From: secustor Date: Thu, 11 Jul 2024 18:35:42 +0200 Subject: [PATCH 01/13] feat(manager): start with bitrise manager --- lib/modules/datasource/api.ts | 2 + lib/modules/datasource/bitrise/index.ts | 48 ++++++++++++++++++ lib/modules/datasource/bitrise/schema.ts | 3 ++ lib/modules/datasource/bitrise/util.ts | 63 ++++++++++++++++++++++++ lib/modules/platform/github/schema.ts | 39 +++++++++++++++ 5 files changed, 155 insertions(+) create mode 100644 lib/modules/datasource/bitrise/index.ts create mode 100644 lib/modules/datasource/bitrise/schema.ts create mode 100644 lib/modules/datasource/bitrise/util.ts create mode 100644 lib/modules/platform/github/schema.ts diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts index eff7da102edd81..e336c3eacbfb60 100644 --- a/lib/modules/datasource/api.ts +++ b/lib/modules/datasource/api.ts @@ -5,6 +5,7 @@ import { AzureBicepResourceDatasource } from './azure-bicep-resource'; import { AzurePipelinesTasksDatasource } from './azure-pipelines-tasks'; import { BazelDatasource } from './bazel'; import { BitbucketTagsDatasource } from './bitbucket-tags'; +import { BitriseDatasource } from './bitrise'; import { CdnJsDatasource } from './cdnjs'; import { ClojureDatasource } from './clojure'; import { ConanDatasource } from './conan'; @@ -73,6 +74,7 @@ api.set(AzureBicepResourceDatasource.id, new AzureBicepResourceDatasource()); api.set(AzurePipelinesTasksDatasource.id, new AzurePipelinesTasksDatasource()); api.set(BazelDatasource.id, new BazelDatasource()); api.set(BitbucketTagsDatasource.id, new BitbucketTagsDatasource()); +api.set(BitriseDatasource.id, new BitriseDatasource()); api.set(CdnJsDatasource.id, new CdnJsDatasource()); api.set(ClojureDatasource.id, new ClojureDatasource()); api.set(ConanDatasource.id, new ConanDatasource()); diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts new file mode 100644 index 00000000000000..36417e6c3f8404 --- /dev/null +++ b/lib/modules/datasource/bitrise/index.ts @@ -0,0 +1,48 @@ +import { Datasource } from '../datasource'; +import type { GetReleasesConfig, ReleaseResult } from '../types'; +import is from '@sindresorhus/is'; +import { fetchPackages } from './util'; +import { GithubHttp } from '../../../util/http/github'; + +export class BitriseDatasource extends Datasource { + static readonly id = 'bitrise'; + + constructor() { + super(BitriseDatasource.id); + } + + override readonly customRegistrySupport = true; + + override readonly defaultRegistryUrls = ['https://github.com/bitrise-io/bitrise-steplib.git']; + + override readonly releaseTimestampSupport = true; + override readonly releaseTimestampNote = + 'The release timestamp is determined from the `published_at` field in the results.'; + override readonly sourceUrlSupport = 'release'; + override readonly sourceUrlNote = + 'The source URL is determined from the `source_code_url` field of the release object in the results.'; + + async getReleases({ + packageName, + registryUrl, + }: GetReleasesConfig): Promise { + // istanbul ignore if + if (!registryUrl) { + return null; + } + + const client = new GithubHttp(this.id) + const packages = await fetchPackages(client, registryUrl); + if (!packages) { + return null; + } + const releases = packages[packageName]; + + const filtered = releases?.filter(is.truthy); + + if (!filtered?.length) { + return null; + } + return { releases: filtered }; + } +} diff --git a/lib/modules/datasource/bitrise/schema.ts b/lib/modules/datasource/bitrise/schema.ts new file mode 100644 index 00000000000000..87f6aa308ac3dd --- /dev/null +++ b/lib/modules/datasource/bitrise/schema.ts @@ -0,0 +1,3 @@ +import {z} from 'zod' + +export const BitriseStepFile = z.object({}).passthrough() diff --git a/lib/modules/datasource/bitrise/util.ts b/lib/modules/datasource/bitrise/util.ts new file mode 100644 index 00000000000000..0f5f492ca6c730 --- /dev/null +++ b/lib/modules/datasource/bitrise/util.ts @@ -0,0 +1,63 @@ +import type { Release } from '../types'; +import { parseGitUrl } from '../../../util/git/url'; +import { logger } from '../../../logger'; +import { GithubDirectoryResponse } from '../../platform/github/schema'; +import semver from '../../versioning/semver'; +import { joinUrlParts } from '../../../util/url'; +import type { GithubHttp } from '../../../util/http/github'; +import { BitriseStepFile } from './schema'; +import { parseSingleYaml } from '../../../util/yaml'; + +export async function fetchPackages(client: GithubHttp,registryUrl: string): Promise | null> { + + const parsedUrl = parseGitUrl(registryUrl) + if (parsedUrl.source !== 'github.com') { + logger.warn( `${parsedUrl.source} is not a supported Git hoster for this datasource`); + return null; + } + + const result: Record = {} + + const baseUrl = `https://api.github.com/repos/${parsedUrl.full_name}/contents`; + const stepsUrl = `${baseUrl}/steps` + const response = await client.getJson(stepsUrl, ) + const parsed = GithubDirectoryResponse.safeParse(response.body) + + if (!parsed.success) { + logger.error(parsed.error, `Failed to parse content ${stepsUrl}`); + return null; + } + + for (const packageDir of parsed.data.filter((element) => element.type === 'dir')) { + const releases: Release[] = [] + + const response = await client.getJson(packageDir.url ) + const parsed = GithubDirectoryResponse.safeParse(response.body) + if (!parsed.success) { + logger.error(parsed.error, `Failed to parse content ${packageDir.url}`); + continue + } + + + for (const versionFile of parsed.data.filter(element => semver.isValid(element.name))) { + const stepUrl = joinUrlParts(baseUrl, versionFile.path, "step.yml") + const file = await client.get(stepUrl, {headers: { + "Accept": "application/vnd.github.raw+json" + }}); + + + const fileParsed = parseSingleYaml(file.body, { + customSchema: BitriseStepFile + }) + + + releases.push({ + version: versionFile.name, + }) + logger.info(response) + } + result[packageDir.name] = releases; + } + + return result; +} diff --git a/lib/modules/platform/github/schema.ts b/lib/modules/platform/github/schema.ts new file mode 100644 index 00000000000000..60d9b88e465c28 --- /dev/null +++ b/lib/modules/platform/github/schema.ts @@ -0,0 +1,39 @@ +import {z} from 'zod'; + +// https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content +const GithubResponseMetadata = z.object({ + name: z.string(), + path: z.string(), + url: z.string(), +}); + +// if the file is inside a directory content list it does not have an encoding or content field. +// This is the case if RAW is requested as the upstream Github URL reader does. +export const GithubFile = GithubResponseMetadata.extend({ + type: z.literal('file'), + // content: z.string().nullable(), + // encoding: z.string().nullable(), +}); + +// contains +export type GithubFile = z.infer; + +export const GithubDirectory = GithubResponseMetadata.extend({ + type: z.literal('dir'), +}); + +export type GithubDirectory = z.infer; + +export const GithubOtherContent = GithubResponseMetadata.extend({ + type: z.literal('symlink').or(z.literal('submodule')), +}); + +export type GithubOtherContent = z.infer; + +export const GithubElement = GithubFile.or( + GithubDirectory, +).or(GithubOtherContent); +export type GithubElement = z.infer; + +export const GithubDirectoryResponse = z.array(GithubElement); + From d62b1de64a850dbf2466184c3521d83c4f502d74 Mon Sep 17 00:00:00 2001 From: secustor Date: Thu, 11 Jul 2024 20:47:37 +0200 Subject: [PATCH 02/13] feat(datasource): add bitrise datasource --- lib/modules/datasource/bitrise/index.spec.ts | 106 +++++++++++++++++++ lib/modules/datasource/bitrise/index.ts | 71 ++++++++++--- lib/modules/datasource/bitrise/readme.md | 21 ++++ lib/modules/datasource/bitrise/schema.ts | 7 +- lib/modules/datasource/bitrise/util.ts | 63 ----------- lib/modules/platform/github/schema.ts | 11 +- lib/util/cache/package/types.ts | 1 + lib/util/http/github.spec.ts | 20 ++++ lib/util/http/github.ts | 8 ++ 9 files changed, 220 insertions(+), 88 deletions(-) create mode 100644 lib/modules/datasource/bitrise/index.spec.ts create mode 100644 lib/modules/datasource/bitrise/readme.md delete mode 100644 lib/modules/datasource/bitrise/util.ts diff --git a/lib/modules/datasource/bitrise/index.spec.ts b/lib/modules/datasource/bitrise/index.spec.ts new file mode 100644 index 00000000000000..ec482169885eb3 --- /dev/null +++ b/lib/modules/datasource/bitrise/index.spec.ts @@ -0,0 +1,106 @@ +import { codeBlock } from 'common-tags'; +import * as httpMock from '../../../../test/http-mock'; +import { getPkgReleases } from '../index'; +import { BitriseDatasource } from './index'; + +describe('modules/datasource/bitrise/index', () => { + describe('getReleases()', () => { + it('returns null for unsupported registryUrl', async () => { + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'script', + registryUrls: ['https://gitlab.com/bitrise-io/bitrise-steplib'], + }), + ).resolves.toBeNull(); + }); + + it('returns version and filter out asset folder', async () => { + httpMock + .scope( + 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', + ) + .get('/activate-build-cache-for-bazel') + .reply(200, [ + { + type: 'dir', + name: '1.0.0', + path: 'steps/activate-build-cache-for-bazel/1.0.0', + }, + { + type: 'dir', + name: '1.0.1', + path: 'steps/activate-build-cache-for-bazel/1.0.1', + }, + { + type: 'dir', + name: 'assets', + path: 'steps/activate-build-cache-for-bazel/assets', + }, + ]) + .get('/activate-build-cache-for-bazel/1.0.0/step.yml') + .reply( + 200, + codeBlock` + published_at: 2024-03-19T13:54:48.081077+01:00 + source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel + website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel + `, + ) + .get('/activate-build-cache-for-bazel/1.0.1/step.yml') + .reply( + 200, + codeBlock` + published_at: "2024-07-03T08:53:25.668504731Z" + source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel + website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel + `, + ); + + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'activate-build-cache-for-bazel', + }), + ).resolves.toEqual({ + registryUrl: 'https://github.com/bitrise-io/bitrise-steplib.git', + releases: [ + { + releaseTimestamp: '2024-03-19T12:54:48.081Z', + sourceUrl: + 'https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel', + version: '1.0.0', + }, + { + releaseTimestamp: '2024-07-03T08:53:25.668Z', + sourceUrl: + 'https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel', + version: '1.0.1', + }, + ], + }); + }); + + it('returns null if there are no releases', async () => { + httpMock + .scope( + 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', + ) + .get('/activate-build-cache-for-bazel') + .reply(200, [ + { + type: 'dir', + name: 'assets', + path: 'steps/activate-build-cache-for-bazel/assets', + }, + ]); + + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'activate-build-cache-for-bazel', + }), + ).resolves.toBeNull(); + }); + }); +}); diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts index 36417e6c3f8404..b421da852391d0 100644 --- a/lib/modules/datasource/bitrise/index.ts +++ b/lib/modules/datasource/bitrise/index.ts @@ -1,8 +1,16 @@ -import { Datasource } from '../datasource'; -import type { GetReleasesConfig, ReleaseResult } from '../types'; import is from '@sindresorhus/is'; -import { fetchPackages } from './util'; +import { logger } from '../../../logger'; +import { cache } from '../../../util/cache/package/decorator'; +import { detectPlatform } from '../../../util/common'; +import { parseGitUrl } from '../../../util/git/url'; import { GithubHttp } from '../../../util/http/github'; +import { joinUrlParts } from '../../../util/url'; +import { parseSingleYaml } from '../../../util/yaml'; +import { GithubDirectoryResponse } from '../../platform/github/schema'; +import semver from '../../versioning/semver'; +import { Datasource } from '../datasource'; +import type { GetReleasesConfig, ReleaseResult } from '../types'; +import { BitriseStepFile } from './schema'; export class BitriseDatasource extends Datasource { static readonly id = 'bitrise'; @@ -13,7 +21,9 @@ export class BitriseDatasource extends Datasource { override readonly customRegistrySupport = true; - override readonly defaultRegistryUrls = ['https://github.com/bitrise-io/bitrise-steplib.git']; + override readonly defaultRegistryUrls = [ + 'https://github.com/bitrise-io/bitrise-steplib.git', + ]; override readonly releaseTimestampSupport = true; override readonly releaseTimestampNote = @@ -22,27 +32,58 @@ export class BitriseDatasource extends Datasource { override readonly sourceUrlNote = 'The source URL is determined from the `source_code_url` field of the release object in the results.'; + @cache({ + namespace: `datasource-${BitriseDatasource.id}`, + key: ({ packageName, registryUrl }: GetReleasesConfig) => + `${registryUrl}/${packageName}`, + }) async getReleases({ - packageName, - registryUrl, - }: GetReleasesConfig): Promise { + packageName, + registryUrl, + }: GetReleasesConfig): Promise { // istanbul ignore if if (!registryUrl) { return null; } - const client = new GithubHttp(this.id) - const packages = await fetchPackages(client, registryUrl); - if (!packages) { + const client = new GithubHttp(this.id); + + const parsedUrl = parseGitUrl(registryUrl); + if (detectPlatform(registryUrl) !== 'github') { + logger.warn( + `${parsedUrl.source} is not a supported Git hoster for this datasource`, + ); return null; } - const releases = packages[packageName]; - const filtered = releases?.filter(is.truthy); + const result: ReleaseResult = { + releases: [], + }; - if (!filtered?.length) { - return null; + const massagedPackageName = encodeURIComponent(packageName); + const packageUrl = `https://api.${parsedUrl.resource}/repos/${parsedUrl.full_name}/contents/steps/${massagedPackageName}`; + + const response = await client.getJson(packageUrl, GithubDirectoryResponse); + + for (const versionDir of response.body.filter((element) => + semver.isValid(element.name), + )) { + const stepUrl = joinUrlParts(packageUrl, versionDir.name, 'step.yml'); + const file = await client.getRawFile(stepUrl); + const { published_at, source_code_url } = parseSingleYaml(file.body, { + customSchema: BitriseStepFile, + }); + + const releaseTimestamp = is.string(published_at) + ? published_at + : published_at.toISOString(); + result.releases.push({ + version: versionDir.name, + releaseTimestamp, + sourceUrl: source_code_url, + }); } - return { releases: filtered }; + + return result; } } diff --git a/lib/modules/datasource/bitrise/readme.md b/lib/modules/datasource/bitrise/readme.md new file mode 100644 index 00000000000000..87fc40c8023bfb --- /dev/null +++ b/lib/modules/datasource/bitrise/readme.md @@ -0,0 +1,21 @@ +This datasource allows to fetch Bitrise steps from Git repositories. + +**Currently only Github is support** + +As `packageName` the step name expected e.g. for the following snippet the `packageName` would be `script`. + +`registryUrl` expects a GitHub HTTP Git Url as used below by Bitrise. See the `default_step_lib_source` field for an example. +```yaml +format_version: 11 +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +project_type: android +app: + envs: + - MY_NAME: My Name +workflows: + test: + steps: + - script@1.1.5: + inputs: + - content: echo "Hello ${MY_NAME}!" +``` diff --git a/lib/modules/datasource/bitrise/schema.ts b/lib/modules/datasource/bitrise/schema.ts index 87f6aa308ac3dd..8b04b3cbe7ae96 100644 --- a/lib/modules/datasource/bitrise/schema.ts +++ b/lib/modules/datasource/bitrise/schema.ts @@ -1,3 +1,6 @@ -import {z} from 'zod' +import { z } from 'zod'; -export const BitriseStepFile = z.object({}).passthrough() +export const BitriseStepFile = z.object({ + published_at: z.date().or(z.string()), + source_code_url: z.string().optional(), +}); diff --git a/lib/modules/datasource/bitrise/util.ts b/lib/modules/datasource/bitrise/util.ts deleted file mode 100644 index 0f5f492ca6c730..00000000000000 --- a/lib/modules/datasource/bitrise/util.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { Release } from '../types'; -import { parseGitUrl } from '../../../util/git/url'; -import { logger } from '../../../logger'; -import { GithubDirectoryResponse } from '../../platform/github/schema'; -import semver from '../../versioning/semver'; -import { joinUrlParts } from '../../../util/url'; -import type { GithubHttp } from '../../../util/http/github'; -import { BitriseStepFile } from './schema'; -import { parseSingleYaml } from '../../../util/yaml'; - -export async function fetchPackages(client: GithubHttp,registryUrl: string): Promise | null> { - - const parsedUrl = parseGitUrl(registryUrl) - if (parsedUrl.source !== 'github.com') { - logger.warn( `${parsedUrl.source} is not a supported Git hoster for this datasource`); - return null; - } - - const result: Record = {} - - const baseUrl = `https://api.github.com/repos/${parsedUrl.full_name}/contents`; - const stepsUrl = `${baseUrl}/steps` - const response = await client.getJson(stepsUrl, ) - const parsed = GithubDirectoryResponse.safeParse(response.body) - - if (!parsed.success) { - logger.error(parsed.error, `Failed to parse content ${stepsUrl}`); - return null; - } - - for (const packageDir of parsed.data.filter((element) => element.type === 'dir')) { - const releases: Release[] = [] - - const response = await client.getJson(packageDir.url ) - const parsed = GithubDirectoryResponse.safeParse(response.body) - if (!parsed.success) { - logger.error(parsed.error, `Failed to parse content ${packageDir.url}`); - continue - } - - - for (const versionFile of parsed.data.filter(element => semver.isValid(element.name))) { - const stepUrl = joinUrlParts(baseUrl, versionFile.path, "step.yml") - const file = await client.get(stepUrl, {headers: { - "Accept": "application/vnd.github.raw+json" - }}); - - - const fileParsed = parseSingleYaml(file.body, { - customSchema: BitriseStepFile - }) - - - releases.push({ - version: versionFile.name, - }) - logger.info(response) - } - result[packageDir.name] = releases; - } - - return result; -} diff --git a/lib/modules/platform/github/schema.ts b/lib/modules/platform/github/schema.ts index 60d9b88e465c28..63115db40a2aca 100644 --- a/lib/modules/platform/github/schema.ts +++ b/lib/modules/platform/github/schema.ts @@ -1,18 +1,15 @@ -import {z} from 'zod'; +import { z } from 'zod'; // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content const GithubResponseMetadata = z.object({ name: z.string(), path: z.string(), - url: z.string(), }); // if the file is inside a directory content list it does not have an encoding or content field. // This is the case if RAW is requested as the upstream Github URL reader does. export const GithubFile = GithubResponseMetadata.extend({ type: z.literal('file'), - // content: z.string().nullable(), - // encoding: z.string().nullable(), }); // contains @@ -30,10 +27,8 @@ export const GithubOtherContent = GithubResponseMetadata.extend({ export type GithubOtherContent = z.infer; -export const GithubElement = GithubFile.or( - GithubDirectory, -).or(GithubOtherContent); +export const GithubElement = + GithubFile.or(GithubDirectory).or(GithubOtherContent); export type GithubElement = z.infer; export const GithubDirectoryResponse = z.array(GithubElement); - diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts index e8a479bd1dce51..0187737243bdbc 100644 --- a/lib/util/cache/package/types.ts +++ b/lib/util/cache/package/types.ts @@ -33,6 +33,7 @@ export type PackageCacheNamespace = | 'datasource-azure-pipelines-tasks' | 'datasource-bazel' | 'datasource-bitbucket-tags' + | 'datasource-bitrise' | 'datasource-cdnjs-digest' | 'datasource-cdnjs' | 'datasource-conan-revisions' diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 68af7bc831436d..6220b55921bf3c 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -786,4 +786,24 @@ describe('util/http/github', () => { ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); }); + + describe('getRawFile()', () => { + it('add header and return', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawFile( + `${githubApiHost}/foo/bar/contents/lore/ipsum.txt`, + ), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); + }); }); diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index 3db9d7b4a741dc..d250ed4e610b25 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -493,4 +493,12 @@ export class GithubHttp extends Http { return result; } + + public getRawFile(url: string): Promise { + return this.get(url, { + headers: { + accept: 'application/vnd.github.raw+json', + }, + }); + } } From db379ff5cabe580fd8778e884c6c3daaf30efe83 Mon Sep 17 00:00:00 2001 From: Sebastian Poxhofer Date: Thu, 11 Jul 2024 21:01:17 +0200 Subject: [PATCH 03/13] Update schema.ts --- lib/modules/platform/github/schema.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/modules/platform/github/schema.ts b/lib/modules/platform/github/schema.ts index 63115db40a2aca..f4e731478658b1 100644 --- a/lib/modules/platform/github/schema.ts +++ b/lib/modules/platform/github/schema.ts @@ -6,13 +6,10 @@ const GithubResponseMetadata = z.object({ path: z.string(), }); -// if the file is inside a directory content list it does not have an encoding or content field. -// This is the case if RAW is requested as the upstream Github URL reader does. export const GithubFile = GithubResponseMetadata.extend({ type: z.literal('file'), }); -// contains export type GithubFile = z.infer; export const GithubDirectory = GithubResponseMetadata.extend({ From 0f62d4c10015150509546aa5811ff7d7ac356aea Mon Sep 17 00:00:00 2001 From: secustor Date: Thu, 11 Jul 2024 21:18:01 +0200 Subject: [PATCH 04/13] fix prettier --- lib/modules/datasource/bitrise/readme.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/bitrise/readme.md b/lib/modules/datasource/bitrise/readme.md index 87fc40c8023bfb..39660331aee95e 100644 --- a/lib/modules/datasource/bitrise/readme.md +++ b/lib/modules/datasource/bitrise/readme.md @@ -5,6 +5,7 @@ This datasource allows to fetch Bitrise steps from Git repositories. As `packageName` the step name expected e.g. for the following snippet the `packageName` would be `script`. `registryUrl` expects a GitHub HTTP Git Url as used below by Bitrise. See the `default_step_lib_source` field for an example. + ```yaml format_version: 11 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git @@ -15,7 +16,7 @@ app: workflows: test: steps: - - script@1.1.5: - inputs: - - content: echo "Hello ${MY_NAME}!" + - script@1.1.5: + inputs: + - content: echo "Hello ${MY_NAME}!" ``` From c14ee2a44ee563906218f1448ae09ac269216072 Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 12 Jul 2024 23:41:09 +0200 Subject: [PATCH 05/13] support GHE api path and implement suggestions --- lib/modules/datasource/bitrise/index.spec.ts | 41 ++++++++++++++++++++ lib/modules/datasource/bitrise/index.ts | 25 +++++++++--- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/lib/modules/datasource/bitrise/index.spec.ts b/lib/modules/datasource/bitrise/index.spec.ts index ec482169885eb3..bbcf0eb5c38915 100644 --- a/lib/modules/datasource/bitrise/index.spec.ts +++ b/lib/modules/datasource/bitrise/index.spec.ts @@ -15,6 +15,47 @@ describe('modules/datasource/bitrise/index', () => { ).resolves.toBeNull(); }); + it('support GHE api url', async () => { + httpMock + .scope( + 'https://github.mycompany.com/api/v3/repos/foo/bar/contents/steps', + ) + .get('/script') + .reply(200, [ + { + type: 'dir', + name: '1.0.0', + path: 'steps/script/1.0.0', + }, + ]) + .get('/script/1.0.0/step.yml') + .reply( + 200, + codeBlock` + published_at: 2024-03-19T13:54:48.081077+01:00 + source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel + website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel + `, + ); + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'script', + registryUrls: ['https://github.mycompany.com/foo/bar'], + }), + ).resolves.toEqual({ + registryUrl: 'https://github.mycompany.com/foo/bar', + releases: [ + { + releaseTimestamp: '2024-03-19T12:54:48.081Z', + sourceUrl: + 'https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel', + version: '1.0.0', + }, + ], + }); + }); + it('returns version and filter out asset folder', async () => { httpMock .scope( diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts index b421da852391d0..1f7f3f39ab494b 100644 --- a/lib/modules/datasource/bitrise/index.ts +++ b/lib/modules/datasource/bitrise/index.ts @@ -15,8 +15,12 @@ import { BitriseStepFile } from './schema'; export class BitriseDatasource extends Datasource { static readonly id = 'bitrise'; + override readonly http: GithubHttp; + constructor() { super(BitriseDatasource.id); + + this.http = new GithubHttp(this.id); } override readonly customRegistrySupport = true; @@ -46,8 +50,6 @@ export class BitriseDatasource extends Datasource { return null; } - const client = new GithubHttp(this.id); - const parsedUrl = parseGitUrl(registryUrl); if (detectPlatform(registryUrl) !== 'github') { logger.warn( @@ -61,15 +63,28 @@ export class BitriseDatasource extends Datasource { }; const massagedPackageName = encodeURIComponent(packageName); - const packageUrl = `https://api.${parsedUrl.resource}/repos/${parsedUrl.full_name}/contents/steps/${massagedPackageName}`; + const baseApiURL = + parsedUrl.resource === 'github.com' + ? 'https://api.github.com' + : `https://${parsedUrl.resource}/api/v3`; + const packageUrl = joinUrlParts( + baseApiURL, + 'repos', + parsedUrl.full_name, + 'contents/steps', + massagedPackageName, + ); - const response = await client.getJson(packageUrl, GithubDirectoryResponse); + const response = await this.http.getJson( + packageUrl, + GithubDirectoryResponse, + ); for (const versionDir of response.body.filter((element) => semver.isValid(element.name), )) { const stepUrl = joinUrlParts(packageUrl, versionDir.name, 'step.yml'); - const file = await client.getRawFile(stepUrl); + const file = await this.http.getRawFile(stepUrl); const { published_at, source_code_url } = parseSingleYaml(file.body, { customSchema: BitriseStepFile, }); From 39c1b167756cc1adaef5c07dcfd65071791d12ca Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 19 Jul 2024 15:51:53 +0200 Subject: [PATCH 06/13] use json file request for now till raw fetching is ready, this will only work for files smaller than 1MB --- lib/modules/datasource/bitrise/index.spec.ts | 123 ++++++++++++++++--- lib/modules/datasource/bitrise/index.ts | 38 +++++- 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/lib/modules/datasource/bitrise/index.spec.ts b/lib/modules/datasource/bitrise/index.spec.ts index bbcf0eb5c38915..cd94e923d74062 100644 --- a/lib/modules/datasource/bitrise/index.spec.ts +++ b/lib/modules/datasource/bitrise/index.spec.ts @@ -1,5 +1,6 @@ import { codeBlock } from 'common-tags'; import * as httpMock from '../../../../test/http-mock'; +import { toBase64 } from '../../../util/string'; import { getPkgReleases } from '../index'; import { BitriseDatasource } from './index'; @@ -29,14 +30,17 @@ describe('modules/datasource/bitrise/index', () => { }, ]) .get('/script/1.0.0/step.yml') - .reply( - 200, - codeBlock` + .reply(200, { + type: 'file', + name: 'step.yml', + path: 'steps/script/1.0.0/step.yml', + encoding: 'base64', + content: toBase64(codeBlock` published_at: 2024-03-19T13:54:48.081077+01:00 - source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel - website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel - `, - ); + source_code_url: https://github.com/bitrise-steplib/bitrise-step-script + website: https://github.com/bitrise-steplib/bitrise-step-script + `), + }); await expect( getPkgReleases({ datasource: BitriseDatasource.id, @@ -48,8 +52,7 @@ describe('modules/datasource/bitrise/index', () => { releases: [ { releaseTimestamp: '2024-03-19T12:54:48.081Z', - sourceUrl: - 'https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel', + sourceUrl: 'https://github.com/bitrise-steplib/bitrise-step-script', version: '1.0.0', }, ], @@ -80,23 +83,29 @@ describe('modules/datasource/bitrise/index', () => { }, ]) .get('/activate-build-cache-for-bazel/1.0.0/step.yml') - .reply( - 200, - codeBlock` + .reply(200, { + type: 'file', + name: 'step.yml', + path: 'steps/activate-build-cache-for-bazel/1.0.0/step.yml', + encoding: 'base64', + content: toBase64(codeBlock` published_at: 2024-03-19T13:54:48.081077+01:00 source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel - `, - ) + `), + }) .get('/activate-build-cache-for-bazel/1.0.1/step.yml') - .reply( - 200, - codeBlock` + .reply(200, { + type: 'file', + name: 'step.yml', + path: 'steps/activate-build-cache-for-bazel/1.0.1/step.yml', + encoding: 'base64', + content: toBase64(codeBlock` published_at: "2024-07-03T08:53:25.668504731Z" source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel - `, - ); + `), + }); await expect( getPkgReleases({ @@ -143,5 +152,81 @@ describe('modules/datasource/bitrise/index', () => { }), ).resolves.toBeNull(); }); + + it('returns null if there is an unexpected format a package', async () => { + httpMock + .scope( + 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', + ) + .get('/activate-build-cache-for-bazel') + .reply(200, { + type: 'file', + name: 'assets', + path: 'steps/activate-build-cache-for-bazel/assets', + }); + + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'activate-build-cache-for-bazel', + }), + ).resolves.toBeNull(); + }); + + it('returns null if the file object has no content', async () => { + httpMock + .scope( + 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', + ) + .get('/script') + .reply(200, [ + { + type: 'dir', + name: '1.0.0', + path: 'steps/script/1.0.0', + }, + ]) + .get('/script/1.0.0/step.yml') + .reply(200, { + type: 'file', + name: 'step.yml', + path: 'steps/script/1.0.0/step.yml', + }); + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'script', + }), + ).resolves.toBeNull(); + }); + + it('returns null if the file object has an unexpected encoding', async () => { + httpMock + .scope( + 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', + ) + .get('/script') + .reply(200, [ + { + type: 'dir', + name: '1.0.0', + path: 'steps/script/1.0.0', + }, + ]) + .get('/script/1.0.0/step.yml') + .reply(200, { + type: 'file', + name: 'step.yml', + path: 'steps/script/1.0.0/step.yml', + encoding: 'none', + content: '', + }); + await expect( + getPkgReleases({ + datasource: BitriseDatasource.id, + packageName: 'script', + }), + ).resolves.toBeNull(); + }); }); }); diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts index 1f7f3f39ab494b..2e76d0ab28018c 100644 --- a/lib/modules/datasource/bitrise/index.ts +++ b/lib/modules/datasource/bitrise/index.ts @@ -4,9 +4,10 @@ import { cache } from '../../../util/cache/package/decorator'; import { detectPlatform } from '../../../util/common'; import { parseGitUrl } from '../../../util/git/url'; import { GithubHttp } from '../../../util/http/github'; +import { fromBase64 } from '../../../util/string'; import { joinUrlParts } from '../../../util/url'; import { parseSingleYaml } from '../../../util/yaml'; -import { GithubDirectoryResponse } from '../../platform/github/schema'; +import { GithubContentResponse } from '../../platform/github/schema'; import semver from '../../versioning/semver'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; @@ -75,17 +76,42 @@ export class BitriseDatasource extends Datasource { massagedPackageName, ); - const response = await this.http.getJson( + const { body: packageRaw } = await this.http.getJson( packageUrl, - GithubDirectoryResponse, + GithubContentResponse, ); - for (const versionDir of response.body.filter((element) => + if (!is.array(packageRaw)) { + logger.warn( + { data: packageRaw, url: packageUrl }, + 'Got unexpected response for Bitrise package location', + ); + return null; + } + + for (const versionDir of packageRaw.filter((element) => semver.isValid(element.name), )) { const stepUrl = joinUrlParts(packageUrl, versionDir.name, 'step.yml'); - const file = await this.http.getRawFile(stepUrl); - const { published_at, source_code_url } = parseSingleYaml(file.body, { + // TODO use getRawFile when ready #30155 + const { body } = await this.http.getJson(stepUrl, GithubContentResponse); + if (!('content' in body)) { + logger.warn( + { data: body, url: stepUrl }, + 'Got unexpected response for Bitrise step location', + ); + return null; + } + if (body.encoding !== 'base64') { + logger.warn( + { data: body, url: stepUrl }, + `Got unexpected encoding for Bitrise step location '${body.encoding}'`, + ); + return null; + } + + const content = fromBase64(body.content); + const { published_at, source_code_url } = parseSingleYaml(content, { customSchema: BitriseStepFile, }); From 27760fb013307df26e02c0742960744f29c120cf Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 19 Jul 2024 15:59:10 +0200 Subject: [PATCH 07/13] remove github http util --- lib/util/http/github.spec.ts | 20 -------------------- lib/util/http/github.ts | 8 -------- 2 files changed, 28 deletions(-) diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 6220b55921bf3c..68af7bc831436d 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -786,24 +786,4 @@ describe('util/http/github', () => { ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); }); - - describe('getRawFile()', () => { - it('add header and return', async () => { - httpMock - .scope(githubApiHost) - .get('/foo/bar/contents/lore/ipsum.txt') - .matchHeader( - 'accept', - 'application/vnd.github.raw+json, application/vnd.github.v3+json', - ) - .reply(200, 'foo'); - await expect( - githubApi.getRawFile( - `${githubApiHost}/foo/bar/contents/lore/ipsum.txt`, - ), - ).resolves.toMatchObject({ - body: 'foo', - }); - }); - }); }); diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index d250ed4e610b25..3db9d7b4a741dc 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -493,12 +493,4 @@ export class GithubHttp extends Http { return result; } - - public getRawFile(url: string): Promise { - return this.get(url, { - headers: { - accept: 'application/vnd.github.raw+json', - }, - }); - } } From 113cfb222e8bbedf2033792c60e298bd87a89879 Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 19 Jul 2024 22:30:01 +0200 Subject: [PATCH 08/13] add homepage --- lib/modules/datasource/bitrise/index.spec.ts | 3 +++ lib/modules/datasource/bitrise/index.ts | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/modules/datasource/bitrise/index.spec.ts b/lib/modules/datasource/bitrise/index.spec.ts index cd94e923d74062..f11cdc1bc167fb 100644 --- a/lib/modules/datasource/bitrise/index.spec.ts +++ b/lib/modules/datasource/bitrise/index.spec.ts @@ -48,6 +48,7 @@ describe('modules/datasource/bitrise/index', () => { registryUrls: ['https://github.mycompany.com/foo/bar'], }), ).resolves.toEqual({ + homepage: 'https://bitrise.io/integrations/steps/script', registryUrl: 'https://github.mycompany.com/foo/bar', releases: [ { @@ -113,6 +114,8 @@ describe('modules/datasource/bitrise/index', () => { packageName: 'activate-build-cache-for-bazel', }), ).resolves.toEqual({ + homepage: + 'https://bitrise.io/integrations/steps/activate-build-cache-for-bazel', registryUrl: 'https://github.com/bitrise-io/bitrise-steplib.git', releases: [ { diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts index 2e76d0ab28018c..18b73e65740533 100644 --- a/lib/modules/datasource/bitrise/index.ts +++ b/lib/modules/datasource/bitrise/index.ts @@ -125,6 +125,14 @@ export class BitriseDatasource extends Datasource { }); } - return result; + // if we have no releases return null + if (!result.releases.length) { + return null; + } + + return { + ...result, + homepage: `https://bitrise.io/integrations/steps/${packageName}`, + }; } } From 9f8144308eae240ebba54ad2d5a4df521dcb9dd4 Mon Sep 17 00:00:00 2001 From: secustor Date: Thu, 11 Jul 2024 14:55:06 +0200 Subject: [PATCH 09/13] feat(manager): start with bitrise manager --- lib/modules/manager/api.ts | 2 + lib/modules/manager/bitrise/extract.spec.ts | 47 +++++++++++++++ lib/modules/manager/bitrise/extract.ts | 49 ++++++++++++++++ lib/modules/manager/bitrise/index.ts | 17 ++++++ lib/modules/manager/bitrise/schema.ts | 12 ++++ lib/modules/manager/bitrise/utils.spec.ts | 27 +++++++++ lib/modules/manager/bitrise/utils.ts | 64 +++++++++++++++++++++ 7 files changed, 218 insertions(+) create mode 100644 lib/modules/manager/bitrise/extract.spec.ts create mode 100644 lib/modules/manager/bitrise/extract.ts create mode 100644 lib/modules/manager/bitrise/index.ts create mode 100644 lib/modules/manager/bitrise/schema.ts create mode 100644 lib/modules/manager/bitrise/utils.spec.ts create mode 100644 lib/modules/manager/bitrise/utils.ts diff --git a/lib/modules/manager/api.ts b/lib/modules/manager/api.ts index 726479f4d079ce..40676bb710bb85 100644 --- a/lib/modules/manager/api.ts +++ b/lib/modules/manager/api.ts @@ -10,6 +10,7 @@ import * as bazelModule from './bazel-module'; import * as bazelisk from './bazelisk'; import * as bicep from './bicep'; import * as bitbucketPipelines from './bitbucket-pipelines'; +import * as bitrise from './bitrise'; import * as buildkite from './buildkite'; import * as bun from './bun'; import * as bundler from './bundler'; @@ -108,6 +109,7 @@ api.set('bazel-module', bazelModule); api.set('bazelisk', bazelisk); api.set('bicep', bicep); api.set('bitbucket-pipelines', bitbucketPipelines); +api.set('bitrise', bitrise) api.set('buildkite', buildkite); api.set('bun', bun); api.set('bundler', bundler); diff --git a/lib/modules/manager/bitrise/extract.spec.ts b/lib/modules/manager/bitrise/extract.spec.ts new file mode 100644 index 00000000000000..6700eaef9623ba --- /dev/null +++ b/lib/modules/manager/bitrise/extract.spec.ts @@ -0,0 +1,47 @@ +import { codeBlock } from 'common-tags'; +import { extractPackageFile } from '../fvm'; +import { GithubReleasesDatasource } from '../../datasource/github-releases'; + +describe('modules/manager/bitrise/extract', () => { + describe('extractPackageFile()', () => { + it('returns null on an empty file', () => { + expect(extractPackageFile('')).toBeNull(); + }); + + it('returns valid file', () => { + expect( + extractPackageFile(codeBlock` + format_version: 11 + default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git + project_type: android + app: + envs: + - MY_NAME: My Name + workflows: + test: + steps: + - script@1.1.5: + inputs: + - content: echo "Hello!" + - restore-cache@1.1.2: + foo: bar + `), + ).toEqual({ + deps: [ + { + datasource: GithubReleasesDatasource.id, + depName: "script", + packageName: "bitrise-steplib/steps-script", + currentValue: '1.1.5', + }, + { + datasource: GithubReleasesDatasource.id, + depName: "restore-cache", + packageName: "bitrise-steplib/steps-restore-cache", + currentValue: '1.1.2', + } + ] + }); + }); + }); +}); diff --git a/lib/modules/manager/bitrise/extract.ts b/lib/modules/manager/bitrise/extract.ts new file mode 100644 index 00000000000000..7fdda68e360748 --- /dev/null +++ b/lib/modules/manager/bitrise/extract.ts @@ -0,0 +1,49 @@ +import { parseSingleYaml } from '../../../util/yaml'; +import type { + ExtractConfig, + PackageDependency, + PackageFileContent, +} from '../types'; +import { BitriseFile } from './schema'; +import { logger } from '../../../logger'; +import { parseStep } from './utils'; +import is from '@sindresorhus/is'; + +export function extractPackageFile( + content: string, + packageFile: string, + config: ExtractConfig, +): PackageFileContent | null { + const deps: PackageDependency[] = []; + + try { + + + const parsed = parseSingleYaml(content, { + customSchema: BitriseFile, + }); + + const workflows = Object.values(parsed.workflows) + for (const workflow of workflows) { + const steps = workflow.steps.flatMap((step) => Object.keys(step)) + for (const step of steps) { + const dep = parseStep(step); + + if (!is.nullOrUndefined(dep)) { + deps.push(dep); + } + } + } + + } catch (err) { + logger.debug( + { err, packageFile }, + `Parsing Bitrise config YAML failed`, + ); + } + + if (!deps.length) { + return null; + } + return { deps }; +} diff --git a/lib/modules/manager/bitrise/index.ts b/lib/modules/manager/bitrise/index.ts new file mode 100644 index 00000000000000..6b2daa98edf7ca --- /dev/null +++ b/lib/modules/manager/bitrise/index.ts @@ -0,0 +1,17 @@ +import type { Category } from '../../../constants'; +import { extractPackageFile } from './extract'; +import { GithubReleasesDatasource } from '../../datasource/github-releases'; + +export { extractPackageFile }; + +export const defaultConfig = { + fileMatch: ['(^|/)bitrise\\.ya?ml$'], +}; + +export const categories: Category[] = ['ci']; + +export const supportedDatasources = [GithubReleasesDatasource.id]; + +export const urls = [ + 'https://devcenter.bitrise.io/en/steps-and-workflows/introduction-to-steps.html', +]; diff --git a/lib/modules/manager/bitrise/schema.ts b/lib/modules/manager/bitrise/schema.ts new file mode 100644 index 00000000000000..9ca3fee88df2bc --- /dev/null +++ b/lib/modules/manager/bitrise/schema.ts @@ -0,0 +1,12 @@ +import {z} from 'zod'; + + +export const BitriseStep = z.record(z.string(), z.unknown()) + +export const BitriseWorkflow = z.object({ + steps: z.array(BitriseStep) +}) + +export const BitriseFile = z.object({ + workflows: z.record(z.string(), BitriseWorkflow) +}) diff --git a/lib/modules/manager/bitrise/utils.spec.ts b/lib/modules/manager/bitrise/utils.spec.ts new file mode 100644 index 00000000000000..a6984151cfcd0a --- /dev/null +++ b/lib/modules/manager/bitrise/utils.spec.ts @@ -0,0 +1,27 @@ +import { parseStep } from './utils'; + +describe('modules/manager/bitrise/utils', () => { + describe('parseStep()', () => { + it('returns null on an empty string', () => { + expect(parseStep('')).toBeNull(); + }); + + it('returns dependency for step', () => { + expect(parseStep('restore-gradle-cache@1.1.2')).toEqual({ + "currentValue": "1.1.2", + "datasource": "github-releases", + "depName": "restore-gradle-cache", + "packageName": "bitrise-steplib/steps-restore-gradle-cache" + }); + }) + + it('returns legacy packageName ', () => { + expect(parseStep('share-pipeline-variable@2.1.2')).toEqual({ + "currentValue": "2.1.2", + "datasource": "github-releases", + "depName": "restore-gradle-cache", + "packageName": "bitrise-steplib/bitrise-steps-restore-gradle-cache" + }); + }) + }); +}); diff --git a/lib/modules/manager/bitrise/utils.ts b/lib/modules/manager/bitrise/utils.ts new file mode 100644 index 00000000000000..5eb59ab5896019 --- /dev/null +++ b/lib/modules/manager/bitrise/utils.ts @@ -0,0 +1,64 @@ +import is from '@sindresorhus/is'; +import { GithubReleasesDatasource } from '../../datasource/github-releases'; +import type { PackageDependency } from '../types'; + +const stepsWithOldNamingScheme: string[] = [ + "share-pipeline-variable", + "restore-gradle-cache", + "restore-cache", + "set-java-version", + "pull-intermediate-files", + "save-gradle-cache", + "save-cache", + "s3-download", + "slack", + "build-router-start" +] + +export function parseStep(stepRef: string): PackageDependency | null { + if (is.emptyString(stepRef)) { + return null; + } + + const dep: PackageDependency = { + datasource: GithubReleasesDatasource.id, + replaceString: stepRef + } + + const splitted = stepRef.split('@'); + + // no version + if (splitted.length === 1) { + return { + ...dep, + depName: stepRef, + skipReason: 'unspecified-version', + packageName: createPackageName(stepRef) + } + } + + // too many @ chars + if (splitted.length > 2) { + return { + ...dep, + depName: stepRef, + skipReason: 'invalid-value', + packageName: createPackageName(stepRef) + } + } + + const [depName, currentValue] = splitted; + return { + ...dep, + depName, + currentValue, + packageName: createPackageName(depName), + } +} + +function createPackageName(stepName: string): string { + if (stepsWithOldNamingScheme.includes(stepName)) { + return `bitrise-steplib/bitrise-steps-${stepName}` + } + return `bitrise-steplib/steps-${stepName}` +} From ed3b8a9fbf37a2793324082f5f624bc81101c90f Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 19 Jul 2024 22:26:50 +0200 Subject: [PATCH 10/13] implement bitrise manager --- lib/modules/manager/api.ts | 2 +- lib/modules/manager/bitrise/extract.spec.ts | 29 ++++++------ lib/modules/manager/bitrise/extract.ts | 23 +++------ lib/modules/manager/bitrise/index.ts | 6 ++- lib/modules/manager/bitrise/readme.md | 1 + lib/modules/manager/bitrise/schema.ts | 13 +++--- lib/modules/manager/bitrise/utils.spec.ts | 25 +++++----- lib/modules/manager/bitrise/utils.ts | 52 ++++----------------- 8 files changed, 57 insertions(+), 94 deletions(-) create mode 100644 lib/modules/manager/bitrise/readme.md diff --git a/lib/modules/manager/api.ts b/lib/modules/manager/api.ts index 40676bb710bb85..7c84644cdf2a4f 100644 --- a/lib/modules/manager/api.ts +++ b/lib/modules/manager/api.ts @@ -109,7 +109,7 @@ api.set('bazel-module', bazelModule); api.set('bazelisk', bazelisk); api.set('bicep', bicep); api.set('bitbucket-pipelines', bitbucketPipelines); -api.set('bitrise', bitrise) +api.set('bitrise', bitrise); api.set('buildkite', buildkite); api.set('bun', bun); api.set('bundler', bundler); diff --git a/lib/modules/manager/bitrise/extract.spec.ts b/lib/modules/manager/bitrise/extract.spec.ts index 6700eaef9623ba..1a226cb027f908 100644 --- a/lib/modules/manager/bitrise/extract.spec.ts +++ b/lib/modules/manager/bitrise/extract.spec.ts @@ -1,16 +1,17 @@ import { codeBlock } from 'common-tags'; -import { extractPackageFile } from '../fvm'; -import { GithubReleasesDatasource } from '../../datasource/github-releases'; +import { BitriseDatasource } from '../../datasource/bitrise'; +import { extractPackageFile } from '.'; describe('modules/manager/bitrise/extract', () => { describe('extractPackageFile()', () => { it('returns null on an empty file', () => { - expect(extractPackageFile('')).toBeNull(); + expect(extractPackageFile('', 'bitrise.yml')).toBeNull(); }); it('returns valid file', () => { expect( - extractPackageFile(codeBlock` + extractPackageFile( + codeBlock` format_version: 11 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git project_type: android @@ -25,22 +26,24 @@ describe('modules/manager/bitrise/extract', () => { - content: echo "Hello!" - restore-cache@1.1.2: foo: bar - `), + `, + 'bitrise.yml', + ), ).toEqual({ deps: [ { - datasource: GithubReleasesDatasource.id, - depName: "script", - packageName: "bitrise-steplib/steps-script", + datasource: BitriseDatasource.id, + packageName: 'script', currentValue: '1.1.5', + replaceString: 'script@1.1.5', }, { - datasource: GithubReleasesDatasource.id, - depName: "restore-cache", - packageName: "bitrise-steplib/steps-restore-cache", + datasource: BitriseDatasource.id, + packageName: 'restore-cache', currentValue: '1.1.2', - } - ] + replaceString: 'restore-cache@1.1.2', + }, + ], }); }); }); diff --git a/lib/modules/manager/bitrise/extract.ts b/lib/modules/manager/bitrise/extract.ts index 7fdda68e360748..5f444b9e17f0b9 100644 --- a/lib/modules/manager/bitrise/extract.ts +++ b/lib/modules/manager/bitrise/extract.ts @@ -1,31 +1,24 @@ +import is from '@sindresorhus/is'; +import { logger } from '../../../logger'; import { parseSingleYaml } from '../../../util/yaml'; -import type { - ExtractConfig, - PackageDependency, - PackageFileContent, -} from '../types'; +import type { PackageDependency, PackageFileContent } from '../types'; import { BitriseFile } from './schema'; -import { logger } from '../../../logger'; import { parseStep } from './utils'; -import is from '@sindresorhus/is'; export function extractPackageFile( content: string, packageFile: string, - config: ExtractConfig, ): PackageFileContent | null { const deps: PackageDependency[] = []; try { - - const parsed = parseSingleYaml(content, { customSchema: BitriseFile, }); - const workflows = Object.values(parsed.workflows) + const workflows = Object.values(parsed.workflows); for (const workflow of workflows) { - const steps = workflow.steps.flatMap((step) => Object.keys(step)) + const steps = workflow.steps.flatMap((step) => Object.keys(step)); for (const step of steps) { const dep = parseStep(step); @@ -34,12 +27,8 @@ export function extractPackageFile( } } } - } catch (err) { - logger.debug( - { err, packageFile }, - `Parsing Bitrise config YAML failed`, - ); + logger.debug({ err, packageFile }, `Parsing Bitrise config YAML failed`); } if (!deps.length) { diff --git a/lib/modules/manager/bitrise/index.ts b/lib/modules/manager/bitrise/index.ts index 6b2daa98edf7ca..a3419fab56835e 100644 --- a/lib/modules/manager/bitrise/index.ts +++ b/lib/modules/manager/bitrise/index.ts @@ -1,6 +1,6 @@ import type { Category } from '../../../constants'; +import { BitriseDatasource } from '../../datasource/bitrise'; import { extractPackageFile } from './extract'; -import { GithubReleasesDatasource } from '../../datasource/github-releases'; export { extractPackageFile }; @@ -8,9 +8,11 @@ export const defaultConfig = { fileMatch: ['(^|/)bitrise\\.ya?ml$'], }; +export const displayName = 'Bitrise'; + export const categories: Category[] = ['ci']; -export const supportedDatasources = [GithubReleasesDatasource.id]; +export const supportedDatasources = [BitriseDatasource.id]; export const urls = [ 'https://devcenter.bitrise.io/en/steps-and-workflows/introduction-to-steps.html', diff --git a/lib/modules/manager/bitrise/readme.md b/lib/modules/manager/bitrise/readme.md new file mode 100644 index 00000000000000..cad96aa8b1d6c0 --- /dev/null +++ b/lib/modules/manager/bitrise/readme.md @@ -0,0 +1 @@ +Updates step references of `bitrise.yml` files. diff --git a/lib/modules/manager/bitrise/schema.ts b/lib/modules/manager/bitrise/schema.ts index 9ca3fee88df2bc..5f74923defc740 100644 --- a/lib/modules/manager/bitrise/schema.ts +++ b/lib/modules/manager/bitrise/schema.ts @@ -1,12 +1,11 @@ -import {z} from 'zod'; +import { z } from 'zod'; - -export const BitriseStep = z.record(z.string(), z.unknown()) +export const BitriseStep = z.record(z.string(), z.unknown()); export const BitriseWorkflow = z.object({ - steps: z.array(BitriseStep) -}) + steps: z.array(BitriseStep), +}); export const BitriseFile = z.object({ - workflows: z.record(z.string(), BitriseWorkflow) -}) + workflows: z.record(z.string(), BitriseWorkflow), +}); diff --git a/lib/modules/manager/bitrise/utils.spec.ts b/lib/modules/manager/bitrise/utils.spec.ts index a6984151cfcd0a..d5ed5beb9c26d5 100644 --- a/lib/modules/manager/bitrise/utils.spec.ts +++ b/lib/modules/manager/bitrise/utils.spec.ts @@ -1,3 +1,4 @@ +import { BitriseDatasource } from '../../datasource/bitrise'; import { parseStep } from './utils'; describe('modules/manager/bitrise/utils', () => { @@ -8,20 +9,20 @@ describe('modules/manager/bitrise/utils', () => { it('returns dependency for step', () => { expect(parseStep('restore-gradle-cache@1.1.2')).toEqual({ - "currentValue": "1.1.2", - "datasource": "github-releases", - "depName": "restore-gradle-cache", - "packageName": "bitrise-steplib/steps-restore-gradle-cache" + currentValue: '1.1.2', + datasource: BitriseDatasource.id, + packageName: 'restore-gradle-cache', + replaceString: 'restore-gradle-cache@1.1.2', }); - }) + }); - it('returns legacy packageName ', () => { - expect(parseStep('share-pipeline-variable@2.1.2')).toEqual({ - "currentValue": "2.1.2", - "datasource": "github-releases", - "depName": "restore-gradle-cache", - "packageName": "bitrise-steplib/bitrise-steps-restore-gradle-cache" + it('parses missing version', () => { + expect(parseStep('share-pipeline-variable')).toEqual({ + datasource: BitriseDatasource.id, + packageName: 'share-pipeline-variable', + replaceString: 'share-pipeline-variable', + skipReason: 'unspecified-version', }); - }) + }); }); }); diff --git a/lib/modules/manager/bitrise/utils.ts b/lib/modules/manager/bitrise/utils.ts index 5eb59ab5896019..5437fe0ffc94f8 100644 --- a/lib/modules/manager/bitrise/utils.ts +++ b/lib/modules/manager/bitrise/utils.ts @@ -1,64 +1,32 @@ import is from '@sindresorhus/is'; -import { GithubReleasesDatasource } from '../../datasource/github-releases'; +import { BitriseDatasource } from '../../datasource/bitrise'; import type { PackageDependency } from '../types'; -const stepsWithOldNamingScheme: string[] = [ - "share-pipeline-variable", - "restore-gradle-cache", - "restore-cache", - "set-java-version", - "pull-intermediate-files", - "save-gradle-cache", - "save-cache", - "s3-download", - "slack", - "build-router-start" -] - export function parseStep(stepRef: string): PackageDependency | null { if (is.emptyString(stepRef)) { return null; } const dep: PackageDependency = { - datasource: GithubReleasesDatasource.id, - replaceString: stepRef - } + datasource: BitriseDatasource.id, + replaceString: stepRef, + }; - const splitted = stepRef.split('@'); + const splitted = stepRef.split('@', 2); // no version if (splitted.length === 1) { return { ...dep, - depName: stepRef, + packageName: stepRef, skipReason: 'unspecified-version', - packageName: createPackageName(stepRef) - } - } - - // too many @ chars - if (splitted.length > 2) { - return { - ...dep, - depName: stepRef, - skipReason: 'invalid-value', - packageName: createPackageName(stepRef) - } + }; } - const [depName, currentValue] = splitted; + const [packageName, currentValue] = splitted; return { ...dep, - depName, + packageName, currentValue, - packageName: createPackageName(depName), - } -} - -function createPackageName(stepName: string): string { - if (stepsWithOldNamingScheme.includes(stepName)) { - return `bitrise-steplib/bitrise-steps-${stepName}` - } - return `bitrise-steplib/steps-${stepName}` + }; } From 346cabdd201cf6f63201c29cc8a776e4f2004ee8 Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 19 Jul 2024 23:32:47 +0200 Subject: [PATCH 11/13] add support for advanced step configurations --- lib/modules/manager/bitrise/extract.spec.ts | 90 ++++++++++++++++++++- lib/modules/manager/bitrise/extract.ts | 2 +- lib/modules/manager/bitrise/index.ts | 6 +- lib/modules/manager/bitrise/schema.ts | 1 + lib/modules/manager/bitrise/utils.ts | 63 +++++++++++++-- 5 files changed, 154 insertions(+), 8 deletions(-) diff --git a/lib/modules/manager/bitrise/extract.spec.ts b/lib/modules/manager/bitrise/extract.spec.ts index 1a226cb027f908..ff59059c8551d6 100644 --- a/lib/modules/manager/bitrise/extract.spec.ts +++ b/lib/modules/manager/bitrise/extract.spec.ts @@ -9,11 +9,34 @@ describe('modules/manager/bitrise/extract', () => { }); it('returns valid file', () => { + expect( + extractPackageFile( + codeBlock` + workflows: + test: + steps: + - script@1.1.5: + `, + 'bitrise.yml', + ), + ).toEqual({ + deps: [ + { + datasource: BitriseDatasource.id, + packageName: 'script', + currentValue: '1.1.5', + replaceString: 'script@1.1.5', + }, + ], + }); + }); + + it('returns valid file with custom default_step_lib_source', () => { expect( extractPackageFile( codeBlock` format_version: 11 - default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git + default_step_lib_source: https://github.com/custom/steplib.git project_type: android app: envs: @@ -36,12 +59,77 @@ describe('modules/manager/bitrise/extract', () => { packageName: 'script', currentValue: '1.1.5', replaceString: 'script@1.1.5', + registryUrls: ['https://github.com/custom/steplib.git'], }, { datasource: BitriseDatasource.id, packageName: 'restore-cache', currentValue: '1.1.2', replaceString: 'restore-cache@1.1.2', + registryUrls: ['https://github.com/custom/steplib.git'], + }, + ], + }); + }); + + it('extract git and path prefixes', () => { + expect( + extractPackageFile( + codeBlock` + workflows: + test: + steps: + - git::https://github.com/bitrise-io/steps-script.git@1.1.3: + - path::./relative/path: + - https://github.com/foo/bar.git::script@1: + `, + 'bitrise.yml', + ), + ).toEqual({ + deps: [ + { + currentValue: '1.1.3', + datasource: 'git-tags', + packageName: 'https://github.com/bitrise-io/steps-script.git', + replaceString: + 'git::https://github.com/bitrise-io/steps-script.git@1.1.3', + }, + { + datasource: 'bitrise', + packageName: 'path::./relative/path', + replaceString: 'path::./relative/path', + skipReason: 'unspecified-version', + }, + { + currentValue: '1', + datasource: 'bitrise', + packageName: 'script', + registryUrls: ['https://github.com/foo/bar.git'], + replaceString: 'https://github.com/foo/bar.git::script@1', + }, + ], + }); + }); + + it('extract bitrise library reference', () => { + expect( + extractPackageFile( + codeBlock` + workflows: + test: + steps: + - https://github.com/foo/bar.git::script@1: + `, + 'bitrise.yml', + ), + ).toEqual({ + deps: [ + { + currentValue: '1', + datasource: 'bitrise', + packageName: 'script', + registryUrls: ['https://github.com/foo/bar.git'], + replaceString: 'https://github.com/foo/bar.git::script@1', }, ], }); diff --git a/lib/modules/manager/bitrise/extract.ts b/lib/modules/manager/bitrise/extract.ts index 5f444b9e17f0b9..0f02ea9ee58f17 100644 --- a/lib/modules/manager/bitrise/extract.ts +++ b/lib/modules/manager/bitrise/extract.ts @@ -20,7 +20,7 @@ export function extractPackageFile( for (const workflow of workflows) { const steps = workflow.steps.flatMap((step) => Object.keys(step)); for (const step of steps) { - const dep = parseStep(step); + const dep = parseStep(step, parsed.default_step_lib_source); if (!is.nullOrUndefined(dep)) { deps.push(dep); diff --git a/lib/modules/manager/bitrise/index.ts b/lib/modules/manager/bitrise/index.ts index a3419fab56835e..70a97588c3b3d6 100644 --- a/lib/modules/manager/bitrise/index.ts +++ b/lib/modules/manager/bitrise/index.ts @@ -1,5 +1,6 @@ import type { Category } from '../../../constants'; import { BitriseDatasource } from '../../datasource/bitrise'; +import { GitTagsDatasource } from '../../datasource/git-tags'; import { extractPackageFile } from './extract'; export { extractPackageFile }; @@ -12,7 +13,10 @@ export const displayName = 'Bitrise'; export const categories: Category[] = ['ci']; -export const supportedDatasources = [BitriseDatasource.id]; +export const supportedDatasources = [ + BitriseDatasource.id, + GitTagsDatasource.id, +]; export const urls = [ 'https://devcenter.bitrise.io/en/steps-and-workflows/introduction-to-steps.html', diff --git a/lib/modules/manager/bitrise/schema.ts b/lib/modules/manager/bitrise/schema.ts index 5f74923defc740..f32030d93dc966 100644 --- a/lib/modules/manager/bitrise/schema.ts +++ b/lib/modules/manager/bitrise/schema.ts @@ -7,5 +7,6 @@ export const BitriseWorkflow = z.object({ }); export const BitriseFile = z.object({ + default_step_lib_source: z.string().optional(), workflows: z.record(z.string(), BitriseWorkflow), }); diff --git a/lib/modules/manager/bitrise/utils.ts b/lib/modules/manager/bitrise/utils.ts index 5437fe0ffc94f8..1e84169c54586f 100644 --- a/lib/modules/manager/bitrise/utils.ts +++ b/lib/modules/manager/bitrise/utils.ts @@ -1,8 +1,12 @@ import is from '@sindresorhus/is'; import { BitriseDatasource } from '../../datasource/bitrise'; +import { GitTagsDatasource } from '../../datasource/git-tags'; import type { PackageDependency } from '../types'; -export function parseStep(stepRef: string): PackageDependency | null { +export function parseStep( + stepRef: string, + defaultRegistry?: string, +): PackageDependency | null { if (is.emptyString(stepRef)) { return null; } @@ -12,10 +16,12 @@ export function parseStep(stepRef: string): PackageDependency | null { replaceString: stepRef, }; - const splitted = stepRef.split('@', 2); + const [ref, currentValue] = stepRef.split('@', 2); + + const refDep = parseStepRef(ref, defaultRegistry); // no version - if (splitted.length === 1) { + if (is.nullOrUndefined(currentValue)) { return { ...dep, packageName: stepRef, @@ -23,10 +29,57 @@ export function parseStep(stepRef: string): PackageDependency | null { }; } - const [packageName, currentValue] = splitted; return { ...dep, - packageName, + ...refDep, currentValue, }; } + +export function parseStepRef( + ref: string, + defaultRegistry?: string, +): PackageDependency { + // handle local path + // https://devcenter.bitrise.io/en/references/steps-reference/step-reference-id-format.html + if (ref.startsWith('path::')) { + return { + depName: ref.split('::', 2)[1], + skipReason: 'local-dependency', + }; + } + + // handle git references + // https://devcenter.bitrise.io/en/references/steps-reference/step-reference-id-format.html + if (ref.startsWith('git::')) { + const [, packageName] = ref.split('::'); + return { + packageName, + datasource: GitTagsDatasource.id, + }; + } + + // step library references + // https://devcenter.bitrise.io/en/references/steps-reference/step-reference-id-format.html + const splitted = ref.split('::', 2); + + // reference which uses default registry + // - script: + if (splitted.length === 1) { + const [packageName] = splitted; + return { + packageName, + datasource: BitriseDatasource.id, + registryUrls: defaultRegistry ? [defaultRegistry] : undefined, + }; + } + + // reference which overwrites Bitrise registry + // https://github.com/bitrise-io/bitrise-steplib.git::script@1: + const [registryUrl, packageName] = splitted; + return { + packageName, + datasource: BitriseDatasource.id, + registryUrls: [registryUrl], + }; +} From 71d81c4a54210327a4ee7d01352d2c7a9d0aa433 Mon Sep 17 00:00:00 2001 From: secustor Date: Sat, 20 Jul 2024 17:19:19 +0200 Subject: [PATCH 12/13] add doc and naming suggestions --- lib/modules/datasource/bitrise/index.spec.ts | 6 +++--- lib/modules/datasource/bitrise/readme.md | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/modules/datasource/bitrise/index.spec.ts b/lib/modules/datasource/bitrise/index.spec.ts index f11cdc1bc167fb..54cef5d3ce2447 100644 --- a/lib/modules/datasource/bitrise/index.spec.ts +++ b/lib/modules/datasource/bitrise/index.spec.ts @@ -16,7 +16,7 @@ describe('modules/datasource/bitrise/index', () => { ).resolves.toBeNull(); }); - it('support GHE api url', async () => { + it('support GitHub Enterprise API URL', async () => { httpMock .scope( 'https://github.mycompany.com/api/v3/repos/foo/bar/contents/steps', @@ -60,7 +60,7 @@ describe('modules/datasource/bitrise/index', () => { }); }); - it('returns version and filter out asset folder', async () => { + it('returns version and filters out the asset folder', async () => { httpMock .scope( 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', @@ -156,7 +156,7 @@ describe('modules/datasource/bitrise/index', () => { ).resolves.toBeNull(); }); - it('returns null if there is an unexpected format a package', async () => { + it('returns null if the package has an unexpected format', async () => { httpMock .scope( 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps', diff --git a/lib/modules/datasource/bitrise/readme.md b/lib/modules/datasource/bitrise/readme.md index 39660331aee95e..e80b3fabaaa398 100644 --- a/lib/modules/datasource/bitrise/readme.md +++ b/lib/modules/datasource/bitrise/readme.md @@ -1,10 +1,14 @@ -This datasource allows to fetch Bitrise steps from Git repositories. +Renovate uses this datasource to fetch Bitrise steps from GitHub repositories. -**Currently only Github is support** +| Renovate field | What value to use? | +| -------------- | --------------------------------------- | +| `packageName` | Name of the Bitrise step | +| `registryUrl` | GitHub HTTP Git URL, as used by Bitrise | -As `packageName` the step name expected e.g. for the following snippet the `packageName` would be `script`. +For example, in the YAML snippet below: -`registryUrl` expects a GitHub HTTP Git Url as used below by Bitrise. See the `default_step_lib_source` field for an example. +- `packageName` is `script` +- `registryUrl` is `https://github.com/bitrise-io/bitrise-steplib.git` ```yaml format_version: 11 From 411c740db685eb483dfb4fe247d7670d8c5c454d Mon Sep 17 00:00:00 2001 From: secustor Date: Sat, 20 Jul 2024 17:31:37 +0200 Subject: [PATCH 13/13] add doc and naming suggestions --- lib/modules/manager/bitrise/extract.spec.ts | 8 ++++---- lib/modules/manager/bitrise/extract.ts | 2 +- lib/modules/manager/bitrise/utils.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/modules/manager/bitrise/extract.spec.ts b/lib/modules/manager/bitrise/extract.spec.ts index ff59059c8551d6..a40c8f2997cb0d 100644 --- a/lib/modules/manager/bitrise/extract.spec.ts +++ b/lib/modules/manager/bitrise/extract.spec.ts @@ -8,7 +8,7 @@ describe('modules/manager/bitrise/extract', () => { expect(extractPackageFile('', 'bitrise.yml')).toBeNull(); }); - it('returns valid file', () => { + it('returns a valid file', () => { expect( extractPackageFile( codeBlock` @@ -31,7 +31,7 @@ describe('modules/manager/bitrise/extract', () => { }); }); - it('returns valid file with custom default_step_lib_source', () => { + it('returns a valid file with custom default_step_lib_source', () => { expect( extractPackageFile( codeBlock` @@ -72,7 +72,7 @@ describe('modules/manager/bitrise/extract', () => { }); }); - it('extract git and path prefixes', () => { + it('extracts git and path prefixes', () => { expect( extractPackageFile( codeBlock` @@ -111,7 +111,7 @@ describe('modules/manager/bitrise/extract', () => { }); }); - it('extract bitrise library reference', () => { + it('extracts Bitrise library reference', () => { expect( extractPackageFile( codeBlock` diff --git a/lib/modules/manager/bitrise/extract.ts b/lib/modules/manager/bitrise/extract.ts index 0f02ea9ee58f17..8bc593e922c037 100644 --- a/lib/modules/manager/bitrise/extract.ts +++ b/lib/modules/manager/bitrise/extract.ts @@ -28,7 +28,7 @@ export function extractPackageFile( } } } catch (err) { - logger.debug({ err, packageFile }, `Parsing Bitrise config YAML failed`); + logger.debug({ err, packageFile }, `Failed to parse Bitrise YAML config`); } if (!deps.length) { diff --git a/lib/modules/manager/bitrise/utils.ts b/lib/modules/manager/bitrise/utils.ts index 1e84169c54586f..66904fc347898a 100644 --- a/lib/modules/manager/bitrise/utils.ts +++ b/lib/modules/manager/bitrise/utils.ts @@ -49,7 +49,7 @@ export function parseStepRef( }; } - // handle git references + // handle Git references // https://devcenter.bitrise.io/en/references/steps-reference/step-reference-id-format.html if (ref.startsWith('git::')) { const [, packageName] = ref.split('::');