From 36388bae1aad9b34c8d5cf4f3bce7e2572e34d1e Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Tue, 16 Jan 2024 17:44:16 +0200 Subject: [PATCH 01/11] feat(platform/gitea): autodiscover repos by topic --- lib/config/options/index.ts | 2 +- lib/modules/platform/gitea/index.spec.ts | 41 ++++++++++++++++++++++++ lib/modules/platform/gitea/index.ts | 39 +++++++++++++++------- lib/modules/platform/gitea/types.ts | 2 ++ 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index ed4df4a41519dd..b8e1e27f881299 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -853,7 +853,7 @@ const options: RenovateOptions[] = [ subType: 'string', default: null, globalOnly: true, - supportedPlatforms: ['github', 'gitlab'], + supportedPlatforms: ['gitea', 'github', 'gitlab'], }, { name: 'prCommitsPerRunLimit', diff --git a/lib/modules/platform/gitea/index.spec.ts b/lib/modules/platform/gitea/index.spec.ts index ae9a7a9abbd571..2a901bbe05cd44 100644 --- a/lib/modules/platform/gitea/index.spec.ts +++ b/lib/modules/platform/gitea/index.spec.ts @@ -70,6 +70,10 @@ describe('modules/platform/gitea/index', () => { partial({ full_name: 'e/f', mirror: true }), ]; + const mockTopicRepos: Repo[] = [ + partial({ full_name: 'a/b' }), + ]; + const mockPRs: MockPr[] = [ partial({ number: 1, @@ -351,6 +355,43 @@ describe('modules/platform/gitea/index', () => { expect(repos).toEqual(['a/b', 'c/d']); }); + it('should return an filtered array of repos', async () => { + const scope = httpMock.scope('https://gitea.com/api/v1'); + + scope + .get('/repos/search') + .query({ + uid: 1, + archived: false, + q: 'renovate', + topic: true, + }) + .reply(200, { + ok: true, + data: mockTopicRepos, + }); + + scope + .get('/repos/search') + .query({ + uid: 1, + archived: false, + q: 'renovatebot', + topic: true, + }) + .reply(200, { + ok: true, + data: mockTopicRepos, + }); + + await initFakePlatform(scope); + + const repos = await gitea.getRepos({ + topics: ['renovate', 'renovatebot'] + }); + expect(repos).toEqual(['a/b']); + }); + it('Sorts repos', async () => { process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT = 'updated'; process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER = 'desc'; diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 13a8f50152b020..4705e282afd69a 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -17,6 +17,7 @@ import { sanitize } from '../../../util/sanitize'; import { ensureTrailingSlash } from '../../../util/url'; import { getPrBodyStruct, hashBody } from '../pr-body'; import type { + AutodiscoverConfig, BranchStatusConfig, CreatePRConfig, EnsureCommentConfig, @@ -154,6 +155,24 @@ async function lookupLabelByName(name: string): Promise { return labelList.find((l) => l.name === name)?.id ?? null; } +async function fetchRepositories(topic?: string): Promise { + const repos = await helper.searchRepos({ + uid: botUserID, + archived: false, + ...(topic && { + topic: true, + q: topic, + }), + ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT && { + sort: process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT as RepoSortMethod, + }), + ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER && { + order: process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER as SortMethod, + }), + }); + return repos.filter((r) => !r.mirror).map((r) => r.full_name); +} + const platform: Platform = { async initPlatform({ endpoint, @@ -295,20 +314,16 @@ const platform: Platform = { }; }, - async getRepos(): Promise { + async getRepos(config?: AutodiscoverConfig): Promise { logger.debug('Auto-discovering Gitea repositories'); try { - const repos = await helper.searchRepos({ - uid: botUserID, - archived: false, - ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT && { - sort: process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT as RepoSortMethod, - }), - ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER && { - order: process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER as SortMethod, - }), - }); - return repos.filter((r) => !r.mirror).map((r) => r.full_name); + const topics = config?.topics || [undefined]; + let repos: string[] = []; + for (const topic of topics) { + const r = await fetchRepositories(topic); + repos = [...new Set([...repos, ...r])]; + } + return repos; } catch (err) { logger.error({ err }, 'Gitea getRepos() error'); throw err; diff --git a/lib/modules/platform/gitea/types.ts b/lib/modules/platform/gitea/types.ts index 6d0e9dbbbd122b..877ea4436d0881 100644 --- a/lib/modules/platform/gitea/types.ts +++ b/lib/modules/platform/gitea/types.ts @@ -144,6 +144,8 @@ export type SortMethod = 'asc' | 'desc'; export interface RepoSearchParams { uid?: number; archived?: boolean; + topic?: boolean; + q?: string; /** * Repo sort type, defaults to `alpha`. From 455a4c099721fab8b75b84450a0a029a8a14be9b Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Tue, 16 Jan 2024 17:47:54 +0200 Subject: [PATCH 02/11] Fix lint --- lib/modules/platform/gitea/index.spec.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/modules/platform/gitea/index.spec.ts b/lib/modules/platform/gitea/index.spec.ts index 2a901bbe05cd44..3084f4a3c169b0 100644 --- a/lib/modules/platform/gitea/index.spec.ts +++ b/lib/modules/platform/gitea/index.spec.ts @@ -70,9 +70,7 @@ describe('modules/platform/gitea/index', () => { partial({ full_name: 'e/f', mirror: true }), ]; - const mockTopicRepos: Repo[] = [ - partial({ full_name: 'a/b' }), - ]; + const mockTopicRepos: Repo[] = [partial({ full_name: 'a/b' })]; const mockPRs: MockPr[] = [ partial({ @@ -387,7 +385,7 @@ describe('modules/platform/gitea/index', () => { await initFakePlatform(scope); const repos = await gitea.getRepos({ - topics: ['renovate', 'renovatebot'] + topics: ['renovate', 'renovatebot'], }); expect(repos).toEqual(['a/b']); }); From 1fbd0cf5628a9b589387a64805a705ef9fd8ad5e Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Tue, 16 Jan 2024 17:50:10 +0200 Subject: [PATCH 03/11] Fix eslint warning --- lib/modules/platform/gitea/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 4705e282afd69a..992a500c768a79 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -317,7 +317,7 @@ const platform: Platform = { async getRepos(config?: AutodiscoverConfig): Promise { logger.debug('Auto-discovering Gitea repositories'); try { - const topics = config?.topics || [undefined]; + const topics = config?.topics ?? [undefined]; let repos: string[] = []; for (const topic of topics) { const r = await fetchRepositories(topic); From 32fb90f3d32b5e5b228d219f395c8233afb11da5 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Wed, 17 Jan 2024 12:34:56 +0200 Subject: [PATCH 04/11] Code review suggestions --- lib/modules/platform/gitea/index.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 992a500c768a79..640f4853353cf3 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -1,5 +1,6 @@ import is from '@sindresorhus/is'; import semver from 'semver'; +import pMap from 'p-map'; import { REPOSITORY_ACCESS_FORBIDDEN, REPOSITORY_ARCHIVED, @@ -317,12 +318,15 @@ const platform: Platform = { async getRepos(config?: AutodiscoverConfig): Promise { logger.debug('Auto-discovering Gitea repositories'); try { - const topics = config?.topics ?? [undefined]; + if (!config?.topics) { + return await fetchRepositories(); + } + let repos: string[] = []; - for (const topic of topics) { + await pMap(config.topics, async (topic: string) => { const r = await fetchRepositories(topic); repos = [...new Set([...repos, ...r])]; - } + }); return repos; } catch (err) { logger.error({ err }, 'Gitea getRepos() error'); From 43718df2f1962cb1838ce6952d2ca6b1cf94e500 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Wed, 17 Jan 2024 12:38:45 +0200 Subject: [PATCH 05/11] Fix import order --- lib/modules/platform/gitea/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 640f4853353cf3..3f6d86b0ade3b9 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; -import semver from 'semver'; import pMap from 'p-map'; +import semver from 'semver'; import { REPOSITORY_ACCESS_FORBIDDEN, REPOSITORY_ARCHIVED, From 2072babd51c9365a6e0d9c3c8a3932b4a1a3d907 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Fri, 19 Jan 2024 13:29:53 +0200 Subject: [PATCH 06/11] Add info about sorting when using multiple topics --- docs/usage/self-hosted-experimental.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md index 7ee6a8ee1f39df..c0b0ef870d8285 100644 --- a/docs/usage/self-hosted-experimental.md +++ b/docs/usage/self-hosted-experimental.md @@ -48,6 +48,8 @@ If set to any string, Renovate will use this as the `user-agent` it sends with H The order method for autodiscover server side repository search. +> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. + Allowed values: - `asc` @@ -63,6 +65,8 @@ Default value: `asc`. The sort method for autodiscover server side repository search. +> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. + Allowed values: - `alpha` From b674f18dd6a0fe39dcfd84a89a235f6f847c8293 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Sat, 20 Jan 2024 13:37:24 +0200 Subject: [PATCH 07/11] Use utils instead of pMap directly --- lib/modules/platform/gitea/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 3f6d86b0ade3b9..76110096f03ea5 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -1,5 +1,4 @@ import is from '@sindresorhus/is'; -import pMap from 'p-map'; import semver from 'semver'; import { REPOSITORY_ACCESS_FORBIDDEN, @@ -10,6 +9,7 @@ import { REPOSITORY_MIRRORED, } from '../../../constants/error-messages'; import { logger } from '../../../logger'; +import * as p from '../../../util/promises'; import type { BranchStatus } from '../../../types'; import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; @@ -323,7 +323,7 @@ const platform: Platform = { } let repos: string[] = []; - await pMap(config.topics, async (topic: string) => { + await p.map(config.topics, async (topic: string) => { const r = await fetchRepositories(topic); repos = [...new Set([...repos, ...r])]; }); From 0283110f34585fa7277cea2d761f396547b6918b Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Sat, 20 Jan 2024 13:41:17 +0200 Subject: [PATCH 08/11] Fix import order --- lib/modules/platform/gitea/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 76110096f03ea5..0bcc07bd41aa14 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -9,11 +9,11 @@ import { REPOSITORY_MIRRORED, } from '../../../constants/error-messages'; import { logger } from '../../../logger'; -import * as p from '../../../util/promises'; import type { BranchStatus } from '../../../types'; import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { setBaseUrl } from '../../../util/http/gitea'; +import * as p from '../../../util/promises'; import { sanitize } from '../../../util/sanitize'; import { ensureTrailingSlash } from '../../../util/url'; import { getPrBodyStruct, hashBody } from '../pr-body'; From 63764e8058cd55e15d7a70e7a8e3ed313b904333 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Sat, 20 Jan 2024 17:29:34 +0200 Subject: [PATCH 09/11] Rewrite as per sugestions --- lib/modules/platform/gitea/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 0bcc07bd41aa14..5d81dc3cfc0fb7 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -10,6 +10,7 @@ import { } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import type { BranchStatus } from '../../../types'; +import { deduplicateArray } from '../../../util/array'; import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { setBaseUrl } from '../../../util/http/gitea'; @@ -322,12 +323,11 @@ const platform: Platform = { return await fetchRepositories(); } - let repos: string[] = []; - await p.map(config.topics, async (topic: string) => { - const r = await fetchRepositories(topic); - repos = [...new Set([...repos, ...r])]; - }); - return repos; + const repos = await p.map( + config.topics, + async (topic: string) => await fetchRepositories(topic), + ); + return deduplicateArray(repos.flat()); } catch (err) { logger.error({ err }, 'Gitea getRepos() error'); throw err; From 3dc264c8814fcd1777bde5657778558c7621d6f7 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Sun, 21 Jan 2024 02:27:15 +0200 Subject: [PATCH 10/11] Simplify code Co-authored-by: Michael Kriese --- lib/modules/platform/gitea/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 5d81dc3cfc0fb7..631940161ba151 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -325,7 +325,7 @@ const platform: Platform = { const repos = await p.map( config.topics, - async (topic: string) => await fetchRepositories(topic), + fetchRepositories, ); return deduplicateArray(repos.flat()); } catch (err) { From 05dd3953feccf4bd83aa2aff159eec6f45350c41 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Sun, 21 Jan 2024 02:29:04 +0200 Subject: [PATCH 11/11] Import only needed funcion from utils --- lib/modules/platform/gitea/index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 631940161ba151..595c8402221428 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -14,7 +14,7 @@ import { deduplicateArray } from '../../../util/array'; import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { setBaseUrl } from '../../../util/http/gitea'; -import * as p from '../../../util/promises'; +import { map } from '../../../util/promises'; import { sanitize } from '../../../util/sanitize'; import { ensureTrailingSlash } from '../../../util/url'; import { getPrBodyStruct, hashBody } from '../pr-body'; @@ -323,10 +323,7 @@ const platform: Platform = { return await fetchRepositories(); } - const repos = await p.map( - config.topics, - fetchRepositories, - ); + const repos = await map(config.topics, fetchRepositories); return deduplicateArray(repos.flat()); } catch (err) { logger.error({ err }, 'Gitea getRepos() error');