From 606ab430d1ac897bd4f6eada5cc80ad2a9ddbd90 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 9 Dec 2024 15:58:07 -0300 Subject: [PATCH] refactor(maven): Unified result type for http fetch (#32813) --- lib/modules/datasource/maven/util.spec.ts | 25 ++++- lib/modules/datasource/maven/util.ts | 103 ++++++++++-------- .../datasource/sbt-package/index.spec.ts | 6 +- lib/modules/datasource/sbt-package/index.ts | 26 +++-- lib/modules/datasource/sbt-plugin/index.ts | 30 +++-- 5 files changed, 110 insertions(+), 80 deletions(-) diff --git a/lib/modules/datasource/maven/util.spec.ts b/lib/modules/datasource/maven/util.spec.ts index 0921b7de58be6b..dd9891abc615fb 100644 --- a/lib/modules/datasource/maven/util.spec.ts +++ b/lib/modules/datasource/maven/util.spec.ts @@ -71,7 +71,10 @@ describe('modules/datasource/maven/util', () => { get: () => Promise.reject(httpError({ message: HOST_DISABLED })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toBeNull(); + expect(res.unwrap()).toEqual({ + ok: false, + err: { type: 'host-disabled' } satisfies MavenFetchError, + }); }); it('returns empty for host error', async () => { @@ -79,7 +82,10 @@ describe('modules/datasource/maven/util', () => { get: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toBeNull(); + expect(res.unwrap()).toEqual({ + ok: false, + err: { type: 'host-error' } satisfies MavenFetchError, + }); }); it('returns empty for temporary error', async () => { @@ -87,7 +93,10 @@ describe('modules/datasource/maven/util', () => { get: () => Promise.reject(httpError({ code: 'ECONNRESET' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toBeNull(); + expect(res.unwrap()).toEqual({ + ok: false, + err: { type: 'temporary-error' } satisfies MavenFetchError, + }); }); it('returns empty for connection error', async () => { @@ -95,7 +104,10 @@ describe('modules/datasource/maven/util', () => { get: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toBeNull(); + expect(res.unwrap()).toEqual({ + ok: false, + err: { type: 'connection-error' } satisfies MavenFetchError, + }); }); it('returns empty for unsupported error', async () => { @@ -104,7 +116,10 @@ describe('modules/datasource/maven/util', () => { Promise.reject(httpError({ name: 'UnsupportedProtocolError' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toBeNull(); + expect(res.unwrap()).toEqual({ + ok: false, + err: { type: 'unsupported-host' } satisfies MavenFetchError, + }); }); }); diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts index 3a6d85956b4c2c..1edc357068e5cd 100644 --- a/lib/modules/datasource/maven/util.ts +++ b/lib/modules/datasource/maven/util.ts @@ -71,73 +71,92 @@ export async function downloadHttpProtocol( http: Http, pkgUrl: URL | string, opts: HttpOptions = {}, -): Promise { +): Promise { const url = pkgUrl.toString(); - const res = await Result.wrap(http.get(url, opts)) - .onError((err) => { + const fetchResult = await Result.wrap( + http.get(url, opts), + ) + .transform((res): MavenFetchSuccess => { + const result: MavenFetchSuccess = { data: res.body }; + + if (!res.authorization) { + result.isCacheable = true; + } + + const lastModified = normalizeDate(res?.headers?.['last-modified']); + if (lastModified) { + result.lastModified = lastModified; + } + + return result; + }) + .catch((err): MavenFetchResult => { // istanbul ignore next: never happens, needs for type narrowing if (!(err instanceof HttpError)) { - return; + return Result.err({ type: 'unknown', err }); } const failedUrl = url; if (err.message === HOST_DISABLED) { logger.trace({ failedUrl }, 'Host disabled'); - return; + return Result.err({ type: 'host-disabled' }); } if (isNotFoundError(err)) { logger.trace({ failedUrl }, `Url not found`); - return; + return Result.err({ type: 'not-found' }); } if (isHostError(err)) { logger.debug(`Cannot connect to host ${failedUrl}`); - return; + return Result.err({ type: 'host-error' }); } if (isPermissionsIssue(err)) { logger.debug( `Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`, ); - return; + return Result.err({ type: 'permission-issue' }); } if (isTemporaryError(err)) { logger.debug({ failedUrl, err }, 'Temporary error'); - return; + if (getHost(url) === getHost(MAVEN_REPO)) { + return Result.err({ type: 'maven-central-temporary-error', err }); + } else { + return Result.err({ type: 'temporary-error' }); + } } if (isConnectionError(err)) { logger.debug(`Connection refused to maven registry ${failedUrl}`); - return; + return Result.err({ type: 'connection-error' }); } if (isUnsupportedHostError(err)) { logger.debug(`Unsupported host ${failedUrl}`); - return; + return Result.err({ type: 'unsupported-host' }); } logger.info({ failedUrl, err }, 'Unknown HTTP download error'); - }) - .catch((err): Result => { - if ( - err instanceof HttpError && - isTemporaryError(err) && - getHost(url) === getHost(MAVEN_REPO) - ) { - return Result.err(new ExternalHostError(err)); - } - - return Result.ok('silent-error'); - }) - .unwrapOrThrow(); + return Result.err({ type: 'unknown', err }); + }); - if (res === 'silent-error') { - return null; + const { err } = fetchResult.unwrap(); + if (err?.type === 'maven-central-temporary-error') { + throw new ExternalHostError(err.err); } - return res; + return fetchResult; +} + +export async function downloadHttpContent( + http: Http, + pkgUrl: URL | string, + opts: HttpOptions = {}, +): Promise { + const fetchResult = await downloadHttpProtocol(http, pkgUrl, opts); + return fetchResult.transform(({ data }) => data).unwrapOrNull(); } function isS3NotFound(err: Error): boolean { @@ -228,7 +247,7 @@ export async function downloadS3Protocol( export async function downloadArtifactRegistryProtocol( http: Http, pkgUrl: URL, -): Promise { +): Promise { const opts: HttpOptions = {}; const host = pkgUrl.host; const path = pkgUrl.pathname; @@ -357,25 +376,21 @@ export async function downloadMavenXml( const protocol = pkgUrl.protocol; if (protocol === 'http:' || protocol === 'https:') { - const res = await downloadHttpProtocol(http, pkgUrl); - const body = res?.body; - if (body) { - return { - xml: new XmlDocument(body), - isCacheable: !res.authorization, - }; - } + const rawResult = await downloadHttpProtocol(http, pkgUrl); + const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => { + const xml = new XmlDocument(data); + return { isCacheable, xml }; + }); + return xmlResult.unwrapOr({}); } if (protocol === 'artifactregistry:') { - const res = await downloadArtifactRegistryProtocol(http, pkgUrl); - const body = res?.body; - if (body) { - return { - xml: new XmlDocument(body), - isCacheable: !res.authorization, - }; - } + const rawResult = await downloadArtifactRegistryProtocol(http, pkgUrl); + const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => { + const xml = new XmlDocument(data); + return { isCacheable, xml }; + }); + return xmlResult.unwrapOr({}); } if (protocol === 's3:') { diff --git a/lib/modules/datasource/sbt-package/index.spec.ts b/lib/modules/datasource/sbt-package/index.spec.ts index aa86cc3d12b0a5..c3011e563175ad 100644 --- a/lib/modules/datasource/sbt-package/index.spec.ts +++ b/lib/modules/datasource/sbt-package/index.spec.ts @@ -149,9 +149,9 @@ describe('modules/datasource/sbt-package/index', () => { .get('/org/example/example_2.12/') .reply(200, `1.2.3/`) .get('/org/example/example_2.12/1.2.3/example-1.2.3.pom') - .reply(200, ``) + .reply(404) .get('/org/example/example_2.12/1.2.3/example_2.12-1.2.3.pom') - .reply(200, ``); + .reply(404); const res = await getPkgReleases({ versioning: mavenVersioning.id, @@ -267,7 +267,7 @@ describe('modules/datasource/sbt-package/index', () => { `, ) .get('/org/example/example_2.13/1.2.3/example_2.13-1.2.3.pom') - .reply(200); + .reply(404); const res = await getPkgReleases({ versioning: mavenVersioning.id, diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index 463661b51c7429..c956922097486c 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -10,8 +10,7 @@ import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; import { MavenDatasource } from '../maven'; import { MAVEN_REPO } from '../maven/common'; -import { downloadHttpProtocol } from '../maven/util'; -import { normalizeDate } from '../metadata'; +import { downloadHttpContent, downloadHttpProtocol } from '../maven/util'; import type { GetReleasesConfig, PostprocessReleaseConfig, @@ -88,8 +87,11 @@ export class SbtPackageDatasource extends MavenDatasource { let dependencyUrl: string | undefined; let packageUrls: string[] | undefined; for (const packageRootUrl of packageRootUrls) { - const res = await downloadHttpProtocol(this.http, packageRootUrl); - if (!res) { + const packageRootContent = await downloadHttpContent( + this.http, + packageRootUrl, + ); + if (!packageRootContent) { continue; } @@ -103,7 +105,7 @@ export class SbtPackageDatasource extends MavenDatasource { dependencyUrl = trimTrailingSlash(packageRootUrl); const rootPath = new URL(packageRootUrl).pathname; - const artifactSubdirs = extractPageLinks(res.body, (href) => { + const artifactSubdirs = extractPageLinks(packageRootContent, (href) => { const path = href.replace(rootPath, ''); if ( @@ -149,15 +151,15 @@ export class SbtPackageDatasource extends MavenDatasource { const allVersions = new Set(); for (const pkgUrl of packageUrls) { - const res = await downloadHttpProtocol(this.http, pkgUrl); + const packageContent = await downloadHttpContent(this.http, pkgUrl); // istanbul ignore if - if (!res) { + if (!packageContent) { invalidPackageUrls.add(pkgUrl); continue; } const rootPath = new URL(pkgUrl).pathname; - const versions = extractPageLinks(res.body, (href) => { + const versions = extractPageLinks(packageContent, (href) => { const path = href.replace(rootPath, ''); if (path.startsWith('.')) { return null; @@ -275,20 +277,20 @@ export class SbtPackageDatasource extends MavenDatasource { } const res = await downloadHttpProtocol(this.http, pomUrl); - const content = res?.body; - if (!content) { + const { val } = res.unwrap(); + if (!val) { invalidPomFiles.add(pomUrl); continue; } const result: PomInfo = {}; - const releaseTimestamp = normalizeDate(res.headers['last-modified']); + const releaseTimestamp = val.lastModified; if (releaseTimestamp) { result.releaseTimestamp = releaseTimestamp; } - const pomXml = new XmlDocument(content); + const pomXml = new XmlDocument(val.data); const homepage = pomXml.valueWithPath('url'); if (homepage) { diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts index fc84d656421be0..e1e49ec3718761 100644 --- a/lib/modules/datasource/sbt-plugin/index.ts +++ b/lib/modules/datasource/sbt-plugin/index.ts @@ -7,7 +7,7 @@ import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; import { Datasource } from '../datasource'; import { MAVEN_REPO } from '../maven/common'; -import { downloadHttpProtocol } from '../maven/util'; +import { downloadHttpContent } from '../maven/util'; import { extractPageLinks, getLatestVersion } from '../sbt-package/util'; import type { GetReleasesConfig, @@ -43,8 +43,7 @@ export class SbtPluginDatasource extends Datasource { scalaVersion: string, ): Promise { const pkgUrl = ensureTrailingSlash(searchRoot); - const res = await downloadHttpProtocol(this.http, pkgUrl); - const indexContent = res?.body; + const indexContent = await downloadHttpContent(this.http, pkgUrl); if (indexContent) { const rootPath = new URL(pkgUrl).pathname; let artifactSubdirs = extractPageLinks(indexContent, (href) => { @@ -84,8 +83,7 @@ export class SbtPluginDatasource extends Datasource { const releases: string[] = []; for (const searchSubdir of artifactSubdirs) { const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); - const res = await downloadHttpProtocol(this.http, pkgUrl); - const content = res?.body; + const content = await downloadHttpContent(this.http, pkgUrl); if (content) { const rootPath = new URL(pkgUrl).pathname; const subdirReleases = extractPageLinks(content, (href) => { @@ -133,8 +131,7 @@ export class SbtPluginDatasource extends Datasource { for (const pomFileName of pomFileNames) { const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; - const res = await downloadHttpProtocol(this.http, pomUrl); - const content = res?.body; + const content = await downloadHttpContent(this.http, pomUrl); if (content) { const pomXml = new XmlDocument(content); @@ -173,13 +170,16 @@ export class SbtPluginDatasource extends Datasource { return href; }; - const res = await downloadHttpProtocol( + const searchRootContent = await downloadHttpContent( this.http, ensureTrailingSlash(searchRoot), ); - if (res) { + if (searchRootContent) { const releases: string[] = []; - const scalaVersionItems = extractPageLinks(res.body, hrefFilterMap); + const scalaVersionItems = extractPageLinks( + searchRootContent, + hrefFilterMap, + ); const scalaVersions = scalaVersionItems.map((x) => x.replace(regEx(/^scala_/), ''), ); @@ -188,24 +188,22 @@ export class SbtPluginDatasource extends Datasource { : scalaVersions; for (const searchVersion of searchVersions) { const searchSubRoot = `${searchRoot}/scala_${searchVersion}`; - const subRootRes = await downloadHttpProtocol( + const subRootContent = await downloadHttpContent( this.http, ensureTrailingSlash(searchSubRoot), ); - if (subRootRes) { - const { body: subRootContent } = subRootRes; + if (subRootContent) { const sbtVersionItems = extractPageLinks( subRootContent, hrefFilterMap, ); for (const sbtItem of sbtVersionItems) { const releasesRoot = `${searchSubRoot}/${sbtItem}`; - const releaseIndexRes = await downloadHttpProtocol( + const releasesIndexContent = await downloadHttpContent( this.http, ensureTrailingSlash(releasesRoot), ); - if (releaseIndexRes) { - const { body: releasesIndexContent } = releaseIndexRes; + if (releasesIndexContent) { const releasesParsed = extractPageLinks( releasesIndexContent, hrefFilterMap,