From ea9781b049cf1910f0f0f0047d2f77d68e55c864 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Tue, 9 Jun 2020 12:08:38 +0200 Subject: [PATCH 1/6] fix(crates): Detect stale upload caches and retry --- src/targets/crates.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/targets/crates.ts b/src/targets/crates.ts index e2075d9d..d303cf4a 100644 --- a/src/targets/crates.ts +++ b/src/targets/crates.ts @@ -24,6 +24,13 @@ const DEFAULT_CARGO_BIN = 'cargo'; */ const CARGO_BIN = process.env.CARGO_BIN || DEFAULT_CARGO_BIN; +/** + * A message fragment emitted by cargo when publishing fails due to a missing + * dependency. This sometimes indicates a false positive if the cache has not + * been updated. + */ +const VERSION_ERROR = 'failed to select a version for the requirement'; + /** Options for "crates" target */ export interface CratesTargetOptions extends TargetConfig { /** Crates API token */ @@ -202,9 +209,23 @@ export class CratesTarget extends BaseTarget { '--manifest-path', crate.manifest_path, ]; - return spawnProcess(CARGO_BIN, args, { - env: { ...process.env, CARGO_REGISTRY_TOKEN: this.cratesConfig.apiToken }, - }); + const env = { + ...process.env, + CARGO_REGISTRY_TOKEN: this.cratesConfig.apiToken, + }; + + for (let i = 0; i <= 1; i++) { + try { + return await spawnProcess(CARGO_BIN, args, { env }); + } catch (e) { + if (i == 0 && e.message.includes(VERSION_ERROR)) { + logger.debug('Potential stale cache detected, trying again...'); + continue; + } else { + throw e; + } + } + } } /** From 7d99de9b5378f54e72833ac53be57b59e7587bb3 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Tue, 9 Jun 2020 12:32:04 +0200 Subject: [PATCH 2/6] fix: Use strict equals Co-authored-by: Anton Ovchinnikov --- src/targets/crates.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/targets/crates.ts b/src/targets/crates.ts index d303cf4a..224243be 100644 --- a/src/targets/crates.ts +++ b/src/targets/crates.ts @@ -218,7 +218,7 @@ export class CratesTarget extends BaseTarget { try { return await spawnProcess(CARGO_BIN, args, { env }); } catch (e) { - if (i == 0 && e.message.includes(VERSION_ERROR)) { + if (i === 0 && e.message.includes(VERSION_ERROR)) { logger.debug('Potential stale cache detected, trying again...'); continue; } else { From ebc0ed951e894c22f0b4199d914447ede0602db3 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Tue, 9 Jun 2020 13:04:35 +0200 Subject: [PATCH 3/6] ref(crates): Add more logging --- src/targets/crates.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/targets/crates.ts b/src/targets/crates.ts index 224243be..77db38fe 100644 --- a/src/targets/crates.ts +++ b/src/targets/crates.ts @@ -189,9 +189,9 @@ export class CratesTarget extends BaseTarget { const crates = this.getPublishOrder(unorderedCrates); logger.debug( - `Publishing packages in the following order: [${crates + `Publishing packages in the following order: ${crates .map(c => c.name) - .toString()}]` + .join(', ')}` ); return forEachChained(crates, async crate => this.publishPackage(crate)); } @@ -214,12 +214,15 @@ export class CratesTarget extends BaseTarget { CARGO_REGISTRY_TOKEN: this.cratesConfig.apiToken, }; + logger.info(`Publishing ${crate.name}`); for (let i = 0; i <= 1; i++) { try { return await spawnProcess(CARGO_BIN, args, { env }); } catch (e) { if (i === 0 && e.message.includes(VERSION_ERROR)) { - logger.debug('Potential stale cache detected, trying again...'); + logger.warn( + `Potential stale cache detected while publishing ${crate.name}, trying again...` + ); continue; } else { throw e; From 4cec5fac8b1cc9ce861e095017306848696ed236 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Tue, 9 Jun 2020 16:14:54 +0200 Subject: [PATCH 4/6] ref(crates): Simplify retry logic --- src/targets/crates.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/targets/crates.ts b/src/targets/crates.ts index 77db38fe..879d40fd 100644 --- a/src/targets/crates.ts +++ b/src/targets/crates.ts @@ -11,7 +11,11 @@ import { GithubGlobalConfig, TargetConfig } from '../schemas/project_config'; import { forEachChained } from '../utils/async'; import { ConfigurationError } from '../utils/errors'; import { withTempDir } from '../utils/files'; -import { checkExecutableIsPresent, spawnProcess } from '../utils/system'; +import { + checkExecutableIsPresent, + sleepAsync, + spawnProcess, +} from '../utils/system'; import { BaseTarget } from './base'; import { BaseArtifactProvider } from '../artifact_providers/base'; @@ -31,6 +35,9 @@ const CARGO_BIN = process.env.CARGO_BIN || DEFAULT_CARGO_BIN; */ const VERSION_ERROR = 'failed to select a version for the requirement'; +/** Delay to wait between publish retries in milliseconds. */ +const RETRY_DELAY_MS = 1000; + /** Options for "crates" target */ export interface CratesTargetOptions extends TargetConfig { /** Crates API token */ @@ -215,18 +222,15 @@ export class CratesTarget extends BaseTarget { }; logger.info(`Publishing ${crate.name}`); - for (let i = 0; i <= 1; i++) { - try { - return await spawnProcess(CARGO_BIN, args, { env }); - } catch (e) { - if (i === 0 && e.message.includes(VERSION_ERROR)) { - logger.warn( - `Potential stale cache detected while publishing ${crate.name}, trying again...` - ); - continue; - } else { - throw e; - } + try { + return await spawnProcess(CARGO_BIN, args, { env }); + } catch (e) { + if (e.message.includes(VERSION_ERROR)) { + logger.warn(`Potential stale cache detected, trying again...`); + await sleepAsync(RETRY_DELAY_MS); + return spawnProcess(CARGO_BIN, args, { env }); + } else { + throw e; } } } From 86f43aadc442455cbf44888e783eba119b6e81a5 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Wed, 10 Jun 2020 17:14:02 +0200 Subject: [PATCH 5/6] ref: Retry a couple of times --- src/targets/crates.ts | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/targets/crates.ts b/src/targets/crates.ts index 879d40fd..4a430c1e 100644 --- a/src/targets/crates.ts +++ b/src/targets/crates.ts @@ -35,8 +35,14 @@ const CARGO_BIN = process.env.CARGO_BIN || DEFAULT_CARGO_BIN; */ const VERSION_ERROR = 'failed to select a version for the requirement'; -/** Delay to wait between publish retries in milliseconds. */ -const RETRY_DELAY_MS = 1000; +/** + * Maximum number of attempts including the initial one when publishing fails + * due to a stale cache. After this number of retries, publishing fails. + */ +const MAX_ATTEMPTS = 4; + +/** Delay to wait between publish retries in seconds. */ +const RETRY_DELAY_SECS = 3; /** Options for "crates" target */ export interface CratesTargetOptions extends TargetConfig { @@ -222,15 +228,18 @@ export class CratesTarget extends BaseTarget { }; logger.info(`Publishing ${crate.name}`); - try { - return await spawnProcess(CARGO_BIN, args, { env }); - } catch (e) { - if (e.message.includes(VERSION_ERROR)) { - logger.warn(`Potential stale cache detected, trying again...`); - await sleepAsync(RETRY_DELAY_MS); - return spawnProcess(CARGO_BIN, args, { env }); - } else { - throw e; + for (let i = 0; i <= MAX_ATTEMPTS; i++) { + try { + return await spawnProcess(CARGO_BIN, args, { env }); + } catch (e) { + if (i < MAX_ATTEMPTS && e.message.includes(VERSION_ERROR)) { + logger.warn( + `Potential stale cache detected, trying again in ${RETRY_DELAY_SECS}s...` + ); + await sleepAsync(RETRY_DELAY_SECS * 1000); + } else { + throw e; + } } } } From 59959072e132ebc25f48b66e2171d69b7af09482 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Thu, 18 Jun 2020 19:27:22 +0200 Subject: [PATCH 6/6] ref(crates): Attempt exponential backoff up to 62s --- src/targets/crates.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/targets/crates.ts b/src/targets/crates.ts index df47d69d..032904c6 100644 --- a/src/targets/crates.ts +++ b/src/targets/crates.ts @@ -39,10 +39,18 @@ const VERSION_ERROR = 'failed to select a version for the requirement'; * Maximum number of attempts including the initial one when publishing fails * due to a stale cache. After this number of retries, publishing fails. */ -const MAX_ATTEMPTS = 4; +const MAX_ATTEMPTS = 5; -/** Delay to wait between publish retries in seconds. */ -const RETRY_DELAY_SECS = 3; +/** + * Initial delay to wait between publish retries in seconds. Exponential backoff + * is applied to this delay on retries. + */ +const RETRY_DELAY_SECS = 2; + +/** + * Exponential backoff that is applied to the initial retry delay. + */ +const RETRY_EXP_FACTOR = 2; /** Options for "crates" target */ export interface CratesTargetOptions extends TargetConfig { @@ -252,16 +260,16 @@ export class CratesTarget extends BaseTarget { CARGO_REGISTRY_TOKEN: this.cratesConfig.apiToken, }; + let delay = RETRY_DELAY_SECS; logger.info(`Publishing ${crate.name}`); for (let i = 0; i <= MAX_ATTEMPTS; i++) { try { return await spawnProcess(CARGO_BIN, args, { env }); } catch (e) { if (i < MAX_ATTEMPTS && e.message.includes(VERSION_ERROR)) { - logger.warn( - `Potential stale cache detected, trying again in ${RETRY_DELAY_SECS}s...` - ); - await sleepAsync(RETRY_DELAY_SECS * 1000); + logger.warn(`Publish failed, trying again in ${delay}s...`); + await sleepAsync(delay * 1000); + delay *= RETRY_EXP_FACTOR; } else { throw e; }