From 8095fe7a2b4ac85e04117d80e8a007955801d8a7 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 2 Jun 2022 21:48:12 +0700 Subject: [PATCH] Move saveAndCompressImagesCached step to buildAndUpload --- src/commands/from_github.ts | 12 ++-- src/params.ts | 4 +- src/tasks/buildAndUpload.ts | 102 +++++++++++++++++++--------------- src/tasks/buildWithBuildx.ts | 17 +----- src/tasks/buildWithCompose.ts | 20 +------ src/types.ts | 2 +- 6 files changed, 69 insertions(+), 88 deletions(-) diff --git a/src/commands/from_github.ts b/src/commands/from_github.ts index 981332f6..7ae74efc 100644 --- a/src/commands/from_github.ts +++ b/src/commands/from_github.ts @@ -8,8 +8,8 @@ import moment from "moment"; import { CliGlobalOptions, defaultArch, Manifest } from "../types"; import { contentHashFile, - getImagePath, - getLegacyImagePath, + getImageFilename, + getLegacyImageFilename, releaseFiles, releaseFilesDefaultNames } from "../params"; @@ -99,13 +99,13 @@ export async function fromGithubHandler({ const { name, version }: Manifest = await got( manifestAsset.browser_download_url ).json(); - const legacyImagePath = getLegacyImagePath(name, version); + const imageLegacyFilename = getLegacyImageFilename(name, version); const legacyImageAsset = release.assets.find( - asset => asset.name === legacyImagePath + asset => asset.name === imageLegacyFilename ); if (legacyImageAsset) { - const imageAmdPath = getImagePath(name, version, defaultArch); - release.assets.push({ ...legacyImageAsset, name: imageAmdPath }); + const imageAmdFilename = getImageFilename(name, version, defaultArch); + release.assets.push({ ...legacyImageAsset, name: imageAmdFilename }); } } diff --git a/src/params.ts b/src/params.ts index fed298e4..4a432b29 100644 --- a/src/params.ts +++ b/src/params.ts @@ -129,12 +129,12 @@ export const releaseFilesDefaultNames: { // Single arch images export const getArchTag = (arch: Architecture): string => arch.replace(/\//g, "-"); -export const getImagePath = ( +export const getImageFilename = ( name: string, version: string, arch: Architecture ): string => `${name}_${version}_${getArchTag(arch)}.txz`; -export const getLegacyImagePath = (name: string, version: string): string => +export const getLegacyImageFilename = (name: string, version: string): string => `${name}_${version}.tar.xz`; /** diff --git a/src/tasks/buildAndUpload.ts b/src/tasks/buildAndUpload.ts index d04b3100..216fdb57 100644 --- a/src/tasks/buildAndUpload.ts +++ b/src/tasks/buildAndUpload.ts @@ -10,8 +10,8 @@ import { addReleaseRecord } from "../utils/releaseRecord"; import { releaseFiles, CliError, - getImagePath, - getLegacyImagePath, + getImageFilename, + getLegacyImageFilename, releaseFilesDefaultNames } from "../params"; import { @@ -23,7 +23,11 @@ import { getComposePath, composeDeleteBuildProperties } from "../utils/compose"; -import { ListrContextBuildAndPublish } from "../types"; +import { + Architecture, + defaultArch, + ListrContextBuildAndPublish +} from "../types"; import { parseTimeout } from "../utils/timeout"; import { buildWithBuildx } from "./buildWithBuildx"; import { buildWithCompose } from "./buildWithCompose"; @@ -40,6 +44,7 @@ import { cliArgsToReleaseUploaderProvider, UploadTo } from "../releaseUploader"; +import { saveAndCompressImagesCached } from "./saveAndCompressImages"; // Pretty percent uploaded reporting const percentToMessage = (percent: number) => @@ -103,21 +108,15 @@ as ${releaseFilesDefaultNames.avatar} and then remove the 'manifest.avatar' prop // Get external image tags to pull and re-tag const images = getComposePackageImages(composeForDev, manifest); - const architectures = - manifest.architectures && parseArchitectures(manifest.architectures); - - // get the architecture of the machine where is executed the dappnodesdk - const hardwareArchitecture = getArchitecture(); + const useBuildx = manifest.architectures !== undefined; + const architectures = manifest.architectures + ? parseArchitectures(manifest.architectures) + : [defaultArch]; - const imagePathAmd = path.join( - buildDir, - getImagePath(name, version, hardwareArchitecture) - ); - - const imagePathLegacy = path.join( - buildDir, - getLegacyImagePath(name, version) - ); + /** Returns image destination full path based on architecture */ + function getImageDestPath(architecture: Architecture): string { + return path.join(buildDir, getImageFilename(name, version, architecture)); + } // Bump upstreamVersion if provided const upstreamVersion = @@ -157,13 +156,13 @@ as ${releaseFilesDefaultNames.avatar} and then remove the 'manifest.avatar' prop fs.mkdirSync(buildDir, { recursive: true }); // Ok on existing dir const buildFiles = fs.readdirSync(buildDir); - const imagePaths = architectures - ? architectures.map(arch => getImagePath(name, version, arch)) - : [imagePathAmd]; + const imageFilenames = architectures.map(arch => + getImageFilename(name, version, arch) + ); // Clean all files except the expected target images for (const filepath of buildFiles) - if (!imagePaths.includes(filepath)) + if (!imageFilenames.includes(filepath)) rimraf.sync(path.join(buildDir, filepath)); } }, @@ -209,40 +208,51 @@ as ${releaseFilesDefaultNames.avatar} and then remove the 'manifest.avatar' prop // compatible with DAppNodes that expect a single ".tar.xz" file // which must be amd64, x86_64 // const imageEntry = files.find(file => /\.tar\.xz$/.test(file)); - ...(architectures - ? architectures.map( - (architecture): ListrTask => ({ - title: `Build architecture ${architecture}`, - task: () => - new Listr( - buildWithBuildx({ + ...architectures.map( + (architecture): ListrTask => ({ + title: `Build architecture ${architecture}`, + task: () => + new Listr([ + ...(useBuildx + ? buildWithBuildx({ architecture, images, composePath, - buildTimeout, - skipSave, - destPath: path.join( - buildDir, - getImagePath(name, version, architecture) - ) + buildTimeout }) - ) - }) - ) - : buildWithCompose({ - images, - composePath, - buildTimeout, - skipSave, - destPath: imagePathAmd - })), + : buildWithCompose({ + composePath, + buildTimeout + })), + + // Save images once per architecture, since the image tag is the same? + ...saveAndCompressImagesCached({ + images, + architecture, + destPath: getImageDestPath(architecture), + buildTimeout, + skipSave + }) + ]) + }) + ), { title: `Upload release to ${releaseUploader.networkName}`, skip: () => skipUpload, task: async (ctx, task) => { - if (fs.existsSync(imagePathAmd)) - fs.copyFileSync(imagePathAmd, imagePathLegacy); + // get the architecture of the machine where is executed the dappnodesdk + const architectureHost = getArchitecture(); + const imageHostArchPath = getImageDestPath(architectureHost); + const imageLegacyPath = path.join( + buildDir, + getLegacyImageFilename(name, version) + ); + + // Legacy code for backwards compatibility. + // Old DAppNodes expect a single linux/amd64 image at a different path than `getImageDestPath()`. + if (fs.existsSync(imageHostArchPath)) + fs.copyFileSync(imageHostArchPath, imageLegacyPath); const gitHead = await getGitHeadIfAvailable({ requireGitData }); diff --git a/src/tasks/buildWithBuildx.ts b/src/tasks/buildWithBuildx.ts index 8e94e3a6..4b5308fe 100644 --- a/src/tasks/buildWithBuildx.ts +++ b/src/tasks/buildWithBuildx.ts @@ -2,7 +2,6 @@ import { ListrTask } from "listr"; import semver from "semver"; import { shell } from "../utils/shell"; import { Architecture, PackageImage, PackageImageLocal } from "../types"; -import { saveAndCompressImagesCached } from "./saveAndCompressImages"; import { getDockerVersion } from "../utils/getDockerVersion"; const minimumDockerVersion = "19.3.0"; @@ -17,16 +16,12 @@ export function buildWithBuildx({ architecture, images, composePath, - destPath, - buildTimeout, - skipSave + buildTimeout }: { architecture: Architecture; images: PackageImage[]; composePath: string; - destPath: string; buildTimeout: number; - skipSave?: boolean; }): ListrTask[] { const localImages = images.filter( (image): image is PackageImageLocal => image.type === "local" @@ -102,14 +97,6 @@ export function buildWithBuildx({ } } } - }, - - ...saveAndCompressImagesCached({ - images, - architecture, - destPath, - buildTimeout, - skipSave - }) + } ]; } diff --git a/src/tasks/buildWithCompose.ts b/src/tasks/buildWithCompose.ts index 3a3f8cb7..8a38995e 100644 --- a/src/tasks/buildWithCompose.ts +++ b/src/tasks/buildWithCompose.ts @@ -1,7 +1,5 @@ import { ListrTask } from "listr"; -import { defaultArch, PackageImage } from "../types"; import { shell } from "../utils/shell"; -import { saveAndCompressImagesCached } from "./saveAndCompressImages"; /** * Save docker image @@ -9,17 +7,11 @@ import { saveAndCompressImagesCached } from "./saveAndCompressImages"; * A local cache file will prevent unnecessary compressions if the image hasn't changed */ export function buildWithCompose({ - images, composePath, - destPath, - buildTimeout, - skipSave + buildTimeout }: { - images: PackageImage[]; composePath: string; - destPath: string; buildTimeout: number; - skipSave?: boolean; }): ListrTask[] { return [ { @@ -32,14 +24,6 @@ export function buildWithCompose({ onData: data => (task.output = data) }); } - }, - - ...saveAndCompressImagesCached({ - images, - architecture: defaultArch, - destPath, - buildTimeout, - skipSave - }) + } ]; } diff --git a/src/types.ts b/src/types.ts index 026e46be..9e94e2fd 100644 --- a/src/types.ts +++ b/src/types.ts @@ -32,7 +32,7 @@ export enum ManifestFormat { export type Architecture = "linux/amd64" | "linux/arm64"; export const architectures: Architecture[] = ["linux/amd64", "linux/arm64"]; -export const defaultArch = "linux/amd64"; +export const defaultArch = "linux/amd64" as const; export type ReleaseType = "major" | "minor" | "patch"; export const releaseTypes: ReleaseType[] = ["major", "minor", "patch"];