Move saveAndCompressImagesCached step to buildAndUpload [ON HOLD]

src/commands/from_github.ts

@@ -8,8 +8,8 @@ import moment from "moment";
 import { CliGlobalOptions, defaultArch, Manifest } from "../types";
 import {
   contentHashFile,
-  getImagePath,
-  getLegacyImagePath,
+  getImageFilename,
+  getLegacyImageFilename,
   releaseFiles,
   releaseFilesDefaultNames
 } from "../params";
@@ -99,13 +99,13 @@ export async function fromGithubHandler({
     const { name, version }: Manifest = await got(
       manifestAsset.browser_download_url
     ).json();
-    const legacyImagePath = getLegacyImagePath(name, version);
+    const imageLegacyFilename = getLegacyImageFilename(name, version);
     const legacyImageAsset = release.assets.find(
-      asset => asset.name === legacyImagePath
+      asset => asset.name === imageLegacyFilename
     );
     if (legacyImageAsset) {
-      const imageAmdPath = getImagePath(name, version, defaultArch);
-      release.assets.push({ ...legacyImageAsset, name: imageAmdPath });
+      const imageAmdFilename = getImageFilename(name, version, defaultArch);
+      release.assets.push({ ...legacyImageAsset, name: imageAmdFilename });
     }
   }
 

src/params.ts

@@ -129,12 +129,12 @@ export const releaseFilesDefaultNames: {
 // Single arch images
 export const getArchTag = (arch: Architecture): string =>
   arch.replace(/\//g, "-");
-export const getImagePath = (
+export const getImageFilename = (
   name: string,
   version: string,
   arch: Architecture
 ): string => `${name}_${version}_${getArchTag(arch)}.txz`;
-export const getLegacyImagePath = (name: string, version: string): string =>
+export const getLegacyImageFilename = (name: string, version: string): string =>
   `${name}_${version}.tar.xz`;
 
 /**

src/tasks/buildAndUpload.ts

@@ -10,8 +10,8 @@ import { addReleaseRecord } from "../utils/releaseRecord";
 import {
   releaseFiles,
   CliError,
-  getImagePath,
-  getLegacyImagePath,
+  getImageFilename,
+  getLegacyImageFilename,
   releaseFilesDefaultNames
 } from "../params";
 import {
@@ -23,7 +23,11 @@ import {
   getComposePath,
   composeDeleteBuildProperties
 } from "../utils/compose";
-import { ListrContextBuildAndPublish } from "../types";
+import {
+  Architecture,
+  defaultArch,
+  ListrContextBuildAndPublish
+} from "../types";
 import { parseTimeout } from "../utils/timeout";
 import { buildWithBuildx } from "./buildWithBuildx";
 import { buildWithCompose } from "./buildWithCompose";
@@ -40,6 +44,7 @@ import {
   cliArgsToReleaseUploaderProvider,
   UploadTo
 } from "../releaseUploader";
+import { saveAndCompressImagesCached } from "./saveAndCompressImages";
 
 // Pretty percent uploaded reporting
 const percentToMessage = (percent: number) =>
@@ -103,21 +108,15 @@ as ${releaseFilesDefaultNames.avatar} and then remove the 'manifest.avatar' prop
   // Get external image tags to pull and re-tag
   const images = getComposePackageImages(composeForDev, manifest);
 
-  const architectures =
-    manifest.architectures && parseArchitectures(manifest.architectures);
-
-  // get the architecture of the machine where is executed the dappnodesdk
-  const hardwareArchitecture = getArchitecture();
+  const useBuildx = manifest.architectures !== undefined;
+  const architectures = manifest.architectures
+    ? parseArchitectures(manifest.architectures)
+    : [defaultArch];
 
-  const imagePathAmd = path.join(
-    buildDir,
-    getImagePath(name, version, hardwareArchitecture)
-  );
-
-  const imagePathLegacy = path.join(
-    buildDir,
-    getLegacyImagePath(name, version)
-  );
+  /** Returns image destination full path based on architecture */
+  function getImageDestPath(architecture: Architecture): string {
+    return path.join(buildDir, getImageFilename(name, version, architecture));
+  }
 
   // Bump upstreamVersion if provided
   const upstreamVersion =
@@ -157,13 +156,13 @@ as ${releaseFilesDefaultNames.avatar} and then remove the 'manifest.avatar' prop
         fs.mkdirSync(buildDir, { recursive: true }); // Ok on existing dir
         const buildFiles = fs.readdirSync(buildDir);
 
-        const imagePaths = architectures
-          ? architectures.map(arch => getImagePath(name, version, arch))
-          : [imagePathAmd];
+        const imageFilenames = architectures.map(arch =>
+          getImageFilename(name, version, arch)
+        );
 
         // Clean all files except the expected target images
         for (const filepath of buildFiles)
-          if (!imagePaths.includes(filepath))
+          if (!imageFilenames.includes(filepath))
             rimraf.sync(path.join(buildDir, filepath));
       }
     },
@@ -209,40 +208,51 @@ as ${releaseFilesDefaultNames.avatar} and then remove the 'manifest.avatar' prop
     // compatible with DAppNodes that expect a single ".tar.xz" file
     // which must be amd64, x86_64
     // const imageEntry = files.find(file => /\.tar\.xz$/.test(file));
-    ...(architectures
-      ? architectures.map(
-          (architecture): ListrTask<ListrContextBuildAndPublish> => ({
-            title: `Build architecture ${architecture}`,
-            task: () =>
-              new Listr(
-                buildWithBuildx({
+    ...architectures.map(
+      (architecture): ListrTask<ListrContextBuildAndPublish> => ({
+        title: `Build architecture ${architecture}`,
+        task: () =>
+          new Listr([
+            ...(useBuildx
+              ? buildWithBuildx({
                   architecture,
                   images,
                   composePath,
-                  buildTimeout,
-                  skipSave,
-                  destPath: path.join(
-                    buildDir,
-                    getImagePath(name, version, architecture)
-                  )
+                  buildTimeout
                 })
-              )
-          })
-        )
-      : buildWithCompose({
-          images,
-          composePath,
-          buildTimeout,
-          skipSave,
-          destPath: imagePathAmd
-        })),
+              : buildWithCompose({
+                  composePath,
+                  buildTimeout
+                })),
+
+            // Save images once per architecture, since the image tag is the same?
+            ...saveAndCompressImagesCached({
+              images,
+              architecture,
+              destPath: getImageDestPath(architecture),
+              buildTimeout,
+              skipSave
+            })
+          ])
+      })
+    ),
 
     {
       title: `Upload release to ${releaseUploader.networkName}`,
       skip: () => skipUpload,
       task: async (ctx, task) => {
-        if (fs.existsSync(imagePathAmd))
-          fs.copyFileSync(imagePathAmd, imagePathLegacy);
+        // get the architecture of the machine where is executed the dappnodesdk
+        const architectureHost = getArchitecture();
+        const imageHostArchPath = getImageDestPath(architectureHost);
+        const imageLegacyPath = path.join(
+          buildDir,
+          getLegacyImageFilename(name, version)
+        );
+
+        // Legacy code for backwards compatibility.
+        // Old DAppNodes expect a single linux/amd64 image at a different path than `getImageDestPath()`.
+        if (fs.existsSync(imageHostArchPath))
+          fs.copyFileSync(imageHostArchPath, imageLegacyPath);
 
         const gitHead = await getGitHeadIfAvailable({ requireGitData });
 

src/tasks/buildWithBuildx.ts

@@ -2,7 +2,6 @@ import { ListrTask } from "listr";
 import semver from "semver";
 import { shell } from "../utils/shell";
 import { Architecture, PackageImage, PackageImageLocal } from "../types";
-import { saveAndCompressImagesCached } from "./saveAndCompressImages";
 import { getDockerVersion } from "../utils/getDockerVersion";
 
 const minimumDockerVersion = "19.3.0";
@@ -17,16 +16,12 @@ export function buildWithBuildx({
   architecture,
   images,
   composePath,
-  destPath,
-  buildTimeout,
-  skipSave
+  buildTimeout
 }: {
   architecture: Architecture;
   images: PackageImage[];
   composePath: string;
-  destPath: string;
   buildTimeout: number;
-  skipSave?: boolean;
 }): ListrTask[] {
   const localImages = images.filter(
     (image): image is PackageImageLocal => image.type === "local"
@@ -102,14 +97,6 @@ export function buildWithBuildx({
           }
         }
       }
-    },
-
-    ...saveAndCompressImagesCached({
-      images,
-      architecture,
-      destPath,
-      buildTimeout,
-      skipSave
-    })
+    }
   ];
 }

src/tasks/buildWithCompose.ts

@@ -1,25 +1,17 @@
 import { ListrTask } from "listr";
-import { defaultArch, PackageImage } from "../types";
 import { shell } from "../utils/shell";
-import { saveAndCompressImagesCached } from "./saveAndCompressImages";
 
 /**
  * Save docker image
  * This step is extremely expensive computationally.
  * A local cache file will prevent unnecessary compressions if the image hasn't changed
  */
 export function buildWithCompose({
-  images,
   composePath,
-  destPath,
-  buildTimeout,
-  skipSave
+  buildTimeout
 }: {
-  images: PackageImage[];
   composePath: string;
-  destPath: string;
   buildTimeout: number;
-  skipSave?: boolean;
 }): ListrTask[] {
   return [
     {
@@ -32,14 +24,6 @@ export function buildWithCompose({
           onData: data => (task.output = data)
         });
       }
-    },
-
-    ...saveAndCompressImagesCached({
-      images,
-      architecture: defaultArch,
-      destPath,
-      buildTimeout,
-      skipSave
-    })
+    }
   ];
 }

src/types.ts

@@ -32,7 +32,7 @@ export enum ManifestFormat {
 
 export type Architecture = "linux/amd64" | "linux/arm64";
 export const architectures: Architecture[] = ["linux/amd64", "linux/arm64"];
-export const defaultArch = "linux/amd64";
+export const defaultArch = "linux/amd64" as const;
 
 export type ReleaseType = "major" | "minor" | "patch";
 export const releaseTypes: ReleaseType[] = ["major", "minor", "patch"];