feat: listAssets for deploy with pagination

src/commands/deploy.ts

@@ -1,11 +1,5 @@
 import type {SatelliteConfig} from '@junobuild/config';
-import {
-  listAssets,
-  uploadBlob,
-  type AssetKey,
-  type Assets,
-  type ENCODING_TYPE
-} from '@junobuild/core-peer';
+import {uploadBlob, type Asset, type AssetKey, type ENCODING_TYPE} from '@junobuild/core-peer';
 import {isNullish, nonNullish} from '@junobuild/utils';
 import {Blob} from 'buffer';
 import crypto from 'crypto';
@@ -19,15 +13,15 @@ import {lstatSync} from 'node:fs';
 import {readFile} from 'node:fs/promises';
 import {basename, extname, join, relative} from 'node:path';
 import {junoConfigExist, readJunoConfig} from '../configs/juno.config';
-import {COLLECTION_DAPP, DAPP_COLLECTION, UPLOAD_BATCH_SIZE} from '../constants/constants';
+import {COLLECTION_DAPP, UPLOAD_BATCH_SIZE} from '../constants/constants';
 import {
   DEPLOY_DEFAULT_ENCODING,
   DEPLOY_DEFAULT_GZIP,
   DEPLOY_DEFAULT_IGNORE,
   DEPLOY_DEFAULT_SOURCE
 } from '../constants/deploy.constants';
 import {clear} from '../services/clear.services';
-import {assertSatelliteMemorySize} from '../services/deploy.services';
+import {assertSatelliteMemorySize, listAssets} from '../services/deploy.services';
 import {links} from '../services/links.services';
 import type {SatelliteConfigEnv} from '../types/config';
 import {hasArgs} from '../utils/args.utils';
@@ -160,10 +154,7 @@ const filterFilesToUpload = async ({
   files: FileDetails[];
   sourceAbsolutePath: string;
 } & SatelliteConfigEnv): Promise<FileDetails[]> => {
-  const existingAssets = await listAssets({
-    collection: DAPP_COLLECTION,
-    satellite: satelliteParameters(env)
-  });
+  const existingAssets = await listAssets({env});
 
   const promises = files.map(
     async (file: FileDetails) => await fileNeedUpload({file, sourceAbsolutePath, existingAssets})
@@ -184,15 +175,15 @@ const fileNeedUpload = async ({
   sourceAbsolutePath
 }: {
   file: FileDetails;
-  existingAssets: Assets;
+  existingAssets: Asset[];
   sourceAbsolutePath: string;
 }): Promise<{
   file: FileDetails;
   upload: boolean;
 }> => {
   const effectiveFilePath = file.alternateFile ?? file.file;
 
-  const asset = existingAssets.assets.find(
+  const asset = existingAssets.find(
     ({fullPath: f}) => f === fullPath({file: effectiveFilePath, sourceAbsolutePath})
   );
 

src/constants/deploy.constants.ts

@@ -5,3 +5,6 @@ export const DEPLOY_DEFAULT_GZIP = '**/*.+(css|js|mjs)';
 
 export const MEMORY_SIZE_ENDPOINT_VERSION = '0.0.14';
 export const MEMORY_HEAP_WARNING = 900_000_000n;
+
+// Observed empirical value, the threshold might be higher, but at the time I wrote this, pagination there were a bit more than 700 files deployed on the Juno website without any issues.
+export const DEPLOY_LIST_ASSETS_PAGINATION = 500;

src/services/deploy.services.ts

@@ -1,8 +1,16 @@
 import {satelliteMemorySize, satelliteVersion} from '@junobuild/admin';
+import type {Asset} from '@junobuild/core-peer';
+import {listAssets as listAssetsLib} from '@junobuild/core-peer';
 import {yellow} from 'kleur';
 import {compare} from 'semver';
 import {readJunoConfig} from '../configs/juno.config';
-import {MEMORY_HEAP_WARNING, MEMORY_SIZE_ENDPOINT_VERSION} from '../constants/deploy.constants';
+import {DAPP_COLLECTION} from '../constants/constants';
+import {
+  DEPLOY_LIST_ASSETS_PAGINATION,
+  MEMORY_HEAP_WARNING,
+  MEMORY_SIZE_ENDPOINT_VERSION
+} from '../constants/deploy.constants';
+import type {SatelliteConfigEnv} from '../types/config';
 import {configEnv} from '../utils/config.utils';
 import {NEW_CMD_LINE, confirmAndExit} from '../utils/prompt.utils';
 import {satelliteParameters} from '../utils/satellite.utils';
@@ -56,3 +64,41 @@ export const assertSatelliteMemorySize = async (args?: string[]) => {
     )} MB. Are you sure you want to proceed with the deployment?`
   );
 };
+
+export const listAssets = async ({
+  startAfter,
+  env
+}: {
+  startAfter?: string;
+  env: SatelliteConfigEnv;
+}): Promise<Asset[]> => {
+  const {assets, items_page, matches_pages} = await listAssetsLib({
+    collection: DAPP_COLLECTION,
+    satellite: satelliteParameters(env),
+    filter: {
+      order: {
+        desc: true,
+        field: 'keys'
+      },
+      paginate: {
+        startAfter,
+        limit: DEPLOY_LIST_ASSETS_PAGINATION
+      }
+    }
+  });
+
+  const last = <T>(elements: T[]): T | undefined => {
+    const {length, [length - 1]: last} = elements;
+    return last;
+  };
+
+  if ((items_page ?? 0n) < (matches_pages ?? 0n)) {
+    const nextAssets = await listAssets({
+      startAfter: last(assets)?.fullPath,
+      env
+    });
+    return [...assets, ...nextAssets];
+  }
+
+  return assets;
+};