node: Add yarn2 support

catsout · catsout · commit c278045d7cc7 · 2023-02-28T21:46:43.000+08:00
diff --git a/node/flatpak_node_generator/flatpak-yarn.js b/node/flatpak_node_generator/flatpak-yarn.js
@@ -0,0 +1,262 @@
+const PackageManager = {
+  Yarn1 : `Yarn Classic`,
+  Yarn2 : `Yarn`,
+  Npm : `npm`,
+  Pnpm : `pnpm`,
+}
+
+module.exports = {
+  name: `flatpak-builder`,
+  factory: require => {
+    const { BaseCommand } = require(`@yarnpkg/cli`);
+    const { Configuration, Manifest, scriptUtils, structUtils, tgzUtils, execUtils, miscUtils, hashUtils } = require('@yarnpkg/core')
+    const { Filename, ZipFS, npath, ppath, PortablePath, xfs } = require('@yarnpkg/fslib');
+    const { getLibzipPromise } = require('@yarnpkg/libzip');
+    const { gitUtils } = require('@yarnpkg/plugin-git');
+    const { PassThrough, Readable, Writable } = require('stream');
+    const { Command, Option } = require(`clipanion`);
+    const { YarnVersion } = require('@yarnpkg/core');
+    const fs = require('fs');
+
+    // from https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell/3.2.3/packages/plugin-essentials/sources/commands/set/version.ts#L194 
+    async function setPackageManager(projectCwd) {
+        const bundleVersion = YarnVersion;
+
+        const manifest = (await Manifest.tryFind(projectCwd)) || new Manifest();
+        
+        if(bundleVersion && miscUtils.isTaggedYarnVersion(bundleVersion)) {
+            manifest.packageManager = `yarn@${bundleVersion}`;
+            const data = {};
+            manifest.exportTo(data);
+
+            const path = ppath.join(projectCwd, Manifest.fileName);
+            const content = `${JSON.stringify(data, null, manifest.indent)}\n`;
+
+            await xfs.changeFilePromise(path, content, {
+                automaticNewlines: true,
+            });
+        }
+    }
+
+    // func from https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell/3.2.3/packages/yarnpkg-core/sources/scriptUtils.ts#L215
+    async function prepareExternalProject(cwd, outputPath , {configuration, locator, stdout, yarn_v1, workspace=null}) {
+        const devirtualizedLocator = locator && structUtils.isVirtualLocator(locator)
+            ? structUtils.devirtualizeLocator(locator)
+            : locator;
+
+        const name = devirtualizedLocator
+            ? structUtils.stringifyLocator(devirtualizedLocator)
+            : `an external project`;
+
+        const stderr = stdout;
+
+        stdout.write(`Packing ${name} from sources\n`);
+
+        const packageManagerSelection = await scriptUtils.detectPackageManager(cwd);
+        let effectivePackageManager;
+        if (packageManagerSelection !== null) {
+            stdout.write(`Using ${packageManagerSelection.packageManager} for bootstrap. Reason: ${packageManagerSelection.reason}\n\n`);
+            effectivePackageManager = packageManagerSelection.packageManager;
+        } else {
+            stdout.write(`No package manager configuration detected; defaulting to Yarn\n\n`);
+            effectivePackageManager = PackageManager.Yarn2;
+        }
+        if (effectivePackageManager === PackageManager.Pnpm) {
+            effectivePackageManager = PackageManager.Npm;
+        }
+
+        const workflows = new Map([
+          [PackageManager.Yarn1, async () => {
+            const workspaceCli = workspace !== null
+              ? [`workspace`, workspace]
+              : [];
+
+            await setPackageManager(cwd);
+
+            await Configuration.updateConfiguration(cwd, {
+                yarnPath: yarn_v1,
+            });
+
+            await xfs.appendFilePromise(ppath.join(cwd, `.npmignore`), `/.yarn\n`);
+
+            const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, stdout, stderr});
+            if (pack.code !== 0)
+              return pack.code;
+
+            return 0;
+          }],
+          [PackageManager.Yarn2, async () => {
+            const workspaceCli = workspace !== null
+              ? [`workspace`, workspace]
+              : [];
+            const lockfilePath = ppath.join(cwd, Filename.lockfile);
+            if (!(await xfs.existsPromise(lockfilePath)))
+              await xfs.writeFilePromise(lockfilePath, ``);
+            
+            const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, stdout, stderr});
+            if (pack.code !== 0)
+              return pack.code;
+            return 0;
+          }],
+          [PackageManager.Npm, async () => {
+            const workspaceCli = workspace !== null
+              ? [`--workspace`, workspace]
+              : [];
+            const packStream = new PassThrough();
+            const packPromise = miscUtils.bufferStream(packStream);
+            const pack = await execUtils.pipevp(`npm`, [`pack`, `--silent`, ...workspaceCli], {cwd, stdout: packStream, stderr});
+            if (pack.code !== 0)
+              return pack.code;
+            
+            const packOutput = (await packPromise).toString().trim().replace(/^.*\n/s, ``);
+            const packTarget = ppath.resolve(cwd, npath.toPortablePath(packOutput));
+            await xfs.renamePromise(packTarget, outputPath);
+            return 0;
+          }],
+        ]);
+        const workflow = workflows.get(effectivePackageManager);
+        const code = await workflow();
+        if (code === 0 || typeof code === `undefined`)
+          return;
+        else 
+          throw `Packing the package failed (exit code ${code})`;
+    }
+
+    class convertToZipCommand extends BaseCommand {
+      static paths = [[`convertToZip`]];
+      yarn_v1 = Option.String({required: true});
+
+      async execute() {
+        const configuration = await Configuration.find(this.context.cwd,
+          this.context.plugins);
+        //const lockfile = configuration.get('lockfileFilename');
+        const cacheFolder = configuration.get('cacheFolder');
+        const locatorFolder = `${cacheFolder}/locator`;
+
+        const compressionLevel = configuration.get(`compressionLevel`);
+        const stdout = this.context.stdout;
+        const gitChecksumPatches = []; // {name:, oriHash:, newHash:}
+
+        async function patchLockfileChecksum(cwd, configuration, patches) {
+            const lockfilePath = ppath.join(cwd, configuration.get(`lockfileFilename`));
+
+            let currentContent = ``;
+            try {
+                currentContent = await xfs.readFilePromise(lockfilePath, `utf8`);
+            } catch (error) {
+            }
+            const newContent = patches.reduce((acc, item, i) => {
+                stdout.write(`patch '${item.name}' checksum:\n-${item.oriHash}\n+${item.newHash}\n\n\n`);
+                const regex = new RegExp(item.oriHash, "g");
+                return acc.replace(regex, item.newHash);
+            }, currentContent);
+            
+            await xfs.writeFilePromise(lockfilePath, newContent);
+        }
+
+
+        stdout.write(`yarn cacheFolder: ${cacheFolder}\n`);
+
+        const convertToZip = async (tgz, target, opts) => {
+          const { compressionLevel, ...bufferOpts } = opts;
+          const zipFs = new ZipFS(target, {
+            create: true,
+            libzip: await getLibzipPromise(),
+            level: compressionLevel
+          });
+          const tgzBuffer = fs.readFileSync(tgz);
+          await tgzUtils.extractArchiveTo(tgzBuffer, zipFs,
+            bufferOpts);
+          zipFs.saveAndClose();
+        }
+
+        stdout.write(`converting cache to zip\n`);
+
+        const files = fs.readdirSync(locatorFolder);
+        const tasks = []
+        for (const i in files) {
+          const file = `${files[i]}`;
+          let tgzFile = `${locatorFolder}/${file}`;
+          const match = file.match(/([^-]+)-([^\.]{1,10}).(tgz|git)/);
+          if (!match) {
+            stdout.write(`ignore ${file}\n`);
+            continue;
+          }
+          let resolution, locator;
+          const entry_type = match[3];
+          const sha = match[2];
+          let checksum;
+
+          if(entry_type === 'tgz') {
+            resolution = Buffer.from(match[1], 'base64').toString();
+            locator = structUtils.parseLocator(resolution, true);
+          }
+          else if(entry_type === 'git') {
+            const gitJson = JSON.parse(fs.readFileSync(tgzFile, 'utf8'));
+
+            resolution = gitJson.resolution;
+            locator = structUtils.parseLocator(resolution, true);
+            checksum = gitJson.checksum;
+
+            const repoPathRel = gitJson.repo_dir_rel;
+
+            const cloneTarget = `${cacheFolder}/${repoPathRel}`;
+            
+            const repoUrlParts = gitUtils.splitRepoUrl(locator.reference);
+            const packagePath = ppath.join(cloneTarget, `package.tgz`);
+
+            await prepareExternalProject(cloneTarget, packagePath, {
+                configuration: configuration,
+                stdout,
+                workspace: repoUrlParts.extra.workspace,
+                locator,
+                yarn_v1: this.yarn_v1,
+            });
+
+            tgzFile = packagePath;
+
+          }
+          const filename =
+            `${structUtils.slugifyLocator(locator)}-${sha}.zip`;
+          const targetFile = `${cacheFolder}/${filename}`
+          tasks.push(async () => {
+            await convertToZip(tgzFile, targetFile, {
+              compressionLevel: compressionLevel,
+              prefixPath: `node_modules/${structUtils.stringifyIdent(locator)}`,
+              stripComponents: 1,
+            });
+
+            if (entry_type === 'git') {
+                const file_checksum = await hashUtils.checksumFile(targetFile);
+
+                if (file_checksum !== checksum) {
+                    const newSha = file_checksum.slice(0, 10);
+                    const newTarget = `${cacheFolder}/${structUtils.slugifyLocator(locator)}-${newSha}.zip`;
+                    fs.renameSync(targetFile, newTarget);
+
+                    gitChecksumPatches.push({
+                        name: locator.name,
+                        oriHash: checksum,
+                        newHash: file_checksum,
+                    });
+                }
+            }
+          });
+        }
+        
+        while (tasks.length) {
+          await Promise.all(tasks.splice(0, 128).map(t => t()));
+        }
+
+        patchLockfileChecksum(this.context.cwd, configuration, gitChecksumPatches);
+        stdout.write(`converting finished\n`);
+      }
+    }
+    return {
+      commands: [
+        convertToZipCommand
+      ],
+    };
+  }
+};
+
diff --git a/node/flatpak_node_generator/manifest.py b/node/flatpak_node_generator/manifest.py
@@ -162,9 +162,18 @@ def add_data_source(self, data: Union[str, bytes], destination: Path) -> None:
         self._add_source_with_destination(source, destination, is_dir=False)
 
     def add_git_source(
-        self, url: str, commit: str, destination: Optional[Path] = None
+        self,
+        url: str,
+        commit: Optional[str] = None,
+        destination: Optional[Path] = None,
+        tag: Optional[str] = None,
     ) -> None:
-        source = {'type': 'git', 'url': url, 'commit': commit}
+        source = {'type': 'git', 'url': url}
+        assert commit or tag
+        if commit:
+            source['commit'] = commit
+        if tag:
+            source['tag'] = tag
         self._add_source_with_destination(source, destination, is_dir=True)
 
     def add_script_source(self, commands: List[str], destination: Path) -> None:
diff --git a/node/flatpak_node_generator/providers/yarn.py b/node/flatpak_node_generator/providers/yarn.py
