🌱 initial workflow run collect data

.gitignore

@@ -4,6 +4,7 @@ build/
 dist/
 downloaded_assets/
 *.vsix
+.env
 
 # TypeScript build info
 **/tsconfig.tsbuildinfo

package.json

@@ -52,6 +52,7 @@
     "dev:vscode": "npm run dev -w vscode",
     "build": "npm run build --workspaces --if-present",
     "collect-assets": "rimraf ./downloaded_assets && node ./scripts/collect-assets.js",
+    "collect-assets-workflow": "rimraf ./downloaded_assets && node ./scripts/collect-assets-workflow.js",
     "dist": "rimraf ./dist && node ./scripts/copy-dist.js",
     "package": "cd dist && vsce package",
     "test": "npm run lint && npm run build && npm run test --workspaces --if-present"
@@ -90,6 +91,7 @@
     "unzipper": "^0.12.3"
   },
   "dependencies": {
+    "dotenv": "^16.4.7",
     "fs-extra": "^11.2.0",
     "globby": "^14.0.2",
     "immer": "10.1.1",

scripts/_download.js

@@ -6,7 +6,7 @@ import fs from "fs-extra";
 import unzipper from "unzipper";
 import * as tar from "tar";
 import { bold, green, yellow } from "colorette";
-import { chmodOwnerPlusX } from "./_util.js";
+import { chmodOwnerPlusX, fetchFirstSuccessfulRun, fetchArtifactsForRun } from "./_util.js";
 
 const GITHUB_API = "https://api.github.com";
 
@@ -203,3 +203,169 @@ export async function downloadAndExtractTarGz({ targetDirectory, url, sha256 })
     console.groupEnd();
   }
 }
+
+/**
+ * Download a workflow artifact from GitHub
+ *
+ * @param {string} url - Download URL for the artifact
+ * @param {string} name - Name of the artifact file
+ * @param {string} outputDir - Directory to save the downloaded artifact
+ * @returns {Promise<string>} - Path to the downloaded artifact file
+ */
+export async function downloadArtifact(url, name, outputDir, token) {
+  try {
+    const assetDir = join(outputDir);
+    const assetFileName = join(assetDir, `${name}`);
+    await fs.ensureDir(assetDir);
+
+    console.log(`Downloading asset: ${name}`);
+    const response = await fetch(url, {
+      headers: {
+        Accept: "application/vnd.github.v3+json",
+        Authorization: `Bearer ${token}`,
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to download ${name}: ${response.statusText}`);
+    }
+
+    const sha256 = await streamResponseToFile(assetFileName, response);
+    console.log(`Verifying download of: ${name}`);
+    console.log(`Asset sha256: ${green(sha256)}`);
+
+    console.log(`Downloaded ${name} to ${assetFileName}`);
+    return assetFileName;
+  } catch (error) {
+    console.error(`Error downloading artifact ${name}:`, error.message);
+    throw error;
+  }
+}
+
+/**
+ * Extract an artifact ZIP file.
+ *
+ * @param {string} filePath - Path to the ZIP file
+ * @param {string} tempDir - Temporary directory for extraction
+ * @param {string} finalDir - Final directory for extracted files
+ */
+export async function extractArtifact(filePath, tempDir, finalDir) {
+  try {
+    console.log(`Checking if ${filePath} is a valid ZIP archive.`);
+
+    // First extraction to temporary directory
+    console.log(`Extracting ${filePath} to temporary directory: ${tempDir}`);
+    await unzipper.Open.file(filePath).then((d) => d.extract({ path: tempDir }));
+
+    // Find and extract any nested zip files in the temp directory
+    const files = await fs.readdir(tempDir);
+    for (const file of files) {
+      if (file.endsWith(".zip")) {
+        const nestedZipPath = join(tempDir, file);
+        console.log(`Extracting nested zip: ${nestedZipPath} to ${finalDir}`);
+        await unzipper.Open.file(nestedZipPath).then((d) => d.extract({ path: finalDir }));
+        console.log(`Deleting nested zip: ${nestedZipPath}`);
+        await fs.unlink(nestedZipPath);
+      }
+    }
+
+    // Cleanup temporary directory
+    console.log(`Cleaning up temporary directory: ${tempDir}`);
+    await fs.rm(tempDir, { recursive: true, force: true });
+
+    // Cleanup the original zip file
+    console.log(`Deleting original zip file: ${filePath}`);
+    await fs.unlink(filePath);
+
+    console.log(`Extraction complete: ${filePath} -> ${finalDir}`);
+  } catch (error) {
+    console.error(`Error extracting artifact ${filePath}:`, error.message);
+    throw error;
+  }
+}
+
+/**
+ * Download and process workflow artifacts.
+ *
+ * @param {string} targetDirectory - Directory to store downloaded artifacts
+ * @param {string} metaFile - Path to the metadata file
+ * @param {string} url - Base URL of the GitHub repository
+ * @param {string} workflow - Name of the workflow file
+ * @param {Array} assets - List of assets to downloa
+ */
+export async function downloadWorkflowArtifacts({
+  targetDirectory,
+  metaFile,
+  url,
+  workflow,
+  assets,
+}) {
+  const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
+
+  if (!GITHUB_TOKEN) {
+    console.error("Error: GITHUB_TOKEN environment variable is not set.");
+    process.exit(1);
+  }
+
+  try {
+    console.log("Fetching first successful workflow run...");
+    const workflowRunId = await fetchFirstSuccessfulRun(url, workflow, GITHUB_TOKEN);
+
+    if (!workflowRunId) {
+      throw new Error("No successful workflow runs found.");
+    }
+
+    console.log(`Workflow Run ID: ${workflowRunId}`);
+    const artifactUrls = await fetchArtifactsForRun(url, workflowRunId, GITHUB_TOKEN);
+
+    if (!artifactUrls || artifactUrls.length === 0) {
+      throw new Error("No artifacts found for the workflow run.");
+    }
+
+    const metadata = {
+      workflowRunId,
+      collectedAt: new Date().toISOString(),
+      assets: [],
+    };
+
+    for (const asset of assets) {
+      const { name } = asset;
+      const artifact = artifactUrls.find((a) => a.name === name);
+
+      if (!artifact) {
+        console.warn(`Asset [${name}] was not found in the workflow artifacts.`);
+        continue;
+      }
+
+      try {
+        console.log(`Processing artifact: ${name}`);
+
+        const downloadedFilePath = await downloadArtifact(
+          artifact.url,
+          name,
+          targetDirectory,
+          GITHUB_TOKEN,
+        );
+
+        // Extract the artifact
+        const tempDir = join(targetDirectory, "temp");
+        const extractionDir = join(targetDirectory, name.replace(/\.zip$/, ""));
+        await extractArtifact(downloadedFilePath, tempDir, extractionDir);
+
+        metadata.assets.push({
+          name,
+          extractionDir,
+          downloadedAt: new Date().toISOString(),
+        });
+      } catch (error) {
+        console.error(`Error processing artifact [${name}]:`, error.message);
+      }
+    }
+
+    await fs.writeJson(metaFile, metadata, { spaces: 2 });
+    console.log(`Metadata written to ${metaFile}`);
+  } catch (error) {
+    console.error("Error downloading workflow artifacts:", error.message);
+    process.exit(1);
+  }
+}

scripts/_util.js

@@ -23,3 +23,77 @@ export function chmodOwnerPlusX(path) {
   const { mode = fs.constants.S_IRUSR } = fs.statSync(path);
   fs.chmodSync(path, mode | fs.constants.S_IXUSR);
 }
+
+/**
+ * Fetch the first successful workflow run ID.
+ *
+ * @param {string} baseUrl - Base URL of the GitHub repository.
+ * @param {string} workflowFile - Name of the workflow file.
+ * @param {string} token - GitHub personal access token.
+ * @returns {Promise<string|null>} - ID of the first successful workflow run.
+ */
+export async function fetchFirstSuccessfulRun(baseUrl, workflowFile, token) {
+  try {
+    const response = await fetch(
+      `${baseUrl}/actions/workflows/${workflowFile}/runs?branch=main&status=success`,
+      {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Accept: "application/vnd.github.v3+json",
+        },
+      },
+    );
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch workflow runs: ${response.statusText}`);
+    }
+
+    const data = await response.json();
+    const runs = data.workflow_runs;
+    if (runs.length === 0) {
+      console.log("No successful runs found.");
+      return null;
+    }
+
+    const firstRun = runs[0];
+    console.log(`First Successful Workflow Run ID: ${firstRun.id}`);
+    return firstRun.id;
+  } catch (error) {
+    console.error("Error fetching workflow runs:", error.message);
+    return null;
+  }
+}
+
+/**
+ * Fetch artifacts for a specific workflow run.
+ *
+ * @param {string} baseUrl - Base URL of the GitHub repository.
+ * @param {string} runId - ID of the workflow run.
+ * @param {string} token - GitHub personal access token.
+ * @returns {Promise<Array>} - List of artifacts with download URLs.
+ */
+export async function fetchArtifactsForRun(baseUrl, runId, token) {
+  try {
+    const response = await fetch(`${baseUrl}/actions/runs/${runId}/artifacts`, {
+      headers: {
+        Authorization: `Bearer ${token}`,
+        Accept: "application/vnd.github.v3+json",
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch artifacts: ${response.statusText}`);
+    }
+
+    const data = await response.json();
+    const downloadUrls = data.artifacts.map((artifact) => ({
+      name: artifact.name,
+      url: artifact.archive_download_url,
+    }));
+    console.log("Artifact Download URLs:", downloadUrls);
+    return downloadUrls;
+  } catch (error) {
+    console.error("Error fetching artifacts:", error.message);
+    return [];
+  }
+}

scripts/collect-assets-workflow.js

@@ -0,0 +1,45 @@
+#! /usr/bin/env node
+import { ensureDir } from "fs-extra/esm";
+import { resolve, join } from "path";
+import { cwdToProjectRoot } from "./_util.js";
+import { downloadWorkflowArtifacts, downloadAndExtractTarGz } from "./_download.js";
+import dotenv from "dotenv";
+dotenv.config();
+
+const REPO_OWNER = "konveyor";
+const REPO_NAME = "kai";
+const WORKFLOW_FILE = "build-and-push-binaries.yml";
+
+const GITHUB_API = "https://api.github.com";
+const BASE_URL = `${GITHUB_API}/repos/${REPO_OWNER}/${REPO_NAME}`;
+const DOWNLOAD_DIR = resolve("downloaded_assets");
+const META_FILE = join(DOWNLOAD_DIR, "kai", "collect.json");
+
+cwdToProjectRoot();
+
+await ensureDir(DOWNLOAD_DIR);
+
+// Download Kai assets via workflow artifacts
+await downloadWorkflowArtifacts({
+  targetDirectory: join(DOWNLOAD_DIR, "kai/"),
+  metaFile: META_FILE,
+  url: BASE_URL,
+  workflow: WORKFLOW_FILE,
+  assets: [
+    { name: "java-deps.zip" },
+    { name: "kai-rpc-server.linux-aarch64.zip", platform: "linux", arch: "arm64", chmod: true },
+    { name: "kai-rpc-server.linux-x86_64.zip", platform: "linux", arch: "x64", chmod: true },
+    { name: "kai-rpc-server.macos-arm64.zip", platform: "darwin", arch: "arm64", chmod: true },
+    { name: "kai-rpc-server.macos-x86_64.zip", platform: "darwin", arch: "x64", chmod: true },
+    { name: "kai-rpc-server.windows-Arm64.zip", platform: "win32", arch: "arm64" },
+    { name: "kai-rpc-server.windows-X64.zip", platform: "win32", arch: "x64" },
+  ],
+});
+
+// Download jdt.ls
+// Base release url: https://download.eclipse.org/jdtls/milestones/1.38.0/
+await downloadAndExtractTarGz({
+  targetDirectory: join(DOWNLOAD_DIR, "jdt.ls-1.38.0/"),
+  url: "https://download.eclipse.org/jdtls/milestones/1.38.0/jdt-language-server-1.38.0-202408011337.tar.gz",
+  sha256: "ba697788a19f2ba57b16302aba6b343c649928c95f76b0d170494ac12d17ac78",
+});