ci: Add github workflow for nitro node

.github/scripts/nitro-node/e2e-test-install-nitro-node.js

@@ -0,0 +1,138 @@
+const os = require("node:os");
+const path = require("node:path");
+const fs = require("node:fs");
+const { spawn } = require("node:child_process");
+
+// Test on both npm and yarn
+const PACKAGE_MANAGERS = ["npm", "yarn"];
+const ADD_DEP_CMDS = {
+  // Need to copy dependency instead of linking so test logic can check the bin
+  npm: "install --install-links",
+  yarn: "add",
+};
+// Path to the package to install
+const NITRO_NODE_PKG =
+  process.env.NITRO_NODE_PKG ||
+  path.resolve(path.normalize(path.join(__dirname, "..", "..", "nitro-node")));
+// Prefixes of downloaded nitro bin subdirectories
+const BIN_DIR_PREFIXES = {
+  darwin: "mac",
+  win32: "win",
+  linux: "linux",
+};
+
+// Utility to check for a file with nitro in name in the corresponding directory
+const checkBinaries = (repoDir) => {
+  // FIXME: Check for unsupported platforms
+  const binDirPrefix = BIN_DIR_PREFIXES[process.platform];
+  const searchRoot = path.join(
+    repoDir,
+    "node_modules",
+    "@janhq",
+    "nitro-node",
+    "bin",
+  );
+  // Get the dir and files that indicate successful download of binaries
+  const matched = fs.readdirSync(searchRoot, { recursive: true }).filter(
+    // FIXME: the result of readdirSync with recursive option is filename
+    // with intermediate subdirectories so this logic might not be correct
+    (fname) => fname.startsWith(binDirPrefix) || fname.includes("nitro"),
+  );
+  console.log(`Downloaded bin paths:`, matched);
+
+  // Must have both the directory for the platform and the binary
+  return matched.length > 1;
+};
+
+// Wrapper to wait for child process to finish
+const childProcessPromise = (childProcess) =>
+  new Promise((resolve, reject) => {
+    childProcess.on("exit", (exitCode) => {
+      const exitNum = Number(exitCode);
+      if (0 == exitNum) {
+        resolve();
+      } else {
+        reject(exitNum);
+      }
+    });
+  });
+
+// Create a temporary directory for testing
+const createTestDir = () =>
+  fs.mkdtempSync(path.join(os.tmpdir(), "dummy-project-"));
+
+// First test, create empty project dir and add nitro-node as dependency
+const firstTest = async (packageManager, repoDir) => {
+  console.log(`[First test @ ${repoDir}] install with ${packageManager}`);
+  // Init project with default package.json
+  const cmd1 = `npm init -y`;
+  console.log("🖥️ " + cmd1);
+  await childProcessPromise(
+    spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }),
+  );
+
+  // Add nitro-node as dependency
+  const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${NITRO_NODE_PKG}`;
+  console.log("🖥️ " + cmd2);
+  await childProcessPromise(
+    spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }),
+  );
+
+  // Check that the binaries exists
+  if (!checkBinaries(repoDir)) process.exit(3);
+
+  // Cleanup node_modules after success
+  //fs.rmSync(path.join(repoDir, "node_modules"), { recursive: true });
+};
+
+// Second test, install the wrapper from another project dir
+const secondTest = async (packageManager, repoDir, wrapperDir) => {
+  console.log(
+    `[Second test @ ${repoDir}] install ${wrapperDir} with ${packageManager}`,
+  );
+  // Init project with default package.json
+  const cmd1 = `npm init -y`;
+  console.log("🖥️ " + cmd1);
+  await childProcessPromise(
+    spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }),
+  );
+
+  // Add wrapper as dependency
+  const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${wrapperDir}`;
+  console.log("🖥️ " + cmd2);
+  await childProcessPromise(
+    spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }),
+  );
+
+  // Check that the binaries exists
+  if (!checkBinaries(repoDir)) process.exit(3);
+};
+
+// Run all the tests for the chosen package manger
+const run = async (packageManager) => {
+  const firstRepoDir = createTestDir();
+
+  // Run first test
+  await firstTest(packageManager, firstRepoDir);
+  console.log("First test ran success");
+
+  // FIXME: Currently failed with npm due to wrong path being resolved.
+  //const secondRepoDir = createTestDir();
+
+  // Run second test
+  //await secondTest(packageManager, secondRepoDir, firstRepoDir);
+  //console.log("Second test ran success");
+};
+
+// Main, run tests for npm and yarn
+const main = async () => {
+  await PACKAGE_MANAGERS.reduce(
+    (p, pkgMng) => p.then(() => run(pkgMng)),
+    Promise.resolve(),
+  );
+};
+
+// Run script if called directly instead of import as module
+if (require.main === module) {
+  main();
+}

.github/workflows/build-nitro-node.yml

@@ -0,0 +1,70 @@
+name: CI / nitro-node / build
+
+on:
+  schedule:
+    - cron: "0 20 * * *" # At 0:20 UTC, which is 7:20 AM UTC+7
+  push:
+    branches:
+      - main
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+    paths: [".github/workflows/build-nitro-node.yml", "nitro-node"]
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths: [".github/workflows/build-nitro-node.yml", "nitro-node"]
+  workflow_dispatch:
+
+env:
+  LLM_MODEL_URL: https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf
+  WHISPER_MODEL_URL: https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Restore cached model file
+        id: cache-model-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            nitro-node/test/test_assets/*.gguf
+          key: ${{ runner.os }}-model-gguf
+
+      - uses: suisei-cn/[email protected]
+        id: download-model-file
+        name: Download model file
+        with:
+          url: "The model we are using is [tinyllama-1.1b](${{ env.LLM_MODEL_URL }})!"
+          target: nitro-node/test/test_assets/
+          auto-match: true
+          retry-times: 3
+
+      - name: Save downloaded model file to cache
+        id: cache-model-save
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            nitro-node/test/test_assets/*.gguf
+          key: ${{ steps.cache-model-restore.outputs.cache-primary-key }}
+
+      - name: Run tests
+        id: test_nitro_node
+        run: |
+          cd nitro-node
+          make clean test-ci

.github/workflows/build.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   schedule:
-    - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7
+    - cron: "0 20 * * *" # At 0:20 UTC, which is 7:20 AM UTC+7
   push:
     branches:
       - main
@@ -24,6 +24,7 @@ on:
         "!docs/**",
         "!.gitignore",
         "!README.md",
+        "!nitro-node/**",
       ]
   pull_request:
     types: [opened, synchronize, reopened]
@@ -44,6 +45,7 @@ on:
         "!docs/**",
         "!.gitignore",
         "!README.md",
+        "!nitro-node/**",
       ]
   workflow_dispatch:
 
@@ -113,7 +115,7 @@ jobs:
             }
             # Get the latest release tag from GitHub API
             LATEST_TAG=$(get_latest_tag)
-            
+
             # Remove the 'v' and append the build number to the version
             NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
             echo "New version: $NEW_VERSION"
@@ -345,7 +347,7 @@ jobs:
         run: |
           ./install_deps.sh
           mkdir build && cd build
-          cmake -DWHISPER_COREML=1 -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} .. 
+          cmake -DWHISPER_COREML=1 -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} ..
           CC=gcc-8 make -j $(sysctl -n hw.ncpu)
           ls -la
 
@@ -423,7 +425,7 @@ jobs:
         run: |
           ./install_deps.sh
           mkdir build && cd build
-          cmake -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} -DLLAMA_METAL=OFF .. 
+          cmake -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} -DLLAMA_METAL=OFF ..
           CC=gcc-8 make -j $(sysctl -n hw.ncp)
           ls -la
 
@@ -499,7 +501,7 @@ jobs:
   #       run: |
   #         ./install_deps.sh
   #         mkdir build && cd build
-  #         cmake -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} -DLLAMA_VULKAN=ON -DLLAMA_METAL=OFF .. 
+  #         cmake -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} -DLLAMA_VULKAN=ON -DLLAMA_METAL=OFF ..
   #         CC=gcc-8 make -j $(sysctl -n hw.ncp)
   #         ls -la
 

.github/workflows/codeql-analysis.yml

@@ -34,6 +34,8 @@ on:
         "!docs/**",
         "!.gitignore",
         "!README.md",
+        "!.github/scripts/nitro-node/**",
+        "!nitro-node/**",
       ]
   pull_request:
     types: [opened, synchronize, reopened]
@@ -55,6 +57,8 @@ on:
         "!docs/**",
         "!.gitignore",
         "!README.md",
+        "!.github/scripts/nitro-node/**",
+        "!nitro-node/**",
       ]
 
 jobs:
@@ -96,7 +100,7 @@ jobs:
         run: |
           ./install_deps.sh
           mkdir build && cd build
-          cmake .. 
+          cmake ..
           CC=gcc-8 make -j $(sysctl -n hw.ncp)
           ls -la
 

.github/workflows/test-install-nitro-node.yml

@@ -0,0 +1,100 @@
+name: CI / nitro-node / e2e-test
+
+on:
+  schedule:
+    - cron: "0 20 * * *" # At 0:20 UTC, which is 7:20 AM UTC+7
+  push:
+    branches:
+      - main
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+    paths:
+      - ".github/scripts/e2e-test-install-nitro-node.js"
+      - ".github/workflows/test-install-nitro-node.yml"
+      - "nitro-node/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - ".github/scripts/e2e-test-install-nitro-node.js"
+      - ".github/workflows/test-install-nitro-node.yml"
+      - "nitro-node/**"
+  workflow_dispatch:
+
+jobs:
+  linux-pack-tarball:
+    runs-on: ubuntu-latest
+    outputs:
+      tarball-url: ${{ steps.upload.outputs.artifact-url }}
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Build tarball
+        id: build
+        run: |
+          cd nitro-node
+          make pack
+          find . -type f -name 'janhq-nitro-node-*.tgz' -exec mv {} janhq-nitro-node.tgz \;
+
+      - name: Upload tarball as artifact
+        id: upload
+        uses: actions/upload-artifact@master
+        with:
+          name: janhq-nitro-node
+          path: nitro-node/janhq-nitro-node.tgz
+          if-no-files-found: error
+
+  install:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    needs: [linux-pack-tarball]
+    if: always() && needs.linux-pack-tarball.result == 'success'
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Enable yarn
+        run: |
+          corepack enable
+          corepack prepare yarn@1 --activate
+
+      - name: Download prebuilt tarball
+        uses: actions/download-artifact@master
+        with:
+          name: janhq-nitro-node
+          path: .github/scripts/
+
+      - name: List tarball content
+        id: tar-tf
+        run: |
+          cd .github
+          cd scripts
+          tar tf janhq-nitro-node.tgz
+
+      - name: Run tests
+        id: test_install_nitro_node
+        env:
+          NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz
+        run: |
+          cd .github
+          cd scripts
+          cd nitro-node
+          node e2e-test-install-nitro-node.js