From 79f01f6e42c26ab7a446412d5cb0e6ed57d7b9b5 Mon Sep 17 00:00:00 2001 From: Tine Kondo Date: Fri, 15 Mar 2024 07:56:56 +0000 Subject: [PATCH] chore(ci): setup nx agents and improve other CI workflows --- .../actions/install-dependencies/action.yml | 2 +- .github/actions/setup/action.yml | 56 +++++++++++++++ .github/workflows/ci.yml | 37 ++++++++++ .github/workflows/commintlint.yml | 4 +- .github/workflows/run-e2e-tests.yml | 61 ---------------- .github/workflows/run-smoke-tests.yml | 48 ------------- .github/workflows/run-unit-tests.yml | 33 --------- .github/workflows/smoke-tests.yml | 35 +++++++++ .husky/pre-commit | 1 + .nx/workflow-steps/install-flutter/main.js | 30 ++++++++ .nx/workflow-steps/install-flutter/main.yaml | 6 ++ .nx/workflow-steps/install-java/main.js | 25 +++++++ .nx/workflow-steps/install-java/main.yaml | 6 ++ .nx/workflows/agents.yaml | 72 +++++++++++++++++++ nx.json | 8 +-- package.json | 2 +- 16 files changed, 276 insertions(+), 150 deletions(-) create mode 100644 .github/actions/setup/action.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/run-e2e-tests.yml delete mode 100644 .github/workflows/run-smoke-tests.yml delete mode 100644 .github/workflows/run-unit-tests.yml create mode 100644 .github/workflows/smoke-tests.yml create mode 100644 .nx/workflow-steps/install-flutter/main.js create mode 100644 .nx/workflow-steps/install-flutter/main.yaml create mode 100644 .nx/workflow-steps/install-java/main.js create mode 100644 .nx/workflow-steps/install-java/main.yaml create mode 100644 .nx/workflows/agents.yaml diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml index ad6396b5..f1d1f09f 100644 --- a/.github/actions/install-dependencies/action.yml +++ b/.github/actions/install-dependencies/action.yml @@ -10,7 +10,7 @@ inputs: runs: using: 'composite' steps: - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v3 with: version: 8.2 diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..8fb722db --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,56 @@ +name: 'Setup Build Tools and Dependencies' +description: Setup node, pnpm, java, flutter, cache, and install dependencies + +inputs: + node-version: # id of input + description: 'Node version' + required: false + default: 20 + +runs: + using: 'composite' + steps: + - name: Derive appropriate SHAs for base and head for `nx affected` commands + uses: nrwl/nx-set-shas@v4 + + - name: Setup PNPM + uses: pnpm/action-setup@v3 + with: + version: 8.2 + + - name: Setup Node.js ${{ inputs.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + cache: 'pnpm' + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + - run: java -version + shell: bash + + - name: Setup Flutter + uses: subosito/flutter-action@v2 + with: + channel: 'stable' + cache: true + cache-key: 'flutter-:os:-:channel:-:version:-:arch:-:hash:' # optional, change this to force refresh cache + cache-path: '${{ runner.tool_cache }}/flutter/:channel:-:version:-:arch:' # optional, change this to specify the cache path + architecture: x64 # optional, x64 or arm64 + - run: flutter --version + shell: bash + + - name: Cache node_modules + id: cache-modules + uses: actions/cache@v4 + with: + lookup-only: true + path: '**/node_modules' + key: pnpm-${{ hashFiles('pnpm-lock.yaml') }} + + - name: Install packages + run: pnpm install --frozen-lockfile + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..d45a3ba3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: + - develop + pull_request: + +# Needed for nx-set-shas when run on the main branch +permissions: + actions: read + contents: read + +jobs: + develop: + runs-on: ubuntu-latest + env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + timeout-minutes: 60 + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install node, pnpm, and dependencies + uses: ./.github/actions/install-dependencies + + # Connect your workspace on nx.app and uncomment this to enable task distribution. + # The "--stop-agents-after" is optional, but allows idle agents to shut down once the "e2e" targets have been requested + - run: pnpm exec nx-cloud start-ci-run --distribute-on="5 linux-small-js-java-flutter" --stop-agents-after="e2e" + + # This line is needed for nx affected to work when CI is running on a PR + - run: git branch --track develop origin/develop + + - run: pnpm exec nx-cloud record -- nx format:check + - run: pnpm exec nx affected -t lint test build e2e --exclude=smoke --codeCoverage diff --git a/.github/workflows/commintlint.yml b/.github/workflows/commintlint.yml index 1c0bd318..c93f4054 100644 --- a/.github/workflows/commintlint.yml +++ b/.github/workflows/commintlint.yml @@ -8,6 +8,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: wagoid/commitlint-github-action@v4 + - uses: wagoid/commitlint-github-action@v5 with: - configFile: '.commitlintrc.js' \ No newline at end of file + configFile: '.commitlintrc.js' diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml deleted file mode 100644 index 93cfe588..00000000 --- a/.github/workflows/run-e2e-tests.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Run e2e Tests - -on: - push: - branches: [develop] - pull_request: - branches: [develop] -concurrency: - group: ${{ github.workflow }}-${{ github.event.number || github.ref }} - cancel-in-progress: true -jobs: - run_e2e: - name: Run e2e Tests - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - matrix: - os: [ubuntu-latest] - node-version: [18] - env: - NX_BRANCH: ${{ github.event.number || github.ref }} - NX_RUN_GROUP: ${{ github.run_id }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v3 - with: - main-branch-name: 'develop' - - - run: | - echo "BASE: ${{ env.NX_BASE }}" - echo "HEAD: ${{ env.NX_HEAD }}" - - - name: Use Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - - - name: Use Flutter - uses: subosito/flutter-action@v2 - with: - channel: 'stable' - cache: true - cache-key: 'flutter-:os:-:channel:-:version:-:arch:-:hash:' # optional, change this to force refresh cache - cache-path: '${{ runner.tool_cache }}/flutter/:channel:-:version:-:arch:' # optional, change this to specify the cache path - architecture: x64 # optional, x64 or arm64 - - - name: Install node, pnpm, and dependencies - uses: ./.github/actions/install-dependencies - - - name: Run e2e tests - env: - NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node-version }} - NX_RUN_GROUP: ${{ github.run_id }}-${{ matrix.os }}-${{ matrix.node-version }} - run: | - pnpm e2e diff --git a/.github/workflows/run-smoke-tests.yml b/.github/workflows/run-smoke-tests.yml deleted file mode 100644 index 836ad1ca..00000000 --- a/.github/workflows/run-smoke-tests.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Nightly Smoke Tests - -on: - # Trigger manually or on schedule - workflow_dispatch: - schedule: - - cron: '0 0 * * *' - -jobs: - run_smoke_tests: - name: Run Smoke Tests - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node-version: [18] - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Use Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - - - name: Use Flutter - uses: subosito/flutter-action@v2 - with: - channel: 'stable' - cache: true - cache-key: 'flutter-:os:-:channel:-:version:-:arch:-:hash:' # optional, change this to force refresh cache - cache-path: '${{ runner.tool_cache }}/flutter/:channel:-:version:-:arch:' # optional, change this to specify the cache path - architecture: x64 # optional, x64 or arm64 - - - name: Install node, pnpm, and dependencies - uses: ./.github/actions/install-dependencies - with: - node-version: ${{ matrix.node-version }} - - - name: Run smoke tests - env: - NX_E2E_CI_CACHE_KEY: smoke-gha-${{ matrix.os }}-${{ matrix.node-version }} - NX_RUN_GROUP: ${{ github.run_id }}-${{ matrix.os }}-${{ matrix.node-version }} - run: pnpm nx test smoke --verbose diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml deleted file mode 100644 index 5a85da49..00000000 --- a/.github/workflows/run-unit-tests.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Run Unit Tests - -on: - push: - branches: [develop] - pull_request: - branches: [develop] - -jobs: - run_unit: - name: Run Unit Tests - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - node-version: [18] - env: - NX_BRANCH: ${{ github.event.number }} - NX_RUN_GROUP: ${{ github.run_id }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install node, pnpm, and dependencies - uses: ./.github/actions/install-dependencies - with: - node-version: ${{ matrix.node-version }} - - - name: Run unit tests - run: pnpm run affected:test --base=origin/develop --parallel=6 --exclude=smoke --verbose --codeCoverage diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml new file mode 100644 index 00000000..d73f2ab2 --- /dev/null +++ b/.github/workflows/smoke-tests.yml @@ -0,0 +1,35 @@ +name: Nightly Smoke Tests + +on: + # Trigger manually or on schedule + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +jobs: + run_smoke_tests: + name: Smoke Tests + runs-on: ${{ matrix.os }} + env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + timeout-minutes: 60 + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node-version: [20] + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup node, pnpm, java, flutter, and dependencies + uses: ./.github/actions/setup + with: + node-version: ${{ matrix.node-version }} + + - name: Run smoke tests + env: + NX_E2E_CI_CACHE_KEY: smoke-gha-${{ matrix.os }}-${{ matrix.node-version }} + NX_RUN_GROUP: ${{ github.run_id }}-${{ matrix.os }}-${{ matrix.node-version }} + run: pnpm nx test smoke --verbose diff --git a/.husky/pre-commit b/.husky/pre-commit index 43cee7ba..800824dc 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,6 +1,7 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" +pnpm affected:format pnpm affected:lint pnpm affected:test pnpm sync-preset-schemas diff --git a/.nx/workflow-steps/install-flutter/main.js b/.nx/workflow-steps/install-flutter/main.js new file mode 100644 index 00000000..0d248556 --- /dev/null +++ b/.nx/workflow-steps/install-flutter/main.js @@ -0,0 +1,30 @@ +const { execSync } = require('child_process'); + +const flutterVersion = process.env.FLUTTER_VERSION || '3.19.3'; +const flutterChannel = process.env.FLUTTER_CHANNEL || 'stable'; + +const flutterPkgName = `flutter_linux_${flutterVersion}-${flutterChannel}.tar.xz`; +const flutterPkgUrl = `https://storage.googleapis.com/flutter_infra_release/releases/${flutterChannel}/linux/${flutterPkgName}`; + +runInBash('ls -l /usr/local', { stdio: 'inherit' }); + +console.log(`Installing Flutter v${flutterVersion}-${flutterChannel} manually`); +execSync(`curl -L ${flutterPkgUrl} | tar -xJf - -C /usr/local`, { + stdio: 'inherit', +}); + +console.log('Adding Flutter bin to PATH...'); +runInBash( + `echo 'export PATH="$PATH:/usr/local/flutter/bin"' >> $HOME/.bashrc`, + { + stdio: 'inherit', + } +); +runInBash('source $HOME/.bashrc', { stdio: 'inherit' }); +runInBash('cat $HOME/.bashrc', { stdio: 'inherit' }); + +execSync('flutter version', { stdio: 'inherit' }); + +function runInBash(cmd, opts = { stdio: 'inherit' }) { + execSync(`echo "#!/usr/bin/env bash\n\n${cmd}" | bash`, opts); +} diff --git a/.nx/workflow-steps/install-flutter/main.yaml b/.nx/workflow-steps/install-flutter/main.yaml new file mode 100644 index 00000000..e86d0ed5 --- /dev/null +++ b/.nx/workflow-steps/install-flutter/main.yaml @@ -0,0 +1,6 @@ +name: Install Flutter +description: Installs Flutter using snap + +definition: + using: 'node' + main: 'nx-cloud/.nx/workflow-steps/install-flutter/main.js' diff --git a/.nx/workflow-steps/install-java/main.js b/.nx/workflow-steps/install-java/main.js new file mode 100644 index 00000000..a03c338e --- /dev/null +++ b/.nx/workflow-steps/install-java/main.js @@ -0,0 +1,25 @@ +const { execSync } = require('child_process'); + +const javaVersion = process.env.JAVA_VERSION || '17'; +const javaDistro = process.env.JAVA_DISTRO || 'ms'; + +const javaVersionFull = `${javaVersion}-${javaDistro}`; + +console.log('Installing Utils(zip, unzip, etc)!'); +execSync('sudo apt update', { stdio: 'inherit' }); +execSync('sudo apt install zip unzip', { stdio: 'inherit' }); + +console.log('Installing SDKMan! first'); +execSync('curl -s "https://get.sdkman.io" | bash', { stdio: 'inherit' }); +runInBash('echo "Current terminal is: $0"', { stdio: 'inherit' }); +runInBash('source $HOME/.sdkman/bin/sdkman-init.sh', { stdio: 'inherit' }); +runInBash('sdk version', { stdio: 'inherit' }); + +console.log('Installing Java using SDKMan!'); +runInBash(`sdk install java ${javaVersionFull}`, { stdio: 'inherit' }); +runInBash(`sdk use java ${javaVersionFull}`, { stdio: 'inherit' }); +runInBash('java -version', { stdio: 'inherit' }); + +function runInBash(cmd, opts = { stdio: 'inherit' }) { + execSync(`echo "#!/usr/bin/env bash\n\n${cmd}" | bash`, opts); +} diff --git a/.nx/workflow-steps/install-java/main.yaml b/.nx/workflow-steps/install-java/main.yaml new file mode 100644 index 00000000..a96400e0 --- /dev/null +++ b/.nx/workflow-steps/install-java/main.yaml @@ -0,0 +1,6 @@ +name: Install Java +description: Installs Java using SDKMan! + +definition: + using: 'node' + main: 'nx-cloud/.nx/workflow-steps/install-java/main.js' diff --git a/.nx/workflows/agents.yaml b/.nx/workflows/agents.yaml new file mode 100644 index 00000000..00b78488 --- /dev/null +++ b/.nx/workflows/agents.yaml @@ -0,0 +1,72 @@ +launch-templates: + linux-small-js-java-flutter: + resource-class: 'docker_linux_amd64/small' + image: 'ubuntu22.04-node20.9-v2' + env: + NX_CLOUD_ACCESS_TOKEN: '{{secrets.NX_CLOUD_ACCESS_TOKEN}}' + JAVA_VERSION: '17' + JAVA_DISTRO: 'ms' + init-steps: + - name: Checkout + uses: 'nrwl/nx-cloud-workflows/v3.5/workflow-steps/checkout/main.yaml' + - name: Restore Node Modules Cache + uses: 'nrwl/nx-cloud-workflows/v3.5/workflow-steps/cache/main.yaml' + env: + KEY: 'package-lock.json|yarn.lock|pnpm-lock.yaml' + PATHS: 'node_modules' + BASE_BRANCH: 'main' + - name: Restore Maven/Gradle Cache + uses: 'nrwl/nx-cloud-workflows/v3.5/workflow-steps/cache/main.yaml' + env: + KEY: 'pom.xml|build.gradle|build.gradle.kts|settings.gradle|settings.gradle.kts' + PATHS: '~/.m2/repository' + BASE_BRANCH: 'main' + #- name: Restore Browser Binary Cache + # uses: 'nrwl/nx-cloud-workflows/v3.5/workflow-steps/cache/main.yaml' + # env: + # KEY: 'package-lock.json|yarn.lock|pnpm-lock.yaml|"browsers"' + # PATHS: | + # '../.cache/Cypress' + # '../.cache/ms-playwright' + # BASE_BRANCH: 'main' + - name: Install Node Modules + uses: 'nrwl/nx-cloud-workflows/v3.5/workflow-steps/install-node-modules/main.yaml' + #- name: Install Browsers (if needed) + # uses: 'nrwl/nx-cloud-workflows/v3.5/workflow-steps/install-browsers/main.yaml' + - name: Install Flutter + script: | + flutterVersion="${FLUTTER_VERSION:-3.19.3}" + flutterChannel="${FLUTTER_CHANNEL:-stable}" + + flutterPkgName="flutter_linux_${flutterVersion}-${flutterChannel}.tar.xz" + flutterPkgUrl="https://storage.googleapis.com/flutter_infra_release/releases/${flutterChannel}/linux/${flutterPkgName}" + + echo "⬇️ Downloading and Installing Flutter from $flutterPkgName..." + curl -s -L ${flutterPkgUrl} | tar -xJf - -C $HOME + echo 'export PATH="$PATH:$HOME/flutter/bin"' >> $HOME/.bashrc + source $HOME/.bashrc + echo "✅ Done !" + + # uses: 'tinesoft/nxrocks/chore/nx-cloud/.nx/workflow-steps/install-flutter/main.yaml' + - name: Install Java + script: | + echo "⌛️ Installing required tools (zip, unzip, ...)" + sudo apt-get update + sudo apt-get install zip unzip -y + + echo "⬇️ Downloading and Installing SDKMan!..." + curl -s "https://get.sdkman.io" | bash + source $HOME/.sdkman/bin/sdkman-init.sh + + javaVersion="${JAVA_VERSION:-17.0.10}" + javaDistro="${JAVA_DISTRO:-ms}" + javaVersionFull="${javaVersion}-${javaDistro}" + + echo "⌛️ Installing Java v$javaVersionFull..." + sdk install java $javaVersionFull + sdk use java $javaVersionFull + + echo "Checking Java Installation and Version..." + java -version + echo "✅ Done !" + # uses: 'tinesoft/nxrocks/chore/nx-cloud/.nx/workflow-steps/install-java/main.yaml' diff --git a/nx.json b/nx.json index 001d5e37..b8d2aabc 100644 --- a/nx.json +++ b/nx.json @@ -12,7 +12,7 @@ "changelog": { "projectChangelogs": { "createRelease": "github" - } + } }, "version": { "conventionalCommits": true @@ -60,9 +60,9 @@ "nx-release-publish": { "dependsOn": ["build"], "options": { - "packageRoot": "{workspaceRoot}/dist/packages/{projectName}" + "packageRoot": "{workspaceRoot}/dist/packages/{projectName}" } - } + } }, "namedInputs": { "default": ["{projectRoot}/**/*", "sharedGlobals"], @@ -80,7 +80,7 @@ "!{projectRoot}/src/test-setup.[jt]s" ] }, - "nxCloudAccessToken": "OTMwZTE2ZDgtODg2NS00Y2M1LTg0OTMtNGU3ZmI0NzY4YTUyfHJlYWQtd3JpdGU=", + "nxCloudAccessToken": "ZDRmZDczZWUtMDU2My00YzQ0LTlkMWEtZjM2YjZlZWNmNGIxfHJlYWQ=", "parallel": 1, "plugins": [] } diff --git a/package.json b/package.json index 7fb7c352..f50c5391 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "affected:test": "nx affected --target test --exclude=smoke", "affected:lint": "nx affected --target lint", "affected:graph": "nx affected --graph", + "affected:format": "nx affected --target format --exclude=smoke", "format": "nx format:write", "format:write": "nx format:write", "format:check": "nx format:check", @@ -93,4 +94,3 @@ "includedScripts": [] } } -