[WIP] add eval action

.github/workflows/eval.yml

@@ -0,0 +1,88 @@
+name: Check eval on pull request
+
+on:
+  - workflow_dispatch
+  - push
+
+permissions:
+  contents: read
+
+jobs:
+  tests:
+    name: eval-check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        system: [x86_64-linux, aarch64-linux, aarch64-darwin, x86_64-darwin]
+    steps:
+      - name: Set up Git
+        run: |
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "GitHub Actions"
+
+      - name: Check if Branch is Part of an Open Pull Request
+        id: check_pr
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Extract the current branch name (removes "refs/heads/" from GITHUB_REF)
+          BRANCH_NAME="${GITHUB_REF#refs/heads/}"
+          # Get all open pull requests where the head is this branch in the user's fork
+          PR_INFO=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
+            "https://api.github.com/repos/NixOS/nixpkgs/pulls?head=${{ github.actor }}:${BRANCH_NAME}")
+          if [[ "$PR_INFO" == "[]" ]]; then
+            echo "No open pull request found for this branch."
+            echo "is_pr=false" >> $GITHUB_ENV
+          else
+            echo "This branch is part of an open pull request."
+            PR_NUMBER=$(echo "$PR_INFO" | jq -r '.[0].number')
+            echo "Pull Request #${PR_NUMBER} detected."
+            echo "is_pr=true" >> $GITHUB_ENV
+            echo "pr_number=${PR_NUMBER}" >> $GITHUB_ENV
+          fi
+
+      - name: Wait for PR Merge Commit Ref to Become Available
+        if: env.is_pr == 'true'
+        run: |
+          MAX_ATTEMPTS=10
+          SLEEP_TIME=10
+          ATTEMPT=1
+          while [[ $ATTEMPT -le $MAX_ATTEMPTS ]]; do
+            MERGE_REF=$(git ls-remote https://github.com/NixOS/nixpkgs "refs/pull/${{ env.pr_number }}/merge")
+            if [[ -n "$MERGE_REF" ]]; then
+              echo "Merge commit ref is available."
+              break
+            fi
+            echo "Merge commit ref not available yet. Attempt $ATTEMPT/$MAX_ATTEMPTS. Waiting $SLEEP_TIME seconds..."
+            sleep $SLEEP_TIME
+            ((ATTEMPT++))
+          done
+          if [[ $ATTEMPT -gt $MAX_ATTEMPTS ]]; then
+            echo "Merge commit ref did not become available in time."
+            exit 1
+          fi
+
+      - name: Checkout PR Merge Commit
+        if: env.is_pr == 'true'
+        run: |
+          git clone --depth 1 https://github.com/NixOS/nixpkgs.git nixpkgs
+          cd nixpkgs
+          git fetch --depth 1 https://github.com/NixOS/nixpkgs.git pull/${{ env.pr_number }}/merge
+          git checkout FETCH_HEAD
+
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+        if: env.is_pr == 'true'
+
+      - name: Enable swap
+        if: env.is_pr == 'true'
+        run: |
+          sudo fallocate -l 10G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+
+      - name: Check eval
+        if: env.is_pr == 'true'
+        run: |
+          cd nixpkgs
+          ./ci/eval-nixpkgs.sh --system "${{matrix.system}}"

ci/eval-nixpkgs.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+system="x86_64-linux"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+NIXPKGS_PATH="$(readlink -f $SCRIPT_DIR/..)"
+
+parseArgs() {
+  while [[ $# -gt 0 ]]; do
+    case $1 in
+      --system)
+        system=$2
+        shift 2
+        ;;
+      *)
+        echo "Unknown argument: $1"
+        exit 1
+        ;;
+    esac
+  done
+}
+
+main() {
+    parseArgs "$@"
+    tmpdir=$(mktemp -d)
+    trap 'rm -rf "$tmpdir"' EXIT
+    (
+      set +e
+      nix-env \
+        --arg "supportedSystems" "[\"$system\"]" \
+        -qaP --no-name \
+        --out-path \
+        --arg checkMeta true \
+        --argstr path "$NIXPKGS_PATH" \
+        -f "$SCRIPT_DIR/outpaths.nix" > "$tmpdir/paths"
+      echo $? > "$tmpdir/exit-code"
+    ) &
+    pid=$!
+    while kill -0 "$pid"; do
+        free -g
+        sleep 20
+    done
+    exit "$(cat "$tmpdir/exit-code")"
+}
+
+main "$@"

ci/outpaths.nix

@@ -0,0 +1,72 @@
+#!/usr/bin/env nix-shell
+# When using as a callable script, passing `--argstr path some/path` overrides $PWD.
+#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path --arg checkMeta true --argstr path $PWD -f"
+{
+  checkMeta,
+  path ? ./..,
+  supportedSystems ? [
+    "aarch64-linux"
+    "aarch64-darwin"
+    "x86_64-linux"
+    "x86_64-darwin"
+  ],
+}:
+let
+  lib = import (path + "/lib");
+  hydraJobs =
+    import (path + "/pkgs/top-level/release.nix")
+      # Compromise: accuracy vs. resources needed for evaluation.
+      {
+        inherit supportedSystems;
+
+        nixpkgsArgs = {
+          config = {
+            allowAliases = false;
+            allowBroken = true;
+            allowUnfree = true;
+            allowInsecurePredicate = x: true;
+            checkMeta = checkMeta;
+
+            handleEvalIssue =
+              reason: errormsg:
+              let
+                fatalErrors = [
+                  "unknown-meta"
+                  "broken-outputs"
+                ];
+              in
+              if builtins.elem reason fatalErrors then abort errormsg else true;
+
+            inHydra = true;
+          };
+        };
+      };
+  recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
+
+  # hydraJobs leaves recurseForDerivations as empty attrmaps;
+  # that would break nix-env and we also need to recurse everywhere.
+  tweak = lib.mapAttrs (
+    name: val:
+    if name == "recurseForDerivations" then
+      true
+    else if lib.isAttrs val && val.type or null != "derivation" then
+      recurseIntoAttrs (tweak val)
+    else
+      val
+  );
+
+  # Some of these contain explicit references to platform(s) we want to avoid;
+  # some even (transitively) depend on ~/.nixpkgs/config.nix (!)
+  blacklist = [
+    "tarball"
+    "metrics"
+    "manual"
+    "darwin-tested"
+    "unstable"
+    "stdenvBootstrapTools"
+    "moduleSystem"
+    "lib-tests" # these just confuse the output
+  ];
+
+in
+tweak (builtins.removeAttrs hydraJobs blacklist)