test,tools: update wpt weekly

.github/workflows/update-wpt.yml

@@ -0,0 +1,78 @@
+# This workflow runs every Monday and updates the `test/fixtures/wpt`
+# to the `epochs/weekly` branch of WPT.
+
+name: Weekly WPT roller
+
+on:
+  workflow_dispatch:
+  schedule:
+    # This is 20 minutes after `epochs/weekly` branch is triggered to be created
+    # in WPT repo, every Monday.
+    # https://github.com/web-platform-tests/wpt/blob/master/.github/workflows/epochs.yml
+    - cron: 30 0 * * 1
+
+env:
+  PYTHON_VERSION: '3.11'
+  NODE_VERSION: lts/*
+
+permissions:
+  contents: read
+
+jobs:
+  update-wpt:
+    if: github.repository == 'nodejs/node' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
+        with:
+          persist-credentials: false
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Node.js
+        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65  # v4.0.0
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Install @node-core/utils
+        run: npm install -g @node-core/utils
+      - name: Configure @node-core/utils
+        run: |
+          ncu-config set branch ${GITHUB_REF_NAME}
+          ncu-config set upstream origin
+          ncu-config set username "${{ secrets.GH_USER_NAME }}"
+          ncu-config set token "${{ secrets.GH_USER_TOKEN }}"
+          ncu-config set repo "$(echo ${{ github.repository }} | cut -d/ -f2)"
+          ncu-config set owner "${{ github.repository_owner }}"
+
+      - name: Environment Information
+        run: npx envinfo
+
+      - name: Set env.WPT_REVISION
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          WPT_REVISION=$(gh api /repos/web-platform-tests/wpt/branches/epochs/weekly --jq '.commit.sha')
+          echo "WPT_REVISION=$WPT_REVISION" >> $GITHUB_ENV
+          echo "WPT_SHORT_REVISION=$(echo $WPT_REVISION | cut -c 1-10)" >> $GITHUB_ENV
+      - name: Rolling update wpt fixtures
+        run: ./tools/dep_updaters/update-wpt.sh
+
+      - name: Build
+        run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
+      - name: Update wpt status.json
+        run: make test-wpt-status-update
+
+      - uses: gr2m/create-or-update-pull-request-action@77596e3166f328b24613f7082ab30bf2d93079d5
+        # Creates a PR or update the Action's existing PR, or
+        # no-op if the base branch is already up-to-date.
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_USER_TOKEN }}
+        with:
+          author: Node.js GitHub Bot <[email protected]>
+          body: This is an automated patch update of wpt to https://github.com/web-platform-tests/wpt/commit/${{ env.WPT_REVISION }}.
+          branch: actions/update-wpt  # Custom branch *just* for this Action.
+          commit-message: 'deps: update wpt to ${{ env.WPT_SHORT_REVISION }}'
+          labels: test
+          title: 'deps: update wpt to ${{ env.WPT_SHORT_REVISION }}'
+          update-pull-request-title-and-body: true

Makefile

@@ -595,6 +595,10 @@ test-wpt-report:
 	-WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
 	$(NODE) "$$PWD/tools/merge-wpt-reports.mjs"
 
+.PHONY: test-wpt-status-update
+test-wpt-status-update:
+	-WPT_UPDATE_STATUS=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
+
 .PHONY: test-internet
 test-internet: all
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) internet

test/common/wpt.js

@@ -59,10 +59,16 @@ function codeUnitStr(char) {
 }
 
 class ReportResult {
-  constructor(name) {
+  /**
+   * Construct a ReportResult.
+   * @param {string} name test name shown in wpt.fyi, unique for every variants.
+   * @param {WPTTestSpec} spec
+   */
+  constructor(name, spec) {
     this.test = name;
     this.status = 'OK';
     this.subtests = [];
+    this.spec = spec;
   }
 
   addSubtest(name, status, message) {
@@ -82,14 +88,18 @@ class ReportResult {
   finish(status) {
     this.status = status ?? 'OK';
   }
+
+  toJSON() {
+    return {
+      test: this.test,
+      status: this.status,
+      subtests: this.subtests,
+    };
+  }
 }
 
-// Generates a report that can be uploaded to wpt.fyi.
-// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
-// for more details.
-class WPTReport {
-  constructor(path) {
-    this.filename = `report-${path.replaceAll('/', '-')}.json`;
+class Report {
+  constructor() {
     /** @type {Map<string, ReportResult>} */
     this.results = new Map();
     this.time_start = Date.now();
@@ -104,10 +114,21 @@ class WPTReport {
     if (this.results.has(name)) {
       return this.results.get(name);
     }
-    const result = new ReportResult(name);
+    const result = new ReportResult(name, spec);
     this.results.set(name, result);
     return result;
   }
+}
+
+// Generates a report that can be uploaded to wpt.fyi.
+// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
+// for more details.
+class WPTReport extends Report {
+  constructor(path) {
+    super();
+    this.filename = `report-${path.replaceAll('/', '-')}.json`;
+    this.time_start = Date.now();
+  }
 
   write() {
     this.time_end = Date.now();
@@ -139,6 +160,47 @@ class WPTReport {
   }
 }
 
+/**
+ * Update status files with the results of the tests.
+ */
+class WPTStatusUpdater extends Report {
+  constructor(testPath) {
+    super();
+    this.filepath = path.join('test/wpt/status', `${testPath}.json`);
+    this.statusFile = JSON.parse(fs.readFileSync(this.filepath, 'utf8'));
+  }
+
+  write() {
+    for (const result of this.results.values()) {
+      if (result.status !== 'OK') {
+        this.statusFile[result.spec.filename] ??= {
+          skip: 'WPT test auto rolling',
+        };
+        continue;
+      }
+      const failedSubtests = result.subtests.filter((it) => it.status !== 'PASS');
+      if (failedSubtests.length === 0) {
+        continue;
+      }
+      const testStatus = this.statusFile[result.spec.filename] ?? {};
+      const expectations = new Set(testStatus.fail?.expected);
+      failedSubtests.forEach((subtest) => {
+        expectations.add(subtest.name);
+      });
+      this.statusFile[result.spec.filename] = {
+        ...testStatus,
+        fail: {
+          ...(testStatus.fail ?? {}),
+          note: testStatus.fail?.note ?? 'WPT test auto rolling',
+          expected: [...expectations],
+        },
+      };
+    }
+
+    fs.writeFileSync(this.filepath, JSON.stringify(this.statusFile, null, 2) + '\n');
+  }
+}
+
 // https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
 // TODO: get rid of this half-baked harness in favor of the one
 // pulled from WPT
@@ -513,6 +575,8 @@ class WPTRunner {
 
     if (process.env.WPT_REPORT != null) {
       this.report = new WPTReport(path);
+    } else if (process.env.WPT_UPDATE_STATUS != null) {
+      this.report = new WPTStatusUpdater(path);
     }
   }
 
@@ -624,6 +688,7 @@ class WPTRunner {
     const run = limit(this.concurrency);
 
     for (const spec of queue) {
+      const reportResult = this.report?.getResult(spec);
       const content = spec.getContent();
       const meta = spec.getMeta(content);
 
@@ -632,14 +697,33 @@ class WPTRunner {
       const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
 
       // Scripts specified with the `// META: script=` header
-      const scriptsToRun = meta.script?.map((script) => {
-        const obj = {
-          filename: this.resource.toRealFilePath(relativePath, script),
-          code: this.resource.read(relativePath, script),
-        };
-        this.scriptsModifier?.(obj);
-        return obj;
-      }) ?? [];
+      let scriptsToRun;
+      try {
+        scriptsToRun = meta.script?.map((script) => {
+          const obj = {
+            filename: this.resource.toRealFilePath(relativePath, script),
+            code: this.resource.read(relativePath, script),
+          };
+          this.scriptsModifier?.(obj);
+          return obj;
+        }) ?? [];
+      } catch (err) {
+        // Generate a subtest failure for visibility if resources are missing.
+        // No need to record this synthetic failure with wpt.fyi.
+        this.fail(
+          spec,
+          {
+            status: NODE_UNCAUGHT,
+            name: 'Resource not found',
+            message: err.message,
+            stack: inspect(err),
+          },
+          kUncaught,
+        );
+        // Mark the whole test as failed in wpt.fyi report.
+        reportResult?.finish('ERROR');
+        continue;
+      }
       // The actual test
       const obj = {
         code: content,
@@ -667,7 +751,6 @@ class WPTRunner {
         this.inProgress.add(spec);
         this.workers.set(spec, worker);
 
-        const reportResult = this.report?.getResult(spec);
         worker.on('message', (message) => {
           switch (message.type) {
             case 'result':
@@ -776,6 +859,10 @@ class WPTRunner {
                   `${passed} passed, ${expectedFailures} expected failures,`,
                   `${failures.length} unexpected failures,`,
                   `${unexpectedPasses.length} unexpected passes`);
+      if (process.env.WPT_UPDATE_STATUS) {
+        // Exit the process normally if we are updating the status file.
+        return;
+      }
       if (failures.length > 0) {
         const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
         throw new Error(

tools/dep_updaters/update-wpt.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+# Shell script to update wpt test fixtures in the source tree to the most recent version.
+
+BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
+
+cd "$BASE_DIR"
+# If WPT_REVISION is empty, the latest revision will be used.
+# shellcheck disable=SC2154
+jq -r 'keys[]' "$BASE_DIR/test/fixtures/wpt/versions.json" | \
+  xargs -L 1 git node wpt --commit "$WPT_REVISION"