diff --git a/.github/workflows/update-wpt.yml b/.github/workflows/update-wpt.yml new file mode 100644 index 00000000000000..946e833c08bcf2 --- /dev/null +++ b/.github/workflows/update-wpt.yml @@ -0,0 +1,78 @@ +# This workflow runs every Monday and updates the `test/fixtures/wpt` +# to the `epochs/weekly` branch of WPT. + +name: Weekly WPT roller + +on: + workflow_dispatch: + schedule: + # This is 20 minutes after `epochs/weekly` branch is triggered to be created + # in WPT repo, every Monday. + # https://github.com/web-platform-tests/wpt/blob/master/.github/workflows/epochs.yml + - cron: 30 0 * * 1 + +env: + PYTHON_VERSION: '3.11' + NODE_VERSION: lts/* + +permissions: + contents: read + +jobs: + update-wpt: + if: github.repository == 'nodejs/node' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install Node.js + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + - name: Install @node-core/utils + run: npm install -g @node-core/utils + - name: Configure @node-core/utils + run: | + ncu-config set branch ${GITHUB_REF_NAME} + ncu-config set upstream origin + ncu-config set username "${{ secrets.GH_USER_NAME }}" + ncu-config set token "${{ secrets.GH_USER_TOKEN }}" + ncu-config set repo "$(echo ${{ github.repository }} | cut -d/ -f2)" + ncu-config set owner "${{ github.repository_owner }}" + + - name: Environment Information + run: npx envinfo + + - name: Set env.WPT_REVISION + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + WPT_REVISION=$(gh api /repos/web-platform-tests/wpt/branches/epochs/weekly --jq '.commit.sha') + echo "WPT_REVISION=$WPT_REVISION" >> $GITHUB_ENV + echo "WPT_SHORT_REVISION=$(echo $WPT_REVISION | cut -c 1-10)" >> $GITHUB_ENV + - name: Rolling update wpt fixtures + run: ./tools/dep_updaters/update-wpt.sh + + - name: Build + run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn" + - name: Update wpt status.json + run: make test-wpt-status-update + + - uses: gr2m/create-or-update-pull-request-action@77596e3166f328b24613f7082ab30bf2d93079d5 + # Creates a PR or update the Action's existing PR, or + # no-op if the base branch is already up-to-date. + env: + GITHUB_TOKEN: ${{ secrets.GH_USER_TOKEN }} + with: + author: Node.js GitHub Bot + body: This is an automated patch update of wpt to https://github.com/web-platform-tests/wpt/commit/${{ env.WPT_REVISION }}. + branch: actions/update-wpt # Custom branch *just* for this Action. + commit-message: 'deps: update wpt to ${{ env.WPT_SHORT_REVISION }}' + labels: test + title: 'deps: update wpt to ${{ env.WPT_SHORT_REVISION }}' + update-pull-request-title-and-body: true diff --git a/Makefile b/Makefile index b7871bf218572c..7726faec5b83aa 100644 --- a/Makefile +++ b/Makefile @@ -595,6 +595,10 @@ test-wpt-report: -WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt $(NODE) "$$PWD/tools/merge-wpt-reports.mjs" +.PHONY: test-wpt-status-update +test-wpt-status-update: + -WPT_UPDATE_STATUS=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt + .PHONY: test-internet test-internet: all $(PYTHON) tools/test.py $(PARALLEL_ARGS) internet diff --git a/test/common/wpt.js b/test/common/wpt.js index 34436639dc3185..1881ea424fb691 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -59,10 +59,16 @@ function codeUnitStr(char) { } class ReportResult { - constructor(name) { + /** + * Construct a ReportResult. + * @param {string} name test name shown in wpt.fyi, unique for every variants. + * @param {WPTTestSpec} spec + */ + constructor(name, spec) { this.test = name; this.status = 'OK'; this.subtests = []; + this.spec = spec; } addSubtest(name, status, message) { @@ -82,14 +88,18 @@ class ReportResult { finish(status) { this.status = status ?? 'OK'; } + + toJSON() { + return { + test: this.test, + status: this.status, + subtests: this.subtests, + }; + } } -// Generates a report that can be uploaded to wpt.fyi. -// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation -// for more details. -class WPTReport { - constructor(path) { - this.filename = `report-${path.replaceAll('/', '-')}.json`; +class Report { + constructor() { /** @type {Map} */ this.results = new Map(); this.time_start = Date.now(); @@ -104,10 +114,21 @@ class WPTReport { if (this.results.has(name)) { return this.results.get(name); } - const result = new ReportResult(name); + const result = new ReportResult(name, spec); this.results.set(name, result); return result; } +} + +// Generates a report that can be uploaded to wpt.fyi. +// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation +// for more details. +class WPTReport extends Report { + constructor(path) { + super(); + this.filename = `report-${path.replaceAll('/', '-')}.json`; + this.time_start = Date.now(); + } write() { this.time_end = Date.now(); @@ -139,6 +160,47 @@ class WPTReport { } } +/** + * Update status files with the results of the tests. + */ +class WPTStatusUpdater extends Report { + constructor(testPath) { + super(); + this.filepath = path.join('test/wpt/status', `${testPath}.json`); + this.statusFile = JSON.parse(fs.readFileSync(this.filepath, 'utf8')); + } + + write() { + for (const result of this.results.values()) { + if (result.status !== 'OK') { + this.statusFile[result.spec.filename] ??= { + skip: 'WPT test auto rolling', + }; + continue; + } + const failedSubtests = result.subtests.filter((it) => it.status !== 'PASS'); + if (failedSubtests.length === 0) { + continue; + } + const testStatus = this.statusFile[result.spec.filename] ?? {}; + const expectations = new Set(testStatus.fail?.expected); + failedSubtests.forEach((subtest) => { + expectations.add(subtest.name); + }); + this.statusFile[result.spec.filename] = { + ...testStatus, + fail: { + ...(testStatus.fail ?? {}), + note: testStatus.fail?.note ?? 'WPT test auto rolling', + expected: [...expectations], + }, + }; + } + + fs.writeFileSync(this.filepath, JSON.stringify(this.statusFile, null, 2) + '\n'); + } +} + // https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js // TODO: get rid of this half-baked harness in favor of the one // pulled from WPT @@ -513,6 +575,8 @@ class WPTRunner { if (process.env.WPT_REPORT != null) { this.report = new WPTReport(path); + } else if (process.env.WPT_UPDATE_STATUS != null) { + this.report = new WPTStatusUpdater(path); } } @@ -624,6 +688,7 @@ class WPTRunner { const run = limit(this.concurrency); for (const spec of queue) { + const reportResult = this.report?.getResult(spec); const content = spec.getContent(); const meta = spec.getMeta(content); @@ -632,14 +697,33 @@ class WPTRunner { const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js'); // Scripts specified with the `// META: script=` header - const scriptsToRun = meta.script?.map((script) => { - const obj = { - filename: this.resource.toRealFilePath(relativePath, script), - code: this.resource.read(relativePath, script), - }; - this.scriptsModifier?.(obj); - return obj; - }) ?? []; + let scriptsToRun; + try { + scriptsToRun = meta.script?.map((script) => { + const obj = { + filename: this.resource.toRealFilePath(relativePath, script), + code: this.resource.read(relativePath, script), + }; + this.scriptsModifier?.(obj); + return obj; + }) ?? []; + } catch (err) { + // Generate a subtest failure for visibility if resources are missing. + // No need to record this synthetic failure with wpt.fyi. + this.fail( + spec, + { + status: NODE_UNCAUGHT, + name: 'Resource not found', + message: err.message, + stack: inspect(err), + }, + kUncaught, + ); + // Mark the whole test as failed in wpt.fyi report. + reportResult?.finish('ERROR'); + continue; + } // The actual test const obj = { code: content, @@ -667,7 +751,6 @@ class WPTRunner { this.inProgress.add(spec); this.workers.set(spec, worker); - const reportResult = this.report?.getResult(spec); worker.on('message', (message) => { switch (message.type) { case 'result': @@ -776,6 +859,10 @@ class WPTRunner { `${passed} passed, ${expectedFailures} expected failures,`, `${failures.length} unexpected failures,`, `${unexpectedPasses.length} unexpected passes`); + if (process.env.WPT_UPDATE_STATUS) { + // Exit the process normally if we are updating the status file. + return; + } if (failures.length > 0) { const file = path.join('test', 'wpt', 'status', `${this.path}.json`); throw new Error( diff --git a/tools/dep_updaters/update-wpt.sh b/tools/dep_updaters/update-wpt.sh new file mode 100755 index 00000000000000..2c5c5ff171a032 --- /dev/null +++ b/tools/dep_updaters/update-wpt.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e +# Shell script to update wpt test fixtures in the source tree to the most recent version. + +BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd) + +cd "$BASE_DIR" +# If WPT_REVISION is empty, the latest revision will be used. +# shellcheck disable=SC2154 +jq -r 'keys[]' "$BASE_DIR/test/fixtures/wpt/versions.json" | \ + xargs -L 1 git node wpt --commit "$WPT_REVISION"