Check Echidna publishing results for failure or success (#1821)

dschuff · web-flow · commit 2d480da2357a · 2024-10-04T13:21:40.000-07:00
* The API call does not return its results synchronously, so the status check script polls for the result and exits with an error if it fails.
* The Makefiles and CI workflow introduce the ability to run the publishing step as a dry run, so it can be used with pull requests.
* CI also supports the ability to use W3C username and password secrets instead of Echidna tokens
diff --git a/.github/workflows/w3c-publish.yml b/.github/workflows/w3c-publish.yml
@@ -1,9 +1,11 @@
-name: Publish to W3C TR space
+name: Validate/Publish to W3C TR space
 
 on:
   push:
     branches: [ main ]
     paths: [ .github/**, document/** ]
+  pull_request:
+    paths: [ .github/**, document/** ]
 
   # Allows you to run this workflow manually from the Actions tab, gh CLI tool,
   # or REST API. THe w3c-status options correspond to the valid options for
@@ -23,9 +25,12 @@ on:
           - CRD
           - CR
 
+env:
+  YARN_ENABLE_IMMUTABLE_INSTALLS: false
+  W3C_STATUS: ${{ github.event_name == 'workflow_dispatch' && inputs.w3c-status || 'WD' }}
+
 jobs:
   publish-to-w3c-TR:
-    if: github.repository == 'WebAssembly/spec'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -43,10 +48,16 @@ jobs:
       - name: Setup Sphinx
         run: pip install six && pip install sphinx==5.1.0
       - name: Publish all specs to their https://www.w3.org/TR/ URLs
+        if: env.W3C_ECHIDNA_TOKEN_CORE
         run: cd document && make -e WD-echidna-CI
         env:
-          W3C_STATUS: ${{ github.event_name == 'push' && 'WD' || inputs.w3c-status }}
           W3C_ECHIDNA_TOKEN_CORE: ${{ secrets.W3C_ECHIDNA_TOKEN_CORE }}
           W3C_ECHIDNA_TOKEN_JSAPI: ${{ secrets.W3C_ECHIDNA_TOKEN_JSAPI }}
           W3C_ECHIDNA_TOKEN_WEBAPI: ${{ secrets.W3C_ECHIDNA_TOKEN_WEBAPI }}
-          YARN_ENABLE_IMMUTABLE_INSTALLS: false
+          ECHIDNA_DRYRUN: ${{ !(github.event_name == 'push' && github.repository == 'WebAssembly/spec' && github.ref == 'refs/heads/main') }}
+      - name: Validate all specs with Echidna
+        if: env.W3C_USERNAME
+        run: cd document && make -e WD-echidna
+        env:
+          W3C_USERNAME: ${{ secrets.W3C_USERNAME }}
+          W3C_PASSWORD: ${{ secrets.W3C_PASSWORD }}
diff --git a/document/core/Makefile b/document/core/Makefile
@@ -13,6 +13,7 @@ NAME          = WebAssembly
 DECISION_URL  = https://github.com/WebAssembly/meetings/blob/main/main/2024/WG-06-12.md
 TAR           = tar
 DEADLINE      = $(shell date -d "+30 days" +%Y-%m-%d 2>/dev/null || date -v +30d +%Y-%m-%d)
+ECHIDNA_DRYRUN = true
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
@@ -196,7 +197,9 @@ WD-echidna: WD-tar
 	curl 'https://labs.w3.org/echidna/api/request' \
 		--user '$(W3C_USERNAME):$(W3C_PASSWORD)' \
 		-F "tar=@$(BUILDDIR)/WD.tar" \
-		-F "decision=$(DECISION_URL)" | tee $(BUILDDIR)/WD-echidna-id.txt
+		-F "decision=$(DECISION_URL)" \
+		-F "dry-run=$(ECHIDNA_DRYRUN)" | tee $(BUILDDIR)/WD-echidna-id.txt
+	python3 ../util/check-echidna-status.py $(BUILDDIR)
 	@echo
 	@echo "Published $(W3C_STATUS). Check its status at https://labs.w3.org/echidna/api/status?id=`cat $(BUILDDIR)/WD-echidna-id.txt`"
 
@@ -210,7 +213,9 @@ WD-echidna-CI: WD-tar
 	curl 'https://labs.w3.org/echidna/api/request' \
 		-F "tar=@$(BUILDDIR)/WD.tar" \
 		-F "token=$(W3C_ECHIDNA_TOKEN_CORE)" \
-		-F "decision=$(DECISION_URL)" | tee $(BUILDDIR)/WD-echidna-id.txt
+		-F "decision=$(DECISION_URL)" \
+		-F "dry-run=$(ECHIDNA_DRYRUN)" | tee $(BUILDDIR)/WD-echidna-id.txt
+	python3 ../util/check-echidna-status.py $(BUILDDIR)
 	@echo
 	@echo "Published $(W3C_STATUS). Check its status at https://labs.w3.org/echidna/api/status?id=`cat $(BUILDDIR)/WD-echidna-id.txt`"
 
diff --git a/document/js-api/Makefile b/document/js-api/Makefile
@@ -6,6 +6,7 @@ NAME          = WebAssembly
 DECISION_URL  = https://github.com/WebAssembly/meetings/blob/main/main/2024/WG-06-12.md
 TAR           = tar
 DEADLINE      = $(shell date -d "+30 days" +%Y-%m-%d 2>/dev/null || date -v +30d +%Y-%m-%d)
+ECHIDNA_DRYRUN = true
 
 .PHONY: all
 all:
@@ -49,7 +50,11 @@ WD-echidna: WD-tar
 	curl 'https://labs.w3.org/echidna/api/request' \
 		--user '$(W3C_USERNAME):$(W3C_PASSWORD)' \
 		-F "tar=@$(BUILDDIR)/WD.tar" \
-		-F "decision=$(DECISION_URL)" | tee $(BUILDDIR)/WD-echidna-id.txt
+		-F "decision=$(DECISION_URL)" \
+		-F "dry-run=$(ECHIDNA_DRYRUN)" | tee $(BUILDDIR)/WD-echidna-id.txt
+	python3 ../util/check-echidna-status.py $(BUILDDIR)
+	@echo
+	@echo "Published $(W3C_STATUS). Check its status at https://labs.w3.org/echidna/api/status?id=`cat $(BUILDDIR)/WD-echidna-id.txt`"
 
 .PHONY: WD-echidna-CI
 WD-echidna-CI: WD-tar
@@ -61,6 +66,8 @@ WD-echidna-CI: WD-tar
 	curl 'https://labs.w3.org/echidna/api/request' \
 		-F "tar=@$(BUILDDIR)/WD.tar" \
 		-F "token=$(W3C_ECHIDNA_TOKEN_JSAPI)" \
-		-F "decision=$(DECISION_URL)" | tee $(BUILDDIR)/WD-echidna-id.txt
+		-F "decision=$(DECISION_URL)" \
+		-F "dry-run=$(ECHIDNA_DRYRUN)" | tee $(BUILDDIR)/WD-echidna-id.txt
+	python3 ../util/check-echidna-status.py $(BUILDDIR)
 	@echo
 	@echo "Published $(W3C_STATUS). Check its status at https://labs.w3.org/echidna/api/status?id=`cat $(BUILDDIR)/WD-echidna-id.txt`"
diff --git a/document/util/check-echidna-status.py b/document/util/check-echidna-status.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+from datetime import datetime, timedelta
+import json
+import os
+import requests
+import sys
+import time
+
+ECHIDNA_ID_FILE = 'WD-echidna-id.txt'
+ECHIDNA_STATUS_URL = 'http://labs.w3.org/echidna/api/status?id='
+
+
+def get_echidna_id(directory):
+    id_file = os.path.join(directory, ECHIDNA_ID_FILE)
+    file_time = os.path.getmtime(id_file)
+    if datetime.fromtimestamp(file_time) < datetime.now() - timedelta(hours=1):
+        print(f'Warning: Echidna ID is not recent: timestamp is {file_time}')
+    with open(id_file, 'r') as f:
+        return f.read().strip()
+
+
+def get_current_response(echidna_id):
+    url = ECHIDNA_STATUS_URL + echidna_id
+    print(f'Fetching {url}')
+    response = requests.get(url, allow_redirects=True)
+    if response.status_code != 200:
+        print(f'Got status code {response.status_code}, text:')
+        print(response.text)
+        raise Exception('Failed to fetch echidna result')
+
+    return response.json()
+
+
+def get_echidna_result(echidna_id):
+    response = get_current_response(echidna_id)
+    while response['results']['status'] == 'started':
+        time.sleep(5)
+        print('Echidna run in progress, retrying...')
+        response = get_current_response(echidna_id)
+
+    result = response['results']['status']
+    print(f'Echidna issue {echidna_id} is {result}.')
+    print(json.dumps(response, indent=2))
+    return result == 'success'
+
+
+def main(argv):
+    directory = os.getcwd() if len(argv) < 2 else argv[1]
+    echidna_id = get_echidna_id(directory)
+    print(f'Got echidna id {echidna_id}.')
+    time.sleep(5)
+    if not get_echidna_result(echidna_id):
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/document/web-api/Makefile b/document/web-api/Makefile
@@ -6,6 +6,7 @@ NAME          = WebAssembly
 DECISION_URL  = https://github.com/WebAssembly/meetings/blob/main/main/2024/WG-06-12.md
 TAR           = tar
 DEADLINE      = $(shell date -d "+30 days" +%Y-%m-%d 2>/dev/null || date -v +30d +%Y-%m-%d)
+ECHIDNA_DRYRUN = true
 
 .PHONY: all
 all:
@@ -49,7 +50,11 @@ WD-echidna: WD-tar
 	curl 'https://labs.w3.org/echidna/api/request' \
 		--user '$(W3C_USERNAME):$(W3C_PASSWORD)' \
 		-F "tar=@$(BUILDDIR)/WD.tar" \
-		-F "decision=$(DECISION_URL)" | tee $(BUILDDIR)/WD-echidna-id.txt
+		-F "decision=$(DECISION_URL)" \
+		-F "dry-run=$(ECHIDNA_DRYRUN)" | tee $(BUILDDIR)/WD-echidna-id.txt
+	python3 ../util/check-echidna-status.py $(BUILDDIR)
+	@echo
+	@echo "Published $(W3C_STATUS). Check its status at https://labs.w3.org/echidna/api/status?id=`cat $(BUILDDIR)/WD-echidna-id.txt`"
 
 .PHONY: WD-echidna-CI
 WD-echidna-CI: WD-tar
@@ -61,6 +66,8 @@ WD-echidna-CI: WD-tar
 	curl 'https://labs.w3.org/echidna/api/request' \
 		-F "tar=@$(BUILDDIR)/WD.tar" \
 		-F "token=$(W3C_ECHIDNA_TOKEN_WEBAPI)" \
-		-F "decision=$(DECISION_URL)" | tee $(BUILDDIR)/WD-echidna-id.txt
+		-F "decision=$(DECISION_URL)" \
+		-F "dry-run=$(ECHIDNA_DRYRUN)" | tee $(BUILDDIR)/WD-echidna-id.txt
+	python3 ../util/check-echidna-status.py $(BUILDDIR)
 	@echo
 	@echo "Published $(W3C_STATUS). Check its status at https://labs.w3.org/echidna/api/status?id=`cat $(BUILDDIR)/WD-echidna-id.txt`"
