ci: generate markdown test summary report

.github/workflows/test.yml

@@ -36,18 +36,23 @@ jobs:
 
       - name: Test on Wokwi
         working-directory: test
-        run: pytest test_hello_world.py
+        run: pytest test_hello_world.py --report-log test_hello_world.json
         env:
           IDF_VERSION: ${{ matrix.idf_version }}
           WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
 
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-hello-world-${{ matrix.idf_version }}-results
+          path: test/test_hello_world.json
+
   run-tests:
     name: test-${{ matrix.test_name }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         test_name:
-          - hello_world
           - psram
           - crypto
           - adc
@@ -77,6 +82,26 @@ jobs:
 
       - name: Test on Wokwi
         working-directory: test
-        run: pytest test_${{ matrix.test_name }}.py
+        run: pytest test_${{ matrix.test_name }}.py --report-log test_${{ matrix.test_name }}.json
         env:
           WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-${{ matrix.test_name }}-results
+          path: test/test_${{ matrix.test_name }}.json
+
+  report:
+    needs: [test-hello-world, run-tests]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          path: test
+          merge-multiple: true
+      - name: Generate test report
+        run: python test/generate_report.py >> $GITHUB_STEP_SUMMARY

test/.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 .pytest_cache
+test_*.json

test/generate_report.py

@@ -0,0 +1,95 @@
+import glob
+import json
+import os
+import re
+
+
+def extract_test_results(report_file: str):
+    counts = {}
+    current_device = None
+    with open(report_file, "r") as f:
+        for line in f:
+            try:
+                data = json.loads(line.strip())
+            except json.JSONDecodeError:
+                continue
+
+            if data.get("$report_type") == "TestReport":
+                location = data.get("location")
+                if location:
+                    current_device = location[-1].split("[")[1].split("]")[0]
+
+            if (
+                data.get("$report_type") == "TestReport"
+                and data.get("when") == "teardown"
+            ):
+                sections = data.get("sections", [])
+                stdout_section = next(
+                    filter(lambda x: x[0] == "Captured stdout call", sections), None
+                )
+                if not stdout_section:
+                    continue
+
+                stdout = stdout_section[1]
+                match = re.search(
+                    r"\r\n(\d+) Tests (\d+) Failures (\d+) Ignored", stdout
+                )
+                if match:
+                    total = int(match.group(1))
+                    fail = int(match.group(2))
+                    skip = int(match.group(3))
+                    passed = total - fail - skip
+                else:
+                    total = 1
+                    fail = int(data.get("outcome") == "failed")
+                    skip = int(data.get("outcome") == "skipped")
+                    passed = int(data.get("outcome") == "passed")
+                summary = []
+                if passed:
+                    summary.append(f"✅ {passed}")
+                if fail:
+                    summary.append(f"❌ {fail}")
+                if skip:
+                    summary.append(f"⏭️ {skip}")
+                counts[current_device] = {
+                    "total": total,
+                    "fail": fail,
+                    "skip": skip,
+                    "summary": " ".join(summary),
+                }
+    return counts
+
+
+if __name__ == "__main__":
+    devices = set()
+    reports = []
+    total = 0
+    failed = 0
+    for file in glob.glob(f"{os.path.dirname(__file__)}/test_*.json"):
+        if file.endswith(".json"):
+            report = extract_test_results(file)
+            devices.update(report.keys())
+            test_name = os.path.basename(file).replace(".json", "")
+            reports.append([test_name, report])
+            total += max(
+                [0] + [report.get(device, {}).get("total", 0) for device in devices]
+            )
+            failed += max(
+                [0] + [report.get(device, {}).get("fail", 0) for device in devices]
+            )
+
+    devices = sorted(devices)
+    print(f"# Wokwi ESP-IDF Test Report")
+    print(f"")
+    print(f"Total tests: {total}")
+    print(f"Failed tests: {failed}")
+    print(f"")
+    print(f"| Test | {' | '.join(devices)} |")
+    print(f"| ---- | {' | '.join(['---' for _ in devices])} |")
+
+    # Sort reports by name
+    reports.sort(key=lambda x: x[0])
+    for name, report in reports:
+        print(
+            f"| {name} | {' | '.join([str(report.get(device, {}).get('summary', '')) for device in devices])} |"
+        )

test/requirements.txt

@@ -1 +1,2 @@
 pytest==8.2.2
+pytest-reportlog==0.4.0