diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48197265..783ad3a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,18 +36,23 @@ jobs: - name: Test on Wokwi working-directory: test - run: pytest test_hello_world.py + run: pytest test_hello_world.py --report-log test_hello_world.json env: IDF_VERSION: ${{ matrix.idf_version }} WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }} + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + name: test-hello-world-${{ matrix.idf_version }}-results + path: test/test_hello_world.json + run-tests: name: test-${{ matrix.test_name }} runs-on: ubuntu-latest strategy: matrix: test_name: - - hello_world - psram - crypto - adc @@ -77,6 +82,26 @@ jobs: - name: Test on Wokwi working-directory: test - run: pytest test_${{ matrix.test_name }}.py + run: pytest test_${{ matrix.test_name }}.py --report-log test_${{ matrix.test_name }}.json env: WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }} + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-${{ matrix.test_name }}-results + path: test/test_${{ matrix.test_name }}.json + + report: + needs: [test-hello-world, run-tests] + runs-on: ubuntu-latest + if: always() + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: test + merge-multiple: true + - name: Generate test report + run: python test/generate_report.py >> $GITHUB_STEP_SUMMARY diff --git a/test/.gitignore b/test/.gitignore index 375de919..31e0be67 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,2 +1,3 @@ __pycache__ .pytest_cache +test_*.json diff --git a/test/generate_report.py b/test/generate_report.py new file mode 100644 index 00000000..ef2dab82 --- /dev/null +++ b/test/generate_report.py @@ -0,0 +1,95 @@ +import glob +import json +import os +import re + + +def extract_test_results(report_file: str): + counts = {} + current_device = None + with open(report_file, "r") as f: + for line in f: + try: + data = json.loads(line.strip()) + except json.JSONDecodeError: + continue + + if data.get("$report_type") == "TestReport": + location = data.get("location") + if location: + current_device = location[-1].split("[")[1].split("]")[0] + + if ( + data.get("$report_type") == "TestReport" + and data.get("when") == "teardown" + ): + sections = data.get("sections", []) + stdout_section = next( + filter(lambda x: x[0] == "Captured stdout call", sections), None + ) + if not stdout_section: + continue + + stdout = stdout_section[1] + match = re.search( + r"\r\n(\d+) Tests (\d+) Failures (\d+) Ignored", stdout + ) + if match: + total = int(match.group(1)) + fail = int(match.group(2)) + skip = int(match.group(3)) + passed = total - fail - skip + else: + total = 1 + fail = int(data.get("outcome") == "failed") + skip = int(data.get("outcome") == "skipped") + passed = int(data.get("outcome") == "passed") + summary = [] + if passed: + summary.append(f"✅ {passed}") + if fail: + summary.append(f"❌ {fail}") + if skip: + summary.append(f"⏭️ {skip}") + counts[current_device] = { + "total": total, + "fail": fail, + "skip": skip, + "summary": " ".join(summary), + } + return counts + + +if __name__ == "__main__": + devices = set() + reports = [] + total = 0 + failed = 0 + for file in glob.glob(f"{os.path.dirname(__file__)}/test_*.json"): + if file.endswith(".json"): + report = extract_test_results(file) + devices.update(report.keys()) + test_name = os.path.basename(file).replace(".json", "") + reports.append([test_name, report]) + total += max( + [0] + [report.get(device, {}).get("total", 0) for device in devices] + ) + failed += max( + [0] + [report.get(device, {}).get("fail", 0) for device in devices] + ) + + devices = sorted(devices) + print(f"# Wokwi ESP-IDF Test Report") + print(f"") + print(f"Total tests: {total}") + print(f"Failed tests: {failed}") + print(f"") + print(f"| Test | {' | '.join(devices)} |") + print(f"| ---- | {' | '.join(['---' for _ in devices])} |") + + # Sort reports by name + reports.sort(key=lambda x: x[0]) + for name, report in reports: + print( + f"| {name} | {' | '.join([str(report.get(device, {}).get('summary', '')) for device in devices])} |" + ) diff --git a/test/requirements.txt b/test/requirements.txt index c75731e6..dc691087 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1 +1,2 @@ pytest==8.2.2 +pytest-reportlog==0.4.0