From 7892824416ed9c0a23be6012f773e2330a942f01 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 4 Apr 2025 14:39:28 +0200 Subject: [PATCH 01/12] Enable Github summary report --- .github/workflows/report.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 92b6348..2c72b71 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -48,12 +48,12 @@ jobs: name: metrics.json path: metrics.json - # - name: Generate GitHub Summary - # run: | - # echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY - # poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY - # poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY - # echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY - # poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY - # poetry run tbx security pretty-print >> $GITHUB_STEP_SUMMARY - # poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY + - name: Generate GitHub Summary + run: | + echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY + poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY + poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY + echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY + poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY + poetry run tbx security pretty-print >> $GITHUB_STEP_SUMMARY + poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY From 9209527919c2aee32a669e2237d0f4d7ddc8e8d0 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 16:55:03 +0200 Subject: [PATCH 02/12] Test fix for PTB --- debug_python_toolbox.py | 68 +++++++++++++++++++++++++++++++++++++++++ noxfile.py | 3 ++ 2 files changed, 71 insertions(+) create mode 100644 debug_python_toolbox.py diff --git a/debug_python_toolbox.py b/debug_python_toolbox.py new file mode 100644 index 0000000..6cfd4ff --- /dev/null +++ b/debug_python_toolbox.py @@ -0,0 +1,68 @@ +import nox + +# imports from PTB +from typing import Optional, Union +import datetime +from pathlib import Path +from exasol.toolbox.metrics import Report +import json +from tempfile import TemporaryDirectory +from nox import Session + +# new imports +import subprocess +import sys + +def total_coverage(file: Union[str, Path]) -> float: + with TemporaryDirectory() as tmpdir: + tmp_dir = Path(tmpdir) + report = tmp_dir / "coverage.json" + p = subprocess.run( + ["coverage", "json", f"--data-file={file}", "-o", f"{report}"], + capture_output=True, + check=False, + encoding="utf-8", + ) + stdout = p.stdout.strip() + if (p.returncode == 1) and (stdout == "No data to report."): + print( + f'The following command' + f' returned non-zero exit status {p.returncode}:\n' + f' {" ".join(p.args)}\n' + f'{stdout}\n' + 'Returning total coverage 100 %.', + file=sys.stderr, + ) + return 100.0 + with open(report, encoding="utf-8") as r: + data = json.load(r) + total: float = data["totals"]["percent_covered"] + + return total + + +def create_report( + commit: str, + date: Optional[datetime.datetime] = None, + coverage_report: Union[str, Path] = ".coverage", + pylint_report: Union[str, Path] = ".lint.txt", + bandit_report: Union[str, Path] = ".security.json", +) -> Report: + return total_coverage(coverage_report) + return Report( + commit=commit, + date=date if date is not None else datetime.datetime.now(), + coverage=total_coverage(coverage_report), + maintainability=maintainability(pylint_report), + reliability=reliability(), + security=security(bandit_report), + technical_debt=technical_debt(), + ) + + +@nox.session(name="project:report", python=False) +def report(session: Session) -> None: + sha1 = str( + session.run("git", "rev-parse", "HEAD", external=True, silent=True) + ).strip() + project_report = create_report(commit=sha1) diff --git a/noxfile.py b/noxfile.py index 176dbf9..b0165fa 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,3 +5,6 @@ # default actions to be run if nothing is explicitly specified with the -s option nox.options.sessions = ["project:fix"] + +from debug_python_toolbox import report + From b5acea2f8c97ba94cb077d9a536891ec250ae28b Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 16:56:40 +0200 Subject: [PATCH 03/12] isort --- debug_python_toolbox.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/debug_python_toolbox.py b/debug_python_toolbox.py index 6cfd4ff..30f666f 100644 --- a/debug_python_toolbox.py +++ b/debug_python_toolbox.py @@ -1,17 +1,22 @@ -import nox - -# imports from PTB -from typing import Optional, Union import datetime -from pathlib import Path -from exasol.toolbox.metrics import Report import json -from tempfile import TemporaryDirectory -from nox import Session # new imports import subprocess import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +# imports from PTB +from typing import ( + Optional, + Union, +) + +import nox +from exasol.toolbox.metrics import Report +from nox import Session + def total_coverage(file: Union[str, Path]) -> float: with TemporaryDirectory() as tmpdir: From 427aaca3107788ebb74f10d55559200e1c85126e Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 16:58:00 +0200 Subject: [PATCH 04/12] project:fix --- debug_python_toolbox.py | 8 ++++---- noxfile.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/debug_python_toolbox.py b/debug_python_toolbox.py index 30f666f..b525bf0 100644 --- a/debug_python_toolbox.py +++ b/debug_python_toolbox.py @@ -31,11 +31,11 @@ def total_coverage(file: Union[str, Path]) -> float: stdout = p.stdout.strip() if (p.returncode == 1) and (stdout == "No data to report."): print( - f'The following command' - f' returned non-zero exit status {p.returncode}:\n' + f"The following command" + f" returned non-zero exit status {p.returncode}:\n" f' {" ".join(p.args)}\n' - f'{stdout}\n' - 'Returning total coverage 100 %.', + f"{stdout}\n" + "Returning total coverage 100 %.", file=sys.stderr, ) return 100.0 diff --git a/noxfile.py b/noxfile.py index b0165fa..3356542 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,4 +7,3 @@ nox.options.sessions = ["project:fix"] from debug_python_toolbox import report - From bee962a09ed8627b9d2c792a4a4c5ada54057015 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 17:00:50 +0200 Subject: [PATCH 05/12] Fixed linter errors --- debug_python_toolbox.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debug_python_toolbox.py b/debug_python_toolbox.py index b525bf0..e363703 100644 --- a/debug_python_toolbox.py +++ b/debug_python_toolbox.py @@ -14,7 +14,13 @@ ) import nox -from exasol.toolbox.metrics import Report +from exasol.toolbox.metrics import ( + Report, + maintainability, + reliability, + security, + technical_debt, +) from nox import Session @@ -53,7 +59,6 @@ def create_report( pylint_report: Union[str, Path] = ".lint.txt", bandit_report: Union[str, Path] = ".security.json", ) -> Report: - return total_coverage(coverage_report) return Report( commit=commit, date=date if date is not None else datetime.datetime.now(), From 3a198a8d1863ba2efce65abd657629d265248ef9 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 17:10:26 +0200 Subject: [PATCH 06/12] Fixed github Workflow --- .github/workflows/report.yml | 1 - .gitignore | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 2c72b71..2202d77 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -55,5 +55,4 @@ jobs: poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY - poetry run tbx security pretty-print >> $GITHUB_STEP_SUMMARY poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index fdfb6de..5424663 100644 --- a/.gitignore +++ b/.gitignore @@ -142,4 +142,9 @@ doc/api itde/ # Emacs -TAGS +/TAGS + +# PTB +/.lint.json +/.lint.txt +/.security.json From 08edcffd92ccb2dd12c28324d4f0aaf2fac5c11c Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 17:13:23 +0200 Subject: [PATCH 07/12] Fixed github Workflow --- .github/workflows/report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 2202d77..96c7786 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -55,4 +55,5 @@ jobs: poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY + poetry run -- tbx lint pretty-print >> $GITHUB_STEP_SUMMARY poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY From 740ae0a78564ad68f01c7c3a6795b5c540e68af4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 17:21:11 +0200 Subject: [PATCH 08/12] Ignore status of reporting coverage to summary --- .github/workflows/report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 96c7786..0e17fe5 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -54,6 +54,6 @@ jobs: poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY - poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY + poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY || true poetry run -- tbx lint pretty-print >> $GITHUB_STEP_SUMMARY poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY From 1979e988320d6f9030d9b51953d544b15ff670f4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 17:26:57 +0200 Subject: [PATCH 09/12] Disabled lint report --- .github/workflows/report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 0e17fe5..6fceff1 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -55,5 +55,5 @@ jobs: poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY || true - poetry run -- tbx lint pretty-print >> $GITHUB_STEP_SUMMARY + # poetry run -- tbx lint pretty-print >> $GITHUB_STEP_SUMMARY poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY From 7aeb908d42fe47e29f496e00c47b14874327151a Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 18:22:08 +0200 Subject: [PATCH 10/12] Fixed copying artifacts --- .github/workflows/report.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 6fceff1..7fa182e 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -33,7 +33,7 @@ jobs: # Errors during copying are ignored because they are checked in the next step cp .coverage ../ || true cp lint-python3.9/.lint.txt ../ || true - cp lint-python3.9/.lint.txt ../ || true + cp lint-python3.9/.lint.json ../ || true cp security-python3.9/.security.json ../ || true - name: Validate Artifacts @@ -51,9 +51,9 @@ jobs: - name: Generate GitHub Summary run: | echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY - poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY - poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY + poetry run -- nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY + poetry run -- nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY || true - # poetry run -- tbx lint pretty-print >> $GITHUB_STEP_SUMMARY - poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY + poetry run -- tbx lint pretty-print >> $GITHUB_STEP_SUMMARY + poetry run -- tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY From 1258dd5dc221d2198dbcd6ea752ac411a144b243 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 19:07:03 +0200 Subject: [PATCH 11/12] Updated changelog c --- doc/changes/unreleased.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 65f8f82..f197bfb 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -4,3 +4,14 @@ * #1: Added initial Project Setup + +Findings in `report.yml`, columns: +* DEVDOC: whether the resp. finding affects the repository developer-documentation, +* PTB: whether the finding affects the folder `.github/workflows/` in the PTB +* PTB Template: whether the finding affects the folder `exasol/toolbox/templates/github/workflows/` in the PTB + +| Location | Finding | DEVDOC | PTB | PTB Template | +|----------|---------|--------|-----|---| +| Step "Generate GitHub Summary", `poetry run -- coverage report --format markdown` | must ignore coverage error due to no data | Y | Y | Y | +| Step "Copy Artifacts into Root Folder" | copies 2x `.lint.txt` but must copy `.lint.json`, too | Y | - | Y | +| Step "Generate GitHub Summary" | contains 2x security, but must be 1x lint and 1x security | Y | - | - | From cee904b6e36933c21302a537b25b24b2dac94d6c Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 4 Apr 2025 19:08:20 +0200 Subject: [PATCH 12/12] Updated changelog --- doc/changes/unreleased.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index f197bfb..6aa9efb 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,9 +1,7 @@ # Unreleased -## Features - -* #1: Added initial Project Setup +## Summary Findings in `report.yml`, columns: * DEVDOC: whether the resp. finding affects the repository developer-documentation, @@ -15,3 +13,13 @@ Findings in `report.yml`, columns: | Step "Generate GitHub Summary", `poetry run -- coverage report --format markdown` | must ignore coverage error due to no data | Y | Y | Y | | Step "Copy Artifacts into Root Folder" | copies 2x `.lint.txt` but must copy `.lint.json`, too | Y | - | Y | | Step "Generate GitHub Summary" | contains 2x security, but must be 1x lint and 1x security | Y | - | - | + + +## Features + +* #1: Added initial Project Setup + +## Bug Fixes + +* Fixed findings described above +