Make pytest_upload_assets more xdist-compatible

ref: eu-cdse/openeo-cdse-infra#566

Author: soxofaan

qa/tools/apex_algorithm_qa_tools/pytest/pytest_upload_assets.py

@@ -47,6 +47,7 @@ def test_dummy(upload_assets_on_fail, tmp_path):
 _log = logging.getLogger(__name__)
 
 _UPLOAD_ASSETS_PLUGIN_NAME = "upload_assets"
+_USER_PROPERTY = "upload_assets"
 
 
 def pytest_addoption(parser: pytest.Parser):
@@ -94,6 +95,9 @@ def collect(self, path: Path, name: str):
         """Collect assets to upload"""
         assert self.collected_assets is not None, "No active collection of assets"
         self.collected_assets[name] = path
+        # TODO: this stat won't work properly in pytest-xdist context
+        #       because incrementing stat happens on xdist workers,
+        #       while summary is done from xdist controller).
         self.upload_stats["collected"] += 1
 
     def pytest_runtest_logstart(self, nodeid):
@@ -103,26 +107,40 @@ def pytest_runtest_logstart(self, nodeid):
     def pytest_runtest_logreport(self, report: pytest.TestReport):
         # TODO #22: option to upload on other outcome as well?
         if report.when == "call" and report.outcome == "failed":
-            self._upload_collected_assets(nodeid=report.nodeid)
+            self._upload_collected_assets(nodeid=report.nodeid, report=report)
+
+        # Merge stats
+        for upload_info in (
+            v for k, v in report.user_properties if k == _USER_PROPERTY
+        ):
+            for k, v in upload_info["stats"].items():
+                self.upload_stats[k] += v
+            if upload_info["uploads"]:
+                self.upload_reports[report.nodeid] = upload_info["uploads"]
 
     def pytest_runtest_logfinish(self, nodeid):
         # Reset collection of assets
         self.collected_assets = None
 
-    def _upload_collected_assets(self, nodeid: str):
-        upload_report = {}
+    def _upload_collected_assets(self, nodeid: str, report: pytest.TestReport):
+        upload_info = {
+            "stats": {"uploaded": 0, "failed": 0},
+            "uploads": {},
+        }
         for name, path in self.collected_assets.items():
             try:
                 url = self._upload_asset(nodeid=nodeid, name=name, path=path)
-                upload_report[name] = {"url": url}
-                self.upload_stats["uploaded"] += 1
+                upload_info["uploads"][name] = {"url": url}
+                upload_info["stats"]["uploaded"] += 1
             except Exception as e:
                 _log.error(
                     f"Failed to upload asset {name=} from {path=}: {e=}", exc_info=True
                 )
-                upload_report[name] = {"error": str(e)}
-                self.upload_stats["failed"] += 1
-        self.upload_reports[nodeid] = upload_report
+                upload_info["uploads"][name] = {"error": str(e)}
+                upload_info["stats"]["failed"] += 1
+
+        # Pass upload info through user_properties to be xdist-compatible
+        report.user_properties.append([_USER_PROPERTY, upload_info])
 
     def _upload_asset(self, nodeid: str, name: str, path: Path) -> str:
         safe_nodeid = re.sub(r"[^a-zA-Z0-9_.-]", "_", nodeid)

qa/unittests/requirements.txt

@@ -3,3 +3,4 @@ git+https://github.com/ESA-APEx/esa-apex-toolbox-python.git@main
 pytest>=8.2.0
 moto[s3, server]>=5.0.13
 dirty-equals>=0.8.0
+pytest-xdist>=3.5.0

qa/unittests/tests/pytest/test_pytest_upload_assets.py

@@ -1,7 +1,7 @@
 import pytest
 
 
-def test_basic_upload_on_fail(
+def test_upload_on_fail_basic(
     pytester: pytest.Pytester, moto_server, s3_client, s3_bucket, monkeypatch
 ):
     pytester.makeconftest(
@@ -44,7 +44,7 @@ def test_fail_and_upload(upload_assets_on_fail, tmp_path):
     run_result.stdout.re_match_lines(
         [
             r".*upload_assets summary",
-            r"- stats: \{'uploaded': 1\}",
+            r"- stats: \{.*'uploaded': 1.*\}",
             r"\s+-\s+'hello.txt' uploaded to 'http://.*?/test-bucket-\w+/test-run-123!test_file_maker.py__test_fail_and_upload!hello.txt'",
         ]
     )
@@ -87,6 +87,59 @@ def test_success(upload_assets_on_fail, tmp_path):
     run_result.stdout.re_match_lines(
         [
             r".*upload_assets summary",
-            r"- stats: \{'uploaded': 0\}",
+            r"- stats: \{.*'uploaded': 0.*\}",
+        ]
+    )
+
+
+def test_upload_on_fail_xdist(
+    pytester: pytest.Pytester, moto_server, s3_client, s3_bucket, monkeypatch
+):
+    pytester.makeconftest(
+        """
+        pytest_plugins = [
+            "apex_algorithm_qa_tools.pytest.pytest_upload_assets",
+        ]
+        """
+    )
+    pytester.makepyfile(
+        test_file_maker="""
+            def test_fail_and_upload(upload_assets_on_fail, tmp_path):
+                path = tmp_path / "hello.txt"
+                path.write_text("Hello world.")
+                upload_assets_on_fail(path)
+                assert 3 == 5
+        """
+    )
+
+    monkeypatch.setenv("APEX_ALGORITHMS_S3_ENDPOINT_URL", moto_server)
+    monkeypatch.setenv("APEX_ALGORITHMS_RUN_ID", "test-run-123")
+
+    run_result = pytester.runpytest_subprocess(
+        f"--upload-assets-s3-bucket={s3_bucket}",
+        "--numprocesses=2",
+    )
+    run_result.stdout.re_match_lines(
+        [
+            r"Plugin `upload_assets` is active, with upload to 'test-bucket-",
+            r"2 workers \[1 item\]",
+        ]
+    )
+    run_result.assert_outcomes(failed=1)
+
+    object_listing = s3_client.list_objects(Bucket=s3_bucket)
+    assert len(object_listing["Contents"])
+    keys = [obj["Key"] for obj in object_listing["Contents"]]
+    expected_key = "test-run-123!test_file_maker.py__test_fail_and_upload!hello.txt"
+    assert keys == [expected_key]
+
+    actual = s3_client.get_object(Bucket=s3_bucket, Key=expected_key)
+    assert actual["Body"].read().decode("utf8") == "Hello world."
+
+    run_result.stdout.re_match_lines(
+        [
+            r".*upload_assets summary",
+            r"- stats: \{.*'uploaded': 1.*\}",
+            r"\s+-\s+'hello.txt' uploaded to 'http://.*?/test-bucket-\w+/test-run-123!test_file_maker.py__test_fail_and_upload!hello.txt'",
         ]
     )