test(archive-apis): use .7z format to export and archive studies

tests/core/utils/test_extract_zip.py

@@ -17,7 +17,7 @@
 import py7zr
 import pytest
 
-from antarest.core.utils.utils import BadArchiveContent, extract_zip
+from antarest.core.utils.utils import BadArchiveContent, extract_archive
 
 
 class TestExtractZip:
@@ -33,7 +33,7 @@ def test_extract_zip__with_zip(self, tmp_path: Path):
 
         # Then, call the function
         with open(zip_path, mode="rb") as stream:
-            extract_zip(stream, tmp_path)
+            extract_archive(stream, tmp_path)
 
         # Finally, check the result
         assert (tmp_path / "test.txt").read_text() == "Hello world!"
@@ -46,7 +46,7 @@ def test_extract_zip__with_7z(self, tmp_path: Path):
 
         # Then, call the function
         with open(zip_path, mode="rb") as stream:
-            extract_zip(stream, tmp_path)
+            extract_archive(stream, tmp_path)
 
         # Finally, check the result
         assert (tmp_path / "test.txt").read_text() == "Hello world!"
@@ -55,22 +55,22 @@ def test_extract_zip__empty_file(self):
         stream = io.BytesIO(b"")
 
         with pytest.raises(BadArchiveContent):
-            extract_zip(stream, Path("dummy/path"))
+            extract_archive(stream, Path("dummy/path"))
 
     def test_extract_zip__corrupted_zip(self):
         stream = io.BytesIO(b"PK\x03\x04 BLURP")
 
         with pytest.raises(BadArchiveContent):
-            extract_zip(stream, Path("dummy/path"))
+            extract_archive(stream, Path("dummy/path"))
 
     def test_extract_zip__corrupted_7z(self):
         stream = io.BytesIO(b"7z BLURP")
 
         with pytest.raises(BadArchiveContent):
-            extract_zip(stream, Path("dummy/path"))
+            extract_archive(stream, Path("dummy/path"))
 
     def test_extract_zip__unknown_format(self):
         stream = io.BytesIO(b"ZORRO")
 
         with pytest.raises(BadArchiveContent):
-            extract_zip(stream, Path("dummy/path"))
+            extract_archive(stream, Path("dummy/path"))

tests/integration/studies_blueprint/assets/test_synthesis/raw_study.synthesis.json

@@ -1067,5 +1067,5 @@
   "store_new_set": true,
   "archive_input_series": [],
   "enr_modelling": "aggregated",
-  "zip_path": null
+  "archive_path": null
 }

tests/integration/studies_blueprint/assets/test_synthesis/variant_study.synthesis.json

@@ -983,5 +983,5 @@
   "store_new_set": true,
   "archive_input_series": [],
   "enr_modelling": "aggregated",
-  "zip_path": null
+  "archive_path": null
 }

tests/integration/test_integration.py

@@ -254,7 +254,7 @@ def test_main(client: TestClient, admin_access_token: str) -> None:
         headers={"Authorization": f'Bearer {fred_credentials["refresh_token"]}'},
     )
     fred_credentials = res.json()
-    res = client.post(
+    client.post(
         f"/v1/studies?name=bar&groups={group_id}",
         headers={"Authorization": f'Bearer {george_credentials["access_token"]}'},
     )
@@ -1504,7 +1504,7 @@ def test_archive(client: TestClient, admin_access_token: str, tmp_path: Path) ->
 
     res = client.get(f"/v1/studies/{study_id}")
     assert res.json()["archived"]
-    assert (tmp_path / "archive_dir" / f"{study_id}.zip").exists()
+    assert (tmp_path / "archive_dir" / f"{study_id}.7z").exists()
 
     res = client.put(f"/v1/studies/{study_id}/unarchive")
 
@@ -1513,12 +1513,12 @@ def test_archive(client: TestClient, admin_access_token: str, tmp_path: Path) ->
         lambda: client.get(
             f"/v1/tasks/{task_id}",
         ).json()["status"]
-        == 3
+        == 3,
     )
 
     res = client.get(f"/v1/studies/{study_id}")
     assert not res.json()["archived"]
-    assert not (tmp_path / "archive_dir" / f"{study_id}.zip").exists()
+    assert not (tmp_path / "archive_dir" / f"{study_id}.7z").exists()
 
 
 def test_maintenance(client: TestClient, admin_access_token: str) -> None:

tests/storage/business/test_export.py

@@ -16,6 +16,7 @@
 
 import pytest
 from checksumdir import dirhash
+from py7zr import SevenZipFile
 
 from antarest.core.config import Config, StorageConfig
 from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy
@@ -48,7 +49,13 @@ def test_export_file(tmp_path: Path):
 
 
 @pytest.mark.unit_test
-@pytest.mark.parametrize("outputs", [True, False])
+@pytest.mark.parametrize(
+    "outputs",
+    [
+        True,
+        False,
+    ],
+)
 def test_export_file(tmp_path: Path, outputs: bool):
     root = tmp_path / "folder"
     root.mkdir()
@@ -58,7 +65,7 @@ def test_export_file(tmp_path: Path, outputs: bool):
     (root / "output/results1").mkdir(parents=True)
     (root / "output/results1/file.txt").write_text("42")
 
-    export_path = tmp_path / "study.zip"
+    export_path = tmp_path / "study.7z"
 
     study_factory = Mock()
     study_service = RawStudyService(
@@ -74,10 +81,11 @@ def test_export_file(tmp_path: Path, outputs: bool):
     study_factory.create_from_fs.return_value = study_tree
 
     study_service.export_study(study, export_path, outputs=outputs)
-    with ZipFile(export_path) as zipf:
-        assert "file.txt" in zipf.namelist()
-        assert "test/file.txt" in zipf.namelist()
-        assert ("output/results1/file.txt" in zipf.namelist()) == outputs
+    with SevenZipFile(export_path) as szf:
+        szf_files = set(szf.getnames())
+        assert "file.txt" in szf_files
+        assert "test/file.txt" in szf_files
+        assert ("output/results1/file.txt" in szf_files) == outputs
 
 
 @pytest.mark.unit_test

tests/storage/repository/filesystem/test_folder_node.py

@@ -31,7 +31,7 @@
 def build_tree() -> INode[t.Any, t.Any, t.Any]:
     config = Mock()
     config.path.exist.return_value = True
-    config.zip_path = None
+    config.archive_path = None
     return TestMiddleNode(
         context=Mock(),
         config=config,

tests/storage/repository/filesystem/test_ini_file_node.py

@@ -96,7 +96,7 @@ def test_get(tmp_path: Path) -> None:
             areas={},
             outputs={},
             study_id="id",
-            zip_path=zipped_path,
+            archive_path=zipped_path,
         ),
         types=types,
     )
@@ -147,7 +147,7 @@ def test_get_depth(tmp_path: Path) -> None:
             areas={},
             outputs={},
             study_id="id",
-            zip_path=zipped_path,
+            archive_path=zipped_path,
         ),
         types=types,
     )

tests/storage/test_service.py

@@ -1563,7 +1563,7 @@ def test_get_save_logs(tmp_path: Path) -> None:
 
     output_config = Mock(get_file=Mock(return_value="output_id"), archived=False)
 
-    file_study_config = FileStudyTreeConfig(tmp_path, tmp_path, "study_id", 0, zip_path=None)
+    file_study_config = FileStudyTreeConfig(tmp_path, tmp_path, "study_id", 0, archive_path=None)
     file_study_config.outputs = {"output_id": output_config}
 
     context = Mock()

tests/storage/web/test_studies_bp.py

@@ -356,29 +356,6 @@ def test_edit_study() -> None:
     mock_storage_service.edit_study.assert_called_once_with("my-uuid", "url/to/change", {"Hello": "World"}, PARAMS)
 
 
-# @pytest.mark.unit_test
-# def test_edit_study_fail() -> None:
-#     mock_storage_service = Mock()
-#
-#     app = FastAPI(title=__name__)
-#     build_study_service(
-#         app,
-#         cache=Mock(),
-#         task_service=Mock(),
-#         file_transfer_manager=Mock(),
-#         study_service=mock_storage_service,
-#         config=CONFIG,
-#         user_service=Mock(),
-#         matrix_service=Mock(spec=MatrixService),
-#     )
-#     client = TestClient(app, raise_server_exceptions=False)
-#     res = client.post("/v1/studies/my-uuid/raw?path=url/to/change", json={})
-#
-#     assert res.status_code == 400
-#
-#     mock_storage_service.edit_study.assert_not_called()
-
-
 @pytest.mark.unit_test
 def test_validate() -> None:
     mock_service = Mock()

tests/study/storage/test_abstract_storage_service.py

@@ -11,126 +11,46 @@
 # This file is part of the Antares project.
 
 import datetime
-import zipfile
 from pathlib import Path
-from typing import List, Optional, Sequence
-from unittest.mock import Mock, call
 
-from antarest.core.config import Config, StorageConfig
-from antarest.core.interfaces.cache import ICache
+from py7zr import SevenZipFile
+
 from antarest.core.model import PublicMode
-from antarest.core.requests import RequestParameters
 from antarest.core.utils.fastapi_sqlalchemy import db
 from antarest.login.model import Group, User
 from antarest.study.model import Study
-from antarest.study.storage.abstract_storage_service import AbstractStorageService
-from antarest.study.storage.patch_service import PatchService
-from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO
-from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory
+from antarest.study.storage.rawstudy.raw_study_service import RawStudyService
 from tests.helpers import with_db_context
 
 
-class MyStorageService(AbstractStorageService):
-    """
-    This class is only defined to test `AbstractStorageService` class PUBLIC methods.
-    Abstract methods are not implemented: there are not used or patched with a Mock object.
-    """
-
-    def create(self, metadata: Study) -> Study:
-        raise NotImplementedError
-
-    def exists(self, metadata: Study) -> bool:
-        raise NotImplementedError
-
-    # noinspection SpellCheckingInspection
-    def copy(self, src_meta: Study, dest_name: str, groups: Sequence[str], with_outputs: bool = False) -> Study:
-        raise NotImplementedError
-
-    def get_raw(
-        self,
-        metadata: Study,
-        use_cache: bool = True,
-        output_dir: Optional[Path] = None,
-    ) -> FileStudy:
-        raise NotImplementedError
-
-    def set_reference_output(self, metadata: Study, output_id: str, status: bool) -> None:
-        raise NotImplementedError
-
-    def delete(self, metadata: Study) -> None:
-        raise NotImplementedError
-
-    def delete_output(self, metadata: Study, output_id: str) -> None:
-        raise NotImplementedError
-
-    def get_study_path(self, metadata: Study) -> Path:
-        raise NotImplementedError
-
-    def export_study_flat(
-        self,
-        metadata: Study,
-        dst_path: Path,
-        outputs: bool = True,
-        output_list_filter: Optional[List[str]] = None,
-        denormalize: bool = True,
-    ) -> None:
-        raise NotImplementedError
-
-    def get_synthesis(self, metadata: Study, params: Optional[RequestParameters] = None) -> FileStudyTreeConfigDTO:
-        raise NotImplementedError
-
-    def initialize_additional_data(self, study: Study) -> bool:
-        raise NotImplementedError
-
-
-class TmpCopy(object):
-    """A helper object that compares equal if a folder is a "tmp_copy" folder."""
-
-    def __init__(self, tmp_path: Path):
-        self.tmp_path = tmp_path
-
-    def __eq__(self, other: Path):
-        if isinstance(other, Path) and other.name == "tmp_copy":
-            # `is_relative_to` is not available for Python < 3.9
-            try:
-                other.relative_to(self.tmp_path)
-                return True
-            except ValueError:
-                return False
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __repr__(self):
-        return f"<TmpCopy({self.tmp_path})>"
-
-
 class TestAbstractStorageService:
     @with_db_context
-    def test_export_study(self, tmp_path: Path) -> None:
-        tmp_dir = tmp_path / "tmp"
-        tmp_dir.mkdir()
-        study_path = tmp_path / "My Study"
+    def test_export_study(self, tmp_path: Path, raw_study_service: RawStudyService) -> None:
+        """
+        Test the `export_study` method of the `AbstractStorageService` class.
+        Args:
+            tmp_path: The temporary directory where to store the 7z file.
+            raw_study_service: The `RawStudyService` instance to test the `export_study` method.
 
-        service = MyStorageService(
-            config=Config(storage=StorageConfig(tmp_dir=tmp_dir)),
-            study_factory=Mock(spec=StudyFactory),
-            patch_service=Mock(spec=PatchService),
-            cache=Mock(spec=ICache),
-        )
+        Returns:
 
-        ## Prepare database objects
+        """
+        # Prepare a dummy study with a `study.antares` file, and non-empty input and output folder
+        study_path = tmp_path / "My Study"
+        study_path.mkdir()
+        content_list = ["study.antares", "input/areas/foo.ini", "output/20240424-1200eco/result.log"]
+        for content in content_list:
+            study_path.joinpath(content).parent.mkdir(parents=True, exist_ok=True)
+            study_path.joinpath(content).touch()
 
         # noinspection PyArgumentList
         user = User(id=0, name="admin")
         db.session.add(user)
         db.session.commit()
-
         # noinspection PyArgumentList
         group = Group(id="my-group", name="group")
         db.session.add(group)
         db.session.commit()
-
         # noinspection PyArgumentList
         metadata = Study(
             name="My Study",
@@ -146,18 +66,23 @@ def test_export_study(self, tmp_path: Path) -> None:
         )
         db.session.add(metadata)
         db.session.commit()
+        db.session.refresh(metadata)
 
-        ## Check the `export_study` function
-        service.export_study_flat = Mock(return_value=None)
-        target_path = tmp_path / "export.zip"
-        actual = service.export_study(metadata, target_path, outputs=True)
+        # Check the `export_study` function
+        target_path = tmp_path / "export.7z"
+        actual = raw_study_service.export_study(metadata, target_path, outputs=True)
         assert actual == target_path
 
-        ## Check the call to export_study_flat
-        assert service.export_study_flat.mock_calls == [call(metadata, TmpCopy(tmp_path), True)]
-
-        ## Check that the ZIP file exist and is valid
-        with zipfile.ZipFile(target_path) as zf:
-            # Actually, there is nothing is the ZIP file,
-            # because the Study files doesn't really exist.
-            assert not zf.namelist()
+        # Check that the 7zip file exist and is valid
+        with SevenZipFile(target_path) as szf:
+            # Check that the content of the 7z file is the same as the study folder
+            assert set(content_list) == (
+                set(szf.getnames())
+                - {
+                    ".",
+                    "input",
+                    "output",
+                    "input/areas",
+                    "output/20240424-1200eco",
+                }
+            )