diff --git a/setup.py b/setup.py index a7cfaf5e..0d587855 100644 --- a/setup.py +++ b/setup.py @@ -34,21 +34,14 @@ def get_authors(): "gunicorn~=22.0.0", "pydantic~=2.5.2", "SQLAlchemy~=2.0.23", - "python-cas~=1.6.0" + "python-cas~=1.6.0", + "autosubmit>=4.1.11", ] # Test dependencies -test_requires = [ - "pytest", - "pytest-cov", - "pytest-asyncio", - "ruff" -] +test_requires = ["pytest", "pytest-cov", "pytest-asyncio", "ruff"] -extras_require = { - 'test': test_requires, - 'all': install_requires + test_requires -} +extras_require = {"test": test_requires, "all": install_requires + test_requires} setup( name="autosubmit_api", diff --git a/tests/experiments/a1ve/conf/as_misc.yml b/tests/experiments/a1ve/conf/as_misc.yml new file mode 100755 index 00000000..42f6716e --- /dev/null +++ b/tests/experiments/a1ve/conf/as_misc.yml @@ -0,0 +1,2 @@ +AS_MISC: True +AS_COMMAND: run diff --git a/tests/experiments/a1ve/conf/autosubmit_a1ve.yml b/tests/experiments/a1ve/conf/autosubmit_a1ve.yml new file mode 100644 index 00000000..fd4fbc20 --- /dev/null +++ b/tests/experiments/a1ve/conf/autosubmit_a1ve.yml @@ -0,0 +1,19 @@ +CONFIG: + # Current version of Autosubmit. + AUTOSUBMIT_VERSION: "4.1.11" + # Maximum number of jobs permitted in the waiting status. + MAXWAITINGJOBS: 20 + # Total number of jobs in the workflow. + TOTALJOBS: 20 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +#wrappers: +# wrapper_sim: +# TYPE: "vertical" +# JOBS_IN_WRAPPER: "SIM" +MAIL: + NOTIFICATIONS: False + TO: +STORAGE: + TYPE: pkl + COPY_REMOTE_LOGS: true diff --git a/tests/experiments/a1ve/conf/expdef_a1ve.yml b/tests/experiments/a1ve/conf/expdef_a1ve.yml new file mode 100644 index 00000000..42335a53 --- /dev/null +++ b/tests/experiments/a1ve/conf/expdef_a1ve.yml @@ -0,0 +1,43 @@ +DEFAULT: + # Job experiment ID. + EXPID: "a1ve" + # Default HPC platform name. + HPCARCH: "local" +EXPERIMENT: + # List of start dates + DATELIST: '20000101' + # List of members. + MEMBERS: fc0 + # Unit of the chunk size. Can be hour, day, month, or year. + CHUNKSIZEUNIT: month + # Size of each chunk. + CHUNKSIZE: '4' + # Number of chunks of the experiment. + NUMCHUNKS: '2' + CHUNKINI: '' + # Calendar used for the experiment. Can be standard or noleap. + CALENDAR: standard +PROJECT: + # Type of the project. + PROJECT_TYPE: none + # Folder to hold the project sources. + PROJECT_DESTINATION: '' +GIT: + PROJECT_ORIGIN: '' + PROJECT_BRANCH: '' + PROJECT_COMMIT: '' + PROJECT_SUBMODULES: '' + FETCH_SINGLE_BRANCH: true +SVN: + PROJECT_URL: '' + PROJECT_REVISION: '' +LOCAL: + PROJECT_PATH: '' +PROJECT_FILES: + FILE_PROJECT_CONF: '' + FILE_JOBS_CONF: '' + JOB_SCRIPTS_TYPE: '' +RERUN: + RERUN: false + RERUN_JOBLIST: '' + diff --git a/tests/experiments/a1ve/conf/jobs_a1ve.yml b/tests/experiments/a1ve/conf/jobs_a1ve.yml new file mode 100644 index 00000000..ee2cd578 --- /dev/null +++ b/tests/experiments/a1ve/conf/jobs_a1ve.yml @@ -0,0 +1,27 @@ +JOBS: + ini: + FILE: ini.sh + RUNNING: once + + sim: + FILE: sim.sh + DEPENDENCIES: ini sim-1 + RUNNING: once + + asim: + FILE: asim.sh + DEPENDENCIES: sim + RUNNING: once + SPLITS: 3 + + post: + FILE: post.sh + RUNNING: once + DEPENDENCIES: + asim: + SPLITS_FROM: + 2,3: # [2:3] is also valid + splits_to: 1,2*,3* # 1,[2:3]* is also valid, you can also specify the step with [2:3:step] + SPLITS: 3 + + diff --git a/tests/experiments/a1ve/conf/metadata/experiment_data.yml b/tests/experiments/a1ve/conf/metadata/experiment_data.yml new file mode 100644 index 00000000..16d207a0 --- /dev/null +++ b/tests/experiments/a1ve/conf/metadata/experiment_data.yml @@ -0,0 +1,137 @@ +CONFIG: + AUTOSUBMIT_VERSION: 4.1.11 + MAXWAITINGJOBS: 20 + TOTALJOBS: 20 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +MAIL: + NOTIFICATIONS: false + TO: +STORAGE: + TYPE: pkl + COPY_REMOTE_LOGS: true +DEFAULT: + EXPID: a1ve + HPCARCH: LOCAL +EXPERIMENT: + DATELIST: '20000101' + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 4 + NUMCHUNKS: 2 + CHUNKINI: '' + CALENDAR: standard +PROJECT: + PROJECT_TYPE: none + PROJECT_DESTINATION: '' +GIT: + PROJECT_ORIGIN: '' + PROJECT_BRANCH: '' + PROJECT_COMMIT: '' + PROJECT_SUBMODULES: '' + FETCH_SINGLE_BRANCH: true +SVN: + PROJECT_URL: '' + PROJECT_REVISION: '' +LOCAL: + PROJECT_PATH: '' +PROJECT_FILES: + FILE_PROJECT_CONF: '' + FILE_JOBS_CONF: '' + JOB_SCRIPTS_TYPE: '' +RERUN: + RERUN: false + RERUN_JOBLIST: '' +JOBS: + INI: + FILE: ini.sh + RUNNING: once + DEPENDENCIES: {} + ADDITIONAL_FILES: [] + SIM: + FILE: sim.sh + DEPENDENCIES: + INI: {} + SIM-1: {} + RUNNING: once + ADDITIONAL_FILES: [] + ASIM: + FILE: asim.sh + DEPENDENCIES: + SIM: {} + RUNNING: once + SPLITS: 3 + ADDITIONAL_FILES: [] + POST: + FILE: post.sh + RUNNING: once + DEPENDENCIES: + ASIM: + SPLITS_FROM: + 2,3: + SPLITS_TO: 1,2*,3* + SPLITS: 3 + ADDITIONAL_FILES: [] +PLATFORMS: + MARENOSTRUM4: + TYPE: slurm + HOST: mn1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + MAX_WALLCLOCK: 48:00 + TEMP_DIR: '' + MARENOSTRUM_ARCHIVE: + TYPE: ps + HOST: dt02.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + TEST_SUITE: false + TRANSFER_NODE: + TYPE: ps + HOST: dt01.bsc.es + PROJECT: bsc32 + USER: + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /gpfs/scratch + TRANSFER_NODE_BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + NORD3: + TYPE: SLURM + HOST: nord1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + MAX_WALLCLOCK: 48:00 + ECMWF-XC40: + TYPE: ecaccess + VERSION: pbs + HOST: cca + USER: + PROJECT: spesiccf + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /scratch/ms + QUEUE: np + SERIAL_QUEUE: ns + MAX_WALLCLOCK: 48:00 +ROOTDIR: /home/ltenorio/autosubmit/a1ve +PROJDIR: /home/ltenorio/autosubmit/a1ve/proj/ diff --git a/tests/experiments/a1ve/conf/metadata/experiment_data.yml.bak b/tests/experiments/a1ve/conf/metadata/experiment_data.yml.bak new file mode 100644 index 00000000..16d207a0 --- /dev/null +++ b/tests/experiments/a1ve/conf/metadata/experiment_data.yml.bak @@ -0,0 +1,137 @@ +CONFIG: + AUTOSUBMIT_VERSION: 4.1.11 + MAXWAITINGJOBS: 20 + TOTALJOBS: 20 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +MAIL: + NOTIFICATIONS: false + TO: +STORAGE: + TYPE: pkl + COPY_REMOTE_LOGS: true +DEFAULT: + EXPID: a1ve + HPCARCH: LOCAL +EXPERIMENT: + DATELIST: '20000101' + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 4 + NUMCHUNKS: 2 + CHUNKINI: '' + CALENDAR: standard +PROJECT: + PROJECT_TYPE: none + PROJECT_DESTINATION: '' +GIT: + PROJECT_ORIGIN: '' + PROJECT_BRANCH: '' + PROJECT_COMMIT: '' + PROJECT_SUBMODULES: '' + FETCH_SINGLE_BRANCH: true +SVN: + PROJECT_URL: '' + PROJECT_REVISION: '' +LOCAL: + PROJECT_PATH: '' +PROJECT_FILES: + FILE_PROJECT_CONF: '' + FILE_JOBS_CONF: '' + JOB_SCRIPTS_TYPE: '' +RERUN: + RERUN: false + RERUN_JOBLIST: '' +JOBS: + INI: + FILE: ini.sh + RUNNING: once + DEPENDENCIES: {} + ADDITIONAL_FILES: [] + SIM: + FILE: sim.sh + DEPENDENCIES: + INI: {} + SIM-1: {} + RUNNING: once + ADDITIONAL_FILES: [] + ASIM: + FILE: asim.sh + DEPENDENCIES: + SIM: {} + RUNNING: once + SPLITS: 3 + ADDITIONAL_FILES: [] + POST: + FILE: post.sh + RUNNING: once + DEPENDENCIES: + ASIM: + SPLITS_FROM: + 2,3: + SPLITS_TO: 1,2*,3* + SPLITS: 3 + ADDITIONAL_FILES: [] +PLATFORMS: + MARENOSTRUM4: + TYPE: slurm + HOST: mn1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + MAX_WALLCLOCK: 48:00 + TEMP_DIR: '' + MARENOSTRUM_ARCHIVE: + TYPE: ps + HOST: dt02.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + TEST_SUITE: false + TRANSFER_NODE: + TYPE: ps + HOST: dt01.bsc.es + PROJECT: bsc32 + USER: + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /gpfs/scratch + TRANSFER_NODE_BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + NORD3: + TYPE: SLURM + HOST: nord1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + MAX_WALLCLOCK: 48:00 + ECMWF-XC40: + TYPE: ecaccess + VERSION: pbs + HOST: cca + USER: + PROJECT: spesiccf + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /scratch/ms + QUEUE: np + SERIAL_QUEUE: ns + MAX_WALLCLOCK: 48:00 +ROOTDIR: /home/ltenorio/autosubmit/a1ve +PROJDIR: /home/ltenorio/autosubmit/a1ve/proj/ diff --git a/tests/experiments/a1ve/conf/platforms_a1ve.yml b/tests/experiments/a1ve/conf/platforms_a1ve.yml new file mode 100644 index 00000000..5ae520b5 --- /dev/null +++ b/tests/experiments/a1ve/conf/platforms_a1ve.yml @@ -0,0 +1,66 @@ +PLATFORMS: + MARENOSTRUM4: + TYPE: slurm + HOST: mn1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + MAX_WALLCLOCK: 48:00 + TEMP_DIR: '' + MARENOSTRUM_ARCHIVE: + TYPE: ps + HOST: dt02.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + TEST_SUITE: false + + TRANSFER_NODE: + TYPE: ps + HOST: dt01.bsc.es + PROJECT: bsc32 + USER: + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /gpfs/scratch + + TRANSFER_NODE_BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + + BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + NORD3: + TYPE: SLURM + HOST: nord1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + MAX_WALLCLOCK: 48:00 + + ECMWF-XC40: + TYPE: ecaccess + VERSION: pbs + HOST: cca + USER: + PROJECT: spesiccf + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /scratch/ms + QUEUE: np + SERIAL_QUEUE: ns + MAX_WALLCLOCK: 48:00 + diff --git a/tests/experiments/a1ve/pkl/job_list_a1ve.pkl b/tests/experiments/a1ve/pkl/job_list_a1ve.pkl new file mode 100644 index 00000000..5be56203 Binary files /dev/null and b/tests/experiments/a1ve/pkl/job_list_a1ve.pkl differ diff --git a/tests/experiments/a1ve/pkl/job_packages_a1ve.db b/tests/experiments/a1ve/pkl/job_packages_a1ve.db new file mode 100644 index 00000000..dbb92f1f Binary files /dev/null and b/tests/experiments/a1ve/pkl/job_packages_a1ve.db differ diff --git a/tests/experiments/a1ve/status/a1ve_20241212_1514.txt b/tests/experiments/a1ve/status/a1ve_20241212_1514.txt new file mode 100644 index 00000000..89c3121c --- /dev/null +++ b/tests/experiments/a1ve/status/a1ve_20241212_1514.txt @@ -0,0 +1,8 @@ +a1ve_INI READY +a1ve_SIM WAITING +a1ve_1_ASIM WAITING +a1ve_2_ASIM WAITING +a1ve_3_ASIM WAITING +a1ve_1_POST WAITING +a1ve_2_POST WAITING +a1ve_3_POST WAITING diff --git a/tests/experiments/a1ve/tmp/LOG_a1ve/.gitkeep b/tests/experiments/a1ve/tmp/LOG_a1ve/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/experiments/autosubmit.db b/tests/experiments/autosubmit.db index 47278733..5794fb39 100644 Binary files a/tests/experiments/autosubmit.db and b/tests/experiments/autosubmit.db differ diff --git a/tests/experiments/metadata/data/job_data_a1ve.db b/tests/experiments/metadata/data/job_data_a1ve.db new file mode 100755 index 00000000..a10b2229 Binary files /dev/null and b/tests/experiments/metadata/data/job_data_a1ve.db differ diff --git a/tests/experiments/metadata/graph/graph_data_a1ve.db b/tests/experiments/metadata/graph/graph_data_a1ve.db new file mode 100644 index 00000000..f92d9209 Binary files /dev/null and b/tests/experiments/metadata/graph/graph_data_a1ve.db differ diff --git a/tests/test_persistance.py b/tests/test_persistance.py index c716c3e9..0eb2ad0b 100644 --- a/tests/test_persistance.py +++ b/tests/test_persistance.py @@ -1,17 +1,13 @@ from autosubmit_api.persistance.pkl_reader import PklReader +import pytest class TestPklReader: - - def test_reader(self, fixture_mock_basic_config): - test_cases = [ - {"expid": "a003", "size": 8}, - {"expid": "a007", "size": 8}, - {"expid": "a3tb", "size": 55}, - ] - - for exp in test_cases: - content = PklReader(exp["expid"]).parse_job_list() - assert len(content) == exp["size"] - for item in content: - assert item.name.startswith(exp["expid"]) + @pytest.mark.parametrize( + "expid, size", [("a003", 8), ("a007", 8), ("a3tb", 55), ("a1ve", 8)] + ) + def test_reader(self, fixture_mock_basic_config, expid, size): + content = PklReader(expid).parse_job_list() + assert len(content) == size + for item in content: + assert item.name.startswith(expid)