Ensure networkx pkl file support (#158)

* add 4.1.11 test with networkx pkl file * include autosubmit module
BSC-ES · Dec 13, 2024 · f465b31 · f465b31
1 parent 00c1821
commit f465b31
Show file tree

Hide file tree

Showing 16 changed files with 452 additions and 24 deletions.
diff --git a/setup.py b/setup.py
@@ -34,21 +34,14 @@ def get_authors():
     "gunicorn~=22.0.0",
     "pydantic~=2.5.2",
     "SQLAlchemy~=2.0.23",
-    "python-cas~=1.6.0"
+    "python-cas~=1.6.0",
+    "autosubmit>=4.1.11",
 ]
 
 # Test dependencies
-test_requires = [
-    "pytest",
-    "pytest-cov",
-    "pytest-asyncio",
-    "ruff"
-]
+test_requires = ["pytest", "pytest-cov", "pytest-asyncio", "ruff"]
 
-extras_require = {
-    'test': test_requires,
-    'all': install_requires + test_requires
-}
+extras_require = {"test": test_requires, "all": install_requires + test_requires}
 
 setup(
     name="autosubmit_api",

diff --git a/tests/experiments/a1ve/conf/as_misc.yml b/tests/experiments/a1ve/conf/as_misc.yml
@@ -0,0 +1,2 @@
+AS_MISC: True
+AS_COMMAND: run
diff --git a/tests/experiments/a1ve/conf/autosubmit_a1ve.yml b/tests/experiments/a1ve/conf/autosubmit_a1ve.yml
@@ -0,0 +1,19 @@
+CONFIG:
+  # Current version of Autosubmit.
+  AUTOSUBMIT_VERSION: "4.1.11"
+  # Maximum number of jobs permitted in the waiting status.
+  MAXWAITINGJOBS: 20
+  # Total number of jobs in the workflow.
+  TOTALJOBS: 20
+  SAFETYSLEEPTIME: 10
+  RETRIALS: 0
+#wrappers:
+#  wrapper_sim:
+#    TYPE: "vertical"
+#    JOBS_IN_WRAPPER: "SIM"
+MAIL:
+  NOTIFICATIONS: False
+  TO:
+STORAGE:
+  TYPE: pkl
+  COPY_REMOTE_LOGS: true
diff --git a/tests/experiments/a1ve/conf/expdef_a1ve.yml b/tests/experiments/a1ve/conf/expdef_a1ve.yml
@@ -0,0 +1,43 @@
+DEFAULT:
+  # Job experiment ID.
+  EXPID: "a1ve"
+  # Default HPC platform name.
+  HPCARCH: "local"
+EXPERIMENT:
+  # List of start dates
+  DATELIST: '20000101'
+  # List of members.
+  MEMBERS: fc0
+  # Unit of the chunk size. Can be hour, day, month, or year.
+  CHUNKSIZEUNIT: month
+  # Size of each chunk.
+  CHUNKSIZE: '4'
+  # Number of chunks of the experiment.
+  NUMCHUNKS: '2'
+  CHUNKINI: ''
+  # Calendar used for the experiment. Can be standard or noleap.
+  CALENDAR: standard
+PROJECT:
+  # Type of the project.
+  PROJECT_TYPE: none
+  # Folder to hold the project sources.
+  PROJECT_DESTINATION: ''
+GIT:
+  PROJECT_ORIGIN: ''
+  PROJECT_BRANCH: ''
+  PROJECT_COMMIT: ''
+  PROJECT_SUBMODULES: ''
+  FETCH_SINGLE_BRANCH: true
+SVN:
+  PROJECT_URL: ''
+  PROJECT_REVISION: ''
+LOCAL:
+  PROJECT_PATH: ''
+PROJECT_FILES:
+  FILE_PROJECT_CONF: ''
+  FILE_JOBS_CONF: ''
+  JOB_SCRIPTS_TYPE: ''
+RERUN:
+  RERUN: false
+  RERUN_JOBLIST: ''
+
diff --git a/tests/experiments/a1ve/conf/jobs_a1ve.yml b/tests/experiments/a1ve/conf/jobs_a1ve.yml
@@ -0,0 +1,27 @@
+JOBS:
+  ini:
+      FILE: ini.sh
+      RUNNING: once
+
+  sim:
+      FILE: sim.sh
+      DEPENDENCIES: ini sim-1
+      RUNNING: once
+
+  asim:
+      FILE: asim.sh
+      DEPENDENCIES: sim
+      RUNNING: once
+      SPLITS: 3
+
+  post:
+      FILE: post.sh
+      RUNNING: once
+      DEPENDENCIES:
+          asim:
+              SPLITS_FROM:
+                  2,3: # [2:3] is also valid
+                      splits_to: 1,2*,3* # 1,[2:3]* is also valid, you can also specify the step with [2:3:step]
+      SPLITS: 3
+
+
diff --git a/tests/experiments/a1ve/conf/metadata/experiment_data.yml b/tests/experiments/a1ve/conf/metadata/experiment_data.yml
@@ -0,0 +1,137 @@
+CONFIG:
+  AUTOSUBMIT_VERSION: 4.1.11
+  MAXWAITINGJOBS: 20
+  TOTALJOBS: 20
+  SAFETYSLEEPTIME: 10
+  RETRIALS: 0
+MAIL:
+  NOTIFICATIONS: false
+  TO:
+STORAGE:
+  TYPE: pkl
+  COPY_REMOTE_LOGS: true
+DEFAULT:
+  EXPID: a1ve
+  HPCARCH: LOCAL
+EXPERIMENT:
+  DATELIST: '20000101'
+  MEMBERS: fc0
+  CHUNKSIZEUNIT: month
+  CHUNKSIZE: 4
+  NUMCHUNKS: 2
+  CHUNKINI: ''
+  CALENDAR: standard
+PROJECT:
+  PROJECT_TYPE: none
+  PROJECT_DESTINATION: ''
+GIT:
+  PROJECT_ORIGIN: ''
+  PROJECT_BRANCH: ''
+  PROJECT_COMMIT: ''
+  PROJECT_SUBMODULES: ''
+  FETCH_SINGLE_BRANCH: true
+SVN:
+  PROJECT_URL: ''
+  PROJECT_REVISION: ''
+LOCAL:
+  PROJECT_PATH: ''
+PROJECT_FILES:
+  FILE_PROJECT_CONF: ''
+  FILE_JOBS_CONF: ''
+  JOB_SCRIPTS_TYPE: ''
+RERUN:
+  RERUN: false
+  RERUN_JOBLIST: ''
+JOBS:
+  INI:
+    FILE: ini.sh
+    RUNNING: once
+    DEPENDENCIES: {}
+    ADDITIONAL_FILES: []
+  SIM:
+    FILE: sim.sh
+    DEPENDENCIES:
+      INI: {}
+      SIM-1: {}
+    RUNNING: once
+    ADDITIONAL_FILES: []
+  ASIM:
+    FILE: asim.sh
+    DEPENDENCIES:
+      SIM: {}
+    RUNNING: once
+    SPLITS: 3
+    ADDITIONAL_FILES: []
+  POST:
+    FILE: post.sh
+    RUNNING: once
+    DEPENDENCIES:
+      ASIM:
+        SPLITS_FROM:
+          2,3:
+            SPLITS_TO: 1,2*,3*
+    SPLITS: 3
+    ADDITIONAL_FILES: []
+PLATFORMS:
+  MARENOSTRUM4:
+    TYPE: slurm
+    HOST: mn1.bsc.es
+    PROJECT: bsc32
+    USER:
+    QUEUE: debug
+    SCRATCH_DIR: /gpfs/scratch
+    ADD_PROJECT_TO_HOST: false
+    MAX_WALLCLOCK: 48:00
+    TEMP_DIR: ''
+  MARENOSTRUM_ARCHIVE:
+    TYPE: ps
+    HOST: dt02.bsc.es
+    PROJECT: bsc32
+    USER:
+    SCRATCH_DIR: /gpfs/scratch
+    ADD_PROJECT_TO_HOST: false
+    TEST_SUITE: false
+  TRANSFER_NODE:
+    TYPE: ps
+    HOST: dt01.bsc.es
+    PROJECT: bsc32
+    USER:
+    ADD_PROJECT_TO_HOST: false
+    SCRATCH_DIR: /gpfs/scratch
+  TRANSFER_NODE_BSCEARTH000:
+    TYPE: ps
+    HOST: bscearth000
+    USER:
+    PROJECT: Earth
+    ADD_PROJECT_TO_HOST: false
+    QUEUE: serial
+    SCRATCH_DIR: /esarchive/scratch
+  BSCEARTH000:
+    TYPE: ps
+    HOST: bscearth000
+    USER:
+    PROJECT: Earth
+    ADD_PROJECT_TO_HOST: false
+    QUEUE: serial
+    SCRATCH_DIR: /esarchive/scratch
+  NORD3:
+    TYPE: SLURM
+    HOST: nord1.bsc.es
+    PROJECT: bsc32
+    USER:
+    QUEUE: debug
+    SCRATCH_DIR: /gpfs/scratch
+    MAX_WALLCLOCK: 48:00
+  ECMWF-XC40:
+    TYPE: ecaccess
+    VERSION: pbs
+    HOST: cca
+    USER:
+    PROJECT: spesiccf
+    ADD_PROJECT_TO_HOST: false
+    SCRATCH_DIR: /scratch/ms
+    QUEUE: np
+    SERIAL_QUEUE: ns
+    MAX_WALLCLOCK: 48:00
+ROOTDIR: /home/ltenorio/autosubmit/a1ve
+PROJDIR: /home/ltenorio/autosubmit/a1ve/proj/
diff --git a/tests/experiments/a1ve/conf/metadata/experiment_data.yml.bak b/tests/experiments/a1ve/conf/metadata/experiment_data.yml.bak
@@ -0,0 +1,137 @@
+CONFIG:
+  AUTOSUBMIT_VERSION: 4.1.11
+  MAXWAITINGJOBS: 20
+  TOTALJOBS: 20
+  SAFETYSLEEPTIME: 10
+  RETRIALS: 0
+MAIL:
+  NOTIFICATIONS: false
+  TO:
+STORAGE:
+  TYPE: pkl
+  COPY_REMOTE_LOGS: true
+DEFAULT:
+  EXPID: a1ve
+  HPCARCH: LOCAL
+EXPERIMENT:
+  DATELIST: '20000101'
+  MEMBERS: fc0
+  CHUNKSIZEUNIT: month
+  CHUNKSIZE: 4
+  NUMCHUNKS: 2
+  CHUNKINI: ''
+  CALENDAR: standard
+PROJECT:
+  PROJECT_TYPE: none
+  PROJECT_DESTINATION: ''
+GIT:
+  PROJECT_ORIGIN: ''
+  PROJECT_BRANCH: ''
+  PROJECT_COMMIT: ''
+  PROJECT_SUBMODULES: ''
+  FETCH_SINGLE_BRANCH: true
+SVN:
+  PROJECT_URL: ''
+  PROJECT_REVISION: ''
+LOCAL:
+  PROJECT_PATH: ''
+PROJECT_FILES:
+  FILE_PROJECT_CONF: ''
+  FILE_JOBS_CONF: ''
+  JOB_SCRIPTS_TYPE: ''
+RERUN:
+  RERUN: false
+  RERUN_JOBLIST: ''
+JOBS:
+  INI:
+    FILE: ini.sh
+    RUNNING: once
+    DEPENDENCIES: {}
+    ADDITIONAL_FILES: []
+  SIM:
+    FILE: sim.sh
+    DEPENDENCIES:
+      INI: {}
+      SIM-1: {}
+    RUNNING: once
+    ADDITIONAL_FILES: []
+  ASIM:
+    FILE: asim.sh
+    DEPENDENCIES:
+      SIM: {}
+    RUNNING: once
+    SPLITS: 3
+    ADDITIONAL_FILES: []
+  POST:
+    FILE: post.sh
+    RUNNING: once
+    DEPENDENCIES:
+      ASIM:
+        SPLITS_FROM:
+          2,3:
+            SPLITS_TO: 1,2*,3*
+    SPLITS: 3
+    ADDITIONAL_FILES: []
+PLATFORMS:
+  MARENOSTRUM4:
+    TYPE: slurm
+    HOST: mn1.bsc.es
+    PROJECT: bsc32
+    USER:
+    QUEUE: debug
+    SCRATCH_DIR: /gpfs/scratch
+    ADD_PROJECT_TO_HOST: false
+    MAX_WALLCLOCK: 48:00
+    TEMP_DIR: ''
+  MARENOSTRUM_ARCHIVE:
+    TYPE: ps
+    HOST: dt02.bsc.es
+    PROJECT: bsc32
+    USER:
+    SCRATCH_DIR: /gpfs/scratch
+    ADD_PROJECT_TO_HOST: false
+    TEST_SUITE: false
+  TRANSFER_NODE:
+    TYPE: ps
+    HOST: dt01.bsc.es
+    PROJECT: bsc32
+    USER:
+    ADD_PROJECT_TO_HOST: false
+    SCRATCH_DIR: /gpfs/scratch
+  TRANSFER_NODE_BSCEARTH000:
+    TYPE: ps
+    HOST: bscearth000
+    USER:
+    PROJECT: Earth
+    ADD_PROJECT_TO_HOST: false
+    QUEUE: serial
+    SCRATCH_DIR: /esarchive/scratch
+  BSCEARTH000:
+    TYPE: ps
+    HOST: bscearth000
+    USER:
+    PROJECT: Earth
+    ADD_PROJECT_TO_HOST: false
+    QUEUE: serial
+    SCRATCH_DIR: /esarchive/scratch
+  NORD3:
+    TYPE: SLURM
+    HOST: nord1.bsc.es
+    PROJECT: bsc32
+    USER:
+    QUEUE: debug
+    SCRATCH_DIR: /gpfs/scratch
+    MAX_WALLCLOCK: 48:00
+  ECMWF-XC40:
+    TYPE: ecaccess
+    VERSION: pbs
+    HOST: cca
+    USER:
+    PROJECT: spesiccf
+    ADD_PROJECT_TO_HOST: false
+    SCRATCH_DIR: /scratch/ms
+    QUEUE: np
+    SERIAL_QUEUE: ns
+    MAX_WALLCLOCK: 48:00
+ROOTDIR: /home/ltenorio/autosubmit/a1ve
+PROJDIR: /home/ltenorio/autosubmit/a1ve/proj/