Merge pull request #51 from OpenSourceBrain/feature/create_test_compa…

…tibility_biosimulators_for_first_example_in_sbml_test_suite

Feature/create test compatibility biosimulators for first example in sbml test suite

.github/workflows/non-omv-local.yml

@@ -0,0 +1,49 @@
+
+name: Testing non OMV scripts (locally using docker)
+
+on:
+  push:
+    branches: [ master, development, experimental, test*, feature/* ]
+  pull_request:
+    branches: [ master, development, experimental, test*, feature/* ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ 3.9, "3.10" ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python  ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version:  ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+          pip install git+https://github.com/OpenSourceBrain/osb-model-validation
+          pip install pyNeuroML[annotations]
+          pip install python-libsedml
+          pip install tellurium
+          sudo apt-get install libncurses5 --fix-missing
+          omv install jneuroml
+          pip install "pymetadata>=0.4.2" docker "requests<2.32.0"
+
+    - name: Final version info
+      run: |
+        omv list -V # list installed engines
+        pip list
+        env
+
+    - name: Test local biosimulators compatibility table creation
+      run: |
+          cd SBML/tests
+          sudo rm -rf output
+          python ./test_biosimulators_local.py --output-dir=tmp_plots
+  
+

.github/workflows/non-omv-remote.yml

@@ -0,0 +1,49 @@
+
+name: Testing non OMV scripts (remotely via URLs)
+
+on:
+  push:
+    branches: [ master, development, experimental, test*, feature/* ]
+  pull_request:
+    branches: [ master, development, experimental, test*, feature/* ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ 3.9, "3.10" ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python  ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version:  ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        pip install git+https://github.com/OpenSourceBrain/osb-model-validation
+        pip install pyNeuroML[annotations]
+        pip install python-libsedml
+        pip install tellurium
+        sudo apt-get install libncurses5 --fix-missing
+        omv install jneuroml
+        pip install "pymetadata>=0.4.2" docker "requests<2.32.0"
+
+    - name: Final version info
+      run: |
+        omv list -V # list installed engines
+        pip list
+        env
+
+    - name: Test remote biosimulators compatibility table creation
+      run: |
+        cd SBML/tests
+        sudo rm -rf output
+        python ./test_biosimulators_remote.py --output-dir=tmp_plots
+  
+

.github/workflows/non-omv.yml

@@ -55,21 +55,13 @@ jobs:
         cd SBML/tests    
         python ./test_biosimulators_docker.py
 
-    - name: Test biosimulators compatibility table creation
-      run: |
-        cd SBML/tests
-        sudo rm -rf output
-        python ./test_compatibility_biosimulators.py --output-dir=tmp_plots
-
-
     - name: Test test_suite output regeneration
       run: |
         cd test_suite        
         ./test_results_regeneration.sh
 
-
     - name: Final version info
       run: |
         omv list -V # list installed engines
         pip list
-        env
+        env

SBML/tests/test_biosimulators_local.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+"""
+This script tests the compatibility of different biosimulation engines with a given SBML and SED-ML file.
+It runs each engine and records the result (pass/fail) and any error messages encountered during the simulation.
+The results are then displayed in a table and saved to a markdown file.
+"""
+
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) # to import utils
+import utils
+import argparse
+
+engines = utils.engines
+types_dict = utils.types_dict
+
+# Save the current working directory
+cwd = os.getcwd()
+print('Current working directory:', cwd)
+
+# SBML folder is one folder up relative to cwd
+path_to_sbml_folder = os.path.abspath(os.path.join(cwd, os.pardir))
+print('Path to SBML folder:', path_to_sbml_folder)
+
+# change the working directory to the SBML folder (because here the SBML and SED-ML files are located)
+os.chdir(path_to_sbml_folder)
+print('Changed working directory to:', os.getcwd())
+
+sbml_file_name = 'LEMS_NML2_Ex9_FN.sbml'
+sedml_file_name = 'LEMS_NML2_Ex9_FN_missing_xmlns.sedml' #xmlns:sbml missing
+
+# output_dir is set to 'd1_plots' by default but can be changed using the --output-dir argument (required to deal with GitHub Actions permission issues)
+parser = argparse.ArgumentParser(description='Test compatibility of different biosimulation engines')
+parser.add_argument('--output-dir',action='store',default='d1_plots',help='prefix of the output directory where the d1 plots will be saved')
+args = parser.parse_args()
+
+test_folder = 'tests'
+
+d1_plots_local_dir = os.path.join(test_folder, args.output_dir + '_local')
+
+print('d1 plots will be saved in:', d1_plots_local_dir)
+
+results_local = utils.run_biosimulators_locally(sedml_file_name=sedml_file_name, 
+                                    sbml_file_name=sbml_file_name,
+                                    d1_plots_local_dir=d1_plots_local_dir, 
+                                    engines=engines, test_folder=test_folder)
+

SBML/tests/test_biosimulators_remote.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+"""
+This script tests the compatibility of different biosimulation engines with a given SBML and SED-ML file.
+It runs each engine and records the result (pass/fail) and any error messages encountered during the simulation.
+The results are then displayed in a table and saved to a markdown file.
+"""
+
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) # to import utils
+import utils
+import argparse
+
+engines = utils.engines
+types_dict = utils.types_dict
+
+# Save the current working directory
+cwd = os.getcwd()
+print('Current working directory:', cwd)
+
+# SBML folder is one folder up relative to cwd
+path_to_sbml_folder = os.path.abspath(os.path.join(cwd, os.pardir))
+print('Path to SBML folder:', path_to_sbml_folder)
+
+# change the working directory to the SBML folder (because here the SBML and SED-ML files are located)
+os.chdir(path_to_sbml_folder)
+print('Changed working directory to:', os.getcwd())
+
+sbml_file_name = 'LEMS_NML2_Ex9_FN.sbml'
+sedml_file_name = 'LEMS_NML2_Ex9_FN_missing_xmlns.sedml' #xmlns:sbml missing
+
+# output_dir is set to 'd1_plots' by default but can be changed using the --output-dir argument (required to deal with GitHub Actions permission issues)
+parser = argparse.ArgumentParser(description='Test compatibility of different biosimulation engines')
+parser.add_argument('--output-dir',action='store',default='d1_plots',help='prefix of the output directory where the d1 plots will be saved')
+args = parser.parse_args()
+
+test_folder = 'tests'
+
+d1_plots_remote_dir = os.path.join(test_folder, args.output_dir + '_remote')
+
+print('d1 plots will be saved in:', d1_plots_remote_dir)
+
+results_remote = utils.run_biosimulators_remotely(sedml_file_name=sedml_file_name, 
+                                    sbml_file_name=sbml_file_name,
+                                    d1_plots_remote_dir=d1_plots_remote_dir, 
+                                    engines=engines, test_folder=test_folder)
+

SBML/tests/test_compatibility_biosimulators.py

@@ -46,4 +46,4 @@
                                  sbml_file_name,
                                  d1_plots_remote_dir, 
                                  d1_plots_local_dir,
-                                 engines=engines, test_folder='tests')
+                                 engines=engines, test_folder=test_folder)

test_suite/README.md

@@ -35,7 +35,7 @@ pip install docker
 Navigate to the directory containing the `process_test_suite.py` script and execute it, specifying the path to the extracted test files. For example if the extraction folder is  `C:\Users\Username\Documents\compbiolibs\SBML_test_suite\semantic` and the `process_test_suite.py` script is in the current directory.
 
 ```
- python process_test_suite.py --suite-path C:\Users\Username\Documents\compbiolibs\SBML_test_suite\semantic  --suite-glob '*/*-sbml-l3v2.xml' --output-file ./results_test.md --limit 5       
+ python process_test_suite.py --suite-path C:\Users\Username\Documents\compbiolibs\SBML_test_suite\semantic  --sbml-level_version 'sbml-l3v2' --output-file ./results_test.md --limit 5       
 ```
 
 ## Command Line Options
@@ -52,10 +52,10 @@ The `process_test_suite.py` script provides various command-line options to cust
   **Usage:** `--suite-path <path>`  
   **Default:** `.` (current directory)
 
-- `--suite-glob`  
-  **Description:** Shell-style glob pattern to match SBML files within the specified suite path.  
-  **Usage:** `--suite-glob <pattern>`  
-  **Default:** `000*/*-sbml-l3v2.xml`
+- `--sbml-level_version'
+  **Description:** String that specifies level and version of files to select for processing (e.g. 'l3v2')
+  **Usage:** `--sbml-level_version <string>`  
+  **Default:** `highest`
 
 - `--suite-url-base`  
   **Description:** Base URL for the online test case files to include as links in the results. Set to an empty string to disable links.  

test_suite/process_test_suite.py

@@ -57,8 +57,8 @@ def parse_arguments():
         "--sbml-level_version",
         action="store",
         type=str,
-        default="sbml-l3v2",
-        help="SBML level and version to test, default is 'sbml-l3v2'",
+        default="highest",
+        help="SBML level and version to test (e.g. 'l3v2'), default is '' which will try to find the highest level and version in the folder",
     )
 
     parser.add_argument(
@@ -119,21 +119,21 @@ def process_cases(args):
     subfolders = os.listdir(suite_path_abs) if args.limit == 0 else os.listdir(suite_path_abs)[:args.limit]    
 
     for subfolder in subfolders:
-        # find relevant files in the subfolder
-        sbml_file_name = f"*-{args.sbml_level_version}.xml"
-        sedml_file_name = f"*-{args.sbml_level_version}-sedml.xml"
-        sbml_file_path = glob.glob(os.path.join(subfolder, sbml_file_name))[0] if len(glob.glob(os.path.join(subfolder, sbml_file_name))) > 0 else []
-        sedml_file_path = glob.glob(os.path.join(subfolder, sedml_file_name))[0] if len(glob.glob(os.path.join(subfolder, sedml_file_name))) > 0 else []
-
-        # if no files found with the specified sbml_level_version, try to find any sbml or sedml files
-        if sbml_file_path == [] or sedml_file_path == []:
-            print(f"Folder {subfolder} has no {args.sbml_level_version} SBML or SED-ML files")
+        # if sbml_level_version is empty string (default), find the highest level and version in the folder
+        if args.sbml_level_version == "highest":
             sedml_file_paths = glob.glob(os.path.join(subfolder, "*-sbml-*sedml.xml"))
             # get last entry in list of sedml_file_paths (because it has the highest level and version number considering the alphabetical order and naming convention)
             sedml_file_path = sedml_file_paths[-1] if sedml_file_paths != [] else []
             sbml_file_path = sedml_file_path.replace("-sedml.xml",".xml") if sedml_file_path != [] else []
-            if sbml_file_path == [] or sedml_file_path == []:
-                print(f"Folder {subfolder} has no SBML or SED-ML files")
+        else:
+            # find relevant files in the subfolder
+            sbml_file_name = f"*-sbml-{args.sbml_level_version}.xml"
+            sedml_file_name = f"*-sbml-{args.sbml_level_version}-sedml.xml"
+            sbml_file_path = glob.glob(os.path.join(subfolder, sbml_file_name))[0] if len(glob.glob(os.path.join(subfolder, sbml_file_name))) > 0 else []
+            sedml_file_path = glob.glob(os.path.join(subfolder, sedml_file_name))[0] if len(glob.glob(os.path.join(subfolder, sedml_file_name))) > 0 else []
+
+        if sbml_file_path == [] or sedml_file_path == []:
+                print(f"Folder {subfolder} has no SBML or SED-ML files {args.sbml_level_version}")
                 continue
         print(f"Processing {sbml_file_path} and {sedml_file_path}")
 

test_suite/test_00001/00001-sbml-l3v2-sedml.xml

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Written by libSedML v1.1.5419.25885 see http://libsedml.sf.net -->
+<sedML level="1" version="1" xmlns="http://sed-ml.org/">
+  <listOfSimulations>
+    <uniformTimeCourse id="simulation_1" initialTime="0" outputStartTime="0" outputEndTime="5" numberOfPoints="50">
+      <algorithm kisaoID="KISAO:0000019" />
+    </uniformTimeCourse>
+  </listOfSimulations>
+  <listOfModels>
+    <model id="model_1" language="urn:sedml:language:sbml" source="00001-sbml-l3v2.xml" />
+  </listOfModels>
+  <listOfTasks>
+    <task id="task_1" modelReference="model_1" simulationReference="simulation_1" />
+  </listOfTasks>
+  <listOfDataGenerators>
+    <dataGenerator id="time_1" name="time">
+      <listOfVariables>
+        <variable id="time" taskReference="task_1" symbol="urn:sedml:symbol:time" />
+      </listOfVariables>
+      <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <ci> time </ci>
+      </math>
+    </dataGenerator>
+    <dataGenerator id="S1_1" name="S1">
+      <listOfVariables>
+        <variable id="S1_1_s" taskReference="task_1" target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='S1']" />
+        <variable id="S1_1_c" taskReference="task_1" target="/sbml:sbml/sbml:model/sbml:listOfCompartments/sbml:compartment[@id='compartment']" />
+      </listOfVariables>
+      <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <apply>
+          <times />
+          <ci> S1_1_s </ci>
+          <ci> S1_1_c </ci>
+        </apply>
+      </math>
+    </dataGenerator>
+    <dataGenerator id="S2_1" name="S2">
+      <listOfVariables>
+        <variable id="S2_1_s" taskReference="task_1" target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='S2']" />
+        <variable id="S2_1_c" taskReference="task_1" target="/sbml:sbml/sbml:model/sbml:listOfCompartments/sbml:compartment[@id='compartment']" />
+      </listOfVariables>
+      <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <apply>
+          <times />
+          <ci> S2_1_s </ci>
+          <ci> S2_1_c </ci>
+        </apply>
+      </math>
+    </dataGenerator>
+  </listOfDataGenerators>
+  <listOfOutputs>
+    <plot2D id="plot_1" name="Timecourse for test 00001">
+      <listOfCurves>
+        <curve id="c_S1_1" name="S1" logX="false" logY="false" xDataReference="time_1" yDataReference="S1_1" />
+        <curve id="c_S2_1" name="S2" logX="false" logY="false" xDataReference="time_1" yDataReference="S2_1" />
+      </listOfCurves>
+    </plot2D>
+    <report id="report_1" name="Report for test 00001">
+      <listOfDataSets>
+        <dataSet id="ds_time" dataReference="time_1" label="Time" />
+        <dataSet id="ds_S1_1" dataReference="S1_1" label="S1" />
+        <dataSet id="ds_S2_1" dataReference="S2_1" label="S2" />
+      </listOfDataSets>
+    </report>
+  </listOfOutputs>
+</sedML>