Migrate unit tests to GitHub

.github/workflows/simulation_test.yml

@@ -20,12 +20,13 @@ on:
       LIBSONATA_BRANCH:
         description: 'libsonata branch to use'
         required: false
+      NEURODAMUS_MODELS_BRANCH:
+        description: 'neurodamus-models branch to use'
+        required: false
 
 env:
-  NEURON_COMMIT_ID: '13654b3'
-  NEURON_BRANCH: ${{ inputs.NEURON_BRANCH || 'master' }}
-  LIBSONATA_REPORT_BRANCH: ${{ inputs.LIBSONATA_REPORT_BRANCH || '1.2.3' }}
-  LIBSONATA_BRANCH: ${{ inputs.LIBSONATA_BRANCH || 'v0.1.27' }}
+  NEURON_COMMIT_ID: 'c48d7d5'
+  RDMAV_FORK_SAFE: '1'
 
 jobs:
   simulation:
@@ -55,6 +56,16 @@ jobs:
         restore-keys: |
           ${{ matrix.os }}-pip-
 
+    - name: Get latest tags
+      run: |
+        echo "NEURON_BRANCH=${{ inputs.NEURON_BRANCH || github.event.inputs.NEURON_BRANCH || 'master' }}" >> $GITHUB_ENV
+        LIBSONATA_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonata/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+        echo "LIBSONATA_BRANCH=${{ inputs.LIBSONATA_BRANCH || github.event.inputs.LIBSONATA_BRANCH || '$LIBSONATA_LATEST' }}" >> $GITHUB_ENV
+        LIBSONATA_REPORT_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonatareport/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+        echo "LIBSONATA_REPORT_BRANCH=${{ inputs.LIBSONATA_REPORT_BRANCH || github.event.inputs.LIBSONATA_REPORT_BRANCH || '$LIBSONATA_REPORT_LATEST' }}" >> $GITHUB_ENV
+        NEURODAMUS_MODELS_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/neurodamus-models/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+        echo "NEURODAMUS_MODELS_BRANCH=${{ inputs.NEURODAMUS_MODELS_BRANCH || github.event.inputs.NEURODAMUS_MODELS_BRANCH || '$NEURODAMUS_MODELS_LATEST' }}" >> $GITHUB_ENV
+
     - name: Get HEAD commit message and look for branches
       run: |
         COMMIT_MESSAGE=$(git log -1 --pretty=%B)
@@ -88,11 +99,13 @@ jobs:
         python -m pip install --upgrade pip setuptools
         pip install cython numpy wheel pkgconfig
 
+    - name: Add virtual environment to PATH
+      run: echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH
+
     - name: Install libsonata
       if: steps.cache-venv.outputs.cache-hit != 'true'
       run: |
-        . ./venv/bin/activate
-        CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata
+        CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata@${{ env.LIBSONATA_BRANCH }}
 
     - name: Cache libsonatareport
       id: cache-libsonatareport
@@ -126,7 +139,6 @@ jobs:
       if: steps.cache-neuron.outputs.cache-hit != 'true'
       run: |
         export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
-        sudo apt-get install flex libfl-dev bison ninja-build
         if [[ ${{ env.NEURON_BRANCH }} == 'master' ]]; then
           git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git
           cd nrn
@@ -135,7 +147,6 @@ jobs:
         else
           git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git --depth=1
         fi
-        . ./venv/bin/activate
         python -m pip install --upgrade pip -r nrn/nrn_requirements.txt
         cmake -B nrn/build -S nrn -G Ninja \
           -DPYTHON_EXECUTABLE=$(which python) \
@@ -153,44 +164,66 @@ jobs:
     - name: Install mpi4py
       if: steps.cache-venv.outputs.cache-hit != 'true'
       run: |
-        . ./venv/bin/activate
         pip install mpi4py
 
     - name: Build h5py with the local hdf5
       run: |
-        . ./venv/bin/activate
-        pip install cython numpy wheel pkgconfig
         CC="mpicc" HDF5_MPI="ON" HDF5_INCLUDEDIR=/usr/include/hdf5/mpich HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich \
           pip install --no-cache-dir --no-binary=h5py h5py --no-build-isolation
 
     - name: Install neurodamus
       run: |
-        . ./venv/bin/activate
-        pip install .
+        pip install .[full]
 
-    - name: Build models
+    - name: Build neocortex models
       run: |
         export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
         export PATH=$(pwd)/nrn/build/install/bin:$PATH
-        # copy mod files from the Zenodo link
-        wget --output-document="O1_mods.xz" --quiet "https://zenodo.org/record/8026353/files/O1_mods.xz?download=1"
-        tar -xf O1_mods.xz
-        cp -r mod tests/share/
-        cp neurodamus/data/mod/*.mod tests/share/mod/
-        ./docker/build_neurodamus.sh tests/share/mod
+        # Clone neurodamus-models repository
+        git clone --branch=${{ env.NEURODAMUS_MODELS_BRANCH }} https://github.com/BlueBrain/neurodamus-models.git
+        # Build neocortex model
+        DATADIR=$(python -c "import neurodamus; from pathlib import Path; print(Path(neurodamus.__file__).parent / 'data')")
+        cmake -B neurodamus-models/build -S neurodamus-models/ \
+          -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install \
+          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
+          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
+          -DNEURODAMUS_CORE_DIR=${DATADIR} \
+          -DNEURODAMUS_MECHANISMS=neocortex \
+          -DNEURODAMUS_NCX_V5=ON
+        cmake --build neurodamus-models/build
+        cmake --install neurodamus-models/build
+        echo "NEURODAMUS_NEOCORTEX_ROOT=$(pwd)/neurodamus-models/install" >> $GITHUB_ENV;
+
+        # Build neocortex-multiscale model
+        echo "build neurodamus-neocortex-multiscale model"
+        cmake -B neurodamus-models/build_multiscale -S neurodamus-models/ \
+          -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install_multiscale \
+          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
+          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
+          -DNEURODAMUS_CORE_DIR=${DATADIR} \
+          -DNEURODAMUS_MECHANISMS=neocortex \
+          -DNEURODAMUS_NCX_METABOLISM=ON \
+          -DNEURODAMUS_NCX_NGV=ON \
+          -DNEURODAMUS_ENABLE_CORENEURON=OFF
+        cmake --build neurodamus-models/build_multiscale
+        cmake --install neurodamus-models/build_multiscale
+        echo "NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT=$(pwd)/neurodamus-models/install_multiscale" >> $GITHUB_ENV;
 
-    - name: Example run
+    - name: load neocortex model
       run: |
-        . ./venv/bin/activate
-        if [[ ${{ matrix.python-version }} == 3.9.* ]]; then
-          export RDMAV_FORK_SAFE=1
-        fi;
-        export PYTHONPATH=$(pwd)/nrn/build/lib/python:$PYTHONPATH
-        cp neurodamus/data/hoc/* tests/share/hoc/
-        export HOC_LIBRARY_PATH=$(pwd)/tests/share/hoc
+        export PYTHONPATH=$(pwd)/nrn/build/install/lib/python:$PYTHONPATH
+        export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_ROOT/share/neurodamus_neocortex/hoc
         export NEURODAMUS_PYTHON=$(pwd)/neurodamus/data
-        export CORENEURONLIB=$(pwd)/x86_64/libcorenrnmech.so
-        export PATH=$(pwd)/x86_64:$PATH
+        export CORENEURONLIB=$NEURODAMUS_NEOCORTEX_ROOT/lib/libcorenrnmech.so
+        export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_ROOT/lib/libnrnmech.so
+        export PATH=$NEURODAMUS_NEOCORTEX_ROOT/bin:$PATH
+
+        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV;
+        echo "HOC_LIBRARY_PATH=$HOC_LIBRARY_PATH" >> $GITHUB_ENV;
+        echo "CORENEURONLIB=$CORENEURONLIB" >> $GITHUB_ENV;
+        echo "NRNMECH_LIB_PATH=$NRNMECH_LIB_PATH" >> $GITHUB_ENV;
+        echo "NEURODAMUS_PYTHON=$NEURODAMUS_PYTHON" >> $GITHUB_ENV;
+
         which special
 
         # launch simulation with NEURON
@@ -201,6 +234,35 @@ jobs:
         mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata_coreneuron.json
         ls reporting_coreneuron/*.h5
 
-    # - name: live debug session, comment out
-    #   if: failure()
-    #   uses: mxschmitt/action-tmate@v3
+    - name: Install test dependencies
+      run: |
+        pip install -r tests/requirements.txt
+
+    - name: test-unit
+      run: |
+        pytest tests/unit
+
+    - name: test-integration
+      run: |
+        pytest -x --forked tests/integration
+
+    - name: test-integration-e2e
+      run: |
+        pytest -s -x --forked --durations=5 --durations-min=15 tests/integration-e2e
+
+    - name: test-scientific
+      run: |
+        pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific
+
+    - name: test-scientific-ngv
+      run: |
+        export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/share/neurodamus_neocortex/hoc
+        export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/lib/libnrnmech.so
+        unset CORENEURONLIB
+        pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific-ngv
+
+    - name: live debug session, comment out
+      if: failure()
+      uses: mxschmitt/action-tmate@v3
+      with:
+       timeout-minutes: 60  # or any other value you need

.gitlab-ci.yml

@@ -52,7 +52,6 @@ test-scientific:
     # modules need to be loaded in this certain order to avoid mixing of libraries
     EXTRA_MODULES:
       unstable:intel-oneapi-compilers
-      unstable:py-bluepysnap
       unstable:neurodamus-neocortex
 
 test-scientific-ngv:

docs/api/neurodamus.io.rst

@@ -9,6 +9,6 @@ Sub-Modules
 
 .. autosummary::
 
-   neurodamus.io.cell_readers
-   neurodamus.io.sonata_config
-   neurodamus.io.synapse_reader
+   cell_readers
+   sonata_config
+   synapse_reader

docs/api/neurodamus.rst

@@ -9,24 +9,24 @@ Package-Level Classes
 =====================
 
 .. autosummary::
-   neurodamus.Neurodamus
-   neurodamus.Node
+   Neurodamus
+   Node
 
 
 Sub-Modules
 ===========
 
 .. autosummary::
-   neurodamus.cell_distributor
-   neurodamus.connection_manager
-   neurodamus.connection
-   neurodamus.gap_junction
-   neurodamus.metype
-   neurodamus.modification_manager
-   neurodamus.ngv
-   neurodamus.replay
-   neurodamus.stimulus_manager
-   neurodamus.target_manager
+   cell_distributor
+   connection_manager
+   connection
+   gap_junction
+   metype
+   modification_manager
+   ngv
+   replay
+   stimulus_manager
+   target_manager
 
 
 Module API

docs/api/neurodamus.utils.rst

@@ -9,8 +9,8 @@ Sub-Modules
 
 .. autosummary::
 
-   neurodamus.utils.compat
-   neurodamus.utils.logging
-   neurodamus.utils.multimap
-   neurodamus.utils.progressbar
-   neurodamus.utils.pyutils
+   compat
+   logging
+   multimap
+   progressbar
+   pyutils

neurodamus/core/configuration.py

@@ -1039,7 +1039,7 @@ def _coreneuron_direct_mode(config: _SimConfig, run_conf):
 
 def get_debug_cell_gid(cli_options):
     gid = cli_options.get("dump_cell_state") if cli_options else None
-    if gid:
+    if gid is not None:
         try:
             # Convert to integer and adjust for sonata mode (0-based to 1-based indexing)
             gid = int(gid) + 1

neurodamus/io/sonata_config.py

@@ -336,9 +336,10 @@ def parsedProjections(self):
                         projection["Source"]
                 if projection.get("Type") == ConnectionTypes.GlioVascular:
                     for node_file_info in self._circuit_networks["nodes"]:
-                        for _, pop_info in node_file_info["populations"].items():
+                        for pop_name, pop_info in node_file_info["populations"].items():
                             if pop_info.get("type") == "vasculature":
-                                projection["VasculaturePath"] = node_file_info["nodes_file"]
+                                projection["VasculaturePath"] =\
+                                    self.circuits.node_population_properties(pop_name).elements_path
 
                 proj_name = "{0}__{1.source}-{1.target}".format(edge_pop_name, edge_pop)
                 projections[proj_name] = projection

tests/integration-e2e/test_cli_opts.py

@@ -9,6 +9,7 @@
 
 SIM_DIR = Path(__file__).parent.parent.absolute() / "simulations" / "v5_sonata"
 CONFIG_FILE_MINI = "simulation_config_mini.json"
+CIRCUIT_DIR = "sub_mini5"
 
 
 @pytest.mark.slow
@@ -19,9 +20,8 @@ def test_save_restore_cli():
     for simulator in ("NEURON", "CORENEURON"):
         test_folder = tempfile.TemporaryDirectory("cli-test-" + simulator)  # auto removed
         test_folder_path = Path(test_folder.name)
-        # Copy non-modified files
-        for f in ("node_sets.json", "circuit_config.json", "input.h5"):
-            shutil.copy(SIM_DIR / f, test_folder_path)
+        # Copy local data
+        shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
 
         for action, tstop in (("save", 100), ("restore", 200)):
             sim_config_data["target_simulator"] = simulator
@@ -48,21 +48,23 @@ def test_cli_prcellgid():
     from neurodamus import Neurodamus
     test_folder = tempfile.TemporaryDirectory("cli-test-prcellgid")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5", "simulation_config_mini.json"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copy(SIM_DIR / "simulation_config_mini.json", test_folder_path)
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
 
     os.chdir(test_folder_path)
-    nd = Neurodamus(CONFIG_FILE_MINI, dump_cell_state=62797)
+    nd = Neurodamus(CONFIG_FILE_MINI, dump_cell_state=1)
     nd.run()
-    assert (test_folder_path / "62798_py_Neuron_t0.0.nrndat").is_file()
-    assert (test_folder_path / "62798_py_Neuron_t100.0.nrndat").is_file()
+    assert (test_folder_path / "2_py_Neuron_t0.0.nrndat").is_file()
+    assert (test_folder_path / "2_py_Neuron_t100.0.nrndat").is_file()
 
 
 def test_cli_disable_reports():
     test_folder = tempfile.TemporaryDirectory("cli-test-disable-reports")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5", "simulation_config_mini.json"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copy(SIM_DIR / "simulation_config_mini.json", test_folder_path)
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
 
     with open(SIM_DIR / CONFIG_FILE_MINI, "r") as f:
         sim_config_data = json.load(f)
@@ -100,8 +102,8 @@ def test_cli_keep_build():
 
     test_folder = tempfile.TemporaryDirectory("cli-test-keep-build")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
     with open(test_folder_path / CONFIG_FILE_MINI, "w") as f:
         json.dump(sim_config_data, f, indent=2)
 
@@ -119,8 +121,8 @@ def test_cli_build_model():
 
     test_folder = tempfile.TemporaryDirectory("cli-test-build-model")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
     with open(test_folder_path / CONFIG_FILE_MINI, "w") as f:
         json.dump(sim_config_data, f, indent=2)
 
@@ -164,8 +166,8 @@ def test_cli_shm_transfer():
 
     test_folder = tempfile.TemporaryDirectory("cli-test-shm-transfer")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
     with open(test_folder_path / CONFIG_FILE_MINI, "w") as f:
         json.dump(sim_config_data, f, indent=2)
 
@@ -194,8 +196,9 @@ def test_cli_shm_transfer():
 def test_cli_lb_mode():
     test_folder = tempfile.TemporaryDirectory("cli-test-lb-mode")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5", "simulation_config_mini.json"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copy(SIM_DIR / "simulation_config_mini.json", test_folder_path)
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
 
     for lb_mode in ("WholeCell", "MultiSplit"):
         result = subprocess.run(
@@ -214,8 +217,9 @@ def test_cli_output_path():
     from neurodamus import Neurodamus
     test_folder = tempfile.TemporaryDirectory("cli-test-output-path")  # auto removed
     test_folder_path = Path(test_folder.name)
-    for f in ("node_sets.json", "circuit_config.json", "input.h5", "simulation_config_mini.json"):
-        shutil.copy(SIM_DIR / f, test_folder_path)
+    # Copy local data
+    shutil.copy(SIM_DIR / "simulation_config_mini.json", test_folder_path)
+    shutil.copytree(SIM_DIR / CIRCUIT_DIR, test_folder_path / CIRCUIT_DIR)
 
     with open(SIM_DIR / CONFIG_FILE_MINI, "r") as f:
         sim_config_data = json.load(f)