Tiny CI: Reactivate integration tests with new CI and detach from BB5

Reactivate integration tests with new CI and detach from BB5

- Solved issue #12.
- Updated metabolism manager for new CI.
- Improved CLI to report full error stack on failure.
- Tested and integrated new CI as default.
- Removed `bluepysnap` dependency.
- Re-enabled steps integration tests.
- Integrated metabolism into MultiscaleRun:
  - Organized configs/templates and cleaned `data`.
  - Moved metabolism data to `data/metabolism`.
  - Updated to use `pkg_path` for flexibility.
  - Relocated metabolism model to `metabolism/ODE_system.jl`.
  - Fixed templates and pathing in pytests.
- Updated GitLab CI for the new structure.
- Added young and old ODE_system models.
- Verified new CI is smaller with stats.

.ci/setup.sh

@@ -16,3 +16,12 @@ set_test_environment() {
   fi
   MPIRUN=${MPIRUN:-"srun --overlap"}
 }
+
+download_tiny_CI_neurodamus_data() {
+    local url="https://github.com/BlueBrain/MultiscaleRun/releases/download/0.8.2/tiny_CI_neurodamus_release-v0.8.2.tar.gz"
+    local filename="tiny_CI_neurodamus_release-v0.8.2.tar.gz"
+
+    # Download the file
+    wget -q "$url" -O "$filename"
+    tar -xzf "$filename"
+}

.ci/test_integration.sh

@@ -7,6 +7,19 @@ set_test_environment
 
 pushd "${SCRIPT_DIR}/.." >/dev/null
 
+download_tiny_CI_neurodamus_data
+
+
+MULTISCALE_RUN_PATH=""
+# Loop through the directories in PYTHONPATH and find the one containing the commit hash
+for path in $(echo $PYTHONPATH | tr ':' '\n'); do
+    if [[ "$path" == *"$PY_MULTISCALE_RUN_COMMIT"* ]]; then
+        MULTISCALE_RUN_PATH=$path
+        break
+    fi
+done
+ln -s "$(pwd)/tiny_CI_neurodamus" "$MULTISCALE_RUN_PATH/multiscale_run/templates/tiny_CI"
+
 num_errors=0
 count_errors() {
     ((num_errors++))

.ci/test_pytest.sh

@@ -8,6 +8,9 @@ set_test_environment
 
 pushd "${SCRIPT_DIR}/.." >/dev/null
 
+download_tiny_CI_neurodamus_data
+ln -s "$(pwd)/tiny_CI_neurodamus" "$(pwd)/multiscale_run/templates/tiny_CI"
+
 num_errors=0
 count_errors() {
     local command="$BASH_COMMAND"

.ci/test_simulation.sh

@@ -12,8 +12,8 @@ postproc=${postproc:true}
 steps=${steps:-false}
 metabolism=${metabolism:-false}
 bloodflow=${bloodflow:-false}
-tstop=${tstop:-300}
-circuit=${circuit:-rat_sscxS1HL_V10_CI}
+tstop=${tstop:-290}
+circuit=${circuit:-tiny_CI}
 
 if [ -z ${sim_name:x} ]; then
   fatal_error "expected environment variable 'SIM_NAME'."
@@ -23,6 +23,8 @@ rm -rf "$sim_name"
 multiscale-run init --no-check -f "$sim_name" --circuit $circuit
 
 pushd "$sim_name"
+wget https://github.com/BlueBrain/MultiscaleRun/releases/download/0.8.2/tiny_CI_neurodamus_release-v0.8.2.tar.gz
+tar -xzvf tiny_CI_neurodamus_release-v0.8.2.tar.gz
 /gpfs/bbp.cscs.ch/project/proj12/jenkins/subcellular/bin/jq ".multiscale_run.with_steps = $steps | .multiscale_run.with_bloodflow = $bloodflow | .multiscale_run.with_metabolism = $metabolism | .run.tstop = $tstop " simulation_config.json > simulation_config.json.bak
 mv simulation_config.json.bak simulation_config.json
 cat simulation_config.json

.gitlab-ci.yml

@@ -52,13 +52,12 @@ pytest:
   - !reference [.bbp_gitlab_access, script]
   - !reference [.spack_test, script]
 
-# TODO: reinstate when ratV10 is working
-# integration:
-#   extends: .spack_test
-#   timeout: 10 minutes
-#   variables:
-#     bb5_duration: "00:10:00"
-#     bb5_ntasks: 2
+integration:
+  extends: .spack_test
+  timeout: 10 minutes
+  variables:
+    bb5_duration: "00:10:00"
+    bb5_ntasks: 2
 
 .nrun:
   extends: .spack_test
@@ -70,9 +69,9 @@ pytest:
     expire_in: 4 hours
   variables:
     bb5_memory: 32G
-    bb5_ntasks: 4
+    bb5_ntasks: 2
     bb5_duration: "04:00:00"
-    circuit: "rat_sscxS1HL_V10_CI"
+    circuit: "tiny_CI"
   script:
   - spack ${SPACK_EXTRA_FLAGS} load /${SPACK_INSTALLED_HASH}
   - ${SPACK_SOURCE_DIR}/.ci/test_simulation.sh
@@ -84,26 +83,21 @@ pytest:
     steps: "true"
     metabolism: "false"
     bloodflow: "false"
+    tstop: 105
 
 dualrun 1/16:
   extends: .dualrun
   variables:
     bb5_ntasks: 1
     bb5_duration: "02:00:00"
-    tstop: 105
+
 
 dualrun 2/16:
   extends: .dualrun
   variables:
     bb5_ntasks: 2
-    tstop: 105
-
-dualrun 16/16:
-  extends: .dualrun
-  variables:
-    bb5_ntasks: 16
 
-ratV10_CI_nrun_[steps, metab, bf]:
+tiny_CI_nrun_[steps, metab, bf]:
   extends: .nrun
   parallel:
     matrix:

docs/contributing.rst

@@ -144,8 +144,9 @@ MultiscaleRun relies on ``setuptools-scm`` utility to infer the Python package v
 
 1. If the structure of the JSON configuration changed during this release (key addition, removal, ...), then increment the JSON ``config_format`` key in the files:
 
-  * ``multiscale_run/data/config/rat_sscxS1HL_V6/simulation_config.json``
-  * ``multiscale_run/data/config/rat_sscxS1HL_V10/simulation_config.json``
+  * ``multiscale_run/templates/rat_sscxS1HL_V6/simulation_config.json``
+  * ``multiscale_run/templates/rat_sscxS1HL_V10/simulation_config.json``
+  * ``multiscale_run/templates/tiny_CI/simulation_config.json``
 
 2. Ensure the Sphinx documentation is up-to-date. The fastest way is to check the artifacts of the ``docs`` stage in the CI.
 3. Ensure the *Releases Notes* section is completed for this version.

docs/simulation_config.rst

@@ -51,6 +51,7 @@ Preprocessor
 
   - **filter_neuron**: Boolean. Determines if the neurons connected to astrocytes must be filtered out when generating the mesh.
   - **neuron_population_name**: String. Name of the neuron population to use. Typically: "All".
+  - **astrocyte_population_name**: String. Name of the astrocyte population to use. Typically: "astrocytes".
 
 Connections among simulators
 ============================

multiscale_run/cli.py

@@ -29,18 +29,14 @@
     DEFAULT_CIRCUIT,
     NAMED_CIRCUITS,
 )
-from .data import (
+from .simulation import MsrSimulation
+from .templates import (
     BB5_JULIA_ENV,
     MSR_CONFIG_JSON,
     MSR_POSTPROC,
     SBATCH_TEMPLATE,
 )
-from .simulation import MsrSimulation
-from .utils import (
-    MsrException,
-    merge_dicts,
-    pushd,
-)
+from .utils import MsrException, copy_symlinks, merge_dicts, pushd
 
 
 def _cli_logger():
@@ -129,8 +125,11 @@ def julia(args=None, **kwargs):
 
 
 @command
-def init(directory, circuit, julia="shared", check=True, force=False):
+def init(
+    directory, circuit, julia="shared", check=True, force=False, metabolism="standard"
+):
     """Setup a new simulation"""
+
     if not force:
         dir_content = set(Path(".").iterdir())
         dir_content.discard(Path(".logs"))
@@ -140,6 +139,7 @@ def init(directory, circuit, julia="shared", check=True, force=False):
                 "Use option '-f' to overwrite the content of the directory."
             )
     assert julia in ["shared", "create", "no"]
+    assert metabolism in ["standard", "young", "aged"]
 
     circuit_def = NAMED_CIRCUITS[circuit]
 
@@ -150,12 +150,10 @@ def init(directory, circuit, julia="shared", check=True, force=False):
     sbatch_parameters["loaded_modules"] = loaded_modules
     SBATCH_TEMPLATE.stream(sbatch_parameters).dump("simulation.sbatch")
     shutil.copy(MSR_POSTPROC, MSR_POSTPROC.name)
-    shutil.copytree(
-        str(circuit_def.path),
-        ".",
-        ignore=shutil.ignore_patterns("cache", "msr*"),
-        dirs_exist_ok=True,
-    )
+    shutil.copy(circuit_def.path / "simulation_config.json", ".")
+    shutil.copy(circuit_def.path / "circuit_config.json", ".")
+    shutil.copy(circuit_def.path / "node_sets.json", ".")
+    copy_symlinks(circuit_def.path, ".")
 
     rd = {"msr_version": str(__version__)}
     if julia == "no":
@@ -203,6 +201,14 @@ def check_julia_env():
             check_julia_env()
 
     circuit_config = circuit_def.json()
+    if metabolism != "standard":
+        p_split = circuit_config["multiscale_run"]["metabolism"][
+            "julia_code_path"
+        ].rsplit(".", 1)
+        rd.setdefault("metabolism", {})["julia_code_path"] = (
+            f"{p_split[0]}_{metabolism}.{p_split[1]}"
+        )
+
     circuit_config = merge_dicts(child={"multiscale_run": rd}, parent=circuit_config)
     with open(MSR_CONFIG_JSON, "w") as ostr:
         json.dump(circuit_config, ostr, indent=4)
@@ -216,6 +222,20 @@ def check_julia_env():
         "generated sbatch file. But feel free to browse and tweak "
         "the JSON configuration files at will!"
     )
+
+    LOGGER.warning(
+        f'Set up with circuit: "{circuit}" and metabolism type: "{metabolism}"'
+    )
+    folder_path = os.path.abspath(directory)
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        LOGGER.warning(f"Folder: {os.path.abspath(directory)}")
+        content = os.listdir(folder_path)
+        LOGGER.warning(
+            f"Contents of the folder {folder_path}:\n"
+            + "\n".join(f"- {item}" for item in content)
+        )
+    else:
+        LOGGER.warning(f"Directory: {directory}")
     return directory
 
 
@@ -592,6 +612,12 @@ def argument_parser():
         "'create' to construct a new env locally, "
         "'no' to skip Julia environment (typically if metabolism model is not required)",
     )
+    parser_init.add_argument(
+        "--metabolism",
+        choices=["standard", "young", "aged"],
+        default="standard",
+        help="Choose Metabolism type. ",
+    )
     parser_init.add_argument(
         "--no-check",
         action="store_const",

multiscale_run/config.py

@@ -9,7 +9,7 @@
 import numpy as np
 
 from . import utils
-from .data import CONFIG_DIR, DATA_DIR, MSR_CONFIG_JSON, MSR_SCHEMA_JSON
+from .templates import MSR_CONFIG_JSON, MSR_PKG_DIR, MSR_SCHEMA_JSON, TEMPLATES_DIR
 
 
 class NamedCircuit(
@@ -51,23 +51,31 @@ def json(self):
 
 NAMED_CIRCUITS = {
     "rat_sscxS1HL_V6": NamedCircuit(
-        path=CONFIG_DIR / "rat_sscxS1HL_V6",
+        path=TEMPLATES_DIR / "rat_sscxS1HL_V6",
         sbatch_parameters=dict(
             job_name="msr_ratV6",
             nodes=1,
             time="01:00:00",
         ),
     ),
     "rat_sscxS1HL_V10": NamedCircuit(
-        path=CONFIG_DIR / "rat_sscxS1HL_V10",
+        path=TEMPLATES_DIR / "rat_sscxS1HL_V10",
         sbatch_parameters=dict(
             job_name="msr_ratV10",
             nodes=64,
             time="10:00:00",
         ),
     ),
+    "tiny_CI": NamedCircuit(
+        path=TEMPLATES_DIR / "tiny_CI",
+        sbatch_parameters=dict(
+            job_name="msr_tiny_CI",
+            nodes=1,
+            time="01:00:00",
+        ),
+    ),
     "rat_sscxS1HL_V10_CI": NamedCircuit(
-        path=CONFIG_DIR / "rat_sscxS1HL_V10",
+        path=TEMPLATES_DIR / "rat_sscxS1HL_V10",
         sbatch_parameters=dict(
             job_name="msr_ratV10_CI",
             nodes=1,
@@ -83,7 +91,7 @@ def json(self):
     ),
 }
 
-DEFAULT_CIRCUIT = "rat_sscxS1HL_V10_CI"
+DEFAULT_CIRCUIT = "tiny_CI"
 
 
 class MsrConfigException(Exception):
@@ -237,7 +245,7 @@ def _load(self):
 
         """
         d = utils.load_json(
-            self.config_path, base_subs_d={"pkg_data_path": str(DATA_DIR)}
+            self.config_path, base_subs_d={"pkg_path": str(MSR_PKG_DIR)}
         )
         self.update(MsrConfig._objectify_config(None, d))
 

multiscale_run/connection_manager.py

@@ -473,7 +473,7 @@ def remove_gids(self, failed_cells: list[int]):
         ndam_m = self.managers["neurodamus"]
         metab_m = self.managers.get("metabolism", None)
 
-        gids = ndam_m.gids
+        gids = ndam_m.gids()
         failed_gids = {
             gids[igid]: e for igid, e in enumerate(failed_cells) if e is not None
         }