Feat add model version

.github/workflows/ci-cd.yml

@@ -8,4 +8,4 @@ jobs:
       with:
         workdir: ./python
         pkg_name: bento-meta
-      secrets: inherit
+      secrets: inherit

python/pyproject.toml

@@ -55,7 +55,7 @@ Tools = ["click", "numpy", "pandas", "spacy", "liquichange"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.0"
-pytest-docker = "^1.0.1"
+pytest-docker = "^3.1.1"
 requests = "^2.28.1"
 myst-nb = {version = "^0.17.1", python = "^3.8"}
 sphinx-autoapi = "^2.0.1"

python/tests/conftest.py

@@ -1,6 +1,9 @@
 import pytest
 import requests
 import string
+import subprocess
+import logging
+from warnings import warn
 from requests.exceptions import ConnectionError
 from time import sleep
 from pdb import set_trace
@@ -25,15 +28,15 @@ def bento_neo4j(docker_services, docker_ip):
     http_url = "http://{}:{}".format(docker_ip, http_port)
     sleep(wait)
     docker_services.wait_until_responsive(
-        timeout=5.0, pause=1.0, check=lambda: is_responsive(http_url)
+        timeout=15.0, pause=1.0, check=lambda: is_responsive(http_url)
     )
     return (bolt_url, http_url)
 
-  
+
 @pytest.fixture(scope="session")
-def plain_neo4j(docker_services, docker_ip):
-    bolt_port = docker_services.port_for("plain-neo4j", 7687)
-    http_port = docker_services.port_for("plain-neo4j", 7474)
+def mdb(docker_services, docker_ip):
+    bolt_port = docker_services.port_for("mdb", 7687)
+    http_port = docker_services.port_for("mdb", 7474)
     bolt_url = "bolt://{}:{}".format(docker_ip, bolt_port)
     http_url = "http://{}:{}".format(docker_ip, http_port)
     sleep(wait)
@@ -44,22 +47,17 @@ def plain_neo4j(docker_services, docker_ip):
 
 
 @pytest.fixture(scope="session")
-def mdb(docker_services, docker_ip):
-    bolt_port = docker_services.port_for("mdb", 7687)
-    http_port = docker_services.port_for("mdb", 7474)
+def mdb_versioned(docker_services, docker_ip):
+    bolt_port = docker_services.port_for("mdb-versioned", 7687)
+    http_port = docker_services.port_for("mdb-versioned", 7474)
     bolt_url = "bolt://{}:{}".format(docker_ip, bolt_port)
     http_url = "http://{}:{}".format(docker_ip, http_port)
     sleep(wait)
     docker_services.wait_until_responsive(
-        timeout=30.0, pause=0.5, check=lambda: is_responsive(http_url)
+        timeout=30.0, pause=1.0, check=lambda: is_responsive(http_url)
     )
     return (bolt_url, http_url)
 
-
-@pytest.fixture(scope="session")
-def mdb_local():
-    return ("bolt://localhost", "http://localhost")
-
 @pytest.fixture()
 def test_paths(model="ICDC", handle="diagnosis", phandle="disease_term", key="Class", value="primary", nanoid="abF32k"):
     tpl = [

python/tests/docker-compose.yml

@@ -1,13 +1,13 @@
 version: '2'
 services:
-  bento-neo4j:
-    image: 'maj1/test-db-bento-meta'
+  mdb-versioned:
+    image: 'maj1/mdb-test-versioned' 
     ports:
       - "7474"
       - "7473"
       - "7687"
-  plain-neo4j:
-    image: 'maj1/test-plain-db-bento-meta'
+  bento-neo4j:
+    image: 'maj1/test-db-bento-meta'
     ports:
       - "7474"
       - "7473"
@@ -18,4 +18,3 @@ services:
       - "7474"
       - "7473"
       - "7687"
-

python/tests/samples/version-test/test-model-a-1.0.yml

@@ -0,0 +1,88 @@
+Handle: A
+Version: 1.0
+Nodes:
+  case:
+    Props:
+      - case_id
+      - patient_id
+  sample:
+    Props:
+      - sample_type
+      - amount
+  file:
+    Props:
+      - md5sum
+      - file_name
+      - file_size
+  diagnosis:
+    Props:
+      - disease
+Relationships:
+  of_case:
+    Mul: one_to_one
+    Ends:
+      - Src: sample
+        Dst: case
+        Mul: many_to_one
+        Props:
+          - days_to_sample
+      - Src: diagnosis
+        Dst: case
+  of_sample:
+    Mul: one_to_one
+    Ends:
+      - Src: file
+        Dst: sample
+        Mul: many_to_many
+  derived_from:
+    Mul: one_to_many
+    Props:
+      - id
+    Ends:
+      - Src: file
+        Dst: file
+        Props:
+          - workflow_id
+      - Src: sample
+        Dst: sample
+PropDefinitions:
+  case_id:
+    Type:
+      pattern: "^CASE-[0-9]+$"
+  patient_id:
+    Type: string
+  sample_type:
+    Type:
+      - normal
+      - tumor
+  amount:
+    Type:
+      units:
+        - mg
+      value_type: number
+  md5sum:
+    Type:
+      pattern: "^[a-f0-9]{40}"
+  file_name:
+    Type: string
+  file_size:
+    Type:
+      units:
+        - Gb
+        - Mb
+      value_type: integer
+  disease:
+    Type: url
+  days_to_sample:
+    Type:
+      units:
+        - days
+      value_type: integer
+  id:
+    Type: string
+  workflow_id:
+    Type:
+      # uuid
+      pattern: "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
+
+

python/tests/samples/version-test/test-model-a-1.1.yml

@@ -0,0 +1,112 @@
+Handle: A
+Version: 1.1
+Nodes:
+  case:
+    Props:
+      - case_id
+      - patient_id
+  sample:
+    Props:
+      - sample_type
+      - amount
+  file:
+    Props:
+      - md5sum
+      - file_name
+      - file_size
+      - file_type
+  diagnosis:
+    Props:
+      - disease
+      - comorbidities
+  investigator:
+    Props:
+      - orcid_id
+Relationships:
+  of_case:
+    Mul: one_to_one
+    Ends:
+      - Src: sample
+        Dst: case
+        Mul: many_to_one
+        Props:
+          - days_to_sample
+      - Src: diagnosis
+        Dst: case
+  of_sample:
+    Mul: one_to_one
+    Ends:
+      - Src: file
+        Dst: sample
+        Mul: many_to_many
+      - Src: investigator
+        Dst: sample
+        Mul: many_to_many
+  derived_from:
+    Mul: one_to_many
+    Props:
+      - id
+    Ends:
+      - Src: file
+        Dst: file
+        Props:
+          - workflow_id
+      - Src: sample
+        Dst: sample
+PropDefinitions:
+  case_id:
+    Type:
+      pattern: "^CASE-[0-9]+$"
+  patient_id:
+    Type: string
+  sample_type:
+    Enum:
+      - normal
+      - tumor
+      - CSF
+  amount:
+    Type:
+      units:
+        - mg
+      value_type: number
+  md5sum:
+    Type:
+      pattern: "^[a-f0-9]{40}"
+  file_name:
+    Type: string
+  file_size:
+    Type:
+      units:
+        - Gb
+        - Mb
+      value_type: integer
+  file_type:
+    Enum:
+      - BAM
+      - VCF
+      - FASTQ
+  disease:
+    Type: url
+  comorbidities:
+    Type:
+      value_type: list
+      item_type:
+        Enum:
+          - diabetes
+          - CKD
+          - COPD
+  days_to_sample:
+    Type:
+      units:
+        - days
+      value_type: integer
+  id:
+    Type: string
+  orcid_id:
+    Type: string
+  workflow_id:
+    Type:
+      # uuid
+      pattern: "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
+
+

python/tests/samples/version-test/test-model-a.yml

@@ -0,0 +1,86 @@
+Nodes:
+  case:
+    Props:
+      - case_id
+      - patient_id
+  sample:
+    Props:
+      - sample_type
+      - amount
+  file:
+    Props:
+      - md5sum
+      - file_name
+      - file_size
+  diagnosis:
+    Props:
+      - disease
+Relationships:
+  of_case:
+    Mul: one_to_one
+    Ends:
+      - Src: sample
+        Dst: case
+        Mul: many_to_one
+        Props:
+          - days_to_sample
+      - Src: diagnosis
+        Dst: case
+  of_sample:
+    Mul: one_to_one
+    Ends:
+      - Src: file
+        Dst: sample
+        Mul: many_to_many
+  derived_from:
+    Mul: one_to_many
+    Props:
+      - id
+    Ends:
+      - Src: file
+        Dst: file
+        Props:
+          - workflow_id
+      - Src: sample
+        Dst: sample
+PropDefinitions:
+  case_id:
+    Type:
+      pattern: "^CASE-[0-9]+$"
+  patient_id:
+    Type: string
+  sample_type:
+    Type:
+      - normal
+      - tumor
+  amount:
+    Type:
+      units:
+        - mg
+      value_type: number
+  md5sum:
+    Type:
+      pattern: "^[a-f0-9]{40}"
+  file_name:
+    Type: string
+  file_size:
+    Type:
+      units:
+        - Gb
+        - Mb
+      value_type: integer
+  disease:
+    Type: url
+  days_to_sample:
+    Type:
+      units:
+        - days
+      value_type: integer
+  id:
+    Type: string
+  workflow_id:
+    Type:
+      # uuid
+      pattern: "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
+
+