Sequence of events (#27)

* first integration batch of sequence of events * fixed concurrency error in survival query subgroup * added sequence of events API docs * added tests for sequence of events * added sequence of events API docs * Small comment typo Co-authored-by: Romain Bouyé <[email protected]>
tuneinsight · Oct 25, 2022 · c13b528 · c13b528
1 parent 2befaea
commit c13b528
Show file tree

Hide file tree

Showing 11 changed files with 557 additions and 126 deletions.
diff --git a/API.md b/API.md
@@ -132,7 +132,7 @@ Retrieve patient IDs from i2b2 based on explore query terms.
 {
   "id": "99999999-9999-9999-9999-999999999999",
   "definition": {
-    "panels": [{
+    "selectionPanels": [{
       "not": false,
       "timing": "any|samevisit|sameinstancenum",
       "cohortItems": ["cohortName0", "cohortName1"],
@@ -147,14 +147,28 @@ Retrieve patient IDs from i2b2 based on explore query terms.
         }
       }]
     }],
+    "sequentialPanels": [{
+    }],
+    "sequentialOperators": [{
+      "whichDateFirst":          "STARTDATE|ENDDATE",
+      "whichObservationFirst":   "FIRST|LAST|ANY",
+      "when":                    "LESS|LESSEQUAL|EQUAL",
+      "whichDateSecond":         "STARTDATE|ENDDATE",
+      "whichObservationSecond":  "FIRST|LAST|ANY",
+      "spans": [{
+        "value": 5,
+        "units": "HOUR|DAY|MONTH|YEAR",
+        "operator": "LESS|LESSEQUAL|EQUAL|GREATEREQUAL|GREATER"
+      }]
+    }],
     "timing": "any|samevisit|sameinstancenum"
   }
 }
 ```
 
 - `id`: ID of the query, must be an UUID
 - `definition`: definition of the explore query
-  - `panels`: panels of the explore query (linked together by an AND)
+  - `selectionPanels`: panels of the explore query (linked together by an AND)
     - `not`: true if the panel is inverted
     - `timing`: timing of the panel
       - `any`: no constrain (default)
@@ -187,6 +201,44 @@ Retrieve patient IDs from i2b2 based on explore query terms.
     - `any`: no constrain (default)
     - `samevisit`: constrain to the same visit
     - `sameinstancenum`: constrain to the same instance number
+  - `sequentialPanels`: sequential panels of the explore query (linked together by a sequential operator, and with the `selectionPanels` by an AND)
+  - `sequentialOperators`: operators determining the temporal relations between the `sequentialPanels`. The element at position `i` determines the relation between the panels at positions `i` and `i + 1`.  
+    The observations identified by the first panel occur before the observations identified by the second panel if  
+    the `whichDateFirst` of the `whichObservationFirst` observation in the first panel  
+    occurs `when` [by `spans[0]` [and `spans[1]`]] than  
+    the `whichDateSecond` of the `whichObservationSecond` observation in the second panel
+    - `whichDateFirst`: the date to be considered to determine the time of the first panel
+      - `STARTDATE`: the start date of the observation (default)
+      - `ENDDATE`: the end date of the observation
+    - `whichObservationFirst`: the observation to be considered to determine the time of the first panel
+      - `FIRST`: the first observation (default)
+      - `LAST`: the last observation
+      - `ANY`: any observation
+    - `when`: the relation between the time of the first panel and the time of the second panel
+      - `LESS`: before (default)
+      - `LESSEQUAL`: before or at the same time
+      - `EQUAL`: at the same time
+    - `whichDateSecond`: the date to be considered to determine the time of the second panel
+      - `STARTDATE`: the start date of the observation (default)
+      - `ENDDATE`: the end date of the observation
+    - `whichObservationSecond`: the observation to be considered to determine the time of the second panel
+      - `FIRST`: the first observation (default)
+      - `LAST`: the last observation
+      - `ANY`: any observation
+    - `spans`: optionally add a time constraint to `when`, e.g. it specifies the difference between the time of the first panel and the time of the second panel (e.g. by 1 and 3 months).  
+    It contains max 2 elements, the first one being the left endpoint of the time constraint, the second the right one.
+      - `value`: numeric value of one of the endpoint of the time constraint
+      - `units`: the units of the time constraint
+        -  `HOUR`
+        -  `DAY`
+        -  `MONTH`
+        -  `YEAR`
+      - `operator`:
+        -  `LESS`
+        -  `LESSEQUAL`
+        -  `EQUAL`
+        -  `GREATEREQUAL`
+        -  `GREATER`
 
 ## Output Data Objects Shared IDs
 - `count`: integer containing the count of patients
@@ -297,12 +349,9 @@ Run survival query.
   "subGroupsDefinitions": [
     {
       "name": "xxxx",
-      "timing": "any|samevisit|sameinstance",
-      "panels": [
-        {
-
-        }
-      ]
+      "constraint": {
+
+      }
     }
   ]
 }
@@ -319,8 +368,7 @@ Run survival query.
 - `timeLimit`: time limit (how many bins) to consider for the survival query
 - `subGroupsDefinitions`: subgroups definitions
   - `name`: name of the subgroup
-  - `timing`: timing of the subgroup
-  - `panels`: panels defining the subgroup, as defined in exploreQuery
+  - `constraint`: `definition` as defined in exploreQuery parameters
 
 ## Output Data Objects Shared IDs
 - `survivalQueryResult`: vector of integers containing the flattened event groups
@@ -338,10 +386,9 @@ Run statistics query.
 ```json
 {
   "id": "99999999-9999-9999-9999-999999999999",
-  "panels": [
+  "constraint": [
 
   ],
-  "timing": "any|samevisit|sameinstancenum",
   "analytes": [
 
   ],
@@ -351,8 +398,7 @@ Run statistics query.
 ```
 
 - `id`: ID of the statistics query, must be an UUID
-- `panels`: array of panels (see "exploreQuery") defining the analyzed population
-- `timing`: `panels`' timing (see "exploreQuery)
+- `constraint`: `definition` as defined in exploreQuery parameters
 - `analytes`: the concepts (see `conceptItems` in "exploreQuery") used as analytes of the statistics query
 - `bucketSize`: bucket size for each analyte (float64)
 - `minObservations`: the total minimal number of observations for each analyte.

diff --git a/pkg/datasource/datasource_test.go b/pkg/datasource/datasource_test.go
@@ -71,7 +71,7 @@ func TestQueryDataObject(t *testing.T) {
 	ds := getDataSource(t)
 	defer dataSourceCleanUp(t, ds)
 
-	params := `{"id": "99999999-9999-1122-0000-999999999999", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}`
+	params := `{"id": "99999999-9999-1122-0000-999999999999", "definition": {"selectionPanels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}`
 	sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{
 		outputNameExploreQueryCount:       "99999999-9999-9999-1111-999999999999",
 		outputNameExploreQueryPatientList: "99999999-9999-9999-0000-999999999999",
@@ -149,7 +149,7 @@ func testWorkflow(t *testing.T, ds *I2b2DataSource) {
 	params = fmt.Sprintf(`{
 		"id": "%v",
 		"definition": {
-			"panels": [
+			"selectionPanels": [
 				{
 					"conceptItems": [
 						{

diff --git a/pkg/datasource/explore_query.go b/pkg/datasource/explore_query.go
@@ -94,7 +94,7 @@ func (ds I2b2DataSource) ExploreQuery(userID string, params *models.ExploreQuery
 func (ds I2b2DataSource) doCrcPsmQuery(userID string, params models.ExploreQueryParameters) (patientCount, patientSetID string, err error) {
 
 	// retrieve patient set IDs for cohort items and replace them in the panels
-	for _, panel := range params.Definition.Panels {
+	for _, panel := range params.Definition.SelectionPanels {
 		for i, cohortItem := range panel.CohortItems {
 			cohort, err := ds.db.GetCohort(userID, cohortItem)
 			if err != nil || cohort == nil {
@@ -105,13 +105,18 @@ func (ds I2b2DataSource) doCrcPsmQuery(userID string, params models.ExploreQuery
 	}
 
 	// build query
-	panels, timing := params.Definition.ToI2b2APIModel()
+	panels, subQueries, subQueriesConstraints, timing, err := params.Definition.ToI2b2APIModel()
+	if err != nil {
+		return "", "", fmt.Errorf("while converting explore query definition to i2b2 model: %v", err)
+	}
 
 	psmReq := i2b2clientmodels.NewCrcPsmReqFromQueryDef(
 		ds.i2b2Client.Ci,
 		params.ID,
 		panels,
 		timing,
+		subQueries,
+		subQueriesConstraints,
 		[]i2b2clientmodels.ResultOutputName{i2b2clientmodels.ResultOutputPatientSet, i2b2clientmodels.ResultOutputCount},
 	)
 

diff --git a/pkg/datasource/explore_query_test.go b/pkg/datasource/explore_query_test.go
@@ -2,8 +2,10 @@ package datasource
 
 import (
 	"fmt"
+	"strconv"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/tuneinsight/geco-i2b2-data-source/pkg/datasource/models"
 	gecomodels "github.com/tuneinsight/sdk-datasource/pkg/models"
@@ -17,7 +19,7 @@ func TestExploreQueryConcept(t *testing.T) {
 	_, count, patientList, err := ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "0",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -33,7 +35,7 @@ func TestExploreQueryConcept(t *testing.T) {
 	_, count, patientList, err = ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "1",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/3/",
@@ -49,7 +51,7 @@ func TestExploreQueryConcept(t *testing.T) {
 	_, count, patientList, err = ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "2",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -75,7 +77,7 @@ func TestExploreQueryConcept(t *testing.T) {
 	_, count, patientList, err = ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "3",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -98,7 +100,7 @@ func TestExploreQueryConceptValue(t *testing.T) {
 	_, count, patientList, err := ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "0",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -117,7 +119,7 @@ func TestExploreQueryConceptValue(t *testing.T) {
 	_, count, patientList, err = ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "1",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -146,7 +148,7 @@ func TestExploreQueryModifier(t *testing.T) {
 	_, count, patientList, err := ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "0",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -169,7 +171,7 @@ func TestExploreQueryModifier(t *testing.T) {
 	_, count, patientList, err = ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "1",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/1/",
@@ -206,7 +208,7 @@ func TestExploreQueryModifierValue(t *testing.T) {
 	_, count, patientList, err := ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "0",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/2/",
@@ -232,7 +234,7 @@ func TestExploreQueryModifierValue(t *testing.T) {
 	_, count, patientList, err = ds.ExploreQuery("testuser1", &models.ExploreQueryParameters{
 		ID: "1",
 		Definition: models.ExploreQueryDefinition{
-			Panels: []models.Panel{{
+			SelectionPanels: []models.Panel{{
 				Not: false,
 				ConceptItems: []models.ConceptItem{{
 					QueryTerm: "/TEST/test/3/",
@@ -279,7 +281,7 @@ func TestExploreQueryDatabase(t *testing.T) {
 	countSharedID := "44444444-7777-8888-4444-444444444444"
 	patientListSharedID := "44444444-7777-4444-7121-444444444444"
 
-	params := fmt.Sprintf(`{"id": "%v", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}`, queryID)
+	params := fmt.Sprintf(`{"id": "%v", "definition": {"selectionPanels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}`, queryID)
 	sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{
 		outputNameExploreQueryCount:       gecomodels.DataObjectSharedID(countSharedID),
 		outputNameExploreQueryPatientList: gecomodels.DataObjectSharedID(patientListSharedID),
@@ -293,3 +295,81 @@ func TestExploreQueryDatabase(t *testing.T) {
 	require.EqualValues(t, countSharedID, query.ResultGecoSharedIDCount.String)
 	require.EqualValues(t, patientListSharedID, query.ResultGecoSharedIDPatientList.String)
 }
+
+func TestExploreQueryWithSequence(t *testing.T) {
+
+	ds := getDataSource(t)
+	defer dataSourceCleanUp(t, ds)
+
+	patientSetID, patientCount, _, err := ds.ExploreQuery(
+		"testUser",
+		&models.ExploreQueryParameters{
+			ID: "",
+			Definition: models.ExploreQueryDefinition{
+				Timing: models.TimingAny,
+				SelectionPanels: []models.Panel{
+					{ConceptItems: []models.ConceptItem{
+						{
+							QueryTerm: "/TEST/test/1/",
+						},
+					},
+						Not: false,
+					}},
+				SequentialPanels: []models.Panel{
+					{ConceptItems: []models.ConceptItem{
+						{
+							QueryTerm: "/TEST/test/1/",
+						},
+					},
+						Not: false,
+					},
+					{ConceptItems: []models.ConceptItem{
+						{
+							QueryTerm: "/TEST/test/2/",
+						},
+					},
+						Not: false,
+					}},
+				SequentialOperators: []models.SequentialOperator{
+					{
+						When:                   models.SequentialOperatorWhenLess,
+						WhichDateFirst:         models.SequentialOperatorWhichDateStart,
+						WhichDateSecond:        models.SequentialOperatorWhichDateStart,
+						WhichObservationFirst:  models.SequentialOperatorWhichObservationFirst,
+						WhichObservationSecond: models.SequentialOperatorWhichObservationFirst,
+					}},
+			},
+		})
+
+	assert.NoError(t, err)
+	t.Log("count:"+strconv.FormatInt(patientCount, 10), "set ID:"+strconv.FormatInt(patientSetID, 10))
+
+	// not a correct number of sequence panels for the number of sequence operators
+	patientCount, patientSetID, _, err = ds.ExploreQuery(
+		"testUser",
+		&models.ExploreQueryParameters{
+			ID: "",
+			Definition: models.ExploreQueryDefinition{
+				Timing: models.TimingAny,
+				SequentialPanels: []models.Panel{
+					{ConceptItems: []models.ConceptItem{
+						{
+							QueryTerm: "/TEST/test/1/",
+						},
+					},
+						Not: false,
+					}},
+				SequentialOperators: []models.SequentialOperator{
+					{
+						When:                   models.SequentialOperatorWhenLess,
+						WhichDateFirst:         models.SequentialOperatorWhichDateStart,
+						WhichDateSecond:        models.SequentialOperatorWhichDateStart,
+						WhichObservationFirst:  models.SequentialOperatorWhichObservationFirst,
+						WhichObservationSecond: models.SequentialOperatorWhichObservationFirst,
+					}},
+			},
+		})
+
+	assert.Error(t, err)
+	t.Log("count:"+strconv.FormatInt(patientCount, 10), "set ID:"+strconv.FormatInt(patientSetID, 10))
+}