From ba0ea1d21032e6e05bc87280104fc361c0934c8e Mon Sep 17 00:00:00 2001
From: "Zhaoyuan (Ryan) Fu" <fuzhaoyuan@gmail.com>
Date: Thu, 7 Nov 2024 13:24:41 -0500
Subject: [PATCH] Patient level filtering works for non-NA

---
 .../helper/StudyViewFilterHelper.java         | 18 +++--
 .../StudyViewFilterMapper.xml                 | 79 +++++++++++++------
 .../mybatisclickhouse/StudyViewMapper.xml     |  4 +-
 3 files changed, 72 insertions(+), 29 deletions(-)

diff --git a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java
index d10d85d98c4..2a06b059d42 100644
--- a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java
+++ b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java
@@ -63,11 +63,7 @@ private CategorizedGenericAssayDataCountFilter extractGenericAssayDataCountFilte
 
         CategorizedGenericAssayDataCountFilter.Builder builder = CategorizedGenericAssayDataCountFilter.getBuilder();
 
-        // TODO: Support patient level profiles and data filtering
-        List<String> sampleCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE)
-            .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY"))
-            .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
-            .toList();
+        // TODO: Support data filtering
         List<String> sampleNumericalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE)
             .stream().filter(profile -> profile.getDatatype().equals("LIMIT-VALUE"))
             .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
@@ -75,9 +71,21 @@ private CategorizedGenericAssayDataCountFilter extractGenericAssayDataCountFilte
         builder.setSampleNumericalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream()
             .filter(genericAssayDataFilter -> sampleNumericalProfileTypes.contains(genericAssayDataFilter.getProfileType()))
             .toList());
+        List<String> sampleCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE)
+            .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY"))
+            .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
+            .toList();
         builder.setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream()
             .filter(genericAssayDataFilter -> sampleCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType()))
             .toList());
+        // Patient level profiles only have CATEGORICAL datatype for now
+        List<String> patientCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.PATIENT)
+            .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL"))
+            .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", ""))
+            .toList();
+        builder.setPatientCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream()
+            .filter(genericAssayDataFilter -> patientCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType()))
+            .toList());
         return builder.build();
     }
 
diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
index 841fd4abf6a..f7d34a34b03 100644
--- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
+++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
@@ -138,7 +138,6 @@
                 <include refid="applySampleTreatmentFilter"/>
             </if>
             <!-- Apply Genomic Data Filter -->
-            <!-- TODO: add patient level data filtering -->
             <if test="studyViewFilterHelper.studyViewFilter.genomicDataFilters != null and !studyViewFilterHelper.studyViewFilter.genomicDataFilters.isEmpty()">
                 <foreach item="genomicDataFilter" collection="studyViewFilterHelper.studyViewFilter.genomicDataFilters" open="INTERSECT" separator="INTERSECT">
                     (
@@ -154,25 +153,9 @@
                 </foreach>
             </if>
             <!-- Apply Generic Assay Data Filter -->
-            <if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters().isEmpty()">
-                <foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
-                    (
-                    <include refid="numericalGenericAssayDataFilter">
-                        <property name="unique_id" value="sample_unique_id"/>
-                        <property name="table_name" value="generic_assay_data_derived"/>
-                    </include>
-                    )
-                </foreach>
-            </if>
-            <if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters().isEmpty()">
-                <foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
-                    <include refid="categoricalGenericAssayDataCountFilter">
-                        <property name="unique_id" value="sample_unique_id"/>
-                        <property name="table_name" value="generic_assay_data_derived"/>
-                    </include>
-                </foreach>
-            </if>
-            
+            <if test="studyViewFilterHelper.studyViewFilter.genericAssayDataFilters != null and !studyViewFilterHelper.studyViewFilter.genericAssayDataFilters.isEmpty()">
+                <include refid="applyGenericAssayDataFilter"/>
+            </if>            
             <!-- Apply Clinical Data Filter -->
             <if test="studyViewFilterHelper.studyViewFilter.clinicalDataFilters != null and !studyViewFilterHelper.studyViewFilter.clinicalDataFilters.isEmpty()">
                 <include refid="applyClinicalDataCountFilter"/>
@@ -549,9 +532,61 @@
             </foreach>
     </sql>
 
+    <sql id="applyGenericAssayDataFilter">
+        <if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters().isEmpty()">
+            <foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleNumericalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
+                (
+                    <include refid="numericalGenericAssayDataFilter">
+                        <property name="unique_id" value="sample_unique_id"/>
+                        <property name="table_name" value="generic_assay_data_derived"/>
+                    </include>
+                )
+            </foreach>
+        </if>
+        <if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters().isEmpty()">
+            <foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getSampleCategoricalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
+                (
+                    <include refid="categoricalGenericAssayDataCountFilter">
+                        <property name="unique_id" value="sample_unique_id"/>
+                        <property name="table_name" value="generic_assay_data_derived"/>
+                    </include>
+                )
+            </foreach>
+        </if>
+        -- patient level profile only have category for now
+        <if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientNumericalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientNumericalGenericAssayDataFilters().isEmpty()">
+            <foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientNumericalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
+                (
+                    SELECT sample_unique_id
+                    FROM sample_derived
+                    WHERE patient_unique_id IN (
+                        <include refid="numericalGenericAssayDataFilter">
+                            <property name="unique_id" value="patient_unique_id"/>
+                            <property name="table_name" value="generic_assay_data_derived"/>
+                        </include>
+                    )
+                )
+            </foreach>
+        </if>
+        <if test="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientCategoricalGenericAssayDataFilters() != null and !studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientCategoricalGenericAssayDataFilters().isEmpty()">
+            <foreach item="genericAssayDataFilter" collection="studyViewFilterHelper.categorizedGenericAssayDataCountFilter.getPatientCategoricalGenericAssayDataFilters()" open="INTERSECT" separator="INTERSECT">
+                (
+                    SELECT sample_unique_id
+                    FROM sample_derived
+                    WHERE patient_unique_id IN (
+                        <include refid="categoricalGenericAssayDataCountFilter">
+                            <property name="unique_id" value="patient_unique_id"/>
+                            <property name="table_name" value="generic_assay_data_derived"/>
+                        </include>
+                    )
+                )
+            </foreach>
+        </if>
+    </sql>
+    
     <sql id="selectAllGenericAssays">
-        SELECT sample_unique_id, value, datatype
-        FROM generic_assay_data_derived
+        SELECT sample_unique_id, patient_unique_id, value, datatype
+        FROM generic_assay_data_derived_patient
         WHERE profile_type = #{genericAssayDataFilter.profileType}
             AND entity_stable_id = #{genericAssayDataFilter.stableId}
     </sql>
diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml
index d714eedfc20..2ae5ced9a2b 100644
--- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml
+++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml
@@ -315,7 +315,7 @@
                 entity_stable_id AS stableId,
                 value,
                 cast(count(*) AS INTEGER) AS count
-            FROM generic_assay_data_derived
+            FROM generic_assay_data_derived_patient
             <where>
                 <!-- Table creation in clickhouse.sql has ensured no NA values but extra caution is always appreciated -->
                 <include refid="normalizeAttributeValue">
@@ -723,7 +723,7 @@
                     <property name="attribute_value" value="value"/>
                 </include> AS value,
                 cast(count(value) as INTEGER) AS count
-            FROM generic_assay_data_derived
+            FROM generic_assay_data_derived_patient
             <where>
                 <!-- Need to ensure no NA values -->
                 <include refid="normalizeAttributeValue">