fix: CSV data parsing fixes

- expand the lists of expected column headers for units and observations.
- set non-numeric observations to 0.0.

Author: eatyourgreens

frontend-v2/src/features/data/normaliseDataHeaders.ts

@@ -3,7 +3,7 @@ const normalisation = {
     'Additional Doses': ['additional doses', 'additional dose', 'addl'],
     'Administration ID': ['administration id', 'cmt', 'adm'],
     'Amount': ['amount', 'amt' ],
-    'Amount Unit': ['amount unit', 'amt_unit', 'amt_units', 'amtunit', 'amtunits'],
+    'Amount Unit': ['amount unit', 'amt_unit', 'amt_units', 'amtunit', 'amtunits', 'units_amt'],
     'Cat Covariate': ['cat covariate', 'cat', 'sex', 'gender', 'group', 'cohort', 'study', 'route', 'matrix'],
     'Censoring': ['cens', 'blq', 'lloq'],
     'Cont Covariate': ['cont covariate', 'weight', 'wt', 'bw', 'age', 'dose'],
@@ -15,12 +15,12 @@ const normalisation = {
     'Infusion Rate': ['infusion rate', 'rate'],
     'Interdose Interval': ['interdose interval', 'ii', 'tau', 'dosing interval'],
     'Observation': ['observation', 'dv', 'c', 'y', 'conc', 'cobs', 'obs'],
-    'Observation Unit': ['observation unit', 'dv_units', 'c_units', 'y_units', 'yunit', 'cunit', 'obs_units', 'obs_unit', 'obsunit'],
+    'Observation Unit': ['observation unit', 'dv_units', 'c_units', 'y_units', 'yunit', 'cunit', 'obs_units', 'obs_unit', 'obsunit', 'units_conc'],
     'Observation ID': ['observation id', 'ytype', 'dvid'],
     'Occasion': ['occasion', 'occ'],
     'Regressor': ['x', 'regressor'],
     'Time': ['time', 't', 'ivar'],
-    'Time Unit': ['time_unit', 'time_units', 't_units', 'tunit'],
+    'Time Unit': ['time_unit', 'time_units', 't_units', 'tunit', 'units_time'],
     'Unit': ['unit', 'units'],
 }
 

pkpdapp/pkpdapp/models/dataset.py

@@ -156,6 +156,10 @@ def replace_data(self, data: pd.DataFrame):
             subject = subjects[subject_id]
 
             if observation != ".":  # measurement observation
+                try:
+                    observation = float(observation)
+                except ValueError:
+                    observation = 0.0
                 Biomarker.objects.create(
                     time=time,
                     subject=subject,

pkpdapp/pkpdapp/utils/data_parser.py

@@ -19,30 +19,30 @@ class DataParser:
             "SUBJECT_ID"
         ],
         "TIME": [
-            "Time", "TIME", "TIMEPOINT", "t", "T", "time"
+            "Time", "TIME", "TIMEPOINT", "t", "T", "time",
         ],
         "TIME_UNIT": [
-            "Time_unit", "Time_units", "TIMEUNIT", "TIME_UNIT"
+            "Time_unit", "Time_units", "TIMEUNIT", "TIME_UNIT", "Units_Time"
         ],
         "AMOUNT": [
             "Amt", "Amount", "AMT", "AMOUNT"
         ],
         "AMOUNT_UNIT": [
-            "Amt_unit", "Amt_units", "AMTUNIT", "UNIT", "AMOUNT_UNIT"
+            "Amt_unit", "Amt_units", "AMTUNIT", "UNIT", "AMOUNT_UNIT", "Units_AMT"
         ],
         "AMOUNT_VARIABLE": [
             "Amount Variable", "AMOUNT_VARIABLE"
         ],
         "OBSERVATION": [
-            "DV", "Observation", "Y", "YVAL", "OBSERVATION",
+            "DV", "Observation", "Y", "YVAL", "OBSERVATION", "Conc",
             "OBSERVATION_VALUE", "OBSERVATIONVALUE"
         ],
         "OBSERVATION_NAME": [
             "Observation_id", "YDESC", "YNAME", "YTYPE", "OBSERVATION_ID",
             "OBSERVATION_NAME", "OBSERVATIONID", "OBSERVATIONNAME"
         ],
         "OBSERVATION_UNIT": [
-            "DV_units", "Observation_unit", "YUNIT", "UNIT",
+            "DV_units", "Observation_unit", "YUNIT", "UNIT", "Units_Conc",
             "OBSERVATION_UNIT",
             "OBSERVATIONUNIT"
         ],