Add config to include data from health records only + update experime…

…nt commands to include new data config

didactic/config/task/data/records.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+
+task:
+  tabular_attrs:
+    # Attributes from patient records unrelated to echocardiogram exams
+      - age
+      - sex
+      - bmi
+      - hf
+      - cad
+      - pad
+      - stroke
+      - tobacco
+      - diabetes
+      - dyslipidemia
+      - etiology
+      - ddd
+      - bradycardic
+      - ace_inhibitor
+      - arb
+      - tz_diuretic
+      - central_acting
+      - beta_blocker
+      - spironolactone
+      - alpha_blocker
+      - ccb
+      - sbp_24
+      - dbp_24
+      - pp_24
+      - ht_severity
+      - ht_grade
+      - creat
+      - gfr
+      - nt_probnp
+      - nt_probnp_group
+
+  time_series_attrs: []

didactic/scripts/experiments.sh

@@ -4,25 +4,27 @@ CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/resu
 # w/ time-series data + w/ ordinal constraint
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-xtab-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=5 +experiment=cardinal/xtab-finetune trainer.enable_progress_bar=False task/data=tab-13+time-series,tabular+time-series task.contrastive_loss_weight=0,0.2,1 task/time_series_tokenizer/model=linear-embedding,transformer 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn ckpt=$HOME/data/models/xtab/iter_2k_patch.ckpt ~callbacks.learning_rate_finder '+trial=range(10)' >>$HOME/data/didactic/results/xtab-finetune,data=tab-13+ts,tab+ts,ordinal=True.log 2>&1
 # w/o time-series data + w/o ordinal constraint
-CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-xtab-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/xtab-finetune trainer.enable_progress_bar=False task/data=tab-13,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False ckpt=$HOME/data/models/xtab/iter_2k_patch.ckpt ~callbacks.learning_rate_finder '+trial=range(10)' >>$HOME/data/didactic/results/xtab-finetune,data=tab-13,tab,ordinal=False.log 2>&1
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-xtab-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/xtab-finetune trainer.enable_progress_bar=False task/data=tab-13,records,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False ckpt=$HOME/data/models/xtab/iter_2k_patch.ckpt ~callbacks.learning_rate_finder '+trial=range(10)' >>$HOME/data/didactic/results/xtab-finetune,data=tab-13,tab,ordinal=False.log 2>&1
 # w/o time-series data + w/ ordinal constraint
-CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-xtab-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/xtab-finetune trainer.enable_progress_bar=False task/data=tab-13,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn ckpt=$HOME/data/models/xtab/iter_2k_patch.ckpt ~callbacks.learning_rate_finder '+trial=range(10)' >>$HOME/data/didactic/results/xtab-finetune,data=tab-13,tab,ordinal=True.log 2>&1
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-xtab-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/xtab-finetune trainer.enable_progress_bar=False task/data=tab-13,records,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn ckpt=$HOME/data/models/xtab/iter_2k_patch.ckpt ~callbacks.learning_rate_finder '+trial=range(10)' >>$HOME/data/didactic/results/xtab-finetune,data=tab-13,tab,ordinal=True.log 2>&1
 
 # scratch
 # w/ time-series data + w/o ordinal constraint
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-scratch didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-scratch trainer.enable_progress_bar=False task/data=tab-13+time-series,tabular+time-series task.contrastive_loss_weight=0,0.2,1 task/time_series_tokenizer/model=linear-embedding,transformer 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-scratch,data=tab-13+ts,tab+ts,ordinal=False.log 2>&1
 # w/ time-series data + w/ ordinal constraint
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-scratch didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-scratch trainer.enable_progress_bar=False task/data=tab-13+time-series,tabular+time-series task.contrastive_loss_weight=0,0.2,1 task/time_series_tokenizer/model=linear-embedding,transformer 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-scratch,data=tab-13+ts,tab+ts,ordinal=True.log 2>&1
 # w/o time-series data + w/o ordinal constraint
-CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-scratch didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-scratch trainer.enable_progress_bar=False task/data=tab-13,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-scratch,data=tab-13,tab,ordinal=False.log 2>&1
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-scratch didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-scratch trainer.enable_progress_bar=False task/data=tab-13,records,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-scratch,data=tab-13,tab,ordinal=False.log 2>&1
 # w/o time-series data + w/ ordinal constraint
-CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-scratch didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-scratch trainer.enable_progress_bar=False task/data=tab-13,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-scratch,data=tab-13,tab,ordinal=True.log 2>&1
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-scratch didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-scratch trainer.enable_progress_bar=False task/data=tab-13,records,tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-scratch,data=tab-13,tab,ordinal=True.log 2>&1
 
 # pretrain
 # tab-13
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-pretrain didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-pretrain trainer.enable_progress_bar=False task/data=tab-13 '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-pretrain,data=tab-13.log 2>&1
 # tab-13+time-series
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-pretrain didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-pretrain trainer.enable_progress_bar=False task/data=tab-13+time-series task/time_series_tokenizer/model=linear-embedding,transformer '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-pretrain,data=tab-13+ts.log 2>&1
+# records
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-pretrain didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-pretrain trainer.enable_progress_bar=False task/data=records exclude_tabular_attrs=[ht_severity,ht_grade] '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-pretrain,data=records.log 2>&1
 # tabular
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-pretrain didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-pretrain trainer.enable_progress_bar=False task/data=tabular exclude_tabular_attrs=[ht_severity,ht_grade] '+trial=range(10)' >>$HOME/data/didactic/results/multimodal-xformer-pretrain,data=tab.log 2>&1
 # tabular+time-series
@@ -37,6 +39,10 @@ CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/resu
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-finetune trainer.enable_progress_bar=False task/data=tab-13+time-series task.contrastive_loss_weight=0,0.2,1 task/time_series_tokenizer/model=linear-embedding,transformer 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False '+trial=range(10)' 'ckpt=/home/local/USHERBROOKE/pain5474/data/didactic/results/cardiac-multimodal-representation/pretrain/${hydra:runtime.choices.task/data}/${hydra:runtime.choices.task/time_series_tokenizer/model}/${trial}.ckpt' >>$HOME/data/didactic/results/multimodal-xformer-finetune,data=tab-13+ts,ordinal=False.log 2>&1
 # tab-13+time-series + w/ ordinal constraint
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-finetune trainer.enable_progress_bar=False task/data=tab-13+time-series task.contrastive_loss_weight=0,0.2,1 task/time_series_tokenizer/model=linear-embedding,transformer 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn '+trial=range(10)' 'ckpt=/home/local/USHERBROOKE/pain5474/data/didactic/results/cardiac-multimodal-representation/pretrain/${hydra:runtime.choices.task/data}/${hydra:runtime.choices.task/time_series_tokenizer/model}/${trial}.ckpt' >>$HOME/data/didactic/results/multimodal-xformer-finetune,data=tab-13+ts,ordinal=True.log 2>&1
+# records + w/o ordinal constraint
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-finetune trainer.enable_progress_bar=False task/data=records task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False '+trial=range(10)' 'ckpt=/home/local/USHERBROOKE/pain5474/data/didactic/results/cardiac-multimodal-representation/pretrain/${hydra:runtime.choices.task/data}/None/ht_severity/${trial}.ckpt' >>$HOME/data/didactic/results/multimodal-xformer-finetune,data=records,ordinal=False.log 2>&1
+# records + w/ ordinal constraint
+CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-finetune trainer.enable_progress_bar=False task/data=records task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=True task.model.ordinal_head.distribution=poisson,binomial task.model.ordinal_head.tau_mode=learn_sigm,learn_fn '+trial=range(10)' 'ckpt=/home/local/USHERBROOKE/pain5474/data/didactic/results/cardiac-multimodal-representation/pretrain/${hydra:runtime.choices.task/data}/None/ht_severity/${trial}.ckpt' >>$HOME/data/didactic/results/multimodal-xformer-finetune,data=records,ordinal=True.log 2>&1
 # tabular + w/o ordinal constraint
 CARDIAC_MULTIMODAL_REPR_PATH=/home/local/USHERBROOKE/pain5474/data/didactic/results/multirun/cardiac-multimodal-representation COMET_PROJECT_NAME=didactic-multimodal-xformer-finetune didactic-runner -m hydra/launcher=joblib hydra.launcher.n_jobs=10 +experiment=cardinal/multimodal-xformer-finetune trainer.enable_progress_bar=False task/data=tabular task.contrastive_loss_weight=0,0.2,1 'task.predict_losses={ht_severity:{_target_:torch.nn.CrossEntropyLoss}}' exclude_tabular_attrs=[ht_severity,ht_grade] task.ordinal_mode=False '+trial=range(10)' 'ckpt=/home/local/USHERBROOKE/pain5474/data/didactic/results/cardiac-multimodal-representation/pretrain/${hydra:runtime.choices.task/data}/None/ht_severity/${trial}.ckpt' >>$HOME/data/didactic/results/multimodal-xformer-finetune,data=tab,ordinal=False.log 2>&1
 # tabular + w/ ordinal constraint
@@ -52,6 +58,7 @@ declare -A time_series_tokenizers
 time_series_tokenizers=(
   [tab-13]=None
   [tab-13+time-series]="linear-embedding transformer"
+  [records]=None
   [tabular]=None
   [tabular+time-series]="linear-embedding transformer"
 )