diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a57376f..7bb134a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,12 @@ ## NEW CONTENT +* Add Predict Modality benchmark page (PR #320). + +# openproblems.bio v2.3.6 + +## NEW CONTENT + * Add an event page for the Weekly wednesday work meeting (PR #299). * Add `Advanced_topics` pages to documentation (PR #300). diff --git a/results/predict_modality/data/dataset_info.json b/results/predict_modality/data/dataset_info.json new file mode 100644 index 00000000..93c45c55 --- /dev/null +++ b/results/predict_modality/data/dataset_info.json @@ -0,0 +1,68 @@ +[ + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "dataset_name": "NeurIPS2021 CITE-Seq (GEX2ADT)", + "dataset_summary": "Single-cell CITE-Seq (GEX+ADT) data collected from bone marrow mononuclear cells of 12 healthy human donors.", + "dataset_description": "Single-cell CITE-Seq data collected from bone marrow mononuclear cells of 12 healthy human donors using the 10X 3 prime Single-Cell Gene Expression kit with Feature Barcoding in combination with the BioLegend TotalSeq B Universal Human Panel v1.0. The dataset was generated to support Multimodal Single-Cell Data Integration Challenge at NeurIPS 2021. Samples were prepared using a standard protocol at four sites. The resulting data was then annotated to identify cell types and remove doublets. The dataset was designed with a nested batch layout such that some donor samples were measured at multiple sites with some donors measured at a single site.", + "data_reference": "luecken2021neurips", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE194122", + "date_created": "25-11-2024", + "file_size": 704994, + "common_dataset_id": "openproblems_neurips2021/bmmc_cite" + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "dataset_name": "NeurIPS2021 Multiome (GEX2ATAC)", + "dataset_summary": "Single-cell Multiome (GEX+ATAC) data collected from bone marrow mononuclear cells of 12 healthy human donors.", + "dataset_description": "Single-cell CITE-Seq data collected from bone marrow mononuclear cells of 12 healthy human donors using the 10X Multiome Gene Expression and Chromatin Accessibility kit. The dataset was generated to support Multimodal Single-Cell Data Integration Challenge at NeurIPS 2021. Samples were prepared using a standard protocol at four sites. The resulting data was then annotated to identify cell types and remove doublets. The dataset was designed with a nested batch layout such that some donor samples were measured at multiple sites with some donors measured at a single site.", + "data_reference": "luecken2021neurips", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE194122", + "date_created": "25-11-2024", + "file_size": 31080807, + "common_dataset_id": "openproblems_neurips2021/bmmc_multiome" + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "dataset_name": "NeurIPS2021 Multiome (ATAC2GEX)", + "dataset_summary": "Single-cell Multiome (GEX+ATAC) data collected from bone marrow mononuclear cells of 12 healthy human donors.", + "dataset_description": "Single-cell CITE-Seq data collected from bone marrow mononuclear cells of 12 healthy human donors using the 10X Multiome Gene Expression and Chromatin Accessibility kit. The dataset was generated to support Multimodal Single-Cell Data Integration Challenge at NeurIPS 2021. Samples were prepared using a standard protocol at four sites. The resulting data was then annotated to identify cell types and remove doublets. The dataset was designed with a nested batch layout such that some donor samples were measured at multiple sites with some donors measured at a single site.", + "data_reference": "luecken2021neurips", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE194122", + "date_created": "25-11-2024", + "file_size": 7883109, + "common_dataset_id": "openproblems_neurips2021/bmmc_multiome" + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "dataset_name": "OpenProblems NeurIPS2022 CITE-Seq (GEX2ADT)", + "dataset_summary": "Single-cell CITE-Seq (GEX+ADT) data collected from bone marrow mononuclear cells of 12 healthy human donors.", + "dataset_description": "Single-cell CITE-Seq data collected from bone marrow mononuclear cells of 12 healthy human donors using the 10X 3 prime Single-Cell Gene Expression kit with Feature Barcoding in combination with the BioLegend TotalSeq B Universal Human Panel v1.0. The dataset was generated to support Multimodal Single-Cell Data Integration Challenge at NeurIPS 2022. Samples were prepared using a standard protocol at four sites. The resulting data was then annotated to identify cell types and remove doublets. The dataset was designed with a nested batch layout such that some donor samples were measured at multiple sites with some donors measured at a single site.", + "data_reference": "lance2024predicting", + "data_url": "https://www.kaggle.com/competitions/open-problems-multimodal/data", + "date_created": "25-11-2024", + "file_size": 591886, + "common_dataset_id": "openproblems_neurips2022/pbmc_cite" + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "dataset_name": "OpenProblems NeurIPS2022 CITE-Seq (ADT2GEX)", + "dataset_summary": "Single-cell CITE-Seq (GEX+ADT) data collected from bone marrow mononuclear cells of 12 healthy human donors.", + "dataset_description": "Single-cell CITE-Seq data collected from bone marrow mononuclear cells of 12 healthy human donors using the 10X 3 prime Single-Cell Gene Expression kit with Feature Barcoding in combination with the BioLegend TotalSeq B Universal Human Panel v1.0. The dataset was generated to support Multimodal Single-Cell Data Integration Challenge at NeurIPS 2022. Samples were prepared using a standard protocol at four sites. The resulting data was then annotated to identify cell types and remove doublets. The dataset was designed with a nested batch layout such that some donor samples were measured at multiple sites with some donors measured at a single site.", + "data_reference": "lance2024predicting", + "data_url": "https://www.kaggle.com/competitions/open-problems-multimodal/data", + "date_created": "25-11-2024", + "file_size": 32551804, + "common_dataset_id": "openproblems_neurips2022/pbmc_cite" + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "dataset_name": "NeurIPS2021 CITE-Seq (ADT2GEX)", + "dataset_summary": "Single-cell CITE-Seq (GEX+ADT) data collected from bone marrow mononuclear cells of 12 healthy human donors.", + "dataset_description": "Single-cell CITE-Seq data collected from bone marrow mononuclear cells of 12 healthy human donors using the 10X 3 prime Single-Cell Gene Expression kit with Feature Barcoding in combination with the BioLegend TotalSeq B Universal Human Panel v1.0. The dataset was generated to support Multimodal Single-Cell Data Integration Challenge at NeurIPS 2021. Samples were prepared using a standard protocol at four sites. The resulting data was then annotated to identify cell types and remove doublets. The dataset was designed with a nested batch layout such that some donor samples were measured at multiple sites with some donors measured at a single site.", + "data_reference": "luecken2021neurips", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE194122", + "date_created": "25-11-2024", + "file_size": 13467880, + "common_dataset_id": "openproblems_neurips2021/bmmc_cite" + } +] diff --git a/results/predict_modality/data/method_info.json b/results/predict_modality/data/method_info.json new file mode 100644 index 00000000..f902f86f --- /dev/null +++ b/results/predict_modality/data/method_info.json @@ -0,0 +1,130 @@ +[ + { + "task_id": "control_methods", + "method_id": "mean_per_gene", + "method_name": "Mean per gene", + "method_summary": "Returns the mean expression value per gene.", + "method_description": "Returns the mean expression value per gene.", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/control_methods/mean_per_gene:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/control_methods/mean_per_gene", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "control_methods", + "method_id": "random_predict", + "method_name": "Random predictions", + "method_summary": "Returns random training profiles.", + "method_description": "Returns random training profiles.", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/control_methods/random_predict:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/control_methods/random_predict", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "control_methods", + "method_id": "zeros", + "method_name": "Zeros", + "method_summary": "Returns a prediction consisting of all zeros.", + "method_description": "Returns a prediction consisting of all zeros.", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/control_methods/zeros:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/control_methods/zeros", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "control_methods", + "method_id": "solution", + "method_name": "Solution", + "method_summary": "Returns the ground-truth solution.", + "method_description": "Returns the ground-truth solution.", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/control_methods/solution:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/control_methods/solution", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "methods", + "method_id": "knnr_py", + "method_name": "KNNR (Py)", + "method_summary": "K-nearest neighbor regression in Python.", + "method_description": "K-nearest neighbor regression in Python.", + "is_baseline": false, + "references_doi": "10.2307/1403797", + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/methods/knnr_py:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/methods/knnr_py", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "methods", + "method_id": "knnr_r", + "method_name": "KNNR (R)", + "method_summary": "K-nearest neighbor regression in R.", + "method_description": "K-nearest neighbor regression in R.", + "is_baseline": false, + "references_doi": "10.2307/1403797", + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/methods/knnr_r:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/methods/knnr_r", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "methods", + "method_id": "lm", + "method_name": "Linear Model", + "method_summary": "Linear model regression.", + "method_description": "A linear model regression method.", + "is_baseline": false, + "references_doi": "10.2307/2346786", + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/methods/lm:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/methods/lm", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + }, + { + "task_id": "methods", + "method_id": "guanlab_dengkw_pm", + "method_name": "Guanlab-dengkw", + "method_summary": "A kernel ridge regression method with RBF kernel.", + "method_description": "This is a solution developed by Team Guanlab - dengkw in the Neurips 2021 competition to predict one modality\nfrom another using kernel ridge regression (KRR) with RBF kernel. Truncated SVD is applied on the combined\ntraining and test data from modality 1 followed by row-wise z-score normalization on the reduced matrix. The\ntruncated SVD of modality 2 is predicted by training a KRR model on the normalized training matrix of modality 1.\nPredictions on the normalized test matrix are then re-mapped to the modality 2 feature space via the right\nsingular vectors.\n", + "is_baseline": false, + "references_doi": "10.1101/2022.04.11.487796", + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_predict_modality", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/methods/guanlab_dengkw_pm:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/methods/guanlab_dengkw_pm", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197" + } +] diff --git a/results/predict_modality/data/metric_execution_info.json b/results/predict_modality/data/metric_execution_info.json new file mode 100644 index 00000000..8e5c6f85 --- /dev/null +++ b/results/predict_modality/data/metric_execution_info.json @@ -0,0 +1,1346 @@ +[ + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:57:45", + "exit_code": 0, + "duration_sec": 34.2, + "cpu_pct": 164.2, + "peak_memory_mb": 1844, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:57:45", + "exit_code": 0, + "duration_sec": 4.2, + "cpu_pct": 212.7, + "peak_memory_mb": 772, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "knnr_py", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:48:35", + "exit_code": 0, + "duration_sec": 32.4, + "cpu_pct": 285.3, + "peak_memory_mb": 3277, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "knnr_py", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:48:35", + "exit_code": 0, + "duration_sec": 4.4, + "cpu_pct": 188, + "peak_memory_mb": 774, + "disk_read_mb": 40, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "knnr_r", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:48:35", + "exit_code": 0, + "duration_sec": 77.4, + "cpu_pct": 69.6, + "peak_memory_mb": 1844, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "knnr_r", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:48:35", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 107.4, + "peak_memory_mb": 781, + "disk_read_mb": 40, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "lm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:49:55", + "exit_code": 0, + "duration_sec": 36.6, + "cpu_pct": 157.1, + "peak_memory_mb": 1844, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "lm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:49:55", + "exit_code": 0, + "duration_sec": 14.6, + "cpu_pct": 63.6, + "peak_memory_mb": 1536, + "disk_read_mb": 40, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "mean_per_gene", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:45", + "exit_code": 0, + "duration_sec": 60, + "cpu_pct": 156, + "peak_memory_mb": 5940, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "mean_per_gene", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:45", + "exit_code": 0, + "duration_sec": 3.6, + "cpu_pct": 416.1, + "peak_memory_mb": 1434, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "random_predict", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:25", + "exit_code": 0, + "duration_sec": 33, + "cpu_pct": 168.1, + "peak_memory_mb": 1946, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "random_predict", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:25", + "exit_code": 0, + "duration_sec": 13.2, + "cpu_pct": 58.8, + "peak_memory_mb": 1536, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "solution", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:25", + "exit_code": 0, + "duration_sec": 70.2, + "cpu_pct": 84.9, + "peak_memory_mb": 1844, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "solution", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:25", + "exit_code": 0, + "duration_sec": 4, + "cpu_pct": 162.3, + "peak_memory_mb": 772, + "disk_read_mb": 40, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "zeros", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:35", + "exit_code": 0, + "duration_sec": 54, + "cpu_pct": 108.5, + "peak_memory_mb": 1844, + "disk_read_mb": 210, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "zeros", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:35", + "exit_code": 0, + "duration_sec": 13.4, + "cpu_pct": 64.6, + "peak_memory_mb": 1536, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:49:55", + "exit_code": 0, + "duration_sec": 281.4, + "cpu_pct": 100.7, + "peak_memory_mb": 4506, + "disk_read_mb": 588, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:49:55", + "exit_code": 0, + "duration_sec": 12.8, + "cpu_pct": 204, + "peak_memory_mb": 6452, + "disk_read_mb": 162, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "knnr_py", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:46:35", + "exit_code": 0, + "duration_sec": 243.6, + "cpu_pct": 107.1, + "peak_memory_mb": 8500, + "disk_read_mb": 630, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "knnr_py", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:46:35", + "exit_code": 0, + "duration_sec": 30.2, + "cpu_pct": 90.3, + "peak_memory_mb": 2765, + "disk_read_mb": 178, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "knnr_r", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:45", + "exit_code": 0, + "duration_sec": 211.2, + "cpu_pct": 125.3, + "peak_memory_mb": 5735, + "disk_read_mb": 534, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "knnr_r", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:45", + "exit_code": 0, + "duration_sec": 14.8, + "cpu_pct": 132.8, + "peak_memory_mb": 5530, + "disk_read_mb": 146, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "lm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 14:55:25", + "exit_code": 0, + "duration_sec": 201.6, + "cpu_pct": 104.1, + "peak_memory_mb": 5940, + "disk_read_mb": 912, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "lm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 14:55:25", + "exit_code": 0, + "duration_sec": 4.8, + "cpu_pct": 286.7, + "peak_memory_mb": 3380, + "disk_read_mb": 270, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "mean_per_gene", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:46:15", + "exit_code": 0, + "duration_sec": 162.6, + "cpu_pct": 129.8, + "peak_memory_mb": 5940, + "disk_read_mb": 306, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "mean_per_gene", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:46:15", + "exit_code": 0, + "duration_sec": 6.2, + "cpu_pct": 179.9, + "peak_memory_mb": 1536, + "disk_read_mb": 68, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "random_predict", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:45:55", + "exit_code": 0, + "duration_sec": 106.2, + "cpu_pct": 124.8, + "peak_memory_mb": 5530, + "disk_read_mb": 324, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "random_predict", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:45:55", + "exit_code": 0, + "duration_sec": 17.6, + "cpu_pct": 71.7, + "peak_memory_mb": 2868, + "disk_read_mb": 74, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "solution", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:45:45", + "exit_code": 0, + "duration_sec": 99, + "cpu_pct": 152.6, + "peak_memory_mb": 8192, + "disk_read_mb": 360, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "solution", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:45:45", + "exit_code": 0, + "duration_sec": 18, + "cpu_pct": 75.9, + "peak_memory_mb": 2868, + "disk_read_mb": 88, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "zeros", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:45:55", + "exit_code": 0, + "duration_sec": 70.2, + "cpu_pct": 161.1, + "peak_memory_mb": 5428, + "disk_read_mb": 288, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "zeros", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:45:55", + "exit_code": 0, + "duration_sec": 4, + "cpu_pct": 329, + "peak_memory_mb": 1434, + "disk_read_mb": 64, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 19:36:45", + "exit_code": 0, + "duration_sec": 187.2, + "cpu_pct": 144.5, + "peak_memory_mb": 7373, + "disk_read_mb": 606, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 19:36:45", + "exit_code": 0, + "duration_sec": 6.6, + "cpu_pct": 307.5, + "peak_memory_mb": 6247, + "disk_read_mb": 170, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "knnr_py", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 148.8, + "cpu_pct": 113.6, + "peak_memory_mb": 3277, + "disk_read_mb": 576, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "knnr_py", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 29.2, + "cpu_pct": 92.8, + "peak_memory_mb": 2765, + "disk_read_mb": 160, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "knnr_r", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 118.2, + "cpu_pct": 111.2, + "peak_memory_mb": 3175, + "disk_read_mb": 486, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "knnr_r", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 6, + "cpu_pct": 224.9, + "peak_memory_mb": 2868, + "disk_read_mb": 130, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "lm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 14:05:05", + "exit_code": 0, + "duration_sec": 186.6, + "cpu_pct": 111, + "peak_memory_mb": 3380, + "disk_read_mb": 834, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "lm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 14:05:05", + "exit_code": 0, + "duration_sec": 6.6, + "cpu_pct": 171.1, + "peak_memory_mb": 2253, + "disk_read_mb": 246, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "mean_per_gene", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:46:35", + "exit_code": 0, + "duration_sec": 163.2, + "cpu_pct": 108.9, + "peak_memory_mb": 7476, + "disk_read_mb": 402, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "mean_per_gene", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:46:35", + "exit_code": 0, + "duration_sec": 29.4, + "cpu_pct": 82.4, + "peak_memory_mb": 2765, + "disk_read_mb": 102, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "random_predict", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:46:15", + "exit_code": 0, + "duration_sec": 86.4, + "cpu_pct": 158.7, + "peak_memory_mb": 4404, + "disk_read_mb": 402, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "random_predict", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:46:15", + "exit_code": 0, + "duration_sec": 5.6, + "cpu_pct": 197.1, + "peak_memory_mb": 2868, + "disk_read_mb": 102, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "solution", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:25", + "exit_code": 0, + "duration_sec": 96.6, + "cpu_pct": 111.2, + "peak_memory_mb": 3175, + "disk_read_mb": 564, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "solution", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:25", + "exit_code": 0, + "duration_sec": 17.2, + "cpu_pct": 62.6, + "peak_memory_mb": 1536, + "disk_read_mb": 154, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "zeros", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:46:35", + "exit_code": 0, + "duration_sec": 135, + "cpu_pct": 107.9, + "peak_memory_mb": 7168, + "disk_read_mb": 390, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "zeros", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:46:35", + "exit_code": 0, + "duration_sec": 28.2, + "cpu_pct": 76.1, + "peak_memory_mb": 2765, + "disk_read_mb": 96, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:55:05", + "exit_code": 0, + "duration_sec": 240.6, + "cpu_pct": 109, + "peak_memory_mb": 3892, + "disk_read_mb": 546, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:55:05", + "exit_code": 0, + "duration_sec": 5.8, + "cpu_pct": 180.7, + "peak_memory_mb": 1639, + "disk_read_mb": 148, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "knnr_py", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:53:25", + "exit_code": 0, + "duration_sec": 248.4, + "cpu_pct": 104.5, + "peak_memory_mb": 4199, + "disk_read_mb": 606, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "knnr_py", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:53:25", + "exit_code": 0, + "duration_sec": 17, + "cpu_pct": 62.2, + "peak_memory_mb": 2356, + "disk_read_mb": 168, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "knnr_r", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:51:35", + "exit_code": 0, + "duration_sec": 207, + "cpu_pct": 110.4, + "peak_memory_mb": 4199, + "disk_read_mb": 510, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "knnr_r", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:51:35", + "exit_code": 0, + "duration_sec": 4.6, + "cpu_pct": 291.7, + "peak_memory_mb": 3482, + "disk_read_mb": 138, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "lm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 14:17:45", + "exit_code": 0, + "duration_sec": 255.6, + "cpu_pct": 108.5, + "peak_memory_mb": 3994, + "disk_read_mb": 852, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "lm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 14:17:45", + "exit_code": 0, + "duration_sec": 7, + "cpu_pct": 166.3, + "peak_memory_mb": 2458, + "disk_read_mb": 250, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "mean_per_gene", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:49:15", + "exit_code": 0, + "duration_sec": 196.8, + "cpu_pct": 94.4, + "peak_memory_mb": 3994, + "disk_read_mb": 270, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "mean_per_gene", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:49:15", + "exit_code": 0, + "duration_sec": 12.6, + "cpu_pct": 89.2, + "peak_memory_mb": 1536, + "disk_read_mb": 58, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "random_predict", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:50:15", + "exit_code": 0, + "duration_sec": 120.6, + "cpu_pct": 110.4, + "peak_memory_mb": 5325, + "disk_read_mb": 288, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "random_predict", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:50:15", + "exit_code": 0, + "duration_sec": 15.4, + "cpu_pct": 65.4, + "peak_memory_mb": 1639, + "disk_read_mb": 64, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "solution", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:48:15", + "exit_code": 0, + "duration_sec": 172.8, + "cpu_pct": 88.3, + "peak_memory_mb": 3994, + "disk_read_mb": 300, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "solution", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:48:15", + "exit_code": 0, + "duration_sec": 12.2, + "cpu_pct": 76.7, + "peak_memory_mb": 1536, + "disk_read_mb": 66, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "zeros", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:50:15", + "exit_code": 0, + "duration_sec": 94.8, + "cpu_pct": 124.3, + "peak_memory_mb": 3892, + "disk_read_mb": 258, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "zeros", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:50:15", + "exit_code": 0, + "duration_sec": 3.2, + "cpu_pct": 409, + "peak_memory_mb": 1434, + "disk_read_mb": 52, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 14:34:05", + "exit_code": 0, + "duration_sec": 22.2, + "cpu_pct": 428.6, + "peak_memory_mb": 5940, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 14:34:05", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 638, + "peak_memory_mb": 2765, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "knnr_py", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 14:00:55", + "exit_code": 0, + "duration_sec": 45, + "cpu_pct": 133.3, + "peak_memory_mb": 1946, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "knnr_py", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 14:00:55", + "exit_code": 0, + "duration_sec": 7.4, + "cpu_pct": 122.5, + "peak_memory_mb": 1536, + "disk_read_mb": 40, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "knnr_r", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:58:15", + "exit_code": 0, + "duration_sec": 33, + "cpu_pct": 170.9, + "peak_memory_mb": 1946, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "knnr_r", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:58:15", + "exit_code": 0, + "duration_sec": 3.8, + "cpu_pct": 212.9, + "peak_memory_mb": 770, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "lm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 14:09:35", + "exit_code": 0, + "duration_sec": 34.8, + "cpu_pct": 157.1, + "peak_memory_mb": 1844, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "lm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 14:09:35", + "exit_code": 0, + "duration_sec": 4, + "cpu_pct": 218.7, + "peak_memory_mb": 772, + "disk_read_mb": 40, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "mean_per_gene", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:52:05", + "exit_code": 0, + "duration_sec": 32.4, + "cpu_pct": 159.7, + "peak_memory_mb": 1946, + "disk_read_mb": 210, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "mean_per_gene", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:52:05", + "exit_code": 0, + "duration_sec": 3.8, + "cpu_pct": 221.8, + "peak_memory_mb": 773, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "random_predict", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:52:05", + "exit_code": 0, + "duration_sec": 22.2, + "cpu_pct": 415.4, + "peak_memory_mb": 5940, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "random_predict", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:52:05", + "exit_code": 0, + "duration_sec": 4, + "cpu_pct": 215.5, + "peak_memory_mb": 770, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "solution", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:52:05", + "exit_code": 0, + "duration_sec": 22.8, + "cpu_pct": 307.7, + "peak_memory_mb": 3175, + "disk_read_mb": 216, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "solution", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:52:05", + "exit_code": 0, + "duration_sec": 2.8, + "cpu_pct": 414.6, + "peak_memory_mb": 1434, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "zeros", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:51:45", + "exit_code": 0, + "duration_sec": 33, + "cpu_pct": 169.7, + "peak_memory_mb": 1844, + "disk_read_mb": 210, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "zeros", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:51:45", + "exit_code": 0, + "duration_sec": 3.8, + "cpu_pct": 222.9, + "peak_memory_mb": 774, + "disk_read_mb": 38, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 19:36:25", + "exit_code": 0, + "duration_sec": 438, + "cpu_pct": 119.6, + "peak_memory_mb": 10957, + "disk_read_mb": 858, + "disk_write_mb": 12 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "guanlab_dengkw_pm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 19:36:25", + "exit_code": 0, + "duration_sec": 8.2, + "cpu_pct": 256.5, + "peak_memory_mb": 5837, + "disk_read_mb": 252, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "knnr_py", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:53:25", + "exit_code": 0, + "duration_sec": 402, + "cpu_pct": 99.4, + "peak_memory_mb": 7885, + "disk_read_mb": 882, + "disk_write_mb": 12 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "knnr_py", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:53:25", + "exit_code": 0, + "duration_sec": 18.4, + "cpu_pct": 65.6, + "peak_memory_mb": 2048, + "disk_read_mb": 260, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "knnr_r", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:58:35", + "exit_code": 0, + "duration_sec": 352.8, + "cpu_pct": 106.4, + "peak_memory_mb": 7783, + "disk_read_mb": 768, + "disk_write_mb": 12 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "knnr_r", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:58:35", + "exit_code": 0, + "duration_sec": 7.6, + "cpu_pct": 152.1, + "peak_memory_mb": 1946, + "disk_read_mb": 222, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "lm", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 18:51:15", + "exit_code": 0, + "duration_sec": 462, + "cpu_pct": 121.4, + "peak_memory_mb": 11060, + "disk_read_mb": 1362, + "disk_write_mb": 12 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "lm", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 18:51:15", + "exit_code": 0, + "duration_sec": 8.6, + "cpu_pct": 313.3, + "peak_memory_mb": 7168, + "disk_read_mb": 422, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "mean_per_gene", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:52:35", + "exit_code": 0, + "duration_sec": 320.4, + "cpu_pct": 99.2, + "peak_memory_mb": 7066, + "disk_read_mb": 420, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "mean_per_gene", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:52:35", + "exit_code": 0, + "duration_sec": 15.8, + "cpu_pct": 78.3, + "peak_memory_mb": 3175, + "disk_read_mb": 106, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "random_predict", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:51:45", + "exit_code": 0, + "duration_sec": 268.2, + "cpu_pct": 108.9, + "peak_memory_mb": 7578, + "disk_read_mb": 528, + "disk_write_mb": 12 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "random_predict", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:51:45", + "exit_code": 0, + "duration_sec": 6.2, + "cpu_pct": 174.3, + "peak_memory_mb": 1639, + "disk_read_mb": 144, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "solution", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:47:35", + "exit_code": 0, + "duration_sec": 306.6, + "cpu_pct": 93.8, + "peak_memory_mb": 7578, + "disk_read_mb": 582, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "solution", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:47:35", + "exit_code": 0, + "duration_sec": 6.2, + "cpu_pct": 145.1, + "peak_memory_mb": 1639, + "disk_read_mb": 160, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "zeros", + "metric_component_name": "correlation", + "resources": { + "submit": "2024-11-25 13:48:35", + "exit_code": 0, + "duration_sec": 176.4, + "cpu_pct": 127, + "peak_memory_mb": 8602, + "disk_read_mb": 402, + "disk_write_mb": 6 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "zeros", + "metric_component_name": "mse", + "resources": { + "submit": "2024-11-25 13:48:35", + "exit_code": 0, + "duration_sec": 6, + "cpu_pct": 160.9, + "peak_memory_mb": 1536, + "disk_read_mb": 100, + "disk_write_mb": 2 + } + } +] diff --git a/results/predict_modality/data/metric_info.json b/results/predict_modality/data/metric_info.json new file mode 100644 index 00000000..bf6c8407 --- /dev/null +++ b/results/predict_modality/data/metric_info.json @@ -0,0 +1,122 @@ +[ + { + "task_id": "metrics", + "component_name": "correlation", + "metric_id": "mean_pearson_per_cell", + "metric_name": "Mean pearson per cell", + "metric_summary": "The mean of the pearson values of per-cell expression value vectors.", + "metric_description": "The mean of the pearson values of per-cell expression value vectors.", + "references_doi": "10.1098/rspl.1895.0041", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/correlation", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/correlation:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "correlation", + "metric_id": "mean_spearman_per_cell", + "metric_name": "Mean spearman per cell", + "metric_summary": "The mean of the spearman values of per-cell expression value vectors.", + "metric_description": "The mean of the spearman values of per-cell expression value vectors.", + "references_doi": "10.1093/biomet/30.1-2.81", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/correlation", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/correlation:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "correlation", + "metric_id": "mean_pearson_per_gene", + "metric_name": "Mean pearson per gene", + "metric_summary": "The mean of the pearson values of per-gene expression value vectors.", + "metric_description": "The mean of the pearson values of per-gene expression value vectors.", + "references_doi": "10.1098/rspl.1895.0041", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/correlation", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/correlation:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "correlation", + "metric_id": "mean_spearman_per_gene", + "metric_name": "Mean spearman per gene", + "metric_summary": "The mean of the spearman values of per-gene expression value vectors.", + "metric_description": "The mean of the spearman values of per-gene expression value vectors.", + "references_doi": "10.1093/biomet/30.1-2.81", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/correlation", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/correlation:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "correlation", + "metric_id": "overall_pearson", + "metric_name": "Overall pearson", + "metric_summary": "The mean of the pearson values of vectorized expression matrices.", + "metric_description": "The mean of the pearson values of vectorized expression matrices.", + "references_doi": "10.1098/rspl.1895.0041", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/correlation", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/correlation:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "correlation", + "metric_id": "overall_spearman", + "metric_name": "Overall spearman", + "metric_summary": "The mean of the spearman values of vectorized expression matrices.", + "metric_description": "The mean of the spearman values of vectorized expression matrices.", + "references_doi": "10.1093/biomet/30.1-2.81", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/correlation", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/correlation:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "mse", + "metric_id": "rmse", + "metric_name": "RMSE", + "metric_summary": "The root mean squared error.", + "metric_description": "The square root of the mean of the square of all of the error.", + "references_doi": "10.5194/gmdd-7-1525-2014", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/mse", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/mse:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": false + }, + { + "task_id": "metrics", + "component_name": "mse", + "metric_id": "mae", + "metric_name": "MAE", + "metric_summary": "The mean absolute error.", + "metric_description": "The average difference between the expression values and the predicted expression values.", + "references_doi": "10.5194/gmdd-7-1525-2014", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_predict_modality/blob/0bd597e201b39fbcbc1fcd7047f7654a9713a197/src/metrics/mse", + "image": "https://ghcr.io/openproblems-bio/task_predict_modality/metrics/mse:build_main", + "code_version": "build_main", + "commit_sha": "0bd597e201b39fbcbc1fcd7047f7654a9713a197", + "maximize": false + } +] diff --git a/results/predict_modality/data/quality_control.json b/results/predict_modality/data/quality_control.json new file mode 100644 index 00000000..5c238a19 --- /dev/null +++ b/results/predict_modality/data/quality_control.json @@ -0,0 +1,1942 @@ +[ + { + "task_id": "task_predict_modality", + "category": "Task info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_id' should be defined\n Task id: task_predict_modality\n Field: task_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Task info", + "name": "Pct 'task_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_name' should be defined\n Task id: task_predict_modality\n Field: task_name\n" + }, + { + "task_id": "task_predict_modality", + "category": "Task info", + "name": "Pct 'task_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_summary' should be defined\n Task id: task_predict_modality\n Field: task_summary\n" + }, + { + "task_id": "task_predict_modality", + "category": "Task info", + "name": "Pct 'task_description' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing([task_info], field)", + "message": "Task metadata field 'task_description' should be defined\n Task id: task_predict_modality\n Field: task_description\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'task_id' should be defined\n Task id: task_predict_modality\n Field: task_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'commit_sha' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'commit_sha' should be defined\n Task id: task_predict_modality\n Field: commit_sha\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'method_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_id' should be defined\n Task id: task_predict_modality\n Field: method_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'method_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_name' should be defined\n Task id: task_predict_modality\n Field: method_name\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'method_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'method_summary' should be defined\n Task id: task_predict_modality\n Field: method_summary\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'paper_reference' missing", + "value": 0.5555555555555556, + "severity": 2, + "severity_value": 3.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'paper_reference' should be defined\n Task id: task_predict_modality\n Field: paper_reference\n" + }, + { + "task_id": "task_predict_modality", + "category": "Method info", + "name": "Pct 'is_baseline' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(method_info, field)", + "message": "Method metadata field 'is_baseline' should be defined\n Task id: task_predict_modality\n Field: is_baseline\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'task_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'task_id' should be defined\n Task id: task_predict_modality\n Field: task_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'commit_sha' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'commit_sha' should be defined\n Task id: task_predict_modality\n Field: commit_sha\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'metric_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_id' should be defined\n Task id: task_predict_modality\n Field: metric_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'metric_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_name' should be defined\n Task id: task_predict_modality\n Field: metric_name\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'metric_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'metric_summary' should be defined\n Task id: task_predict_modality\n Field: metric_summary\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'paper_reference' missing", + "value": 1.0, + "severity": 2, + "severity_value": 3.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'paper_reference' should be defined\n Task id: task_predict_modality\n Field: paper_reference\n" + }, + { + "task_id": "task_predict_modality", + "category": "Metric info", + "name": "Pct 'maximize' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(metric_info, field)", + "message": "Metric metadata field 'maximize' should be defined\n Task id: task_predict_modality\n Field: maximize\n" + }, + { + "task_id": "task_predict_modality", + "category": "Dataset info", + "name": "Pct 'task_id' missing", + "value": 1.0, + "severity": 2, + "severity_value": 3.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'task_id' should be defined\n Task id: task_predict_modality\n Field: task_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Dataset info", + "name": "Pct 'dataset_id' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: task_predict_modality\n Field: dataset_id\n" + }, + { + "task_id": "task_predict_modality", + "category": "Dataset info", + "name": "Pct 'dataset_name' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: task_predict_modality\n Field: dataset_name\n" + }, + { + "task_id": "task_predict_modality", + "category": "Dataset info", + "name": "Pct 'dataset_summary' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: task_predict_modality\n Field: dataset_summary\n" + }, + { + "task_id": "task_predict_modality", + "category": "Dataset info", + "name": "Pct 'data_reference' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'data_reference' should be defined\n Task id: task_predict_modality\n Field: data_reference\n" + }, + { + "task_id": "task_predict_modality", + "category": "Dataset info", + "name": "Pct 'data_url' missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "percent_missing(dataset_info, field)", + "message": "Dataset metadata field 'data_url' should be defined\n Task id: task_predict_modality\n Field: data_url\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw data", + "name": "Number of results", + "value": 72, + "severity": 0, + "severity_value": 0.0, + "code": "len(results) == len(method_info) * len(metric_info) * len(dataset_info)", + "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: task_predict_modality\n Number of results: 72\n Number of methods: 9\n Number of metrics: 8\n Number of datasets: 8\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'mean_pearson_per_cell' %missing", + "value": 0.05555555555555558, + "severity": 0, + "severity_value": 0.5555555555555558, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: mean_pearson_per_cell\n Percentage missing: 6%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'mean_spearman_per_cell' %missing", + "value": 0.05555555555555558, + "severity": 0, + "severity_value": 0.5555555555555558, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: mean_spearman_per_cell\n Percentage missing: 6%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'mean_pearson_per_gene' %missing", + "value": 0.05555555555555558, + "severity": 0, + "severity_value": 0.5555555555555558, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: mean_pearson_per_gene\n Percentage missing: 6%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'mean_spearman_per_gene' %missing", + "value": 0.05555555555555558, + "severity": 0, + "severity_value": 0.5555555555555558, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: mean_spearman_per_gene\n Percentage missing: 6%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'overall_pearson' %missing", + "value": 0.16666666666666663, + "severity": 1, + "severity_value": 1.6666666666666663, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: overall_pearson\n Percentage missing: 17%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'overall_spearman' %missing", + "value": 0.16666666666666663, + "severity": 1, + "severity_value": 1.6666666666666663, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: overall_spearman\n Percentage missing: 17%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'rmse' %missing", + "value": 0.05555555555555558, + "severity": 0, + "severity_value": 0.5555555555555558, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: rmse\n Percentage missing: 6%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Metric 'mae' %missing", + "value": 0.05555555555555558, + "severity": 0, + "severity_value": 0.5555555555555558, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n Metric id: mae\n Percentage missing: 6%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'mean_per_gene' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: mean_per_gene\n Percentage missing: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'random_predict' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: random_predict\n Percentage missing: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'zeros' %missing", + "value": 0.25, + "severity": 2, + "severity_value": 2.5, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: zeros\n Percentage missing: 25%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'solution' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: solution\n Percentage missing: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'knnr_py' %missing", + "value": 0.125, + "severity": 1, + "severity_value": 1.25, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: knnr_py\n Percentage missing: 12%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'knnr_r' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: knnr_r\n Percentage missing: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'lm' %missing", + "value": 0.125, + "severity": 1, + "severity_value": 1.25, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: lm\n Percentage missing: 12%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'lmds_irlba_rf' %missing", + "value": 0.0, + "severity": 0, + "severity_value": 0.0, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: lmds_irlba_rf\n Percentage missing: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Method 'guanlab_dengkw_pm' %missing", + "value": 0.25, + "severity": 2, + "severity_value": 2.5, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n method id: guanlab_dengkw_pm\n Percentage missing: 25%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2021/bmmc_cite/normal' %missing", + "value": 0.02777777777777779, + "severity": 0, + "severity_value": 0.2777777777777779, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2021/bmmc_cite/normal\n Percentage missing: 3%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2021/bmmc_multiome/normal' %missing", + "value": 0.02777777777777779, + "severity": 0, + "severity_value": 0.2777777777777779, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2021/bmmc_multiome/normal\n Percentage missing: 3%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2022/pbmc_multiome/swap' %missing", + "value": 0.36111111111111116, + "severity": 3, + "severity_value": 3.6111111111111116, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2022/pbmc_multiome/swap\n Percentage missing: 36%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2021/bmmc_multiome/swap' %missing", + "value": 0.02777777777777779, + "severity": 0, + "severity_value": 0.2777777777777779, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2021/bmmc_multiome/swap\n Percentage missing: 3%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2022/pbmc_multiome/normal' %missing", + "value": 0.13888888888888884, + "severity": 1, + "severity_value": 1.3888888888888884, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2022/pbmc_multiome/normal\n Percentage missing: 14%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2022/pbmc_cite/normal' %missing", + "value": 0.02777777777777779, + "severity": 0, + "severity_value": 0.2777777777777779, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2022/pbmc_cite/normal\n Percentage missing: 3%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2022/pbmc_cite/swap' %missing", + "value": 0.02777777777777779, + "severity": 0, + "severity_value": 0.2777777777777779, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2022/pbmc_cite/swap\n Percentage missing: 3%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Raw results", + "name": "Dataset 'openproblems_neurips2021/bmmc_cite/swap' %missing", + "value": 0.02777777777777779, + "severity": 0, + "severity_value": 0.2777777777777779, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_predict_modality\n dataset id: openproblems_neurips2021/bmmc_cite/swap\n Percentage missing: 3%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene mean_pearson_per_cell", + "value": 0.1425, + "severity": 0, + "severity_value": -0.1425, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_pearson_per_cell\n Worst score: 0.1425%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene mean_pearson_per_cell", + "value": 0.849, + "severity": 0, + "severity_value": 0.4245, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_pearson_per_cell\n Best score: 0.849%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict mean_pearson_per_cell", + "value": 0.0202, + "severity": 0, + "severity_value": -0.0202, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_pearson_per_cell\n Worst score: 0.0202%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict mean_pearson_per_cell", + "value": 0.7556, + "severity": 0, + "severity_value": 0.3778, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_pearson_per_cell\n Best score: 0.7556%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros mean_pearson_per_cell", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_pearson_per_cell\n Worst score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros mean_pearson_per_cell", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_pearson_per_cell\n Best score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution mean_pearson_per_cell", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_pearson_per_cell\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution mean_pearson_per_cell", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_pearson_per_cell\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py mean_pearson_per_cell", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_pearson_per_cell\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py mean_pearson_per_cell", + "value": 0.8764, + "severity": 0, + "severity_value": 0.4382, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_pearson_per_cell\n Best score: 0.8764%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r mean_pearson_per_cell", + "value": 0.1076, + "severity": 0, + "severity_value": -0.1076, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_pearson_per_cell\n Worst score: 0.1076%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r mean_pearson_per_cell", + "value": 0.8737, + "severity": 0, + "severity_value": 0.43685, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_pearson_per_cell\n Best score: 0.8737%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm mean_pearson_per_cell", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_pearson_per_cell\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm mean_pearson_per_cell", + "value": 0.7078, + "severity": 0, + "severity_value": 0.3539, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_pearson_per_cell\n Best score: 0.7078%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf mean_pearson_per_cell", + "value": 0.0729, + "severity": 0, + "severity_value": -0.0729, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_pearson_per_cell\n Worst score: 0.0729%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf mean_pearson_per_cell", + "value": 0.7061, + "severity": 0, + "severity_value": 0.35305, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_pearson_per_cell\n Best score: 0.7061%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm mean_pearson_per_cell", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_pearson_per_cell\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm mean_pearson_per_cell", + "value": 0.8843, + "severity": 0, + "severity_value": 0.44215, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_pearson_per_cell\n Best score: 0.8843%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene mean_spearman_per_cell", + "value": 0.1574, + "severity": 0, + "severity_value": -0.1574, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_spearman_per_cell\n Worst score: 0.1574%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene mean_spearman_per_cell", + "value": 0.5996, + "severity": 0, + "severity_value": 0.2998, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_spearman_per_cell\n Best score: 0.5996%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict mean_spearman_per_cell", + "value": 0.0723, + "severity": 0, + "severity_value": -0.0723, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_spearman_per_cell\n Worst score: 0.0723%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict mean_spearman_per_cell", + "value": 0.5185, + "severity": 0, + "severity_value": 0.25925, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_spearman_per_cell\n Best score: 0.5185%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros mean_spearman_per_cell", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_spearman_per_cell\n Worst score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros mean_spearman_per_cell", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_spearman_per_cell\n Best score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution mean_spearman_per_cell", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_spearman_per_cell\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution mean_spearman_per_cell", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_spearman_per_cell\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py mean_spearman_per_cell", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_spearman_per_cell\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py mean_spearman_per_cell", + "value": 0.6911, + "severity": 0, + "severity_value": 0.34555, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_spearman_per_cell\n Best score: 0.6911%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r mean_spearman_per_cell", + "value": 0.1941, + "severity": 0, + "severity_value": -0.1941, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_spearman_per_cell\n Worst score: 0.1941%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r mean_spearman_per_cell", + "value": 0.6708, + "severity": 0, + "severity_value": 0.3354, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_spearman_per_cell\n Best score: 0.6708%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm mean_spearman_per_cell", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_spearman_per_cell\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm mean_spearman_per_cell", + "value": 0.6098, + "severity": 0, + "severity_value": 0.3049, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_spearman_per_cell\n Best score: 0.6098%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf mean_spearman_per_cell", + "value": 0.0228, + "severity": 0, + "severity_value": -0.0228, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_spearman_per_cell\n Worst score: 0.0228%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf mean_spearman_per_cell", + "value": 0.589, + "severity": 0, + "severity_value": 0.2945, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_spearman_per_cell\n Best score: 0.589%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm mean_spearman_per_cell", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_spearman_per_cell\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm mean_spearman_per_cell", + "value": 0.686, + "severity": 0, + "severity_value": 0.343, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_spearman_per_cell\n Best score: 0.686%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene mean_pearson_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_pearson_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene mean_pearson_per_gene", + "value": 0.0157, + "severity": 0, + "severity_value": 0.00785, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_pearson_per_gene\n Best score: 0.0157%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict mean_pearson_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_pearson_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict mean_pearson_per_gene", + "value": 0.0034, + "severity": 0, + "severity_value": 0.0017, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_pearson_per_gene\n Best score: 0.0034%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros mean_pearson_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_pearson_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros mean_pearson_per_gene", + "value": 0.0157, + "severity": 0, + "severity_value": 0.00785, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_pearson_per_gene\n Best score: 0.0157%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution mean_pearson_per_gene", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_pearson_per_gene\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution mean_pearson_per_gene", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_pearson_per_gene\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py mean_pearson_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_pearson_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py mean_pearson_per_gene", + "value": 0.601, + "severity": 0, + "severity_value": 0.3005, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_pearson_per_gene\n Best score: 0.601%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r mean_pearson_per_gene", + "value": 0.0207, + "severity": 0, + "severity_value": -0.0207, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_pearson_per_gene\n Worst score: 0.0207%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r mean_pearson_per_gene", + "value": 0.5439, + "severity": 0, + "severity_value": 0.27195, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_pearson_per_gene\n Best score: 0.5439%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm mean_pearson_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_pearson_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm mean_pearson_per_gene", + "value": 0.5264, + "severity": 0, + "severity_value": 0.2632, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_pearson_per_gene\n Best score: 0.5264%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf mean_pearson_per_gene", + "value": 0.0619, + "severity": 0, + "severity_value": -0.0619, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_pearson_per_gene\n Worst score: 0.0619%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf mean_pearson_per_gene", + "value": 0.5398, + "severity": 0, + "severity_value": 0.2699, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_pearson_per_gene\n Best score: 0.5398%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm mean_pearson_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_pearson_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm mean_pearson_per_gene", + "value": 0.6474, + "severity": 0, + "severity_value": 0.3237, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_pearson_per_gene\n Best score: 0.6474%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene mean_spearman_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_spearman_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene mean_spearman_per_gene", + "value": 0.0193, + "severity": 0, + "severity_value": 0.00965, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mean_spearman_per_gene\n Best score: 0.0193%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict mean_spearman_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_spearman_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict mean_spearman_per_gene", + "value": 0.0009, + "severity": 0, + "severity_value": 0.00045, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mean_spearman_per_gene\n Best score: 0.0009%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros mean_spearman_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_spearman_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros mean_spearman_per_gene", + "value": 0.0193, + "severity": 0, + "severity_value": 0.00965, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mean_spearman_per_gene\n Best score: 0.0193%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution mean_spearman_per_gene", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_spearman_per_gene\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution mean_spearman_per_gene", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mean_spearman_per_gene\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py mean_spearman_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_spearman_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py mean_spearman_per_gene", + "value": 0.499, + "severity": 0, + "severity_value": 0.2495, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mean_spearman_per_gene\n Best score: 0.499%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r mean_spearman_per_gene", + "value": 0.0208, + "severity": 0, + "severity_value": -0.0208, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_spearman_per_gene\n Worst score: 0.0208%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r mean_spearman_per_gene", + "value": 0.4419, + "severity": 0, + "severity_value": 0.22095, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mean_spearman_per_gene\n Best score: 0.4419%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm mean_spearman_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_spearman_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm mean_spearman_per_gene", + "value": 0.4398, + "severity": 0, + "severity_value": 0.2199, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mean_spearman_per_gene\n Best score: 0.4398%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf mean_spearman_per_gene", + "value": 0.0631, + "severity": 0, + "severity_value": -0.0631, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_spearman_per_gene\n Worst score: 0.0631%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf mean_spearman_per_gene", + "value": 0.4441, + "severity": 0, + "severity_value": 0.22205, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mean_spearman_per_gene\n Best score: 0.4441%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm mean_spearman_per_gene", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_spearman_per_gene\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm mean_spearman_per_gene", + "value": 0.5135, + "severity": 0, + "severity_value": 0.25675, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mean_spearman_per_gene\n Best score: 0.5135%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene overall_pearson", + "value": 0.1203, + "severity": 0, + "severity_value": -0.1203, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: overall_pearson\n Worst score: 0.1203%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene overall_pearson", + "value": 0.4197, + "severity": 0, + "severity_value": 0.20985, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: overall_pearson\n Best score: 0.4197%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict overall_pearson", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: overall_pearson\n Worst score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict overall_pearson", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: overall_pearson\n Best score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros overall_pearson", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: overall_pearson\n Worst score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros overall_pearson", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: overall_pearson\n Best score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution overall_pearson", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: overall_pearson\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution overall_pearson", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: overall_pearson\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py overall_pearson", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: overall_pearson\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py overall_pearson", + "value": 0.7082, + "severity": 0, + "severity_value": 0.3541, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: overall_pearson\n Best score: 0.7082%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r overall_pearson", + "value": 0.0864, + "severity": 0, + "severity_value": -0.0864, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: overall_pearson\n Worst score: 0.0864%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r overall_pearson", + "value": 0.6654, + "severity": 0, + "severity_value": 0.3327, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: overall_pearson\n Best score: 0.6654%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm overall_pearson", + "value": -0.6151, + "severity": 0, + "severity_value": 0.6151, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: overall_pearson\n Worst score: -0.6151%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm overall_pearson", + "value": 0.6541, + "severity": 0, + "severity_value": 0.32705, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: overall_pearson\n Best score: 0.6541%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf overall_pearson", + "value": -2.4102, + "severity": 2, + "severity_value": 2.4102, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: overall_pearson\n Worst score: -2.4102%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf overall_pearson", + "value": 0.6371, + "severity": 0, + "severity_value": 0.31855, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: overall_pearson\n Best score: 0.6371%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm overall_pearson", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: overall_pearson\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm overall_pearson", + "value": 0.7483, + "severity": 0, + "severity_value": 0.37415, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: overall_pearson\n Best score: 0.7483%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene overall_spearman", + "value": 0.0908, + "severity": 0, + "severity_value": -0.0908, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: overall_spearman\n Worst score: 0.0908%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene overall_spearman", + "value": 0.1824, + "severity": 0, + "severity_value": 0.0912, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: overall_spearman\n Best score: 0.1824%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict overall_spearman", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: overall_spearman\n Worst score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict overall_spearman", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: overall_spearman\n Best score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros overall_spearman", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: overall_spearman\n Worst score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros overall_spearman", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: overall_spearman\n Best score: 0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution overall_spearman", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: overall_spearman\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution overall_spearman", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: overall_spearman\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py overall_spearman", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: overall_spearman\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py overall_spearman", + "value": 0.61, + "severity": 0, + "severity_value": 0.305, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: overall_spearman\n Best score: 0.61%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r overall_spearman", + "value": 0.1103, + "severity": 0, + "severity_value": -0.1103, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: overall_spearman\n Worst score: 0.1103%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r overall_spearman", + "value": 0.5663, + "severity": 0, + "severity_value": 0.28315, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: overall_spearman\n Best score: 0.5663%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm overall_spearman", + "value": -0.2449, + "severity": 0, + "severity_value": 0.2449, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: overall_spearman\n Worst score: -0.2449%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm overall_spearman", + "value": 0.5364, + "severity": 0, + "severity_value": 0.2682, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: overall_spearman\n Best score: 0.5364%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf overall_spearman", + "value": -0.9923, + "severity": 0, + "severity_value": 0.9923, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: overall_spearman\n Worst score: -0.9923%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf overall_spearman", + "value": 0.5143, + "severity": 0, + "severity_value": 0.25715, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: overall_spearman\n Best score: 0.5143%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm overall_spearman", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: overall_spearman\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm overall_spearman", + "value": 0.6234, + "severity": 0, + "severity_value": 0.3117, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: overall_spearman\n Best score: 0.6234%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene rmse", + "value": 0.2007, + "severity": 0, + "severity_value": -0.2007, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: rmse\n Worst score: 0.2007%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene rmse", + "value": 0.5005, + "severity": 0, + "severity_value": 0.25025, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: rmse\n Best score: 0.5005%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict rmse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: rmse\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict rmse", + "value": 0.3408, + "severity": 0, + "severity_value": 0.1704, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: rmse\n Best score: 0.3408%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros rmse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: rmse\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros rmse", + "value": 0.3061, + "severity": 0, + "severity_value": 0.15305, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: rmse\n Best score: 0.3061%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution rmse", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: rmse\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution rmse", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: rmse\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py rmse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: rmse\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py rmse", + "value": 0.5465, + "severity": 0, + "severity_value": 0.27325, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: rmse\n Best score: 0.5465%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r rmse", + "value": 0.2044, + "severity": 0, + "severity_value": -0.2044, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: rmse\n Worst score: 0.2044%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r rmse", + "value": 0.5417, + "severity": 0, + "severity_value": 0.27085, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: rmse\n Best score: 0.5417%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm rmse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: rmse\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm rmse", + "value": 0.4537, + "severity": 0, + "severity_value": 0.22685, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: rmse\n Best score: 0.4537%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf rmse", + "value": 0.0089, + "severity": 0, + "severity_value": -0.0089, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: rmse\n Worst score: 0.0089%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf rmse", + "value": 0.3061, + "severity": 0, + "severity_value": 0.15305, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: rmse\n Best score: 0.3061%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm rmse", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: rmse\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm rmse", + "value": 0.561, + "severity": 0, + "severity_value": 0.2805, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: rmse\n Best score: 0.561%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score mean_per_gene mae", + "value": 0.0259, + "severity": 0, + "severity_value": -0.0259, + "code": "worst_score >= -1", + "message": "Method mean_per_gene performs much worse than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mae\n Worst score: 0.0259%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score mean_per_gene mae", + "value": 0.4307, + "severity": 0, + "severity_value": 0.21535, + "code": "best_score <= 2", + "message": "Method mean_per_gene performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: mean_per_gene\n Metric id: mae\n Best score: 0.4307%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score random_predict mae", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_predict performs much worse than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mae\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score random_predict mae", + "value": 0.2591, + "severity": 0, + "severity_value": 0.12955, + "code": "best_score <= 2", + "message": "Method random_predict performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: random_predict\n Metric id: mae\n Best score: 0.2591%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score zeros mae", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method zeros performs much worse than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mae\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score zeros mae", + "value": 0.46, + "severity": 0, + "severity_value": 0.23, + "code": "best_score <= 2", + "message": "Method zeros performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: zeros\n Metric id: mae\n Best score: 0.46%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score solution mae", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method solution performs much worse than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mae\n Worst score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score solution mae", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method solution performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: solution\n Metric id: mae\n Best score: 1%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_py mae", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method knnr_py performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mae\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_py mae", + "value": 0.5267, + "severity": 0, + "severity_value": 0.26335, + "code": "best_score <= 2", + "message": "Method knnr_py performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_py\n Metric id: mae\n Best score: 0.5267%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score knnr_r mae", + "value": 0.0167, + "severity": 0, + "severity_value": -0.0167, + "code": "worst_score >= -1", + "message": "Method knnr_r performs much worse than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mae\n Worst score: 0.0167%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score knnr_r mae", + "value": 0.5132, + "severity": 0, + "severity_value": 0.2566, + "code": "best_score <= 2", + "message": "Method knnr_r performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: knnr_r\n Metric id: mae\n Best score: 0.5132%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lm mae", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method lm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mae\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lm mae", + "value": 0.4771, + "severity": 0, + "severity_value": 0.23855, + "code": "best_score <= 2", + "message": "Method lm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lm\n Metric id: mae\n Best score: 0.4771%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score lmds_irlba_rf mae", + "value": -0.0588, + "severity": 0, + "severity_value": 0.0588, + "code": "worst_score >= -1", + "message": "Method lmds_irlba_rf performs much worse than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mae\n Worst score: -0.0588%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score lmds_irlba_rf mae", + "value": 0.278, + "severity": 0, + "severity_value": 0.139, + "code": "best_score <= 2", + "message": "Method lmds_irlba_rf performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: lmds_irlba_rf\n Metric id: mae\n Best score: 0.278%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Worst score guanlab_dengkw_pm mae", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method guanlab_dengkw_pm performs much worse than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mae\n Worst score: 0.0%\n" + }, + { + "task_id": "task_predict_modality", + "category": "Scaling", + "name": "Best score guanlab_dengkw_pm mae", + "value": 0.5124, + "severity": 0, + "severity_value": 0.2562, + "code": "best_score <= 2", + "message": "Method guanlab_dengkw_pm performs a lot better than baselines.\n Task id: task_predict_modality\n Method id: guanlab_dengkw_pm\n Metric id: mae\n Best score: 0.5124%\n" + } +] \ No newline at end of file diff --git a/results/predict_modality/data/results.json b/results/predict_modality/data/results.json new file mode 100644 index 00000000..ad3cf37f --- /dev/null +++ b/results/predict_modality/data/results.json @@ -0,0 +1,1634 @@ +[ + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "guanlab_dengkw_pm", + "metric_values": { + "mae": 0.5442, + "mean_pearson_per_cell": 0.787, + "mean_pearson_per_gene": 0.6418, + "mean_spearman_per_cell": 0.686, + "mean_spearman_per_gene": 0.5039, + "overall_pearson": 0.7518, + "overall_spearman": 0.6366, + "rmse": 0.7305 + }, + "scaled_scores": { + "mae": 0.466, + "mean_pearson_per_cell": 0.787, + "mean_pearson_per_gene": 0.6474, + "mean_spearman_per_cell": 0.686, + "mean_spearman_per_gene": 0.5135, + "overall_pearson": 0.7483, + "overall_spearman": 0.6234, + "rmse": 0.4548 + }, + "mean_score": 0.6158, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 588, + "cpu_pct": 406, + "peak_memory_mb": 46285, + "disk_read_mb": 715, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "knnr_py", + "metric_values": { + "mae": 0.5223, + "mean_pearson_per_cell": 0.7772, + "mean_pearson_per_gene": 0.5946, + "mean_spearman_per_cell": 0.6911, + "mean_spearman_per_gene": 0.4891, + "overall_pearson": 0.7122, + "overall_spearman": 0.6237, + "rmse": 0.7245 + }, + "scaled_scores": { + "mae": 0.4875, + "mean_pearson_per_cell": 0.7772, + "mean_pearson_per_gene": 0.601, + "mean_spearman_per_cell": 0.6911, + "mean_spearman_per_gene": 0.499, + "overall_pearson": 0.7082, + "overall_spearman": 0.61, + "rmse": 0.4593 + }, + "mean_score": 0.6042, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 51.4, + "cpu_pct": 315.1, + "peak_memory_mb": 10138, + "disk_read_mb": 715, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "knnr_r", + "metric_values": { + "mae": 0.5593, + "mean_pearson_per_cell": 0.7563, + "mean_pearson_per_gene": 0.5367, + "mean_spearman_per_cell": 0.6708, + "mean_spearman_per_gene": 0.4309, + "overall_pearson": 0.67, + "overall_spearman": 0.5816, + "rmse": 0.7776 + }, + "scaled_scores": { + "mae": 0.4512, + "mean_pearson_per_cell": 0.7563, + "mean_pearson_per_gene": 0.5439, + "mean_spearman_per_cell": 0.6708, + "mean_spearman_per_gene": 0.4419, + "overall_pearson": 0.6654, + "overall_spearman": 0.5663, + "rmse": 0.4197 + }, + "mean_score": 0.5644, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 80, + "cpu_pct": 105.3, + "peak_memory_mb": 11981, + "disk_read_mb": 718, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "lm", + "metric_values": { + "mae": 0.566, + "mean_pearson_per_cell": 0.7078, + "mean_pearson_per_gene": 0.5188, + "mean_spearman_per_cell": 0.6098, + "mean_spearman_per_gene": 0.4288, + "overall_pearson": 0.6589, + "overall_spearman": 0.5528, + "rmse": 0.7754 + }, + "scaled_scores": { + "mae": 0.4446, + "mean_pearson_per_cell": 0.7078, + "mean_pearson_per_gene": 0.5264, + "mean_spearman_per_cell": 0.6098, + "mean_spearman_per_gene": 0.4398, + "overall_pearson": 0.6541, + "overall_spearman": 0.5364, + "rmse": 0.4213 + }, + "mean_score": 0.5425, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 125, + "cpu_pct": 106.5, + "peak_memory_mb": 11162, + "disk_read_mb": 719, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "mean_per_gene", + "metric_values": { + "mae": 0.6606, + "mean_pearson_per_cell": 0.1661, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.1574, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.1411, + "overall_spearman": 0.1229, + "rmse": 0.9598 + }, + "scaled_scores": { + "mae": 0.3518, + "mean_pearson_per_cell": 0.1661, + "mean_pearson_per_gene": 0.0157, + "mean_spearman_per_cell": 0.1574, + "mean_spearman_per_gene": 0.0193, + "overall_pearson": 0.1291, + "overall_spearman": 0.0908, + "rmse": 0.2837 + }, + "mean_score": 0.1517, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 7, + "cpu_pct": 136, + "peak_memory_mb": 5632, + "disk_read_mb": 69, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "random_predict", + "metric_values": { + "mae": 0.8256, + "mean_pearson_per_cell": 0.021, + "mean_pearson_per_gene": -0.016, + "mean_spearman_per_cell": 0.0723, + "mean_spearman_per_gene": -0.0197, + "overall_pearson": 0.0138, + "overall_spearman": 0.0353, + "rmse": 1.1688 + }, + "scaled_scores": { + "mae": 0.1899, + "mean_pearson_per_cell": 0.021, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.0723, + "mean_spearman_per_gene": 0, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0.1277 + }, + "mean_score": 0.0514, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 12.5, + "cpu_pct": 93.9, + "peak_memory_mb": 3175, + "disk_read_mb": 78, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "solution", + "metric_values": { + "mae": 0, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 0 + }, + "scaled_scores": { + "mae": 1, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 11.1, + "cpu_pct": 98.1, + "peak_memory_mb": 3175, + "disk_read_mb": 28, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/normal", + "method_id": "zeros", + "metric_values": { + "mae": 1.0191, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": "NA", + "overall_spearman": "NA", + "rmse": 1.3399 + }, + "scaled_scores": { + "mae": 0, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0.0157, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0.0193, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0 + }, + "mean_score": 0.0044, + "resources": { + "submit": "2024-11-25 13:44:25", + "exit_code": 0, + "duration_sec": 5.9, + "cpu_pct": 126.5, + "peak_memory_mb": 2868, + "disk_read_mb": 69, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "guanlab_dengkw_pm", + "metric_values": { + "mae": 0.1664, + "mean_pearson_per_cell": 0.7444, + "mean_pearson_per_gene": 0.2093, + "mean_spearman_per_cell": 0.4099, + "mean_spearman_per_gene": 0.2081, + "overall_pearson": 0.7419, + "overall_spearman": 0.4276, + "rmse": 0.3421 + }, + "scaled_scores": { + "mae": 0.1886, + "mean_pearson_per_cell": 0.7444, + "mean_pearson_per_gene": 0.2096, + "mean_spearman_per_cell": 0.4099, + "mean_spearman_per_gene": 0.2084, + "overall_pearson": 0.549, + "overall_spearman": 0.2608, + "rmse": 0.3721 + }, + "mean_score": 0.3678, + "resources": { + "submit": "2024-11-25 13:43:57", + "exit_code": 0, + "duration_sec": 314, + "cpu_pct": 669.1, + "peak_memory_mb": 46080, + "disk_read_mb": 703, + "disk_write_mb": 51 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "knnr_py", + "metric_values": { + "mae": 0.1666, + "mean_pearson_per_cell": 0.7362, + "mean_pearson_per_gene": 0.1745, + "mean_spearman_per_cell": 0.4034, + "mean_spearman_per_gene": 0.1608, + "overall_pearson": 0.7318, + "overall_spearman": 0.409, + "rmse": 0.3477 + }, + "scaled_scores": { + "mae": 0.1874, + "mean_pearson_per_cell": 0.7362, + "mean_pearson_per_gene": 0.1748, + "mean_spearman_per_cell": 0.4034, + "mean_spearman_per_gene": 0.161, + "overall_pearson": 0.5314, + "overall_spearman": 0.2367, + "rmse": 0.3619 + }, + "mean_score": 0.3491, + "resources": { + "submit": "2024-11-25 13:43:57", + "exit_code": 0, + "duration_sec": 40.9, + "cpu_pct": 186.7, + "peak_memory_mb": 29287, + "disk_read_mb": 703, + "disk_write_mb": 58 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "knnr_r", + "metric_values": { + "mae": 0.1682, + "mean_pearson_per_cell": 0.725, + "mean_pearson_per_gene": 0.1458, + "mean_spearman_per_cell": 0.3916, + "mean_spearman_per_gene": 0.1528, + "overall_pearson": 0.7227, + "overall_spearman": 0.402, + "rmse": 0.3529 + }, + "scaled_scores": { + "mae": 0.1799, + "mean_pearson_per_cell": 0.725, + "mean_pearson_per_gene": 0.146, + "mean_spearman_per_cell": 0.3916, + "mean_spearman_per_gene": 0.153, + "overall_pearson": 0.5155, + "overall_spearman": 0.2277, + "rmse": 0.3524 + }, + "mean_score": 0.3364, + "resources": { + "submit": "2024-11-25 13:43:57", + "exit_code": 0, + "duration_sec": 94, + "cpu_pct": 103, + "peak_memory_mb": 13517, + "disk_read_mb": 706, + "disk_write_mb": 42 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "lm", + "metric_values": { + "mae": 0.1787, + "mean_pearson_per_cell": 0.616, + "mean_pearson_per_gene": 0.1074, + "mean_spearman_per_cell": 0.33, + "mean_spearman_per_gene": 0.0645, + "overall_pearson": 0.6095, + "overall_spearman": 0.2786, + "rmse": 0.4098 + }, + "scaled_scores": { + "mae": 0.1288, + "mean_pearson_per_cell": 0.616, + "mean_pearson_per_gene": 0.1076, + "mean_spearman_per_cell": 0.33, + "mean_spearman_per_gene": 0.0645, + "overall_pearson": 0.3177, + "overall_spearman": 0.0683, + "rmse": 0.248 + }, + "mean_score": 0.2351, + "resources": { + "submit": "2024-11-25 13:43:56", + "exit_code": 0, + "duration_sec": 4248, + "cpu_pct": 100.2, + "peak_memory_mb": 8602, + "disk_read_mb": 706, + "disk_write_mb": 105 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "mean_per_gene", + "metric_values": { + "mae": 0.1853, + "mean_pearson_per_cell": 0.6681, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.3705, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.6679, + "overall_spearman": 0.3669, + "rmse": 0.3801 + }, + "scaled_scores": { + "mae": 0.0964, + "mean_pearson_per_cell": 0.6681, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.3705, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.4197, + "overall_spearman": 0.1824, + "rmse": 0.3026 + }, + "mean_score": 0.2549, + "resources": { + "submit": "2024-11-25 13:43:56", + "exit_code": 0, + "duration_sec": 23.8, + "cpu_pct": 84.4, + "peak_memory_mb": 9626, + "disk_read_mb": 653, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "random_predict", + "metric_values": { + "mae": 0.2051, + "mean_pearson_per_cell": 0.4271, + "mean_pearson_per_gene": 0.0001, + "mean_spearman_per_cell": 0.2433, + "mean_spearman_per_gene": 0.0009, + "overall_pearson": 0.4277, + "overall_spearman": 0.2257, + "rmse": 0.5449 + }, + "scaled_scores": { + "mae": 0, + "mean_pearson_per_cell": 0.4271, + "mean_pearson_per_gene": 0.0001, + "mean_spearman_per_cell": 0.2433, + "mean_spearman_per_gene": 0.0009, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0 + }, + "mean_score": 0.0839, + "resources": { + "submit": "2024-11-25 13:43:56", + "exit_code": 0, + "duration_sec": 19.5, + "cpu_pct": 90.2, + "peak_memory_mb": 4916, + "disk_read_mb": 663, + "disk_write_mb": 7 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "solution", + "metric_values": { + "mae": 0, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 0.9986, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 0.9986, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 0 + }, + "scaled_scores": { + "mae": 1, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2024-11-25 13:43:55", + "exit_code": 0, + "duration_sec": 12.3, + "cpu_pct": 83.3, + "peak_memory_mb": 3175, + "disk_read_mb": 40, + "disk_write_mb": 13 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_cite/swap", + "method_id": "zeros", + "metric_values": { + "mae": 0.1608, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": "NA", + "overall_spearman": "NA", + "rmse": 0.5347 + }, + "scaled_scores": { + "mae": 0.216, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0.0187 + }, + "mean_score": 0.0293, + "resources": { + "submit": "2024-11-25 13:43:56", + "exit_code": 0, + "duration_sec": 12.6, + "cpu_pct": 93.7, + "peak_memory_mb": 4608, + "disk_read_mb": 653, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "guanlab_dengkw_pm", + "metric_values": { + "mae": 0.1236, + "mean_pearson_per_cell": 0.1841, + "mean_pearson_per_gene": 0.0875, + "mean_spearman_per_cell": 0.214, + "mean_spearman_per_gene": 0.0908, + "overall_pearson": 0.1679, + "overall_spearman": 0.2199, + "rmse": 0.512 + }, + "scaled_scores": { + "mae": 0.1278, + "mean_pearson_per_cell": 0.1841, + "mean_pearson_per_gene": 0.0881, + "mean_spearman_per_cell": 0.214, + "mean_spearman_per_gene": 0.0915, + "overall_pearson": 0.1522, + "overall_spearman": 0.1441, + "rmse": 0.281 + }, + "mean_score": 0.1603, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 21050, + "cpu_pct": 3007.6, + "peak_memory_mb": 23143, + "disk_read_mb": 1127, + "disk_write_mb": 37 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "knnr_py", + "metric_values": { + "mae": 0.1328, + "mean_pearson_per_cell": 0.1656, + "mean_pearson_per_gene": 0.0798, + "mean_spearman_per_cell": 0.2072, + "mean_spearman_per_gene": 0.0811, + "overall_pearson": 0.1648, + "overall_spearman": 0.2169, + "rmse": 0.5116 + }, + "scaled_scores": { + "mae": 0.0631, + "mean_pearson_per_cell": 0.1656, + "mean_pearson_per_gene": 0.0804, + "mean_spearman_per_cell": 0.2072, + "mean_spearman_per_gene": 0.0818, + "overall_pearson": 0.1489, + "overall_spearman": 0.1407, + "rmse": 0.2816 + }, + "mean_score": 0.1462, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 45.8, + "cpu_pct": 491.9, + "peak_memory_mb": 22221, + "disk_read_mb": 1127, + "disk_write_mb": 32 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "knnr_r", + "metric_values": { + "mae": 0.1341, + "mean_pearson_per_cell": 0.1163, + "mean_pearson_per_gene": 0.0485, + "mean_spearman_per_cell": 0.1966, + "mean_spearman_per_gene": 0.0552, + "overall_pearson": 0.115, + "overall_spearman": 0.2018, + "rmse": 0.5213 + }, + "scaled_scores": { + "mae": 0.0538, + "mean_pearson_per_cell": 0.1163, + "mean_pearson_per_gene": 0.0491, + "mean_spearman_per_cell": 0.1966, + "mean_spearman_per_gene": 0.0559, + "overall_pearson": 0.0982, + "overall_spearman": 0.1242, + "rmse": 0.2679 + }, + "mean_score": 0.1203, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 44.9, + "cpu_pct": 111.8, + "peak_memory_mb": 9728, + "disk_read_mb": 1127, + "disk_write_mb": 17 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "lm", + "metric_values": { + "mae": 0.1351, + "mean_pearson_per_cell": 0.1804, + "mean_pearson_per_gene": 0.0892, + "mean_spearman_per_cell": 0.2039, + "mean_spearman_per_gene": 0.089, + "overall_pearson": 0.1739, + "overall_spearman": 0.2157, + "rmse": 0.5109 + }, + "scaled_scores": { + "mae": 0.0471, + "mean_pearson_per_cell": 0.1804, + "mean_pearson_per_gene": 0.0899, + "mean_spearman_per_cell": 0.2039, + "mean_spearman_per_gene": 0.0897, + "overall_pearson": 0.1583, + "overall_spearman": 0.1395, + "rmse": 0.2825 + }, + "mean_score": 0.1489, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 1189, + "cpu_pct": 100.6, + "peak_memory_mb": 7271, + "disk_read_mb": 1127, + "disk_write_mb": 75 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "mean_per_gene", + "metric_values": { + "mae": 0.1381, + "mean_pearson_per_cell": 0.1425, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.1936, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.1366, + "overall_spearman": 0.1968, + "rmse": 0.5135 + }, + "scaled_scores": { + "mae": 0.0259, + "mean_pearson_per_cell": 0.1425, + "mean_pearson_per_gene": 0.0005, + "mean_spearman_per_cell": 0.1936, + "mean_spearman_per_gene": 0.0006, + "overall_pearson": 0.1203, + "overall_spearman": 0.1187, + "rmse": 0.2788 + }, + "mean_score": 0.1101, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 13.4, + "cpu_pct": 109.1, + "peak_memory_mb": 4608, + "disk_read_mb": 824, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "random_predict", + "metric_values": { + "mae": 0.1417, + "mean_pearson_per_cell": 0.0203, + "mean_pearson_per_gene": -0.0005, + "mean_spearman_per_cell": 0.0896, + "mean_spearman_per_gene": -0.0006, + "overall_pearson": 0.0186, + "overall_spearman": 0.0886, + "rmse": 0.7121 + }, + "scaled_scores": { + "mae": 0, + "mean_pearson_per_cell": 0.0203, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.0896, + "mean_spearman_per_gene": 0, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0 + }, + "mean_score": 0.0137, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 15.2, + "cpu_pct": 164.6, + "peak_memory_mb": 7066, + "disk_read_mb": 833, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "solution", + "metric_values": { + "mae": 0, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 0.9984, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 0.9984, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 0 + }, + "scaled_scores": { + "mae": 1, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 11.8, + "cpu_pct": 107.5, + "peak_memory_mb": 3277, + "disk_read_mb": 57, + "disk_write_mb": 30 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/normal", + "method_id": "zeros", + "metric_values": { + "mae": 0.0825, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": "NA", + "overall_spearman": "NA", + "rmse": 0.5247 + }, + "scaled_scores": { + "mae": 0.4181, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0.0005, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0.0006, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0.2631 + }, + "mean_score": 0.0853, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 14.4, + "cpu_pct": 118.3, + "peak_memory_mb": 6759, + "disk_read_mb": 824, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "guanlab_dengkw_pm", + "metric_values": { + "mae": 0.2482, + "mean_pearson_per_cell": 0.4629, + "mean_pearson_per_gene": 0.1552, + "mean_spearman_per_cell": 0.3016, + "mean_spearman_per_gene": 0.1477, + "overall_pearson": 0.4691, + "overall_spearman": 0.3188, + "rmse": 0.5178 + }, + "scaled_scores": { + "mae": 0.089, + "mean_pearson_per_cell": 0.4629, + "mean_pearson_per_gene": 0.1556, + "mean_spearman_per_cell": 0.3016, + "mean_spearman_per_gene": 0.1479, + "overall_pearson": 0.3519, + "overall_spearman": 0.2018, + "rmse": 0.3101 + }, + "mean_score": 0.2526, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 353, + "cpu_pct": 590.8, + "peak_memory_mb": 22119, + "disk_read_mb": 1946, + "disk_write_mb": 49 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "knnr_py", + "metric_values": { + "mae": 0.2508, + "mean_pearson_per_cell": 0.4598, + "mean_pearson_per_gene": 0.1409, + "mean_spearman_per_cell": 0.3001, + "mean_spearman_per_gene": 0.1309, + "overall_pearson": 0.4648, + "overall_spearman": 0.3148, + "rmse": 0.5204 + }, + "scaled_scores": { + "mae": 0.0793, + "mean_pearson_per_cell": 0.4598, + "mean_pearson_per_gene": 0.1413, + "mean_spearman_per_cell": 0.3001, + "mean_spearman_per_gene": 0.1311, + "overall_pearson": 0.3466, + "overall_spearman": 0.1971, + "rmse": 0.3066 + }, + "mean_score": 0.2452, + "resources": { + "submit": "2024-11-25 13:50:44", + "exit_code": 0, + "duration_sec": 100, + "cpu_pct": 165.5, + "peak_memory_mb": 30208, + "disk_read_mb": 1946, + "disk_write_mb": 59 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "knnr_r", + "metric_values": { + "mae": 0.2679, + "mean_pearson_per_cell": 0.3783, + "mean_pearson_per_gene": 0.0328, + "mean_spearman_per_cell": 0.2574, + "mean_spearman_per_gene": 0.0397, + "overall_pearson": 0.3809, + "overall_spearman": 0.2663, + "rmse": 0.5475 + }, + "scaled_scores": { + "mae": 0.0167, + "mean_pearson_per_cell": 0.3783, + "mean_pearson_per_gene": 0.0331, + "mean_spearman_per_cell": 0.2574, + "mean_spearman_per_gene": 0.0398, + "overall_pearson": 0.2441, + "overall_spearman": 0.1403, + "rmse": 0.2706 + }, + "mean_score": 0.1726, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 101, + "cpu_pct": 100.7, + "peak_memory_mb": 10343, + "disk_read_mb": 2048, + "disk_write_mb": 43 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "lm", + "metric_values": { + "mae": 0.2554, + "mean_pearson_per_cell": 0.4392, + "mean_pearson_per_gene": 0.107, + "mean_spearman_per_cell": 0.293, + "mean_spearman_per_gene": 0.1125, + "overall_pearson": 0.439, + "overall_spearman": 0.3033, + "rmse": 0.5279 + }, + "scaled_scores": { + "mae": 0.0625, + "mean_pearson_per_cell": 0.4392, + "mean_pearson_per_gene": 0.1074, + "mean_spearman_per_cell": 0.293, + "mean_spearman_per_gene": 0.1127, + "overall_pearson": 0.3151, + "overall_spearman": 0.1837, + "rmse": 0.2967 + }, + "mean_score": 0.2263, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 1714, + "cpu_pct": 100.3, + "peak_memory_mb": 12391, + "disk_read_mb": 2048, + "disk_write_mb": 100 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "mean_per_gene", + "metric_values": { + "mae": 0.2639, + "mean_pearson_per_cell": 0.3926, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.2762, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.3931, + "overall_spearman": 0.28, + "rmse": 0.5381 + }, + "scaled_scores": { + "mae": 0.0313, + "mean_pearson_per_cell": 0.3926, + "mean_pearson_per_gene": 0.0003, + "mean_spearman_per_cell": 0.2762, + "mean_spearman_per_gene": 0.0001, + "overall_pearson": 0.259, + "overall_spearman": 0.1563, + "rmse": 0.283 + }, + "mean_score": 0.1749, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 12.3, + "cpu_pct": 137.8, + "peak_memory_mb": 6554, + "disk_read_mb": 347, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "random_predict", + "metric_values": { + "mae": 0.2724, + "mean_pearson_per_cell": 0.1809, + "mean_pearson_per_gene": -0.0003, + "mean_spearman_per_cell": 0.1505, + "mean_spearman_per_gene": -0.0001, + "overall_pearson": 0.1809, + "overall_spearman": 0.1466, + "rmse": 0.7505 + }, + "scaled_scores": { + "mae": 0, + "mean_pearson_per_cell": 0.1809, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.1505, + "mean_spearman_per_gene": 0, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0 + }, + "mean_score": 0.0414, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 17.1, + "cpu_pct": 90.9, + "peak_memory_mb": 3687, + "disk_read_mb": 357, + "disk_write_mb": 7 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "solution", + "metric_values": { + "mae": 0, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 0.9995, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 0.9995, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 0 + }, + "scaled_scores": { + "mae": 1, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 7.6, + "cpu_pct": 139.6, + "peak_memory_mb": 3175, + "disk_read_mb": 35, + "disk_write_mb": 8 + } + }, + { + "dataset_id": "openproblems_neurips2021/bmmc_multiome/swap", + "method_id": "zeros", + "metric_values": { + "mae": 0.1593, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": "NA", + "overall_spearman": "NA", + "rmse": 0.6039 + }, + "scaled_scores": { + "mae": 0.4151, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0.0003, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0.0001, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0.1953 + }, + "mean_score": 0.0764, + "resources": { + "submit": "2024-11-25 13:47:05", + "exit_code": 0, + "duration_sec": 12.2, + "cpu_pct": 86.8, + "peak_memory_mb": 2560, + "disk_read_mb": 347, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "guanlab_dengkw_pm", + "metric_values": { + "mae": 0.2864, + "mean_pearson_per_cell": 0.6017, + "mean_pearson_per_gene": 0.5, + "mean_spearman_per_cell": 0.5123, + "mean_spearman_per_gene": 0.4332, + "overall_pearson": 0.68, + "overall_spearman": 0.5456, + "rmse": 0.3819 + }, + "scaled_scores": { + "mae": 0.5124, + "mean_pearson_per_cell": 0.6017, + "mean_pearson_per_gene": 0.5, + "mean_spearman_per_cell": 0.5123, + "mean_spearman_per_gene": 0.4337, + "overall_pearson": 0.651, + "overall_spearman": 0.4876, + "rmse": 0.5 + }, + "mean_score": 0.5248, + "resources": { + "submit": "2024-11-25 13:48:06", + "exit_code": 0, + "duration_sec": 2614, + "cpu_pct": 2090.5, + "peak_memory_mb": 65536, + "disk_read_mb": 3380, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "knnr_py", + "metric_values": { + "mae": 0.278, + "mean_pearson_per_cell": 0.5932, + "mean_pearson_per_gene": 0.4528, + "mean_spearman_per_cell": 0.513, + "mean_spearman_per_gene": 0.3894, + "overall_pearson": 0.6524, + "overall_spearman": 0.5258, + "rmse": 0.3758 + }, + "scaled_scores": { + "mae": 0.5267, + "mean_pearson_per_cell": 0.5932, + "mean_pearson_per_gene": 0.4528, + "mean_spearman_per_cell": 0.513, + "mean_spearman_per_gene": 0.3901, + "overall_pearson": 0.6209, + "overall_spearman": 0.4652, + "rmse": 0.5079 + }, + "mean_score": 0.5087, + "resources": { + "submit": "2024-11-25 13:52:34", + "exit_code": 0, + "duration_sec": 441, + "cpu_pct": 108.5, + "peak_memory_mb": 36557, + "disk_read_mb": 3380, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "knnr_r", + "metric_values": { + "mae": 0.2859, + "mean_pearson_per_cell": 0.5852, + "mean_pearson_per_gene": 0.4306, + "mean_spearman_per_cell": 0.5012, + "mean_spearman_per_gene": 0.3627, + "overall_pearson": 0.6443, + "overall_spearman": 0.5081, + "rmse": 0.3866 + }, + "scaled_scores": { + "mae": 0.5132, + "mean_pearson_per_cell": 0.5852, + "mean_pearson_per_gene": 0.4306, + "mean_spearman_per_cell": 0.5012, + "mean_spearman_per_gene": 0.3634, + "overall_pearson": 0.6121, + "overall_spearman": 0.4453, + "rmse": 0.4938 + }, + "mean_score": 0.4931, + "resources": { + "submit": "2024-11-25 13:52:24", + "exit_code": 0, + "duration_sec": 295, + "cpu_pct": 101.1, + "peak_memory_mb": 32768, + "disk_read_mb": 3380, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "lm", + "metric_values": { + "mae": 0.3071, + "mean_pearson_per_cell": 0.5354, + "mean_pearson_per_gene": 0.3609, + "mean_spearman_per_cell": 0.4539, + "mean_spearman_per_gene": 0.3513, + "overall_pearson": 0.5511, + "overall_spearman": 0.4652, + "rmse": 0.4172 + }, + "scaled_scores": { + "mae": 0.4771, + "mean_pearson_per_cell": 0.5354, + "mean_pearson_per_gene": 0.3609, + "mean_spearman_per_cell": 0.4539, + "mean_spearman_per_gene": 0.3519, + "overall_pearson": 0.5104, + "overall_spearman": 0.3969, + "rmse": 0.4537 + }, + "mean_score": 0.4425, + "resources": { + "submit": "2024-11-25 13:48:06", + "exit_code": 0, + "duration_sec": 1093, + "cpu_pct": 101.5, + "peak_memory_mb": 39424, + "disk_read_mb": 3380, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "mean_per_gene", + "metric_values": { + "mae": 0.3344, + "mean_pearson_per_cell": 0.2697, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.219, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.2118, + "overall_spearman": 0.1996, + "rmse": 0.482 + }, + "scaled_scores": { + "mae": 0.4307, + "mean_pearson_per_cell": 0.2697, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.219, + "mean_spearman_per_gene": 0.001, + "overall_pearson": 0.1404, + "overall_spearman": 0.0973, + "rmse": 0.3689 + }, + "mean_score": 0.1909, + "resources": { + "submit": "2024-11-25 13:48:05", + "exit_code": 0, + "duration_sec": 12, + "cpu_pct": 107.6, + "peak_memory_mb": 5530, + "disk_read_mb": 103, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "random_predict", + "metric_values": { + "mae": 0.4352, + "mean_pearson_per_cell": 0.1143, + "mean_pearson_per_gene": 0.0034, + "mean_spearman_per_cell": 0.1291, + "mean_spearman_per_gene": -0.001, + "overall_pearson": 0.0831, + "overall_spearman": 0.1133, + "rmse": 0.6318 + }, + "scaled_scores": { + "mae": 0.2591, + "mean_pearson_per_cell": 0.1143, + "mean_pearson_per_gene": 0.0034, + "mean_spearman_per_cell": 0.1291, + "mean_spearman_per_gene": 0, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0.1727 + }, + "mean_score": 0.0848, + "resources": { + "submit": "2024-11-25 13:48:06", + "exit_code": 0, + "duration_sec": 14.6, + "cpu_pct": 137.9, + "peak_memory_mb": 3175, + "disk_read_mb": 113, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "solution", + "metric_values": { + "mae": 0, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 0 + }, + "scaled_scores": { + "mae": 1, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2024-11-25 13:48:06", + "exit_code": 0, + "duration_sec": 12.6, + "cpu_pct": 157.1, + "peak_memory_mb": 3175, + "disk_read_mb": 28, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/normal", + "method_id": "zeros", + "metric_values": { + "mae": 0.5874, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": "NA", + "overall_spearman": "NA", + "rmse": 0.7637 + }, + "scaled_scores": { + "mae": 0, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0.001, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0 + }, + "mean_score": 0.0001, + "resources": { + "submit": "2024-11-25 13:48:06", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 75, + "peak_memory_mb": 1844, + "disk_read_mb": 103, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "guanlab_dengkw_pm", + "metric_values": { + "mae": 0.0984, + "mean_pearson_per_cell": 0.8843, + "mean_pearson_per_gene": 0.149, + "mean_spearman_per_cell": 0.6144, + "mean_spearman_per_gene": 0.1402, + "overall_pearson": 0.8834, + "overall_spearman": 0.6151, + "rmse": 0.1926 + }, + "scaled_scores": { + "mae": 0.3605, + "mean_pearson_per_cell": 0.8843, + "mean_pearson_per_gene": 0.1648, + "mean_spearman_per_cell": 0.6144, + "mean_spearman_per_gene": 0.1551, + "overall_pearson": 0.5274, + "overall_spearman": 0.2092, + "rmse": 0.561 + }, + "mean_score": 0.4346, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 21011, + "cpu_pct": 3220.7, + "peak_memory_mb": 65332, + "disk_read_mb": 3277, + "disk_write_mb": 77 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "knnr_py", + "metric_values": { + "mae": 0.1012, + "mean_pearson_per_cell": 0.8764, + "mean_pearson_per_gene": 0.1169, + "mean_spearman_per_cell": 0.6173, + "mean_spearman_per_gene": 0.1074, + "overall_pearson": 0.8753, + "overall_spearman": 0.6171, + "rmse": 0.199 + }, + "scaled_scores": { + "mae": 0.3418, + "mean_pearson_per_cell": 0.8764, + "mean_pearson_per_gene": 0.1293, + "mean_spearman_per_cell": 0.6173, + "mean_spearman_per_gene": 0.1189, + "overall_pearson": 0.4946, + "overall_spearman": 0.2133, + "rmse": 0.5465 + }, + "mean_score": 0.4173, + "resources": { + "submit": "2024-11-25 13:50:34", + "exit_code": 0, + "duration_sec": 78, + "cpu_pct": 127.8, + "peak_memory_mb": 48128, + "disk_read_mb": 3277, + "disk_write_mb": 82 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "knnr_r", + "metric_values": { + "mae": 0.1017, + "mean_pearson_per_cell": 0.8737, + "mean_pearson_per_gene": 0.0957, + "mean_spearman_per_cell": 0.6252, + "mean_spearman_per_gene": 0.0889, + "overall_pearson": 0.8726, + "overall_spearman": 0.6251, + "rmse": 0.2011 + }, + "scaled_scores": { + "mae": 0.3388, + "mean_pearson_per_cell": 0.8737, + "mean_pearson_per_gene": 0.1059, + "mean_spearman_per_cell": 0.6252, + "mean_spearman_per_gene": 0.0984, + "overall_pearson": 0.4837, + "overall_spearman": 0.2298, + "rmse": 0.5417 + }, + "mean_score": 0.4122, + "resources": { + "submit": "2024-11-25 13:48:54", + "exit_code": 0, + "duration_sec": 360, + "cpu_pct": 100.5, + "peak_memory_mb": 35840, + "disk_read_mb": 3277, + "disk_write_mb": 62 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "lm", + "metric_values": { + "mae": 0.1335, + "mean_pearson_per_cell": 0.6058, + "mean_pearson_per_gene": 0.075, + "mean_spearman_per_cell": 0.4077, + "mean_spearman_per_gene": 0.0795, + "overall_pearson": 0.6015, + "overall_spearman": 0.3941, + "rmse": 0.345 + }, + "scaled_scores": { + "mae": 0.1322, + "mean_pearson_per_cell": 0.6058, + "mean_pearson_per_gene": 0.083, + "mean_spearman_per_cell": 0.4077, + "mean_spearman_per_gene": 0.088, + "overall_pearson": -0.6151, + "overall_spearman": -0.2449, + "rmse": 0.2138 + }, + "mean_score": 0.1913, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 14491, + "cpu_pct": 100.1, + "peak_memory_mb": 31949, + "disk_read_mb": 3277, + "disk_write_mb": 162 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "mean_per_gene", + "metric_values": { + "mae": 0.1094, + "mean_pearson_per_cell": 0.849, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.5996, + "mean_spearman_per_gene": 0, + "overall_pearson": 0.8471, + "overall_spearman": 0.5988, + "rmse": 0.2192 + }, + "scaled_scores": { + "mae": 0.2887, + "mean_pearson_per_cell": 0.849, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0.5996, + "mean_spearman_per_gene": 0.0001, + "overall_pearson": 0.3804, + "overall_spearman": 0.1758, + "rmse": 0.5005 + }, + "mean_score": 0.3493, + "resources": { + "submit": "2024-11-25 13:47:55", + "exit_code": 0, + "duration_sec": 43.3, + "cpu_pct": 105.2, + "peak_memory_mb": 26215, + "disk_read_mb": 3277, + "disk_write_mb": 4 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "random_predict", + "metric_values": { + "mae": 0.1307, + "mean_pearson_per_cell": 0.7556, + "mean_pearson_per_gene": 0.0001, + "mean_spearman_per_cell": 0.5185, + "mean_spearman_per_gene": -0.0001, + "overall_pearson": 0.7533, + "overall_spearman": 0.5133, + "rmse": 0.2893 + }, + "scaled_scores": { + "mae": 0.1501, + "mean_pearson_per_cell": 0.7556, + "mean_pearson_per_gene": 0.0001, + "mean_spearman_per_cell": 0.5185, + "mean_spearman_per_gene": 0, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0.3408 + }, + "mean_score": 0.2206, + "resources": { + "submit": "2024-11-25 13:48:14", + "exit_code": 0, + "duration_sec": 65, + "cpu_pct": 98.8, + "peak_memory_mb": 25908, + "disk_read_mb": 3277, + "disk_write_mb": 23 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "solution", + "metric_values": { + "mae": 0, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 0.9041, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 0.9041, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 0 + }, + "scaled_scores": { + "mae": 1, + "mean_pearson_per_cell": 1, + "mean_pearson_per_gene": 1, + "mean_spearman_per_cell": 1, + "mean_spearman_per_gene": 1, + "overall_pearson": 1, + "overall_spearman": 1, + "rmse": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2024-11-25 13:44:05", + "exit_code": 0, + "duration_sec": 14, + "cpu_pct": 150.3, + "peak_memory_mb": 5837, + "disk_read_mb": 59, + "disk_write_mb": 32 + } + }, + { + "dataset_id": "openproblems_neurips2022/pbmc_cite/swap", + "method_id": "zeros", + "metric_values": { + "mae": 0.1538, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0, + "overall_pearson": "NA", + "overall_spearman": "NA", + "rmse": 0.4388 + }, + "scaled_scores": { + "mae": 0, + "mean_pearson_per_cell": 0, + "mean_pearson_per_gene": 0, + "mean_spearman_per_cell": 0, + "mean_spearman_per_gene": 0.0001, + "overall_pearson": 0, + "overall_spearman": 0, + "rmse": 0 + }, + "mean_score": 9.2796e-06, + "resources": { + "submit": "2024-11-25 13:44:06", + "exit_code": 0, + "duration_sec": 34.2, + "cpu_pct": 103.8, + "peak_memory_mb": 13210, + "disk_read_mb": 3277, + "disk_write_mb": 1 + } + } +] diff --git a/results/predict_modality/data/state.yaml b/results/predict_modality/data/state.yaml new file mode 100644 index 00000000..abbb0fc1 --- /dev/null +++ b/results/predict_modality/data/state.yaml @@ -0,0 +1,9 @@ +id: process +output_scores: !file results.json +output_method_info: !file method_info.json +output_metric_info: !file metric_info.json +output_dataset_info: !file dataset_info.json +output_task_info: !file task_info.json +output_qc: !file quality_control.json +output_metric_execution_info: !file metric_execution_info.json + diff --git a/results/predict_modality/data/task_info.json b/results/predict_modality/data/task_info.json new file mode 100644 index 00000000..72d050de --- /dev/null +++ b/results/predict_modality/data/task_info.json @@ -0,0 +1,72 @@ +{ + "task_id": "task_predict_modality", + "commit_sha": null, + "task_name": "Predict Modality", + "task_summary": "Predicting the profiles of one modality (e.g. protein abundance) from another (e.g. mRNA expression).", + "task_description": "Experimental techniques to measure multiple modalities within the same single cell are increasingly becoming available. \nThe demand for these measurements is driven by the promise to provide a deeper insight into the state of a cell. \nYet, the modalities are also intrinsically linked. We know that DNA must be accessible (ATAC data) to produce mRNA \n(expression data), and mRNA in turn is used as a template to produce protein (protein abundance). These processes \nare regulated often by the same molecules that they produce: for example, a protein may bind DNA to prevent the production \nof more mRNA. Understanding these regulatory processes would be transformative for synthetic biology and drug target discovery. \nAny method that can predict a modality from another must have accounted for these regulatory processes, but the demand for \nmulti-modal data shows that this is not trivial.\n", + "repo": "https://github.com/openproblems-bio/task_predict_modality", + "issue_tracker": "https://github.com/openproblems-bio/task_predict_modality/issues", + "authors": [ + { + "name": "Alejandro Granados", + "roles": "author", + "info": { + "github": "agranado" + } + }, + { + "name": "Alex Tong", + "roles": "author", + "info": { + "github": "atong01" + } + }, + { + "name": "Bastian Rieck", + "roles": "author", + "info": { + "github": "Pseudomanifold" + } + }, + { + "name": "Daniel Burkhardt", + "roles": "author", + "info": { + "github": "dburkhardt" + } + }, + { + "name": "Kai Waldrant", + "roles": "contributor", + "info": { + "github": "KaiWaldrant", + "orcid": "0009-0003-8555-1361" + } + }, + { + "name": "Kaiwen Deng", + "roles": "contributor", + "info": { + "email": "dengkw@umich.edu", + "github": "nonztalk" + } + }, + { + "name": "Louise Deconinck", + "roles": "author", + "info": { + "github": "LouiseDck" + } + }, + { + "name": "Robrecht Cannoodt", + "roles": ["author", "maintainer"], + "info": { + "github": "rcannood", + "orcid": "0000-0003-3641-729X" + } + } + ], + "version": "build_main", + "license": "MIT" +} diff --git a/results/predict_modality/index.qmd b/results/predict_modality/index.qmd new file mode 100644 index 00000000..97b947de --- /dev/null +++ b/results/predict_modality/index.qmd @@ -0,0 +1,20 @@ +--- +title: "Predict Modality" +subtitle: "Predicting the profiles of one modality (e.g. protein abundance) from another (e.g. mRNA expression)." +image: thumbnail.svg +page-layout: full +css: ../_include/task_template.css +engine: knitr +fig-cap-location: bottom +citation-location: document +bibliography: + - library.bib + - ../../library.bib +toc: false +--- +```{r} +#| include: false +params <- list(data_dir = "results/predict_modality/data") +params <- list(data_dir = "./data") +``` +{{< include ../_include/_task_template.qmd >}} diff --git a/results/predict_modality/thumbnail.svg b/results/predict_modality/thumbnail.svg new file mode 100644 index 00000000..59436e61 --- /dev/null +++ b/results/predict_modality/thumbnail.svg @@ -0,0 +1,666 @@ + + + + + + + + Gene + Expression + A + B + C + + + + + + True + Predicted + + + + + + + + + + + Chromatin Accessibility + Gene Expression + + Cell 1 + Cell 2 + + + + + + + + + + + + + + + Cell 3 + + + + + + + + A + B + C + Gene + + + + + + + + Task + Metric + Root mean square error + + + + + + + + + + + + + + A + B + C + Gene + + + + + + + + + + + + + + Ground-truth + Predicted + + Value Type + + + + + + Gene A + Genes + Gene B + Gene C + +