From d3f340b8ffc25c77d685ff3553e9807974232ddf Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Wed, 8 Jan 2025 21:14:20 +0100 Subject: [PATCH] update results --- _core | 2 +- _openproblems | 2 +- _task_template | 2 +- .../label_projection/data/dataset_info.json | 218 +- .../label_projection/data/method_info.json | 462 +- .../data/metric_execution_info.json | 4146 ++++++++ .../label_projection/data/metric_info.json | 82 +- .../data/quality_control.json | 1872 ++-- results/label_projection/data/results.json | 8946 +++++++++++------ results/label_projection/data/state.yaml | 9 + results/label_projection/data/task_info.json | 13 +- 11 files changed, 11402 insertions(+), 4352 deletions(-) create mode 100644 results/label_projection/data/metric_execution_info.json create mode 100644 results/label_projection/data/state.yaml diff --git a/_core b/_core index 3ada7662..405c288a 160000 --- a/_core +++ b/_core @@ -1 +1 @@ -Subproject commit 3ada76624ec63cd1e751041f186605e9de600456 +Subproject commit 405c288a53c9a011b41688a47a84c249aa7ba586 diff --git a/_openproblems b/_openproblems index bf274d82..f52c8c4a 160000 --- a/_openproblems +++ b/_openproblems @@ -1 +1 @@ -Subproject commit bf274d8279e2686df64b75a799c08aa9bf311101 +Subproject commit f52c8c4ace3696767da9aec211fc6e24f9f246c0 diff --git a/_task_template b/_task_template index 8461f1fb..8c60c7c3 160000 --- a/_task_template +++ b/_task_template @@ -1 +1 @@ -Subproject commit 8461f1fb05336fd1f9f93f883679193a7a260ecc +Subproject commit 8c60c7c30c4c33797ab201667457bcf8849f83b4 diff --git a/results/label_projection/data/dataset_info.json b/results/label_projection/data/dataset_info.json index 078b8b8d..156f0d4e 100644 --- a/results/label_projection/data/dataset_info.json +++ b/results/label_projection/data/dataset_info.json @@ -1,98 +1,122 @@ [ - { - "dataset_name": "CeNGEN (split by batch)", - "image": "openproblems", - "data_url": "https://github.com/Munfred/wormcells-data/releases/download/taylor2020/taylor2020.h5ad", - "data_reference": "hammarlund2018cengen", - "dataset_summary": "100k FACS-isolated C. elegans neurons from 17 experiments sequenced on 10x Genomics. Split into train/test by experimental batch. Dimensions: 100955 cells, 22469 genes. 169 cell types (avg. 597\u00b1800 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "cengen_batch", - "source_dataset_id": "openproblems_v1/cengen", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/cengen.py" - }, - { - "dataset_name": "CeNGEN (random split)", - "image": "openproblems", - "data_url": "https://github.com/Munfred/wormcells-data/releases/download/taylor2020/taylor2020.h5ad", - "data_reference": "hammarlund2018cengen", - "dataset_summary": "100k FACS-isolated C. elegans neurons from 17 experiments sequenced on 10x Genomics. Split into train/test randomly. Dimensions: 100955 cells, 22469 genes. 169 cell types avg. 597\u00b1800 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "cengen_random", - "source_dataset_id": "openproblems_v1/cengen", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/cengen.py" - }, - { - "dataset_name": "Pancreas (by batch)", - "image": "openproblems", - "data_url": "https://ndownloader.figshare.com/files/36086813", - "data_reference": "luecken2022benchmarking", - "dataset_summary": "Human pancreatic islet scRNA-seq data from 6 datasets across technologies (CEL-seq, CEL-seq2, Smart-seq2, inDrop, Fluidigm C1, and SMARTER-seq). Split into train/test by experimental batch. Dimensions: 16382 cells, 18771 genes. 14 cell types (avg. 1170\u00b11703 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "pancreas_batch", - "source_dataset_id": "openproblems_v1/pancreas", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/pancreas.py" - }, - { - "dataset_name": "Pancreas (random split)", - "image": "openproblems", - "data_url": "https://ndownloader.figshare.com/files/36086813", - "data_reference": "luecken2022benchmarking", - "dataset_summary": "Human pancreatic islet scRNA-seq data from 6 datasets across technologies (CEL-seq, CEL-seq2, Smart-seq2, inDrop, Fluidigm C1, and SMARTER-seq). Split into train/test randomly. Dimensions: 16382 cells, 18771 genes. 14 cell types (avg. 1170\u00b11703 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "pancreas_random", - "source_dataset_id": "openproblems_v1/pancreas", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/pancreas.py" - }, - { - "dataset_name": "Pancreas (random split with label noise)", - "image": "openproblems", - "data_url": "https://ndownloader.figshare.com/files/36086813", - "data_reference": "luecken2022benchmarking", - "dataset_summary": "Human pancreatic islet scRNA-seq data from 6 datasets across technologies (CEL-seq, CEL-seq2, Smart-seq2, inDrop, Fluidigm C1, and SMARTER-seq). Split into train/test randomly with 20% label noise. Dimensions: 16382 cells, 18771 genes. 14 cell types (avg. 1170\u00b11703 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "pancreas_random_label_noise", - "source_dataset_id": "openproblems_v1/pancreas", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/pancreas.py" - }, - { - "dataset_name": "Tabula Muris Senis Lung (random split)", - "image": "openproblems", - "data_url": "https://tabula-muris-senis.ds.czbiohub.org/", - "data_reference": "tabula2020single", - "dataset_summary": "All lung cells from Tabula Muris Senis, a 500k cell-atlas from 18 organs and tissues across the mouse lifespan. Split into train/test randomly. Dimensions: 24540 cells, 17985 genes. 39 cell types (avg. 629\u00b1999 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "tabula_muris_senis_lung_random", - "source_dataset_id": "openproblems_v1/tabula_muris_senis_droplet_lung", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/tabula_muris_senis.py" - }, - { - "dataset_name": "Zebrafish (by laboratory)", - "image": "openproblems", - "data_url": "https://ndownloader.figshare.com/files/24566651?private_link=e3921450ec1bd0587870", - "data_reference": "wagner2018single", - "dataset_summary": "90k cells from zebrafish embryos throughout the first day of development, with and without a knockout of chordin, an important developmental gene. Split into train/test by laboratory. Dimensions: 26022 cells, 25258 genes. 24 cell types (avg. 1084\u00b11156 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "zebrafish_labs", - "source_dataset_id": "openproblems_v1/zebrafish", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/zebrafish.py" - }, - { - "dataset_name": "Zebrafish (random split)", - "image": "openproblems", - "data_url": "https://ndownloader.figshare.com/files/24566651?private_link=e3921450ec1bd0587870", - "data_reference": "wagner2018single", - "dataset_summary": "90k cells from zebrafish embryos throughout the first day of development, with and without a knockout of chordin, an important developmental gene. Split into train/test randomly. Dimensions: 26022 cells, 25258 genes. 24 cell types (avg. 1084\u00b11156 cells per cell type).", - "task_id": "label_projection", - "commit_sha": "9d1665076cf6215a31f89ed2be8be20a02502887", - "dataset_id": "zebrafish_random", - "source_dataset_id": "openproblems_v1/zebrafish", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/main/openproblems/tasks/label_projection/datasets/zebrafish.py" - } -] \ No newline at end of file + { + "dataset_id": "cellxgene_census/gtex_v9", + "dataset_name": "GTEX v9", + "dataset_summary": "Single-nucleus cross-tissue molecular reference maps to decipher disease gene function", + "dataset_description": "Understanding the function of genes and their regulation in tissue homeostasis and disease requires knowing the cellular context in which genes are expressed in tissues across the body. Single cell genomics allows the generation of detailed cellular atlases in human tissues, but most efforts are focused on single tissue types. Here, we establish a framework for profiling multiple tissues across the human body at single-cell resolution using single nucleus RNA-Seq (snRNA-seq), and apply it to 8 diverse, archived, frozen tissue types (three donors per tissue). We apply four snRNA-seq methods to each of 25 samples from 16 donors, generating a cross-tissue atlas of 209,126 nuclei profiles, and benchmark them vs. scRNA-seq of comparable fresh tissues. We use a conditional variational autoencoder (cVAE) to integrate an atlas across tissues, donors, and laboratory methods. We highlight shared and tissue-specific features of tissue-resident immune cells, identifying tissue-restricted and non-restricted resident myeloid populations. These include a cross-tissue conserved dichotomy between LYVE1- and HLA class II-expressing macrophages, and the broad presence of LAM-like macrophages across healthy tissues that is also observed in disease. For rare, monogenic muscle diseases, we identify cell types that likely underlie the neuromuscular, metabolic, and immune components of these diseases, and biological processes involved in their pathology. For common complex diseases and traits analyzed by GWAS, we identify the cell types and gene modules that potentially underlie disease mechanisms. The experimental and analytical frameworks we describe will enable the generation of large-scale studies of how cellular and molecular processes vary across individuals and populations.", + "data_reference": "eraslan2022singlenucleus", + "data_url": "https://cellxgene.cziscience.com/collections/a3ffde6c-7ad2-498a-903c-d58e732f7470", + "date_created": "08-01-2025", + "file_size": 206108150 + }, + { + "dataset_id": "openproblems_v1/cengen", + "dataset_name": "CeNGEN", + "dataset_summary": "Complete Gene Expression Map of an Entire Nervous System", + "dataset_description": "100k FACS-isolated C. elegans neurons from 17 experiments sequenced on 10x Genomics.", + "data_reference": "hammarlund2018cengen", + "data_url": "https://www.cengen.org", + "date_created": "08-01-2025", + "file_size": 8339122 + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "dataset_name": "Tabula Sapiens", + "dataset_summary": "A multiple-organ, single-cell transcriptomic atlas of humans", + "dataset_description": "Tabula Sapiens is a benchmark, first-draft human cell atlas of nearly 500,000 cells from 24 organs of 15 normal human subjects. This work is the product of the Tabula Sapiens Consortium. Taking the organs from the same individual controls for genetic background, age, environment, and epigenetic effects and allows detailed analysis and comparison of cell types that are shared between tissues. Our work creates a detailed portrait of cell types as well as their distribution and variation in gene expression across tissues and within the endothelial, epithelial, stromal and immune compartments.", + "data_reference": "consortium2022tabula", + "data_url": "https://cellxgene.cziscience.com/collections/e5f58829-1a66-40b5-a624-9046778e74f5", + "date_created": "08-01-2025", + "file_size": 1727821930 + }, + { + "dataset_id": "cellxgene_census/hypomap", + "dataset_name": "HypoMap", + "dataset_summary": "A unified single cell gene expression atlas of the murine hypothalamus", + "dataset_description": "The hypothalamus plays a key role in coordinating fundamental body functions. Despite recent progress in single-cell technologies, a unified catalogue and molecular characterization of the heterogeneous cell types and, specifically, neuronal subtypes in this brain region are still lacking. Here we present an integrated reference atlas “HypoMap” of the murine hypothalamus consisting of 384,925 cells, with the ability to incorporate new additional experiments. We validate HypoMap by comparing data collected from SmartSeq2 and bulk RNA sequencing of selected neuronal cell types with different degrees of cellular heterogeneity.", + "data_reference": "steuernagel2022hypomap", + "data_url": "https://cellxgene.cziscience.com/collections/d86517f0-fa7e-4266-b82e-a521350d6d36", + "date_created": "08-01-2025", + "file_size": 23568346 + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "dataset_name": "Immune Cell Atlas", + "dataset_summary": "Cross-tissue immune cell analysis reveals tissue-specific features in humans", + "dataset_description": "Despite their crucial role in health and disease, our knowledge of immune cells within human tissues remains limited. We surveyed the immune compartment of 16 tissues from 12 adult donors by single-cell RNA sequencing and VDJ sequencing generating a dataset of ~360,000 cells. To systematically resolve immune cell heterogeneity across tissues, we developed CellTypist, a machine learning tool for rapid and precise cell type annotation. Using this approach, combined with detailed curation, we determined the tissue distribution of finely phenotyped immune cell types, revealing hitherto unappreciated tissue-specific features and clonal architecture of T and B cells. Our multitissue approach lays the foundation for identifying highly resolved immune cell types by leveraging a common reference dataset, tissue-integrated expression analysis, and antigen receptor sequencing.", + "data_reference": "dominguez2022crosstissue", + "data_url": "https://cellxgene.cziscience.com/collections/62ef75e4-cbea-454e-a0ce-998ec40223d3", + "date_created": "08-01-2025", + "file_size": 341174505 + }, + { + "dataset_id": "cellxgene_census/hcla", + "dataset_name": "Human Lung Cell Atlas", + "dataset_summary": "An integrated cell atlas of the human lung in health and disease (core)", + "dataset_description": "The integrated Human Lung Cell Atlas (HLCA) represents the first large-scale, integrated single-cell reference atlas of the human lung. It consists of over 2 million cells from the respiratory tract of 486 individuals, and includes 49 different datasets. It is split into the HLCA core, and the extended or full HLCA. The HLCA core includes data of healthy lung tissue from 107 individuals, and includes manual cell type annotations based on consensus across 6 independent experts, as well as demographic, biological and technical metadata.", + "data_reference": "sikkema2023integrated", + "data_url": "https://cellxgene.cziscience.com/collections/6f6d381a-7701-4781-935c-db10d30de293", + "date_created": "08-01-2025", + "file_size": 197407896 + }, + { + "dataset_id": "cellxgene_census/dkd", + "dataset_name": "Diabetic Kidney Disease", + "dataset_summary": "Multimodal single cell sequencing implicates chromatin accessibility and genetic background in diabetic kidney disease progression", + "dataset_description": "Multimodal single cell sequencing is a powerful tool for interrogating cell-specific changes in transcription and chromatin accessibility. We performed single nucleus RNA (snRNA-seq) and assay for transposase accessible chromatin sequencing (snATAC-seq) on human kidney cortex from donors with and without diabetic kidney disease (DKD) to identify altered signaling pathways and transcription factors associated with DKD. Both snRNA-seq and snATAC-seq had an increased proportion of VCAM1+ injured proximal tubule cells (PT_VCAM1) in DKD samples. PT_VCAM1 has a pro-inflammatory expression signature and transcription factor motif enrichment implicated NFkB signaling. We used stratified linkage disequilibrium score regression to partition heritability of kidney-function-related traits using publicly-available GWAS summary statistics. Cell-specific PT_VCAM1 peaks were enriched for heritability of chronic kidney disease (CKD), suggesting that genetic background may regulate chromatin accessibility and DKD progression. snATAC-seq found cell-specific differentially accessible regions (DAR) throughout the nephron that change accessibility in DKD and these regions were enriched for glucocorticoid receptor (GR) motifs. Changes in chromatin accessibility were associated with decreased expression of insulin receptor, increased gluconeogenesis, and decreased expression of the GR cytosolic chaperone, FKBP5, in the diabetic proximal tubule. Cleavage under targets and release using nuclease (CUT&RUN) profiling of GR binding in bulk kidney cortex and an in vitro model of the proximal tubule (RPTEC) showed that DAR co-localize with GR binding sites. CRISPRi silencing of GR response elements (GRE) in the FKBP5 gene body reduced FKBP5 expression in RPTEC, suggesting that reduced FKBP5 chromatin accessibility in DKD may alter cellular response to GR. We developed an open-source tool for single cell allele specific analysis (SALSA) to model the effect of genetic background on gene expression. Heterozygous germline single nucleotide variants (SNV) in proximal tubule ATAC peaks were associated with allele-specific chromatin accessibility and differential expression of target genes within cis-coaccessibility networks. Partitioned heritability of proximal tubule ATAC peaks with a predicted allele-specific effect was enriched for eGFR, suggesting that genetic background may modify DKD progression in a cell-specific manner.", + "data_reference": "wilson2022multimodal", + "data_url": "https://cellxgene.cziscience.com/collections/b3e2c6e3-9b05-4da9-8f42-da38a664b45b", + "date_created": "08-01-2025", + "file_size": 86763866 + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "dataset_name": "Mouse Pancreatic Islet Atlas", + "dataset_summary": "Mouse pancreatic islet scRNA-seq atlas across sexes, ages, and stress conditions including diabetes", + "dataset_description": "To better understand pancreatic β-cell heterogeneity we generated a mouse pancreatic islet atlas capturing a wide range of biological conditions. The atlas contains scRNA-seq datasets of over 300,000 mouse pancreatic islet cells, of which more than 100,000 are β-cells, from nine datasets with 56 samples, including two previously unpublished datasets. The samples vary in sex, age (ranging from embryonic to aged), chemical stress, and disease status (including T1D NOD model development and two T2D models, mSTZ and db/db) together with different diabetes treatments. Additional information about data fields is available in anndata uns field 'field_descriptions' and on https://github.com/theislab/mm_pancreas_atlas_rep/blob/main/resources/cellxgene.md.", + "data_reference": "hrovatin2023delineating", + "data_url": "https://cellxgene.cziscience.com/collections/296237e2-393d-4e31-b590-b03f74ac5070", + "date_created": "08-01-2025", + "file_size": 133936661 + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "dataset_name": "Human immune", + "dataset_summary": "Human immune cells dataset from the scIB benchmarks", + "dataset_description": "Human immune cells from peripheral blood and bone marrow taken from 5 datasets comprising 10 batches across technologies (10X, Smart-seq2).", + "data_reference": "luecken2022benchmarking", + "data_url": "https://theislab.github.io/scib-reproducibility/dataset_immune_cell_hum.html", + "date_created": "08-01-2025", + "file_size": 38683549 + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "dataset_name": "ABCA Mouse Brain scRNAseq", + "dataset_summary": "A high-resolution scRNAseq atlas of cell types in the whole mouse brain", + "dataset_description": "See dataset_reference for more information. Note that we only took the 10xv2 data from the dataset.", + "data_reference": "10.1038/s41586-023-06812-z", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE246717", + "date_created": "08-01-2025", + "file_size": 317012639 + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "dataset_name": "Zebrafish embryonic cells", + "dataset_summary": "Single-cell mRNA sequencing of zebrafish embryonic cells.", + "dataset_description": "90k cells from zebrafish embryos throughout the first day of development, with and without a knockout of chordin, an important developmental gene.", + "data_reference": "wagner2018single", + "data_url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE112294", + "date_created": "08-01-2025", + "file_size": 291760514 + }, + { + "dataset_id": "openproblems_v1/pancreas", + "dataset_name": "Human pancreas", + "dataset_summary": "Human pancreas cells dataset from the scIB benchmarks", + "dataset_description": "Human pancreatic islet scRNA-seq data from 6 datasets across technologies (CEL-seq, CEL-seq2, Smart-seq2, inDrop, Fluidigm C1, and SMARTER-seq).", + "data_reference": "luecken2022benchmarking", + "data_url": "https://theislab.github.io/scib-reproducibility/dataset_pancreas.html", + "date_created": "08-01-2025", + "file_size": 73130523 + } +] diff --git a/results/label_projection/data/method_info.json b/results/label_projection/data/method_info.json index e61c44a2..e3206f41 100644 --- a/results/label_projection/data/method_info.json +++ b/results/label_projection/data/method_info.json @@ -1,272 +1,290 @@ [ { - "method_name": "K-neighbors classifier (log CP10k)", - "method_summary": "K-neighbors classifier uses the \"k-nearest neighbours\" approach, which is a popular machine learning algorithm for classification and regression tasks. The assumption underlying KNN in this context is that cells with similar gene expression profiles tend to belong to the same cell type. For each unlabelled cell, this method computes the $k$ labelled cells (in this case, 5) with the smallest distance in PCA space, and assigns that cell the most common cell type among its $k$ nearest neighbors.", - "paper_name": "Nearest neighbor pattern classification", - "paper_reference": "cover1967nearest", - "paper_year": 1967, - "code_url": "https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "knn_classifier_log_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/knn_classifier.py" + "task_id": "control_methods", + "method_id": "majority_vote", + "method_name": "Majority Vote", + "method_summary": "A control-type method that predicts all cells to belong to the most abundant cell type in the dataset", + "method_description": "A control-type method that predicts all cells to belong to the most abundant cell type in the dataset", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_label_projection", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_label_projection/control_methods/majority_vote:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/control_methods/majority_vote", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "K-neighbors classifier (log scran)", - "method_summary": "K-neighbors classifier uses the \"k-nearest neighbours\" approach, which is a popular machine learning algorithm for classification and regression tasks. The assumption underlying KNN in this context is that cells with similar gene expression profiles tend to belong to the same cell type. For each unlabelled cell, this method computes the $k$ labelled cells (in this case, 5) with the smallest distance in PCA space, and assigns that cell the most common cell type among its $k$ nearest neighbors.", - "paper_name": "Nearest neighbor pattern classification", - "paper_reference": "cover1967nearest", - "paper_year": 1967, - "code_url": "https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-r-base", - "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "knn_classifier_scran", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/knn_classifier.py" + "task_id": "control_methods", + "method_id": "random_labels", + "method_name": "Random Labels", + "method_summary": "a negative control, where the labels are randomly predicted.", + "method_description": "A negative control, where the labels are randomly predicted without training the data.", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_label_projection", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_label_projection/control_methods/random_labels:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/control_methods/random_labels", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Logistic regression (log CP10k)", - "method_summary": "Logistic Regression estimates parameters of a logistic function for multivariate classification tasks. Here, we use 100-dimensional whitened PCA coordinates as independent variables, and the model minimises the cross entropy loss over all cell type classes. ", - "paper_name": "Applied Logistic Regression", - "paper_reference": "hosmer2013applied", - "paper_year": 2013, - "code_url": "https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "logistic_regression_log_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/logistic_regression.py" + "task_id": "control_methods", + "method_id": "true_labels", + "method_name": "True labels", + "method_summary": "a positive control, solution labels are copied 1 to 1 to the predicted data.", + "method_description": "A positive control, where the solution labels are copied 1 to 1 to the predicted data.", + "is_baseline": true, + "references_doi": null, + "references_bibtex": null, + "code_url": "https://github.com/openproblems-bio/task_label_projection", + "documentation_url": null, + "image": "https://ghcr.io/openproblems-bio/task_label_projection/control_methods/true_labels:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/control_methods/true_labels", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Logistic regression (log scran)", - "method_summary": "Logistic Regression estimates parameters of a logistic function for multivariate classification tasks. Here, we use 100-dimensional whitened PCA coordinates as independent variables, and the model minimises the cross entropy loss over all cell type classes. ", - "paper_name": "Applied Logistic Regression", - "paper_reference": "hosmer2013applied", - "paper_year": 2013, - "code_url": "https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-r-base", + "task_id": "methods", + "method_id": "geneformer", + "method_name": "Geneformer", + "method_summary": "Geneformer is a foundational transformer model pretrained on a large-scale corpus of single cell transcriptomes to enable context-aware predictions in settings with limited data in network biology.", + "method_description": "Geneformer is a context-aware, attention-based deep learning model pretrained on a large-scale corpus of single-cell transcriptomes to enable context-specific predictions in settings with limited data in network biology. Here, a pre-trained Geneformer model is fine-tuned and used to predict cell type labels for an unlabelled dataset.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "logistic_regression_scran", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/logistic_regression.py" + "references_doi": ["10.1038/s41586-023-06139-9", "10.1101/2024.08.16.608180"], + "references_bibtex": null, + "code_url": "https://huggingface.co/ctheodoris/Geneformer", + "documentation_url": "https://geneformer.readthedocs.io/en/latest/index.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/geneformer:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/geneformer", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Majority Vote", - "method_summary": "Assignment of all predicted labels as the most common label in the training data", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", + "task_id": "methods", + "method_id": "knn", + "method_name": "KNN", + "method_summary": "Assumes cells with similar gene expression belong to the same cell type, and assigns an unlabelled cell the most common cell type among its k nearest neighbors in PCA space.", + "method_description": "Using the \"k-nearest neighbours\" approach, which is a\npopular machine learning algorithm for classification and regression tasks.\nThe assumption underlying KNN in this context is that cells with similar gene\nexpression profiles tend to belong to the same cell type. For each unlabelled\ncell, this method computes the $k$ labelled cells (in this case, 5) with the\nsmallest distance in PCA space, and assigns that cell the most common cell\ntype among its $k$ nearest neighbors.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "majority_vote", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/baseline.py" + "references_doi": "10.1109/tit.1967.1053964", + "references_bibtex": null, + "code_url": "https://github.com/scikit-learn/scikit-learn", + "documentation_url": "https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/knn:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/knn", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Multilayer perceptron (log CP10k)", - "method_summary": "MLP or \"Multi-Layer Perceptron\" is a type of artificial neural network that consists of multiple layers of interconnected neurons. Each neuron computes a weighted sum of all neurons in the previous layer and transforms it with nonlinear activation function. The output layer provides the final prediction, and network weights are updated by gradient descent to minimize the cross entropy loss. Here, the input data is 100-dimensional whitened PCA coordinates for each cell, and we use two hidden layers of 100 neurons each.", - "paper_name": "Connectionist learning procedures", - "paper_reference": "hinton1989connectionist", - "paper_year": 1990, - "code_url": "https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", + "task_id": "methods", + "method_id": "logistic_regression", + "method_name": "Logistic Regression", + "method_summary": "Logistic Regression with 100-dimensional PCA coordinates estimates parameters for multivariate classification by minimizing cross entropy loss over cell type classes.", + "method_description": "Logistic Regression estimates parameters of a logistic function for\nmultivariate classification tasks. Here, we use 100-dimensional whitened PCA\ncoordinates as independent variables, and the model minimises the cross\nentropy loss over all cell type classes.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "mlp_log_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/mlp.py" + "references_doi": null, + "references_bibtex": "@book{hosmer2013applied,\n title = {Applied logistic regression},\n author = {Hosmer Jr, D.W. and Lemeshow, S. and Sturdivant, R.X.},\n year = {2013},\n publisher = {John Wiley \\& Sons},\n volume = {398}\n}\n", + "code_url": "https://github.com/scikit-learn/scikit-learn", + "documentation_url": "https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/logistic_regression:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/logistic_regression", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Multilayer perceptron (log scran)", - "method_summary": "MLP or \"Multi-Layer Perceptron\" is a type of artificial neural network that consists of multiple layers of interconnected neurons. Each neuron computes a weighted sum of all neurons in the previous layer and transforms it with nonlinear activation function. The output layer provides the final prediction, and network weights are updated by gradient descent to minimize the cross entropy loss. Here, the input data is 100-dimensional whitened PCA coordinates for each cell, and we use two hidden layers of 100 neurons each.", - "paper_name": "Connectionist learning procedures", - "paper_reference": "hinton1989connectionist", - "paper_year": 1990, - "code_url": "https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-r-base", + "task_id": "methods", + "method_id": "mlp", + "method_name": "Multilayer perceptron", + "method_summary": "A neural network with 100-dimensional PCA input, two hidden layers, and gradient descent weight updates to minimize cross entropy loss.", + "method_description": "Multi-Layer Perceptron is a type of artificial neural network that\nconsists of multiple layers of interconnected neurons. Each neuron computes a\nweighted sum of all neurons in the previous layer and transforms it with\nnonlinear activation function. The output layer provides the final\nprediction, and network weights are updated by gradient descent to minimize\nthe cross entropy loss. Here, the input data is 100-dimensional whitened PCA\ncoordinates for each cell, and we use two hidden layers of 100 neurons each.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "mlp_scran", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/mlp.py" + "references_doi": "10.1016/0004-3702(89)90049-0", + "references_bibtex": null, + "code_url": "https://github.com/scikit-learn/scikit-learn", + "documentation_url": "https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/mlp:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/mlp", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Random Labels", - "method_summary": "Random assignment of predicted labels proportionate to label abundance in training data", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "is_baseline": true, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "random_labels", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/baseline.py" + "task_id": "methods", + "method_id": "naive_bayes", + "method_name": "Naive Bayesian Classifier", + "method_summary": "Naive Bayes classification using feature probabilities to project cell type labels from a reference dataset.", + "method_description": "Naive Bayes classification leverages probabilistic models based on Bayes' theorem\nto classify cells into different types. In the context of single-cell datasets, this method\nutilizes the probabilities of features to project cell type labels from a reference dataset\nto new datasets. The algorithm assumes independence between features, making it computationally\nefficient and well-suited for high-dimensional data. It is particularly useful for annotating\ncells in atlas-scale datasets, ensuring consistency and alignment with existing reference annotations.\n", + "is_baseline": false, + "references_doi": null, + "references_bibtex": "@book{hosmer2013applied,\n title={Applied logistic regression},\n author={Hosmer, David W and Lemeshow, Stanley and Sturdivant, Rodney X},\n year={2013},\n publisher={John Wiley \\& Sons}\n}\n", + "code_url": "https://github.com/scikit-learn/scikit-learn", + "documentation_url": "https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/naive_bayes:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/naive_bayes", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "scANVI (All genes)", - "method_summary": "scANVI or \"single-cell ANnotation using Variational Inference\" is a semi-supervised variant of the scVI(Lopez et al. 2018) algorithm. Like scVI, scANVI uses deep neural networks and stochastic optimization to model uncertainty caused by technical noise and bias in single - cell transcriptomics measurements. However, scANVI also leverages cell type labels in the generative modelling. In this approach, scANVI is used to predict the cell type labels of the unlabelled test data.", - "paper_name": "Probabilistic harmonization and annotation of single-cell transcriptomics data with deep generative models", - "paper_reference": "xu2021probabilistic", - "paper_year": 2021, - "code_url": "https://github.com/YosefLab/scvi-tools/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-pytorch", + "task_id": "methods", + "method_id": "scanvi", + "method_name": "scANVI", + "method_summary": "scANVI predicts cell type labels for unlabelled test data by leveraging cell type labels, modelling uncertainty and using deep neural networks with stochastic optimization.", + "method_description": "single-cell ANnotation using Variational Inference is a\nsemi-supervised variant of the scVI(Lopez et al. 2018) algorithm. Like scVI,\nscANVI uses deep neural networks and stochastic optimization to model\nuncertainty caused by technical noise and bias in single - cell\ntranscriptomics measurements. However, scANVI also leverages cell type labels\nin the generative modelling. In this approach, scANVI is used to predict the\ncell type labels of the unlabelled test data.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "scanvi_all_genes", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/scvi_tools.py" + "references_doi": "10.1101/2020.07.16.205997", + "references_bibtex": null, + "code_url": "https://github.com/scverse/scvi-tools", + "documentation_url": "https://scarches.readthedocs.io/en/latest/scanvi_surgery_pipeline.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/scanvi:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/scanvi", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "scANVI (Seurat v3 2000 HVG)", - "method_summary": "scANVI or \"single-cell ANnotation using Variational Inference\" is a semi-supervised variant of the scVI(Lopez et al. 2018) algorithm. Like scVI, scANVI uses deep neural networks and stochastic optimization to model uncertainty caused by technical noise and bias in single - cell transcriptomics measurements. However, scANVI also leverages cell type labels in the generative modelling. In this approach, scANVI is used to predict the cell type labels of the unlabelled test data.", - "paper_name": "Probabilistic harmonization and annotation of single-cell transcriptomics data with deep generative models", - "paper_reference": "xu2021probabilistic", - "paper_year": 2021, - "code_url": "https://github.com/YosefLab/scvi-tools/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-pytorch", + "task_id": "methods", + "method_id": "scanvi_scarches", + "method_name": "scANVI+scArches", + "method_summary": "Query to reference single-cell integration with transfer learning with scANVI and scArches", + "method_description": "scArches+scANVI or \"Single-cell architecture surgery\" is a deep learning method for mapping new datasets onto a pre-existing reference model, using transfer learning and parameter optimization. It first uses scANVI to build a reference model from the training data, and then apply scArches to map the test data onto the reference model and make predictions.", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "scanvi_hvg", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/scvi_tools.py" + "references_doi": "10.1101/2020.07.16.205997", + "references_bibtex": null, + "code_url": "https://github.com/scverse/scvi-tools", + "documentation_url": "https://docs.scvi-tools.org", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/scanvi_scarches:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/scanvi_scarches", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "scArches+scANVI (All genes)", - "method_summary": "scArches+scANVI or \"Single-cell architecture surgery\" is a deep learning method for mapping new datasets onto a pre-existing reference model, using transfer learning and parameter optimization. It first uses scANVI to build a reference model from the training data, and then apply scArches to map the test data onto the reference model and make predictions.", - "paper_name": "Query to reference single-cell integration with transfer learning", - "paper_reference": "lotfollahi2020query", - "paper_year": 2021, - "code_url": "https://github.com/YosefLab/scvi-tools/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-pytorch", + "task_id": "methods", + "method_id": "scgpt_zero_shot", + "method_name": "scGPT zero-shot", + "method_summary": "Reference mapping using cell embedding by pretrained scGPT model.", + "method_description": "scGPT is a foundation model for single-cell biology based on a generative pre-trained transformer and trained on a repository of over 33 million cells. Following the zero-shot approach, a pre-trained scGPT model is used to embed cells and map unlabelled cells in a query set to the reference dataset with provided annotations based on a nearest neighbor similarity search. \n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "scarches_scanvi_all_genes", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/scvi_tools.py" + "references_doi": "10.1038/s41592-024-02201-0", + "references_bibtex": null, + "code_url": "https://github.com/bowang-lab/scGPT", + "documentation_url": "https://scgpt.readthedocs.io/en/latest/", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/scgpt_zero_shot:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/scgpt_zero_shot", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "scArches+scANVI (Seurat v3 2000 HVG)", - "method_summary": "scArches+scANVI or \"Single-cell architecture surgery\" is a deep learning method for mapping new datasets onto a pre-existing reference model, using transfer learning and parameter optimization. It first uses scANVI to build a reference model from the training data, and then apply scArches to map the test data onto the reference model and make predictions.", - "paper_name": "Query to reference single-cell integration with transfer learning", - "paper_reference": "lotfollahi2020query", - "paper_year": 2021, - "code_url": "https://github.com/YosefLab/scvi-tools/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-pytorch", + "task_id": "methods", + "method_id": "scimilarity", + "method_name": "SCimilarity", + "method_summary": "SCimilarity provides unifying representation of single cell expression profiles", + "method_description": "SCimilarity is a unifying representation of single cell expression profiles\nthat quantifies similarity between expression states and generalizes to\nrepresent new studies without additional training.\n\nThis method uses the SCimilarity cell annotation module. As labels in the\nSCimilarity model are likely to be different to those in the dataset we use\na combination of exact string matching and iterative linear sum assignment to\nmatch generate a mapping between them using the training data.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "scarches_scanvi_hvg", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/scvi_tools.py" + "references_doi": "10.1101/2023.07.18.549537", + "references_bibtex": null, + "code_url": "https://github.com/Genentech/scimilarity", + "documentation_url": "https://genentech.github.io/scimilarity/index.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/scimilarity:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/scimilarity", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "scArches+scANVI+xgboost (All genes)", - "method_summary": "scArches+scANVI or \"Single-cell architecture surgery\" is a deep learning method for mapping new datasets onto a pre-existing reference model, using transfer learning and parameter optimization. It first uses scANVI to build a reference model from the training data, and then apply scArches to map the test data onto the reference model and make predictions.", - "paper_name": "Query to reference single-cell integration with transfer learning", - "paper_reference": "lotfollahi2020query", - "paper_year": 2021, - "code_url": "https://github.com/YosefLab/scvi-tools/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-pytorch", + "task_id": "methods", + "method_id": "scimilarity_knn", + "method_name": "SCimilarity (kNN)", + "method_summary": "SCimilarity provides unifying representation of single cell expression profiles", + "method_description": "SCimilarity is a unifying representation of single cell expression profiles\nthat quantifies similarity between expression states and generalizes to\nrepresent new studies without additional training.\n\nThis method trains a kNN classifier using cell embeddings from SCimilarity.\nThe classifier is trained on embeddings for the training data and used to\npredict labels for the test data. This does not use the SCimilarity cell\nannotation model but avoids needing to match SCimilarity labels to dataset\nlabels.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "scarches_scanvi_xgb_all_genes", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/scvi_tools.py" + "references_doi": "10.1101/2023.07.18.549537", + "references_bibtex": null, + "code_url": "https://github.com/Genentech/scimilarity", + "documentation_url": "https://genentech.github.io/scimilarity/index.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/scimilarity_knn:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/scimilarity_knn", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "scArches+scANVI+xgboost (Seurat v3 2000 HVG)", - "method_summary": "scArches+scANVI or \"Single-cell architecture surgery\" is a deep learning method for mapping new datasets onto a pre-existing reference model, using transfer learning and parameter optimization. It first uses scANVI to build a reference model from the training data, and then apply scArches to map the test data onto the reference model and make predictions.", - "paper_name": "Query to reference single-cell integration with transfer learning", - "paper_reference": "lotfollahi2020query", - "paper_year": 2021, - "code_url": "https://github.com/YosefLab/scvi-tools/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-pytorch", + "task_id": "methods", + "method_id": "scprint", + "method_name": "scPRINT", + "method_summary": "scPRINT is a large transformer model built for the inference of gene networks", + "method_description": "scPRINT is a large transformer model built for the inference of gene networks\n(connections between genes explaining the cell's expression profile) from\nscRNAseq data.\n\nIt uses novel encoding and decoding of the cell expression profile and new\npre-training methodologies to learn a cell model.\n\nscPRINT can be used to perform the following analyses:\n - expression denoising: increase the resolution of your scRNAseq data\n - cell embedding: generate a low-dimensional representation of your dataset\n - label prediction: predict the cell type, disease, sequencer, sex, and\n ethnicity of your cells\n - gene network inference: generate a gene network from any cell or cell\n cluster in your scRNAseq dataset\n\nThis method uses the zero-shot ability of scPRINT for cell type prediction. \nAs some predicted labels are likely to be different to those in the reference \ndataset, we use a combination of exact string matching and iterative linear sum \nassignment to generate a mapping between them using the training data.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "scarches_scanvi_xgb_hvg", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/scvi_tools.py" + "references_doi": "10.1101/2024.07.29.605556", + "references_bibtex": null, + "code_url": "https://github.com/cantinilab/scPRINT", + "documentation_url": "https://cantinilab.github.io/scPRINT/", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/scprint:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/scprint", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "Seurat reference mapping (SCTransform)", - "method_summary": "Seurat reference mapping is a cell type label transfer method provided by the Seurat package. Gene expression counts are first normalised by SCTransform before computing PCA. Then it finds mutual nearest neighbours, known as transfer anchors, between the labelled and unlabelled part of the data in PCA space, and computes each cell\u2019s distance to each of the anchor pairs. Finally, it uses the labelled anchors to predict cell types for unlabelled cells based on these distances.", - "paper_name": "Integrated analysis of multimodal single-cell data", - "paper_reference": "hao2021integrated", - "paper_year": 2021, - "code_url": "https://github.com/satijalab/seurat/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-r-extras", + "task_id": "methods", + "method_id": "seurat_transferdata", + "method_name": "Seurat TransferData", + "method_summary": "Seurat reference mapping predicts cell types for unlabelled cells using PCA distances, labelled anchors, and transfer anchors from Seurat, with SCTransform normalization.", + "method_description": "Seurat reference mapping is a cell type label transfer method provided by the\nSeurat package. Gene expression counts are first normalised by SCTransform\nbefore computing PCA. Then it finds mutual nearest neighbours, known as\ntransfer anchors, between the labelled and unlabelled part of the data in PCA\nspace, and computes each cell's distance to each of the anchor pairs.\nFinally, it uses the labelled anchors to predict cell types for unlabelled\ncells based on these distances.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "seurat", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/seurat.py" + "references_doi": "10.1016/j.cell.2021.04.048", + "references_bibtex": null, + "code_url": "https://github.com/satijalab/seurat", + "documentation_url": "https://satijalab.org/seurat/articles/integration_mapping.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/seurat_transferdata:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/seurat_transferdata", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "True Labels", - "method_summary": "Perfect assignment of the predicted labels from the test labels", - "paper_name": "Open Problems for Single Cell Analysis", - "paper_reference": "openproblems", - "paper_year": 2022, - "code_url": "https://github.com/openproblems-bio/openproblems/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "is_baseline": true, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "method_id": "true_labels", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/baseline.py" + "task_id": "methods", + "method_id": "singler", + "method_name": "SingleR", + "method_summary": "Reference-Based Single-Cell RNA-Seq Annotation", + "method_description": "Performs unbiased cell type recognition from single-cell RNA sequencing data,\nby leveraging reference transcriptomic datasets of pure cell types to infer the\ncell of origin of each single cell independently.\n", + "is_baseline": false, + "references_doi": "10.1038/s41590-018-0276-y", + "references_bibtex": null, + "code_url": "https://www.bioconductor.org/packages/release/bioc/html/SingleR.html", + "documentation_url": "https://www.bioconductor.org/packages/release/bioc/vignettes/SingleR/inst/doc/SingleR.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/singler:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/singler", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "XGBoost (log CP10k)", - "method_summary": "XGBoost is a gradient boosting decision tree model that learns multiple tree structures in the form of a series of input features and their values, leading to a prediction decision, and averages predictions from all its trees. Here, input features are normalised gene expression values.", - "paper_name": "XGBoost: A Scalable Tree Boosting System", - "paper_reference": "chen2016xgboost", - "paper_year": 2016, - "code_url": "https://xgboost.readthedocs.io/en/stable/index.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-python-extras", + "task_id": "methods", + "method_id": "uce", + "method_name": "UCE", + "method_summary": "UCE offers a unified biological latent space that can represent any cell", + "method_description": "Universal Cell Embedding (UCE) is a single-cell foundation model that offers a\nunified biological latent space that can represent any cell, regardless of\ntissue or species.\n\nThis method trains a logistic regression classifier on the UCE embedding\nspace.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "xgboost_log_cp10k", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/xgboost.py" + "references_doi": "10.1101/2023.11.28.568918", + "references_bibtex": null, + "code_url": "https://github.com/snap-stanford/UCE", + "documentation_url": "https://github.com/snap-stanford/UCE/blob/main/README.md", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/uce:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/uce", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" }, { - "method_name": "XGBoost (log scran)", - "method_summary": "XGBoost is a gradient boosting decision tree model that learns multiple tree structures in the form of a series of input features and their values, leading to a prediction decision, and averages predictions from all its trees. Here, input features are normalised gene expression values.", - "paper_name": "XGBoost: A Scalable Tree Boosting System", - "paper_reference": "chen2016xgboost", - "paper_year": 2016, - "code_url": "https://xgboost.readthedocs.io/en/stable/index.html/tree/v1.0.0/openproblems/tasks", - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems-r-extras", + "task_id": "methods", + "method_id": "xgboost", + "method_name": "XGBoost", + "method_summary": "XGBoost is a decision tree model that averages multiple trees with gradient boosting.", + "method_description": "XGBoost is a gradient boosting decision tree model that learns multiple tree\nstructures in the form of a series of input features and their values,\nleading to a prediction decision, and averages predictions from all its\ntrees. Here, input features are normalised gene expression values.\n", "is_baseline": false, - "code_version": "v1.0.0", - "task_id": "label_projection", - "commit_sha": "cef4e5cac0b51d454d45e22e354988e77540c40d", - "method_id": "xgboost_scran", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/methods/xgboost.py" + "references_doi": "10.1145/2939672.2939785", + "references_bibtex": null, + "code_url": "https://github.com/dmlc/xgboost", + "documentation_url": "https://xgboost.readthedocs.io/en/stable/index.html", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/methods/xgboost:build_main", + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/methods/xgboost", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d" } -] \ No newline at end of file +] diff --git a/results/label_projection/data/metric_execution_info.json b/results/label_projection/data/metric_execution_info.json new file mode 100644 index 00000000..d4cc5bea --- /dev/null +++ b/results/label_projection/data/metric_execution_info.json @@ -0,0 +1,4146 @@ +[ + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:22:05", + "exit_code": 0, + "duration_sec": 12.7, + "cpu_pct": 166.9, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:22:05", + "exit_code": 0, + "duration_sec": 61.2, + "cpu_pct": 110, + "peak_memory_mb": 6964, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:15:15", + "exit_code": 0, + "duration_sec": 14, + "cpu_pct": 155.7, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:15:15", + "exit_code": 0, + "duration_sec": 42, + "cpu_pct": 150.3, + "peak_memory_mb": 6964, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:31:55", + "exit_code": 0, + "duration_sec": 12.5, + "cpu_pct": 163.4, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:31:55", + "exit_code": 0, + "duration_sec": 36.9, + "cpu_pct": 165.8, + "peak_memory_mb": 6964, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:33:45", + "exit_code": 0, + "duration_sec": 19.2, + "cpu_pct": 123.5, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:33:45", + "exit_code": 0, + "duration_sec": 48.3, + "cpu_pct": 138.6, + "peak_memory_mb": 7066, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:27:35", + "exit_code": 0, + "duration_sec": 19.9, + "cpu_pct": 115.6, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:27:35", + "exit_code": 0, + "duration_sec": 59.4, + "cpu_pct": 112.5, + "peak_memory_mb": 6964, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:28:05", + "exit_code": 0, + "duration_sec": 19, + "cpu_pct": 123.8, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:28:05", + "exit_code": 0, + "duration_sec": 37.5, + "cpu_pct": 165.4, + "peak_memory_mb": 6964, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:35:45", + "exit_code": 0, + "duration_sec": 8.4, + "cpu_pct": 165.5, + "peak_memory_mb": 6554, + "disk_read_mb": 339, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:35:45", + "exit_code": 0, + "duration_sec": 18.9, + "cpu_pct": 256.1, + "peak_memory_mb": 6452, + "disk_read_mb": 1017, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:48:05", + "exit_code": 0, + "duration_sec": 12.9, + "cpu_pct": 169.2, + "peak_memory_mb": 6964, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:48:05", + "exit_code": 0, + "duration_sec": 56.1, + "cpu_pct": 127.8, + "peak_memory_mb": 4301, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:16:35", + "exit_code": 0, + "duration_sec": 8.6, + "cpu_pct": 206.4, + "peak_memory_mb": 6554, + "disk_read_mb": 337, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:16:35", + "exit_code": 0, + "duration_sec": 19.5, + "cpu_pct": 243.7, + "peak_memory_mb": 6452, + "disk_read_mb": 1014, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:30:06", + "exit_code": 0, + "duration_sec": 22.3, + "cpu_pct": 99.6, + "peak_memory_mb": 7066, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:30:05", + "exit_code": 0, + "duration_sec": 40.5, + "cpu_pct": 164.3, + "peak_memory_mb": 6964, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:37:55", + "exit_code": 0, + "duration_sec": 21.5, + "cpu_pct": 112.8, + "peak_memory_mb": 4404, + "disk_read_mb": 507, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:37:55", + "exit_code": 0, + "duration_sec": 66.3, + "cpu_pct": 110.7, + "peak_memory_mb": 4404, + "disk_read_mb": 1524, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:07:05", + "exit_code": 0, + "duration_sec": 9.4, + "cpu_pct": 133.1, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:07:05", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 419.2, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:04:45", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 459.4, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:04:45", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 484.1, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:06:55", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 433.3, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:06:55", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 465.6, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:07:15", + "exit_code": 0, + "duration_sec": 3.9, + "cpu_pct": 328.9, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:07:15", + "exit_code": 0, + "duration_sec": 6.9, + "cpu_pct": 443.8, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:04:35", + "exit_code": 0, + "duration_sec": 59.7, + "cpu_pct": 19.6, + "peak_memory_mb": 5632, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:04:36", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 452.8, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:06:55", + "exit_code": 0, + "duration_sec": 6.9, + "cpu_pct": 132.9, + "peak_memory_mb": 2868, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:06:55", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 353.4, + "peak_memory_mb": 5530, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:16:45", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 482.2, + "peak_memory_mb": 5632, + "disk_read_mb": 112, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:16:45", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 426.2, + "peak_memory_mb": 5632, + "disk_read_mb": 339, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:33:25", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 293.9, + "peak_memory_mb": 2868, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:33:25", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 464.7, + "peak_memory_mb": 5530, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scimilarity", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:31:35", + "exit_code": 0, + "duration_sec": 7.7, + "cpu_pct": 170.5, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scimilarity", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:31:35", + "exit_code": 0, + "duration_sec": 32.7, + "cpu_pct": 121.6, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scimilarity_knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:33:35", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 394.3, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scimilarity_knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:33:35", + "exit_code": 0, + "duration_sec": 27.9, + "cpu_pct": 134.3, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:07:35", + "exit_code": 0, + "duration_sec": 9.4, + "cpu_pct": 141, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:07:35", + "exit_code": 0, + "duration_sec": 28.5, + "cpu_pct": 147.3, + "peak_memory_mb": 5530, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:12:05", + "exit_code": 0, + "duration_sec": 4.4, + "cpu_pct": 196.1, + "peak_memory_mb": 2970, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:12:05", + "exit_code": 0, + "duration_sec": 9, + "cpu_pct": 307.2, + "peak_memory_mb": 2868, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:07:15", + "exit_code": 0, + "duration_sec": 3.9, + "cpu_pct": 302, + "peak_memory_mb": 5530, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:07:15", + "exit_code": 0, + "duration_sec": 13.8, + "cpu_pct": 279.8, + "peak_memory_mb": 5632, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "uce", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:34:15", + "exit_code": 0, + "duration_sec": 8, + "cpu_pct": 128.6, + "peak_memory_mb": 2970, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "uce", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:34:15", + "exit_code": 0, + "duration_sec": 9.6, + "cpu_pct": 272.5, + "peak_memory_mb": 2970, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:08:05", + "exit_code": 0, + "duration_sec": 4.5, + "cpu_pct": 263.5, + "peak_memory_mb": 5632, + "disk_read_mb": 111, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:08:05", + "exit_code": 0, + "duration_sec": 11.7, + "cpu_pct": 317.3, + "peak_memory_mb": 5530, + "disk_read_mb": 336, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:19:55", + "exit_code": 0, + "duration_sec": 11.2, + "cpu_pct": 111.6, + "peak_memory_mb": 5530, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:19:55", + "exit_code": 0, + "duration_sec": 34.5, + "cpu_pct": 112.4, + "peak_memory_mb": 5530, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:10:15", + "exit_code": 0, + "duration_sec": 3.4, + "cpu_pct": 251.6, + "peak_memory_mb": 5632, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:10:15", + "exit_code": 0, + "duration_sec": 10.8, + "cpu_pct": 160.7, + "peak_memory_mb": 5632, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:20:15", + "exit_code": 0, + "duration_sec": 4.7, + "cpu_pct": 271.2, + "peak_memory_mb": 5530, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:20:15", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 410.4, + "peak_memory_mb": 5530, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:25:25", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 405.2, + "peak_memory_mb": 5530, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:25:25", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 233.5, + "peak_memory_mb": 2868, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:20:25", + "exit_code": 0, + "duration_sec": 72, + "cpu_pct": 18.1, + "peak_memory_mb": 5837, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:20:25", + "exit_code": 0, + "duration_sec": 14.7, + "cpu_pct": 261.4, + "peak_memory_mb": 5837, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:08:35", + "exit_code": 0, + "duration_sec": 13.1, + "cpu_pct": 99.5, + "peak_memory_mb": 5530, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:08:35", + "exit_code": 0, + "duration_sec": 13.8, + "cpu_pct": 275.3, + "peak_memory_mb": 5632, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:27:25", + "exit_code": 0, + "duration_sec": 3.1, + "cpu_pct": 221.6, + "peak_memory_mb": 2868, + "disk_read_mb": 226, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:27:25", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 361.1, + "peak_memory_mb": 5632, + "disk_read_mb": 681, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:44:45", + "exit_code": 0, + "duration_sec": 12.1, + "cpu_pct": 108.7, + "peak_memory_mb": 5632, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:44:45", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 386.9, + "peak_memory_mb": 5632, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scimilarity", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:36:45", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 433.9, + "peak_memory_mb": 5632, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scimilarity", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:36:45", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 286.4, + "peak_memory_mb": 2868, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scimilarity_knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:24:15", + "exit_code": 0, + "duration_sec": 11.3, + "cpu_pct": 106, + "peak_memory_mb": 5632, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scimilarity_knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:24:15", + "exit_code": 0, + "duration_sec": 33.9, + "cpu_pct": 107.2, + "peak_memory_mb": 5632, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:14:35", + "exit_code": 0, + "duration_sec": 11.9, + "cpu_pct": 117, + "peak_memory_mb": 5530, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:14:35", + "exit_code": 0, + "duration_sec": 12.6, + "cpu_pct": 257.9, + "peak_memory_mb": 5837, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:10:35", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 139.6, + "peak_memory_mb": 3072, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:10:35", + "exit_code": 0, + "duration_sec": 30.9, + "cpu_pct": 90.4, + "peak_memory_mb": 2868, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:09:15", + "exit_code": 0, + "duration_sec": 12.8, + "cpu_pct": 100.1, + "peak_memory_mb": 5530, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:09:15", + "exit_code": 0, + "duration_sec": 114.3, + "cpu_pct": 35.4, + "peak_memory_mb": 5632, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:21:25", + "exit_code": 0, + "duration_sec": 9.7, + "cpu_pct": 125.9, + "peak_memory_mb": 5632, + "disk_read_mb": 225, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:21:25", + "exit_code": 0, + "duration_sec": 225, + "cpu_pct": 18.8, + "peak_memory_mb": 5632, + "disk_read_mb": 678, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:33:15", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 428.1, + "peak_memory_mb": 5530, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:33:15", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 290.6, + "peak_memory_mb": 2868, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:30:05", + "exit_code": 0, + "duration_sec": 4.6, + "cpu_pct": 266.4, + "peak_memory_mb": 5632, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:30:05", + "exit_code": 0, + "duration_sec": 25.8, + "cpu_pct": 139, + "peak_memory_mb": 5632, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:34:55", + "exit_code": 0, + "duration_sec": 4.8, + "cpu_pct": 269.4, + "peak_memory_mb": 5632, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:34:55", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 420.5, + "peak_memory_mb": 5530, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:51:15", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 481.2, + "peak_memory_mb": 5530, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:51:15", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 369.6, + "peak_memory_mb": 5632, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:38:15", + "exit_code": 0, + "duration_sec": 9.2, + "cpu_pct": 111.1, + "peak_memory_mb": 5632, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:38:16", + "exit_code": 0, + "duration_sec": 46.5, + "cpu_pct": 70.3, + "peak_memory_mb": 5530, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:41:45", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 395.6, + "peak_memory_mb": 5530, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:41:45", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 443.2, + "peak_memory_mb": 5632, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:18:15", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 402, + "peak_memory_mb": 5530, + "disk_read_mb": 218, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:18:15", + "exit_code": 0, + "duration_sec": 24.9, + "cpu_pct": 130.6, + "peak_memory_mb": 3072, + "disk_read_mb": 657, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 10:03:55", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 482.7, + "peak_memory_mb": 5632, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 10:03:55", + "exit_code": 0, + "duration_sec": 10.2, + "cpu_pct": 252.9, + "peak_memory_mb": 3072, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 11:35:15", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 447.1, + "peak_memory_mb": 5530, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 11:35:15", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 423.2, + "peak_memory_mb": 5530, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:28:45", + "exit_code": 0, + "duration_sec": 3.7, + "cpu_pct": 273.4, + "peak_memory_mb": 5530, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:28:45", + "exit_code": 0, + "duration_sec": 11.4, + "cpu_pct": 258.1, + "peak_memory_mb": 5632, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:26:15", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 440.9, + "peak_memory_mb": 5530, + "disk_read_mb": 217, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:26:15", + "exit_code": 0, + "duration_sec": 8.7, + "cpu_pct": 394.9, + "peak_memory_mb": 5530, + "disk_read_mb": 654, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:08:55", + "exit_code": 0, + "duration_sec": 12.8, + "cpu_pct": 105.5, + "peak_memory_mb": 5530, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:08:55", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 234.5, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:11:55", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 302.8, + "peak_memory_mb": 2868, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:11:55", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 444.6, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:11:45", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 512.9, + "peak_memory_mb": 5632, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:11:45", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 459.4, + "peak_memory_mb": 5632, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:14:25", + "exit_code": 0, + "duration_sec": 11.7, + "cpu_pct": 110.2, + "peak_memory_mb": 5632, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:14:26", + "exit_code": 0, + "duration_sec": 36, + "cpu_pct": 112.9, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:10:55", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 399.4, + "peak_memory_mb": 5632, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:10:55", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 387.5, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:10:25", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 309.8, + "peak_memory_mb": 5530, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:10:25", + "exit_code": 0, + "duration_sec": 9, + "cpu_pct": 251.5, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:16:25", + "exit_code": 0, + "duration_sec": 5.2, + "cpu_pct": 190.4, + "peak_memory_mb": 5632, + "disk_read_mb": 54, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:16:25", + "exit_code": 0, + "duration_sec": 9.6, + "cpu_pct": 258.6, + "peak_memory_mb": 2868, + "disk_read_mb": 162, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:50:15", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 284.8, + "peak_memory_mb": 2868, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:50:15", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 449.9, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:17:45", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 381.1, + "peak_memory_mb": 5632, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:17:45", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 410.9, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:48:45", + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 500.8, + "peak_memory_mb": 5632, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:48:45", + "exit_code": 0, + "duration_sec": 12, + "cpu_pct": 274.2, + "peak_memory_mb": 5530, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:10:45", + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 457.9, + "peak_memory_mb": 5632, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:10:45", + "exit_code": 0, + "duration_sec": 10.5, + "cpu_pct": 218.9, + "peak_memory_mb": 5632, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:17:45", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 452.6, + "peak_memory_mb": 5530, + "disk_read_mb": 52, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:17:45", + "exit_code": 0, + "duration_sec": 9.9, + "cpu_pct": 275.3, + "peak_memory_mb": 2970, + "disk_read_mb": 159, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:31:45", + "exit_code": 0, + "duration_sec": 14.3, + "cpu_pct": 89.6, + "peak_memory_mb": 5530, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:31:45", + "exit_code": 0, + "duration_sec": 14.4, + "cpu_pct": 263.9, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:22:05", + "exit_code": 0, + "duration_sec": 11, + "cpu_pct": 125.7, + "peak_memory_mb": 5632, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:22:05", + "exit_code": 0, + "duration_sec": 34.8, + "cpu_pct": 117.2, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:31:45", + "exit_code": 0, + "duration_sec": 14.2, + "cpu_pct": 95.1, + "peak_memory_mb": 5530, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:31:45", + "exit_code": 0, + "duration_sec": 23.7, + "cpu_pct": 160.7, + "peak_memory_mb": 5940, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:34:15", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 374.1, + "peak_memory_mb": 5530, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:34:15", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 361.6, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:24:45", + "exit_code": 0, + "duration_sec": 11.3, + "cpu_pct": 105.5, + "peak_memory_mb": 5632, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:24:45", + "exit_code": 0, + "duration_sec": 34.2, + "cpu_pct": 106.1, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:25:45", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 428.2, + "peak_memory_mb": 5530, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:25:45", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 429.9, + "peak_memory_mb": 5530, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:36:05", + "exit_code": 0, + "duration_sec": 4.6, + "cpu_pct": 280.6, + "peak_memory_mb": 5837, + "disk_read_mb": 355, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:36:06", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 417.9, + "peak_memory_mb": 5530, + "disk_read_mb": 1068, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:59:45", + "exit_code": 0, + "duration_sec": 2.8, + "cpu_pct": 347, + "peak_memory_mb": 5530, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:59:45", + "exit_code": 0, + "duration_sec": 10.2, + "cpu_pct": 259.7, + "peak_memory_mb": 3277, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scimilarity", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:11:05", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 391.9, + "peak_memory_mb": 5530, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scimilarity", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:11:05", + "exit_code": 0, + "duration_sec": 8.4, + "cpu_pct": 403.5, + "peak_memory_mb": 5530, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scimilarity_knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:48:35", + "exit_code": 0, + "duration_sec": 3.1, + "cpu_pct": 302.9, + "peak_memory_mb": 2868, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scimilarity_knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:48:35", + "exit_code": 0, + "duration_sec": 12.3, + "cpu_pct": 249.4, + "peak_memory_mb": 5530, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:37:25", + "exit_code": 0, + "duration_sec": 3.4, + "cpu_pct": 307.9, + "peak_memory_mb": 5837, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:37:25", + "exit_code": 0, + "duration_sec": 26.1, + "cpu_pct": 142.7, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:21:55", + "exit_code": 0, + "duration_sec": 10.4, + "cpu_pct": 108.2, + "peak_memory_mb": 5940, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:21:55", + "exit_code": 0, + "duration_sec": 225, + "cpu_pct": 17.8, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:38:35", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 120, + "peak_memory_mb": 5632, + "disk_read_mb": 354, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:38:35", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 471.3, + "peak_memory_mb": 5632, + "disk_read_mb": 1065, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:42:45", + "exit_code": 0, + "duration_sec": 3.1, + "cpu_pct": 299.8, + "peak_memory_mb": 2868, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:42:45", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 516.7, + "peak_memory_mb": 5632, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:38:15", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 309.3, + "peak_memory_mb": 2868, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:38:15", + "exit_code": 0, + "duration_sec": 18, + "cpu_pct": 209.9, + "peak_memory_mb": 5632, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:43:55", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 346.4, + "peak_memory_mb": 5632, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:43:55", + "exit_code": 0, + "duration_sec": 11.4, + "cpu_pct": 332.2, + "peak_memory_mb": 5632, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:41:35", + "exit_code": 0, + "duration_sec": 5.7, + "cpu_pct": 232.8, + "peak_memory_mb": 5632, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:41:35", + "exit_code": 0, + "duration_sec": 20.7, + "cpu_pct": 153.5, + "peak_memory_mb": 2868, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:41:45", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 455, + "peak_memory_mb": 5735, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:41:46", + "exit_code": 0, + "duration_sec": 28.5, + "cpu_pct": 118.9, + "peak_memory_mb": 5735, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:45:15", + "exit_code": 0, + "duration_sec": 5.4, + "cpu_pct": 220.9, + "peak_memory_mb": 5735, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:45:15", + "exit_code": 0, + "duration_sec": 16.8, + "cpu_pct": 242.4, + "peak_memory_mb": 5632, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:54:05", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 421.2, + "peak_memory_mb": 5530, + "disk_read_mb": 157, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:54:05", + "exit_code": 0, + "duration_sec": 8.4, + "cpu_pct": 421.7, + "peak_memory_mb": 5530, + "disk_read_mb": 474, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:09:05", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 458.5, + "peak_memory_mb": 5530, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:09:05", + "exit_code": 0, + "duration_sec": 13.8, + "cpu_pct": 284.4, + "peak_memory_mb": 5632, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:45:55", + "exit_code": 0, + "duration_sec": 7, + "cpu_pct": 145, + "peak_memory_mb": 3072, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:45:55", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 436, + "peak_memory_mb": 5530, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:43:05", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 428.6, + "peak_memory_mb": 5530, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:43:05", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 522.7, + "peak_memory_mb": 5632, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:11:35", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 437.2, + "peak_memory_mb": 5530, + "disk_read_mb": 156, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:11:35", + "exit_code": 0, + "duration_sec": 8.4, + "cpu_pct": 393.5, + "peak_memory_mb": 5530, + "disk_read_mb": 471, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:50:05", + "exit_code": 0, + "duration_sec": 5.6, + "cpu_pct": 182.6, + "peak_memory_mb": 4608, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:50:05", + "exit_code": 0, + "duration_sec": 18.3, + "cpu_pct": 189.5, + "peak_memory_mb": 6452, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:57:55", + "exit_code": 0, + "duration_sec": 3.2, + "cpu_pct": 339.4, + "peak_memory_mb": 7271, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:57:55", + "exit_code": 0, + "duration_sec": 17.1, + "cpu_pct": 178.6, + "peak_memory_mb": 4608, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:57:55", + "exit_code": 0, + "duration_sec": 12.1, + "cpu_pct": 119.2, + "peak_memory_mb": 5632, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:57:55", + "exit_code": 0, + "duration_sec": 36.9, + "cpu_pct": 118.4, + "peak_memory_mb": 5632, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 11:52:06", + "exit_code": 0, + "duration_sec": 5.5, + "cpu_pct": 119.1, + "peak_memory_mb": 3277, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 11:52:05", + "exit_code": 0, + "duration_sec": 21.9, + "cpu_pct": 152.6, + "peak_memory_mb": 6042, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:53:35", + "exit_code": 0, + "duration_sec": 11.9, + "cpu_pct": 114.9, + "peak_memory_mb": 4608, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:53:35", + "exit_code": 0, + "duration_sec": 10.2, + "cpu_pct": 373.2, + "peak_memory_mb": 7271, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:05:45", + "exit_code": 0, + "duration_sec": 6.7, + "cpu_pct": 146.8, + "peak_memory_mb": 4608, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:05:45", + "exit_code": 0, + "duration_sec": 28.5, + "cpu_pct": 119.5, + "peak_memory_mb": 7271, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:55:45", + "exit_code": 0, + "duration_sec": 4.5, + "cpu_pct": 201.8, + "peak_memory_mb": 4608, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:55:45", + "exit_code": 0, + "duration_sec": 21.9, + "cpu_pct": 176.3, + "peak_memory_mb": 6042, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 11:15:15", + "exit_code": 0, + "duration_sec": 8.5, + "cpu_pct": 129.5, + "peak_memory_mb": 7271, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 11:15:16", + "exit_code": 0, + "duration_sec": 22.5, + "cpu_pct": 162.4, + "peak_memory_mb": 7271, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scimilarity", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 10:09:35", + "exit_code": 0, + "duration_sec": 11.5, + "cpu_pct": 88.7, + "peak_memory_mb": 4199, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scimilarity", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 10:09:35", + "exit_code": 0, + "duration_sec": 37.8, + "cpu_pct": 82.4, + "peak_memory_mb": 2868, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scimilarity_knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:29:35", + "exit_code": 0, + "duration_sec": 7.1, + "cpu_pct": 127.4, + "peak_memory_mb": 4608, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scimilarity_knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:29:35", + "exit_code": 0, + "duration_sec": 29.7, + "cpu_pct": 97.9, + "peak_memory_mb": 4608, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:59:15", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 154.5, + "peak_memory_mb": 6042, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:59:15", + "exit_code": 0, + "duration_sec": 10.2, + "cpu_pct": 377.7, + "peak_memory_mb": 7271, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:51:15", + "exit_code": 0, + "duration_sec": 3.2, + "cpu_pct": 428.8, + "peak_memory_mb": 7271, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:51:15", + "exit_code": 0, + "duration_sec": 10.2, + "cpu_pct": 363.4, + "peak_memory_mb": 7271, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 10:28:55", + "exit_code": 0, + "duration_sec": 8.4, + "cpu_pct": 152.8, + "peak_memory_mb": 6042, + "disk_read_mb": 1639, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 10:28:55", + "exit_code": 0, + "duration_sec": 22.5, + "cpu_pct": 170.4, + "peak_memory_mb": 7271, + "disk_read_mb": 4917, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:52:15", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 444.8, + "peak_memory_mb": 5632, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:52:15", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 459, + "peak_memory_mb": 5530, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:50:35", + "exit_code": 0, + "duration_sec": 4.2, + "cpu_pct": 191.3, + "peak_memory_mb": 5530, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:50:35", + "exit_code": 0, + "duration_sec": 107.1, + "cpu_pct": 35.8, + "peak_memory_mb": 5530, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 506.8, + "peak_memory_mb": 5632, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 29.7, + "cpu_pct": 121.9, + "peak_memory_mb": 5632, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 11:33:25", + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 488.2, + "peak_memory_mb": 5632, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 11:33:25", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 446.5, + "peak_memory_mb": 5530, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:52:15", + "exit_code": 0, + "duration_sec": 34.3, + "cpu_pct": 38.7, + "peak_memory_mb": 5530, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:52:15", + "exit_code": 0, + "duration_sec": 21, + "cpu_pct": 135, + "peak_memory_mb": 2868, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:52:45", + "exit_code": 0, + "duration_sec": 9.5, + "cpu_pct": 141.2, + "peak_memory_mb": 5632, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:52:45", + "exit_code": 0, + "duration_sec": 6.9, + "cpu_pct": 491.1, + "peak_memory_mb": 5632, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:15", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 524.9, + "peak_memory_mb": 5632, + "disk_read_mb": 38, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:15", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 362.7, + "peak_memory_mb": 5632, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:15:55", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 307, + "peak_memory_mb": 2868, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:15:55", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 275.1, + "peak_memory_mb": 2868, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:52:45", + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 490.7, + "peak_memory_mb": 5632, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:52:45", + "exit_code": 0, + "duration_sec": 9, + "cpu_pct": 271.2, + "peak_memory_mb": 2868, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:59:05", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 385.9, + "peak_memory_mb": 5530, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:59:05", + "exit_code": 0, + "duration_sec": 9, + "cpu_pct": 234.8, + "peak_memory_mb": 2868, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:48:25", + "exit_code": 0, + "duration_sec": 8.7, + "cpu_pct": 152.4, + "peak_memory_mb": 5632, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:48:25", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 484.4, + "peak_memory_mb": 5530, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:12:05", + "exit_code": 0, + "duration_sec": 5.7, + "cpu_pct": 222.7, + "peak_memory_mb": 5530, + "disk_read_mb": 37, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:12:05", + "exit_code": 0, + "duration_sec": 26.7, + "cpu_pct": 142.4, + "peak_memory_mb": 5632, + "disk_read_mb": 114, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:44:25", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 440.1, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:44:25", + "exit_code": 0, + "duration_sec": 37.8, + "cpu_pct": 107.3, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:40:45", + "exit_code": 0, + "duration_sec": 7, + "cpu_pct": 189, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:40:45", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 365.1, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:39:56", + "exit_code": 0, + "duration_sec": 17.3, + "cpu_pct": 74.1, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:39:56", + "exit_code": 0, + "duration_sec": 46.2, + "cpu_pct": 85.9, + "peak_memory_mb": 5530, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:39:56", + "exit_code": 0, + "duration_sec": 17.3, + "cpu_pct": 77.8, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:39:55", + "exit_code": 0, + "duration_sec": 45.9, + "cpu_pct": 75.5, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:44:25", + "exit_code": 0, + "duration_sec": 10.1, + "cpu_pct": 124.4, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:44:25", + "exit_code": 0, + "duration_sec": 16.2, + "cpu_pct": 218.9, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:48:25", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 435.7, + "peak_memory_mb": 5632, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:48:25", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 285.3, + "peak_memory_mb": 2868, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:42:55", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 429.1, + "peak_memory_mb": 5632, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:42:55", + "exit_code": 0, + "duration_sec": 13.2, + "cpu_pct": 289.7, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:49:35", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 418.9, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:49:35", + "exit_code": 0, + "duration_sec": 11.7, + "cpu_pct": 272.1, + "peak_memory_mb": 5530, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:41:45", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 415.4, + "peak_memory_mb": 5530, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:41:45", + "exit_code": 0, + "duration_sec": 9, + "cpu_pct": 304.7, + "peak_memory_mb": 2868, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:49:45", + "exit_code": 0, + "duration_sec": 6.6, + "cpu_pct": 153.3, + "peak_memory_mb": 2868, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:49:45", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 431.1, + "peak_memory_mb": 5530, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:39:56", + "exit_code": 0, + "duration_sec": 2.2, + "cpu_pct": 387, + "peak_memory_mb": 5632, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:39:55", + "exit_code": 0, + "duration_sec": 13.2, + "cpu_pct": 288.7, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:47:35", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 446.3, + "peak_memory_mb": 5632, + "disk_read_mb": 66, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:47:35", + "exit_code": 0, + "duration_sec": 16.5, + "cpu_pct": 219.3, + "peak_memory_mb": 5632, + "disk_read_mb": 201, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:50:35", + "exit_code": 0, + "duration_sec": 3.3, + "cpu_pct": 328.9, + "peak_memory_mb": 5530, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:50:35", + "exit_code": 0, + "duration_sec": 12.6, + "cpu_pct": 206.7, + "peak_memory_mb": 2868, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:53:15", + "exit_code": 0, + "duration_sec": 2.8, + "cpu_pct": 411, + "peak_memory_mb": 5530, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:53:15", + "exit_code": 0, + "duration_sec": 7.2, + "cpu_pct": 496.1, + "peak_memory_mb": 5632, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 435.3, + "peak_memory_mb": 5530, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 400.6, + "peak_memory_mb": 5530, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 420.8, + "peak_memory_mb": 5530, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 32.4, + "cpu_pct": 116, + "peak_memory_mb": 5632, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:54:35", + "exit_code": 0, + "duration_sec": 2.3, + "cpu_pct": 452.6, + "peak_memory_mb": 5632, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:54:35", + "exit_code": 0, + "duration_sec": 13.8, + "cpu_pct": 202.9, + "peak_memory_mb": 2868, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 10.3, + "cpu_pct": 122.4, + "peak_memory_mb": 5632, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:53:06", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 433.9, + "peak_memory_mb": 5530, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:26", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 488.3, + "peak_memory_mb": 5632, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:25", + "exit_code": 0, + "duration_sec": 20.7, + "cpu_pct": 147.6, + "peak_memory_mb": 2868, + "disk_read_mb": 300, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:59:35", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 461.8, + "peak_memory_mb": 5632, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:59:35", + "exit_code": 0, + "duration_sec": 6.6, + "cpu_pct": 512.6, + "peak_memory_mb": 5632, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:57:55", + "exit_code": 0, + "duration_sec": 3.1, + "cpu_pct": 288.9, + "peak_memory_mb": 2868, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:57:55", + "exit_code": 0, + "duration_sec": 13.5, + "cpu_pct": 219.8, + "peak_memory_mb": 5632, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:52:45", + "exit_code": 0, + "duration_sec": 3.9, + "cpu_pct": 255.1, + "peak_memory_mb": 5632, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:52:45", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 257.3, + "peak_memory_mb": 2868, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:26", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 347.1, + "peak_memory_mb": 5530, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:25", + "exit_code": 0, + "duration_sec": 7.5, + "cpu_pct": 376.9, + "peak_memory_mb": 5530, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:15", + "exit_code": 0, + "duration_sec": 4.9, + "cpu_pct": 207.7, + "peak_memory_mb": 5632, + "disk_read_mb": 99, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:15", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 343.4, + "peak_memory_mb": 5530, + "disk_read_mb": 297, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "knn", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:57:35", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 428.4, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "knn", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:57:35", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 458.4, + "peak_memory_mb": 5632, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "logistic_regression", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:56:25", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 450, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "logistic_regression", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:56:25", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 345.3, + "peak_memory_mb": 5530, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "majority_vote", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:45", + "exit_code": 0, + "duration_sec": 3.1, + "cpu_pct": 284.4, + "peak_memory_mb": 2868, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "majority_vote", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:45", + "exit_code": 0, + "duration_sec": 8.1, + "cpu_pct": 461.1, + "peak_memory_mb": 5530, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "mlp", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:58:05", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 434.8, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "mlp", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:58:05", + "exit_code": 0, + "duration_sec": 10.8, + "cpu_pct": 340.9, + "peak_memory_mb": 5632, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "naive_bayes", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:35", + "exit_code": 0, + "duration_sec": 5.7, + "cpu_pct": 185.2, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "naive_bayes", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:35", + "exit_code": 0, + "duration_sec": 115.8, + "cpu_pct": 35.4, + "peak_memory_mb": 5632, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "random_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:55:56", + "exit_code": 0, + "duration_sec": 4.5, + "cpu_pct": 296.3, + "peak_memory_mb": 5837, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "random_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:55:55", + "exit_code": 0, + "duration_sec": 33.3, + "cpu_pct": 121.8, + "peak_memory_mb": 5632, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scanvi", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:00:55", + "exit_code": 0, + "duration_sec": 2.5, + "cpu_pct": 464.7, + "peak_memory_mb": 5632, + "disk_read_mb": 309, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scanvi", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:00:55", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 444.7, + "peak_memory_mb": 5632, + "disk_read_mb": 927, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scanvi_scarches", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:05:25", + "exit_code": 0, + "duration_sec": 3.6, + "cpu_pct": 349, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scanvi_scarches", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:05:25", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 177.2, + "peak_memory_mb": 1536, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "seurat_transferdata", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:56:25", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 350.5, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "seurat_transferdata", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:56:25", + "exit_code": 0, + "duration_sec": 8.4, + "cpu_pct": 346, + "peak_memory_mb": 5530, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "singler", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 09:43:15", + "exit_code": 0, + "duration_sec": 6.7, + "cpu_pct": 173, + "peak_memory_mb": 5632, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "singler", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 09:43:15", + "exit_code": 0, + "duration_sec": 10.2, + "cpu_pct": 266.3, + "peak_memory_mb": 3175, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "true_labels", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:56:05", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 314.4, + "peak_memory_mb": 5530, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "true_labels", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:56:05", + "exit_code": 0, + "duration_sec": 33.6, + "cpu_pct": 122.3, + "peak_memory_mb": 5632, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "xgboost", + "metric_component_name": "accuracy", + "resources": { + "submit": "2025-01-08 08:59:25", + "exit_code": 0, + "duration_sec": 2.6, + "cpu_pct": 392.3, + "peak_memory_mb": 5632, + "disk_read_mb": 308, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "xgboost", + "metric_component_name": "f1", + "resources": { + "submit": "2025-01-08 08:59:25", + "exit_code": 0, + "duration_sec": 7.8, + "cpu_pct": 387.4, + "peak_memory_mb": 5632, + "disk_read_mb": 924, + "disk_write_mb": 3 + } + } +] diff --git a/results/label_projection/data/metric_info.json b/results/label_projection/data/metric_info.json index 07b08336..b702b546 100644 --- a/results/label_projection/data/metric_info.json +++ b/results/label_projection/data/metric_info.json @@ -1,38 +1,62 @@ [ { - "metric_name": "Accuracy", - "metric_summary": "Average number of correctly applied labels.", - "paper_reference": "grandini2020metrics", - "maximize": true, - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", + "task_id": "metrics", + "component_name": "accuracy", "metric_id": "accuracy", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/metrics/accuracy.py", - "code_version": "v1.0.0" + "metric_name": "Accuracy", + "metric_summary": "The percentage of correctly predicted labels.", + "metric_description": "The percentage of correctly predicted labels.", + "references_doi": "10.48550/arxiv.2008.05756", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/metrics/accuracy", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/metrics/accuracy:build_main", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d", + "maximize": true }, { - "metric_name": "F1 score", - "metric_summary": "The [F1 score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) is a weighted average of the precision and recall over all class labels, where an F1 score reaches its best value at 1 and worst score at 0, where each class contributes to the score relative to its frequency in the dataset.", - "paper_reference": "grandini2020metrics", - "maximize": true, - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", - "metric_id": "f1", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/metrics/f1.py", - "code_version": "v1.0.0" + "task_id": "metrics", + "component_name": "f1", + "metric_id": "f1_weighted", + "metric_name": "F1 weighted", + "metric_summary": "Average weigthed support between each labels F1 score", + "metric_description": "Calculates the F1 score for each label, and find their average weighted by support (the number of true instances for each label). This alters 'macro' to account for label imbalance; it can result in an F-score that is not between precision and recall.", + "references_doi": "10.48550/arxiv.2008.05756", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/metrics/f1", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/metrics/f1:build_main", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d", + "maximize": true }, { - "metric_name": "Macro F1 score", - "metric_summary": "The macro F1 score is an unweighted F1 score, where each class contributes equally, regardless of its frequency.", - "paper_reference": "grandini2020metrics", - "maximize": true, - "image": "https://github.com/openproblems-bio/openproblems/pkgs/container/openproblems", - "task_id": "label_projection", - "commit_sha": "b3456fd73c04c28516f6df34c57e6e3e8b0dab32", + "task_id": "metrics", + "component_name": "f1", "metric_id": "f1_macro", - "implementation_url": "https://github.com/openproblems-bio/openproblems/blob/v1.0.0/openproblems/tasks/label_projection/metrics/f1.py", - "code_version": "v1.0.0" + "metric_name": "F1 macro", + "metric_summary": "Unweighted mean of each label F1-score", + "metric_description": "Calculates the F1 score for each label, and find their unweighted mean. This does not take label imbalance into account.", + "references_doi": "10.48550/arxiv.2008.05756", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/metrics/f1", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/metrics/f1:build_main", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d", + "maximize": true + }, + { + "task_id": "metrics", + "component_name": "f1", + "metric_id": "f1_micro", + "metric_name": "F1 micro", + "metric_summary": "Calculation of TP, FN and FP.", + "metric_description": "Calculates the F1 score globally by counting the total true positives, false negatives and false positives.", + "references_doi": "10.48550/arxiv.2008.05756", + "references_bibtex": null, + "implementation_url": "https://github.com/openproblems-bio/task_label_projection/blob/0def7c62cbda5cdf52c7406a06f91d9e36a5535d/src/metrics/f1", + "image": "https://ghcr.io/openproblems-bio/task_label_projection/metrics/f1:build_main", + "code_version": "build_main", + "commit_sha": "0def7c62cbda5cdf52c7406a06f91d9e36a5535d", + "maximize": true } -] \ No newline at end of file +] diff --git a/results/label_projection/data/quality_control.json b/results/label_projection/data/quality_control.json index 6db27aee..69162bd2 100644 --- a/results/label_projection/data/quality_control.json +++ b/results/label_projection/data/quality_control.json @@ -1,1492 +1,2032 @@ [ { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Task info", "name": "Pct 'task_id' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_id' should be defined\n Task id: label_projection\n Field: task_id\n" + "message": "Task metadata field 'task_id' should be defined\n Task id: task_label_projection\n Field: task_id\n" }, { - "task_id": "label_projection", - "category": "Task info", - "name": "Pct 'commit_sha' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, - "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'commit_sha' should be defined\n Task id: label_projection\n Field: commit_sha\n" - }, - { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Task info", "name": "Pct 'task_name' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_name' should be defined\n Task id: label_projection\n Field: task_name\n" + "message": "Task metadata field 'task_name' should be defined\n Task id: task_label_projection\n Field: task_name\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Task info", "name": "Pct 'task_summary' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_summary' should be defined\n Task id: label_projection\n Field: task_summary\n" + "message": "Task metadata field 'task_summary' should be defined\n Task id: task_label_projection\n Field: task_summary\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Task info", "name": "Pct 'task_description' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing([task_info], field)", - "message": "Task metadata field 'task_description' should be defined\n Task id: label_projection\n Field: task_description\n" + "message": "Task metadata field 'task_description' should be defined\n Task id: task_label_projection\n Field: task_description\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'task_id' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'task_id' should be defined\n Task id: label_projection\n Field: task_id\n" + "message": "Method metadata field 'task_id' should be defined\n Task id: task_label_projection\n Field: task_id\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'commit_sha' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'commit_sha' should be defined\n Task id: label_projection\n Field: commit_sha\n" + "message": "Method metadata field 'commit_sha' should be defined\n Task id: task_label_projection\n Field: commit_sha\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'method_id' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_id' should be defined\n Task id: label_projection\n Field: method_id\n" + "message": "Method metadata field 'method_id' should be defined\n Task id: task_label_projection\n Field: method_id\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'method_name' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_name' should be defined\n Task id: label_projection\n Field: method_name\n" + "message": "Method metadata field 'method_name' should be defined\n Task id: task_label_projection\n Field: method_name\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'method_summary' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'method_summary' should be defined\n Task id: label_projection\n Field: method_summary\n" + "message": "Method metadata field 'method_summary' should be defined\n Task id: task_label_projection\n Field: method_summary\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'paper_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "value": 0.8333333333333334, + "severity": 2, + "severity_value": 3.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'paper_reference' should be defined\n Task id: label_projection\n Field: paper_reference\n" + "message": "Method metadata field 'paper_reference' should be defined\n Task id: task_label_projection\n Field: paper_reference\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Method info", "name": "Pct 'is_baseline' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(method_info, field)", - "message": "Method metadata field 'is_baseline' should be defined\n Task id: label_projection\n Field: is_baseline\n" + "message": "Method metadata field 'is_baseline' should be defined\n Task id: task_label_projection\n Field: is_baseline\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'task_id' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'task_id' should be defined\n Task id: label_projection\n Field: task_id\n" + "message": "Metric metadata field 'task_id' should be defined\n Task id: task_label_projection\n Field: task_id\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'commit_sha' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'commit_sha' should be defined\n Task id: label_projection\n Field: commit_sha\n" + "message": "Metric metadata field 'commit_sha' should be defined\n Task id: task_label_projection\n Field: commit_sha\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'metric_id' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_id' should be defined\n Task id: label_projection\n Field: metric_id\n" + "message": "Metric metadata field 'metric_id' should be defined\n Task id: task_label_projection\n Field: metric_id\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'metric_name' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_name' should be defined\n Task id: label_projection\n Field: metric_name\n" + "message": "Metric metadata field 'metric_name' should be defined\n Task id: task_label_projection\n Field: metric_name\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'metric_summary' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'metric_summary' should be defined\n Task id: label_projection\n Field: metric_summary\n" + "message": "Metric metadata field 'metric_summary' should be defined\n Task id: task_label_projection\n Field: metric_summary\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'paper_reference' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "value": 1.0, + "severity": 2, + "severity_value": 3.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'paper_reference' should be defined\n Task id: label_projection\n Field: paper_reference\n" + "message": "Metric metadata field 'paper_reference' should be defined\n Task id: task_label_projection\n Field: paper_reference\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Metric info", "name": "Pct 'maximize' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(metric_info, field)", - "message": "Metric metadata field 'maximize' should be defined\n Task id: label_projection\n Field: maximize\n" + "message": "Metric metadata field 'maximize' should be defined\n Task id: task_label_projection\n Field: maximize\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Dataset info", "name": "Pct 'task_id' missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "value": 1.0, + "severity": 2, + "severity_value": 3.0, "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'task_id' should be defined\n Task id: label_projection\n Field: task_id\n" + "message": "Dataset metadata field 'task_id' should be defined\n Task id: task_label_projection\n Field: task_id\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Dataset info", - "name": "Pct 'commit_sha' missing", + "name": "Pct 'dataset_id' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'commit_sha' should be defined\n Task id: label_projection\n Field: commit_sha\n" + "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: task_label_projection\n Field: dataset_id\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Dataset info", - "name": "Pct 'dataset_id' missing", + "name": "Pct 'dataset_name' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_id' should be defined\n Task id: label_projection\n Field: dataset_id\n" + "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: task_label_projection\n Field: dataset_name\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Dataset info", - "name": "Pct 'dataset_name' missing", + "name": "Pct 'dataset_summary' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_name' should be defined\n Task id: label_projection\n Field: dataset_name\n" + "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: task_label_projection\n Field: dataset_summary\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Dataset info", - "name": "Pct 'dataset_summary' missing", + "name": "Pct 'data_reference' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'dataset_summary' should be defined\n Task id: label_projection\n Field: dataset_summary\n" + "message": "Dataset metadata field 'data_reference' should be defined\n Task id: task_label_projection\n Field: data_reference\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Dataset info", - "name": "Pct 'data_reference' missing", + "name": "Pct 'data_url' missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "percent_missing(dataset_info, field)", - "message": "Dataset metadata field 'data_reference' should be defined\n Task id: label_projection\n Field: data_reference\n" + "message": "Dataset metadata field 'data_url' should be defined\n Task id: task_label_projection\n Field: data_url\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw data", "name": "Number of results", - "value": 128, + "value": 216, "severity": 0, "severity_value": 0.0, "code": "len(results) == len(method_info) * len(metric_info) * len(dataset_info)", - "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: label_projection\n Number of results: 128\n Number of methods: 16\n Number of metrics: 3\n Number of datasets: 8\n" + "message": "Number of results should be equal to #methods × #metrics × #datasets.\n Task id: task_label_projection\n Number of results: 216\n Number of methods: 18\n Number of metrics: 4\n Number of datasets: 12\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", "name": "Metric 'accuracy' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "value": 0.31481481481481477, + "severity": 3, + "severity_value": 3.1481481481481475, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n Metric id: accuracy\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n Metric id: accuracy\n Percentage missing: 31%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Metric 'f1' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Metric 'f1_weighted' %missing", + "value": 0.31481481481481477, + "severity": 3, + "severity_value": 3.1481481481481475, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n Metric id: f1\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n Metric id: f1_weighted\n Percentage missing: 31%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", "name": "Metric 'f1_macro' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "value": 0.31481481481481477, + "severity": 3, + "severity_value": 3.1481481481481475, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n Metric id: f1_macro\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n Metric id: f1_macro\n Percentage missing: 31%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'knn_classifier_log_cp10k' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Metric 'f1_micro' %missing", + "value": 0.31481481481481477, + "severity": 3, + "severity_value": 3.1481481481481475, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: knn_classifier_log_cp10k\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n Metric id: f1_micro\n Percentage missing: 31%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'knn_classifier_scran' %missing", + "name": "Method 'majority_vote' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: knn_classifier_scran\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: majority_vote\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'logistic_regression_log_cp10k' %missing", + "name": "Method 'random_labels' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: logistic_regression_log_cp10k\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: random_labels\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'logistic_regression_scran' %missing", + "name": "Method 'true_labels' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: logistic_regression_scran\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: true_labels\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'majority_vote' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'geneformer' %missing", + "value": 1.0, + "severity": 3, + "severity_value": 10.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: majority_vote\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: geneformer\n Percentage missing: 100%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'mlp_log_cp10k' %missing", + "name": "Method 'knn' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: mlp_log_cp10k\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: knn\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'mlp_scran' %missing", + "name": "Method 'logistic_regression' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: mlp_scran\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: logistic_regression\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'random_labels' %missing", + "name": "Method 'mlp' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: random_labels\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: mlp\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'scanvi_all_genes' %missing", + "name": "Method 'naive_bayes' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: scanvi_all_genes\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: naive_bayes\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'scanvi_hvg' %missing", + "name": "Method 'scanvi' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: scanvi_hvg\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: scanvi\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'scarches_scanvi_all_genes' %missing", + "name": "Method 'scanvi_scarches' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: scarches_scanvi_all_genes\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: scanvi_scarches\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'scarches_scanvi_hvg' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'scgpt_zero_shot' %missing", + "value": 1.0, + "severity": 3, + "severity_value": 10.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: scarches_scanvi_hvg\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: scgpt_zero_shot\n Percentage missing: 100%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'seurat' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'scimilarity' %missing", + "value": 0.6666666666666667, + "severity": 3, + "severity_value": 6.666666666666667, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: seurat\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: scimilarity\n Percentage missing: 67%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'true_labels' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'scimilarity_knn' %missing", + "value": 0.6666666666666667, + "severity": 3, + "severity_value": 6.666666666666667, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: true_labels\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: scimilarity_knn\n Percentage missing: 67%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'xgboost_log_cp10k' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'scprint' %missing", + "value": 1.0, + "severity": 3, + "severity_value": 10.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: xgboost_log_cp10k\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: scprint\n Percentage missing: 100%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Method 'xgboost_scran' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'seurat_transferdata' %missing", + "value": 0.16666666666666663, + "severity": 1, + "severity_value": 1.6666666666666663, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n method id: xgboost_scran\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: seurat_transferdata\n Percentage missing: 17%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'cengen_batch' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'singler' %missing", + "value": 0.25, + "severity": 2, + "severity_value": 2.5, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: cengen_batch\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: singler\n Percentage missing: 25%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'cengen_random' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Method 'uce' %missing", + "value": 0.9166666666666666, + "severity": 3, + "severity_value": 9.166666666666666, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: cengen_random\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: uce\n Percentage missing: 92%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'pancreas_batch' %missing", + "name": "Method 'xgboost' %missing", "value": 0.0, "severity": 0, "severity_value": 0.0, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: pancreas_batch\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n method id: xgboost\n Percentage missing: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'pancreas_random' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Dataset 'cellxgene_census/gtex_v9' %missing", + "value": 0.2222222222222222, + "severity": 2, + "severity_value": 2.222222222222222, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: pancreas_random\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/gtex_v9\n Percentage missing: 22%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'pancreas_random_label_noise' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Dataset 'openproblems_v1/cengen' %missing", + "value": 0.33333333333333337, + "severity": 3, + "severity_value": 3.3333333333333335, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: pancreas_random_label_noise\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: openproblems_v1/cengen\n Percentage missing: 33%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'tabula_muris_senis_lung_random' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Dataset 'cellxgene_census/tabula_sapiens' %missing", + "value": 0.2777777777777778, + "severity": 2, + "severity_value": 2.7777777777777777, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: tabula_muris_senis_lung_random\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/tabula_sapiens\n Percentage missing: 28%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'zebrafish_labs' %missing", - "value": 0.0, - "severity": 0, - "severity_value": 0.0, + "name": "Dataset 'cellxgene_census/hypomap' %missing", + "value": 0.33333333333333337, + "severity": 3, + "severity_value": 3.3333333333333335, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/hypomap\n Percentage missing: 33%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'cellxgene_census/immune_cell_atlas' %missing", + "value": 0.2777777777777778, + "severity": 2, + "severity_value": 2.7777777777777777, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/immune_cell_atlas\n Percentage missing: 28%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'cellxgene_census/hcla' %missing", + "value": 0.38888888888888884, + "severity": 3, + "severity_value": 3.8888888888888884, "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: zebrafish_labs\n Percentage missing: 0%\n" + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/hcla\n Percentage missing: 39%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Raw results", - "name": "Dataset 'zebrafish_random' %missing", + "name": "Dataset 'cellxgene_census/dkd' %missing", + "value": 0.16666666666666663, + "severity": 1, + "severity_value": 1.6666666666666663, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/dkd\n Percentage missing: 17%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'cellxgene_census/mouse_pancreas_atlas' %missing", + "value": 0.38888888888888884, + "severity": 3, + "severity_value": 3.8888888888888884, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: cellxgene_census/mouse_pancreas_atlas\n Percentage missing: 39%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'openproblems_v1/immune_cells' %missing", + "value": 0.33333333333333337, + "severity": 3, + "severity_value": 3.3333333333333335, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: openproblems_v1/immune_cells\n Percentage missing: 33%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2' %missing", + "value": 0.38888888888888884, + "severity": 3, + "severity_value": 3.8888888888888884, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2\n Percentage missing: 39%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'openproblems_v1/zebrafish' %missing", + "value": 0.33333333333333337, + "severity": 3, + "severity_value": 3.3333333333333335, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: openproblems_v1/zebrafish\n Percentage missing: 33%\n" + }, + { + "task_id": "task_label_projection", + "category": "Raw results", + "name": "Dataset 'openproblems_v1/pancreas' %missing", + "value": 0.33333333333333337, + "severity": 3, + "severity_value": 3.3333333333333335, + "code": "pct_missing <= .1", + "message": "Percentage of missing results should be less than 10%.\n Task id: task_label_projection\n dataset id: openproblems_v1/pancreas\n Percentage missing: 33%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score majority_vote accuracy", "value": 0.0, "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method majority_vote performs much worse than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: accuracy\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score majority_vote accuracy", + "value": 0.4546, + "severity": 0, + "severity_value": 0.2273, + "code": "best_score <= 2", + "message": "Method majority_vote performs a lot better than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: accuracy\n Best score: 0.4546%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score random_labels accuracy", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: accuracy\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score random_labels accuracy", + "value": 0.0737, + "severity": 0, + "severity_value": 0.03685, + "code": "best_score <= 2", + "message": "Method random_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: accuracy\n Best score: 0.0737%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score true_labels accuracy", + "value": 1, + "severity": 0, + "severity_value": -1.0, + "code": "worst_score >= -1", + "message": "Method true_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: accuracy\n Worst score: 1%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score true_labels accuracy", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method true_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: accuracy\n Best score: 1%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score geneformer accuracy", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method geneformer performs much worse than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: accuracy\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score geneformer accuracy", + "value": 0, + "severity": 0, "severity_value": 0.0, - "code": "pct_missing <= .1", - "message": "Percentage of missing results should be less than 10%.\n Task id: label_projection\n dataset id: zebrafish_random\n Percentage missing: 0%\n" + "code": "best_score <= 2", + "message": "Method geneformer performs a lot better than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: accuracy\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score knn_classifier_log_cp10k accuracy", - "value": 0.1049049049049049, + "name": "Worst score knn accuracy", + "value": 0.2486, "severity": 0, - "severity_value": -0.1049049049049049, + "severity_value": -0.2486, "code": "worst_score >= -1", - "message": "Method knn_classifier_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: knn_classifier_log_cp10k\n Metric id: accuracy\n Worst score: 0.1049049049049049%\n" + "message": "Method knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: accuracy\n Worst score: 0.2486%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score knn_classifier_log_cp10k accuracy", - "value": 0.9551307062036676, + "name": "Best score knn accuracy", + "value": 0.9992, "severity": 0, - "severity_value": 0.4775653531018338, + "severity_value": 0.4996, "code": "best_score <= 2", - "message": "Method knn_classifier_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: knn_classifier_log_cp10k\n Metric id: accuracy\n Best score: 0.9551307062036676%\n" + "message": "Method knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: accuracy\n Best score: 0.9992%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score knn_classifier_scran accuracy", - "value": 0.16823490156823492, + "name": "Worst score logistic_regression accuracy", + "value": -0.0107, "severity": 0, - "severity_value": -0.16823490156823492, + "severity_value": 0.0107, "code": "worst_score >= -1", - "message": "Method knn_classifier_scran performs much worse than baselines.\n Task id: label_projection\n Method id: knn_classifier_scran\n Metric id: accuracy\n Worst score: 0.16823490156823492%\n" + "message": "Method logistic_regression performs much worse than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: accuracy\n Worst score: -0.0107%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score knn_classifier_scran accuracy", - "value": 0.9543503706593836, + "name": "Best score logistic_regression accuracy", + "value": 0.9992, "severity": 0, - "severity_value": 0.4771751853296918, + "severity_value": 0.4996, "code": "best_score <= 2", - "message": "Method knn_classifier_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: knn_classifier_scran\n Metric id: accuracy\n Best score: 0.9543503706593836%\n" + "message": "Method logistic_regression performs a lot better than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: accuracy\n Best score: 0.9992%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score logistic_regression_log_cp10k accuracy", - "value": 0.22649315982649318, + "name": "Worst score mlp accuracy", + "value": 0.2644, "severity": 0, - "severity_value": -0.22649315982649318, + "severity_value": -0.2644, "code": "worst_score >= -1", - "message": "Method logistic_regression_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: logistic_regression_log_cp10k\n Metric id: accuracy\n Worst score: 0.22649315982649318%\n" + "message": "Method mlp performs much worse than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: accuracy\n Worst score: 0.2644%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score logistic_regression_log_cp10k accuracy", - "value": 0.9843932891143191, + "name": "Best score mlp accuracy", + "value": 1.0, "severity": 0, - "severity_value": 0.49219664455715956, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method logistic_regression_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: logistic_regression_log_cp10k\n Metric id: accuracy\n Best score: 0.9843932891143191%\n" + "message": "Method mlp performs a lot better than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: accuracy\n Best score: 1.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score logistic_regression_scran accuracy", - "value": 0.22308975642308976, + "name": "Worst score naive_bayes accuracy", + "value": 0.189, "severity": 0, - "severity_value": -0.22308975642308976, + "severity_value": -0.189, "code": "worst_score >= -1", - "message": "Method logistic_regression_scran performs much worse than baselines.\n Task id: label_projection\n Method id: logistic_regression_scran\n Metric id: accuracy\n Worst score: 0.22308975642308976%\n" + "message": "Method naive_bayes performs much worse than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: accuracy\n Worst score: 0.189%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score logistic_regression_scran accuracy", - "value": 0.9297698010144363, + "name": "Best score naive_bayes accuracy", + "value": 0.9851, "severity": 0, - "severity_value": 0.46488490050721815, + "severity_value": 0.49255, "code": "best_score <= 2", - "message": "Method logistic_regression_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: logistic_regression_scran\n Metric id: accuracy\n Best score: 0.9297698010144363%\n" + "message": "Method naive_bayes performs a lot better than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: accuracy\n Best score: 0.9851%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score majority_vote accuracy", - "value": -0.028628628628628628, + "name": "Worst score scanvi accuracy", + "value": 0.2797, "severity": 0, - "severity_value": 0.028628628628628628, + "severity_value": -0.2797, "code": "worst_score >= -1", - "message": "Method majority_vote performs much worse than baselines.\n Task id: label_projection\n Method id: majority_vote\n Metric id: accuracy\n Worst score: -0.028628628628628628%\n" + "message": "Method scanvi performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: accuracy\n Worst score: 0.2797%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score majority_vote accuracy", - "value": 0.19574944071588368, + "name": "Best score scanvi accuracy", + "value": 0.9967, "severity": 0, - "severity_value": 0.09787472035794184, + "severity_value": 0.49835, "code": "best_score <= 2", - "message": "Method majority_vote performs a lot better than baselines.\n Task id: label_projection\n Method id: majority_vote\n Metric id: accuracy\n Best score: 0.19574944071588368%\n" + "message": "Method scanvi performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: accuracy\n Best score: 0.9967%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score mlp_log_cp10k accuracy", - "value": 0.2155488822155489, + "name": "Worst score scanvi_scarches accuracy", + "value": 0.28, "severity": 0, - "severity_value": -0.2155488822155489, + "severity_value": -0.28, "code": "worst_score >= -1", - "message": "Method mlp_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: mlp_log_cp10k\n Metric id: accuracy\n Worst score: 0.2155488822155489%\n" + "message": "Method scanvi_scarches performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: accuracy\n Worst score: 0.28%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score mlp_log_cp10k accuracy", - "value": 0.9847834568864612, + "name": "Best score scanvi_scarches accuracy", + "value": 0.9983, "severity": 0, - "severity_value": 0.4923917284432306, + "severity_value": 0.49915, "code": "best_score <= 2", - "message": "Method mlp_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: mlp_log_cp10k\n Metric id: accuracy\n Best score: 0.9847834568864612%\n" + "message": "Method scanvi_scarches performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: accuracy\n Best score: 0.9983%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score mlp_scran accuracy", - "value": 0.2167500834167501, + "name": "Worst score scgpt_zero_shot accuracy", + "value": 0, "severity": 0, - "severity_value": -0.2167500834167501, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method mlp_scran performs much worse than baselines.\n Task id: label_projection\n Method id: mlp_scran\n Metric id: accuracy\n Worst score: 0.2167500834167501%\n" + "message": "Method scgpt_zero_shot performs much worse than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: accuracy\n Worst score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score mlp_scran accuracy", - "value": 0.980881779165041, + "name": "Best score scgpt_zero_shot accuracy", + "value": 0, "severity": 0, - "severity_value": 0.4904408895825205, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method mlp_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: mlp_scran\n Metric id: accuracy\n Best score: 0.980881779165041%\n" + "message": "Method scgpt_zero_shot performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: accuracy\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score random_labels accuracy", + "name": "Worst score scimilarity accuracy", "value": 0.0, "severity": 0, "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method random_labels performs much worse than baselines.\n Task id: label_projection\n Method id: random_labels\n Metric id: accuracy\n Worst score: 0.0%\n" + "message": "Method scimilarity performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: accuracy\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score random_labels accuracy", + "name": "Best score scimilarity accuracy", + "value": 0.873, + "severity": 0, + "severity_value": 0.4365, + "code": "best_score <= 2", + "message": "Method scimilarity performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: accuracy\n Best score: 0.873%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scimilarity_knn accuracy", "value": 0.0, "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scimilarity_knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: accuracy\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scimilarity_knn accuracy", + "value": 0.9188, + "severity": 0, + "severity_value": 0.4594, + "code": "best_score <= 2", + "message": "Method scimilarity_knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: accuracy\n Best score: 0.9188%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scprint accuracy", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scprint performs much worse than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: accuracy\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scprint accuracy", + "value": 0, + "severity": 0, "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method random_labels performs a lot better than baselines.\n Task id: label_projection\n Method id: random_labels\n Metric id: accuracy\n Best score: 0.0%\n" + "message": "Method scprint performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: accuracy\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scanvi_all_genes accuracy", - "value": 0.19766433099766434, + "name": "Worst score seurat_transferdata accuracy", + "value": 0.0, "severity": 0, - "severity_value": -0.19766433099766434, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scanvi_all_genes performs much worse than baselines.\n Task id: label_projection\n Method id: scanvi_all_genes\n Metric id: accuracy\n Worst score: 0.19766433099766434%\n" + "message": "Method seurat_transferdata performs much worse than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: accuracy\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scanvi_all_genes accuracy", - "value": 0.9676160749122122, + "name": "Best score seurat_transferdata accuracy", + "value": 0.9925, "severity": 0, - "severity_value": 0.4838080374561061, + "severity_value": 0.49625, "code": "best_score <= 2", - "message": "Method scanvi_all_genes performs a lot better than baselines.\n Task id: label_projection\n Method id: scanvi_all_genes\n Metric id: accuracy\n Best score: 0.9676160749122122%\n" + "message": "Method seurat_transferdata performs a lot better than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: accuracy\n Best score: 0.9925%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scanvi_hvg accuracy", - "value": 0.18745412078745413, + "name": "Worst score singler accuracy", + "value": 0.0, "severity": 0, - "severity_value": -0.18745412078745413, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scanvi_hvg performs much worse than baselines.\n Task id: label_projection\n Method id: scanvi_hvg\n Metric id: accuracy\n Worst score: 0.18745412078745413%\n" + "message": "Method singler performs much worse than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: accuracy\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scanvi_hvg accuracy", - "value": 0.9722980881779164, + "name": "Best score singler accuracy", + "value": 0.9892, "severity": 0, - "severity_value": 0.4861490440889582, + "severity_value": 0.4946, "code": "best_score <= 2", - "message": "Method scanvi_hvg performs a lot better than baselines.\n Task id: label_projection\n Method id: scanvi_hvg\n Metric id: accuracy\n Best score: 0.9722980881779164%\n" + "message": "Method singler performs a lot better than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: accuracy\n Best score: 0.9892%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scarches_scanvi_all_genes accuracy", - "value": 0.20387053720387055, + "name": "Worst score uce accuracy", + "value": 0.0, "severity": 0, - "severity_value": -0.20387053720387055, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scarches_scanvi_all_genes performs much worse than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_all_genes\n Metric id: accuracy\n Worst score: 0.20387053720387055%\n" + "message": "Method uce performs much worse than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: accuracy\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scarches_scanvi_all_genes accuracy", - "value": 0.9430355052672649, + "name": "Best score uce accuracy", + "value": 0.0234, "severity": 0, - "severity_value": 0.47151775263363244, + "severity_value": 0.0117, "code": "best_score <= 2", - "message": "Method scarches_scanvi_all_genes performs a lot better than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_all_genes\n Metric id: accuracy\n Best score: 0.9430355052672649%\n" + "message": "Method uce performs a lot better than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: accuracy\n Best score: 0.0234%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scarches_scanvi_hvg accuracy", - "value": 0.0668693009118541, + "name": "Worst score xgboost accuracy", + "value": 0.2561, "severity": 0, - "severity_value": -0.0668693009118541, + "severity_value": -0.2561, "code": "worst_score >= -1", - "message": "Method scarches_scanvi_hvg performs much worse than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_hvg\n Metric id: accuracy\n Worst score: 0.0668693009118541%\n" + "message": "Method xgboost performs much worse than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: accuracy\n Worst score: 0.2561%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scarches_scanvi_hvg accuracy", - "value": 0.944596176355833, + "name": "Best score xgboost accuracy", + "value": 0.995, "severity": 0, - "severity_value": 0.4722980881779165, + "severity_value": 0.4975, "code": "best_score <= 2", - "message": "Method scarches_scanvi_hvg performs a lot better than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_hvg\n Metric id: accuracy\n Best score: 0.944596176355833%\n" + "message": "Method xgboost performs a lot better than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: accuracy\n Best score: 0.995%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score seurat accuracy", - "value": 0.32552552552552555, + "name": "Worst score majority_vote f1_weighted", + "value": 0.0, "severity": 0, - "severity_value": -0.32552552552552555, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method seurat performs much worse than baselines.\n Task id: label_projection\n Method id: seurat\n Metric id: accuracy\n Worst score: 0.32552552552552555%\n" + "message": "Method majority_vote performs much worse than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: f1_weighted\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score seurat accuracy", - "value": 0.9773702692157628, + "name": "Best score majority_vote f1_weighted", + "value": 0.2066, "severity": 0, - "severity_value": 0.4886851346078814, + "severity_value": 0.1033, "code": "best_score <= 2", - "message": "Method seurat performs a lot better than baselines.\n Task id: label_projection\n Method id: seurat\n Metric id: accuracy\n Best score: 0.9773702692157628%\n" + "message": "Method majority_vote performs a lot better than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: f1_weighted\n Best score: 0.2066%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score true_labels accuracy", - "value": 1.0, + "name": "Worst score random_labels f1_weighted", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: f1_weighted\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score random_labels f1_weighted", + "value": 0.1408, + "severity": 0, + "severity_value": 0.0704, + "code": "best_score <= 2", + "message": "Method random_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: f1_weighted\n Best score: 0.1408%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score true_labels f1_weighted", + "value": 1, "severity": 0, "severity_value": -1.0, "code": "worst_score >= -1", - "message": "Method true_labels performs much worse than baselines.\n Task id: label_projection\n Method id: true_labels\n Metric id: accuracy\n Worst score: 1.0%\n" + "message": "Method true_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: f1_weighted\n Worst score: 1%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score true_labels accuracy", - "value": 1.0, + "name": "Best score true_labels f1_weighted", + "value": 1, "severity": 0, "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method true_labels performs a lot better than baselines.\n Task id: label_projection\n Method id: true_labels\n Metric id: accuracy\n Best score: 1.0%\n" + "message": "Method true_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: f1_weighted\n Best score: 1%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score xgboost_log_cp10k accuracy", - "value": 0.2022022022022022, + "name": "Worst score geneformer f1_weighted", + "value": 0, "severity": 0, - "severity_value": -0.2022022022022022, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method xgboost_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: xgboost_log_cp10k\n Metric id: accuracy\n Worst score: 0.2022022022022022%\n" + "message": "Method geneformer performs much worse than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: f1_weighted\n Worst score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score xgboost_log_cp10k accuracy", - "value": 0.9707374170893485, + "name": "Best score geneformer f1_weighted", + "value": 0, "severity": 0, - "severity_value": 0.48536870854467423, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method xgboost_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: xgboost_log_cp10k\n Metric id: accuracy\n Best score: 0.9707374170893485%\n" + "message": "Method geneformer performs a lot better than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: f1_weighted\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score xgboost_scran accuracy", - "value": 0.21401401401401401, + "name": "Worst score knn f1_weighted", + "value": 0.2459, "severity": 0, - "severity_value": -0.21401401401401401, + "severity_value": -0.2459, "code": "worst_score >= -1", - "message": "Method xgboost_scran performs much worse than baselines.\n Task id: label_projection\n Method id: xgboost_scran\n Metric id: accuracy\n Worst score: 0.21401401401401401%\n" + "message": "Method knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: f1_weighted\n Worst score: 0.2459%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score xgboost_scran accuracy", - "value": 0.9605930550136559, + "name": "Best score knn f1_weighted", + "value": 0.9992, "severity": 0, - "severity_value": 0.48029652750682794, + "severity_value": 0.4996, "code": "best_score <= 2", - "message": "Method xgboost_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: xgboost_scran\n Metric id: accuracy\n Best score: 0.9605930550136559%\n" + "message": "Method knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: f1_weighted\n Best score: 0.9992%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score knn_classifier_log_cp10k f1", - "value": 0.10780750046605095, + "name": "Worst score logistic_regression f1_weighted", + "value": 0.1681, "severity": 0, - "severity_value": -0.10780750046605095, + "severity_value": -0.1681, "code": "worst_score >= -1", - "message": "Method knn_classifier_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: knn_classifier_log_cp10k\n Metric id: f1\n Worst score: 0.10780750046605095%\n" + "message": "Method logistic_regression performs much worse than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: f1_weighted\n Worst score: 0.1681%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score knn_classifier_log_cp10k f1", - "value": 0.9542446392734802, + "name": "Best score logistic_regression f1_weighted", + "value": 0.9992, "severity": 0, - "severity_value": 0.4771223196367401, + "severity_value": 0.4996, "code": "best_score <= 2", - "message": "Method knn_classifier_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: knn_classifier_log_cp10k\n Metric id: f1\n Best score: 0.9542446392734802%\n" + "message": "Method logistic_regression performs a lot better than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: f1_weighted\n Best score: 0.9992%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score knn_classifier_scran f1", - "value": 0.16976683609619148, + "name": "Worst score mlp f1_weighted", + "value": 0.2594, "severity": 0, - "severity_value": -0.16976683609619148, + "severity_value": -0.2594, "code": "worst_score >= -1", - "message": "Method knn_classifier_scran performs much worse than baselines.\n Task id: label_projection\n Method id: knn_classifier_scran\n Metric id: f1\n Worst score: 0.16976683609619148%\n" + "message": "Method mlp performs much worse than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: f1_weighted\n Worst score: 0.2594%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score knn_classifier_scran f1", - "value": 0.9532095976968298, + "name": "Best score mlp f1_weighted", + "value": 1.0, "severity": 0, - "severity_value": 0.4766047988484149, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method knn_classifier_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: knn_classifier_scran\n Metric id: f1\n Best score: 0.9532095976968298%\n" + "message": "Method mlp performs a lot better than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: f1_weighted\n Best score: 1.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score logistic_regression_log_cp10k f1", - "value": 0.2588674947434843, + "name": "Worst score naive_bayes f1_weighted", + "value": 0.2154, "severity": 0, - "severity_value": -0.2588674947434843, + "severity_value": -0.2154, "code": "worst_score >= -1", - "message": "Method logistic_regression_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: logistic_regression_log_cp10k\n Metric id: f1\n Worst score: 0.2588674947434843%\n" + "message": "Method naive_bayes performs much worse than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: f1_weighted\n Worst score: 0.2154%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score logistic_regression_log_cp10k f1", - "value": 0.9843211893874999, + "name": "Best score naive_bayes f1_weighted", + "value": 0.9858, "severity": 0, - "severity_value": 0.49216059469374995, + "severity_value": 0.4929, "code": "best_score <= 2", - "message": "Method logistic_regression_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: logistic_regression_log_cp10k\n Metric id: f1\n Best score: 0.9843211893874999%\n" + "message": "Method naive_bayes performs a lot better than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: f1_weighted\n Best score: 0.9858%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score logistic_regression_scran f1", - "value": 0.26181121454905526, + "name": "Worst score scanvi f1_weighted", + "value": 0.2525, "severity": 0, - "severity_value": -0.26181121454905526, + "severity_value": -0.2525, "code": "worst_score >= -1", - "message": "Method logistic_regression_scran performs much worse than baselines.\n Task id: label_projection\n Method id: logistic_regression_scran\n Metric id: f1\n Worst score: 0.26181121454905526%\n" + "message": "Method scanvi performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: f1_weighted\n Worst score: 0.2525%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score logistic_regression_scran f1", - "value": 0.9241759560319147, + "name": "Best score scanvi f1_weighted", + "value": 0.997, "severity": 0, - "severity_value": 0.46208797801595736, + "severity_value": 0.4985, "code": "best_score <= 2", - "message": "Method logistic_regression_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: logistic_regression_scran\n Metric id: f1\n Best score: 0.9241759560319147%\n" + "message": "Method scanvi performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: f1_weighted\n Best score: 0.997%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score majority_vote f1", - "value": -0.07442059841081748, + "name": "Worst score scanvi_scarches f1_weighted", + "value": 0.2639, "severity": 0, - "severity_value": 0.07442059841081748, + "severity_value": -0.2639, "code": "worst_score >= -1", - "message": "Method majority_vote performs much worse than baselines.\n Task id: label_projection\n Method id: majority_vote\n Metric id: f1\n Worst score: -0.07442059841081748%\n" + "message": "Method scanvi_scarches performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: f1_weighted\n Worst score: 0.2639%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score majority_vote f1", - "value": -0.00876806656780211, + "name": "Best score scanvi_scarches f1_weighted", + "value": 0.9985, "severity": 0, - "severity_value": -0.004384033283901055, + "severity_value": 0.49925, "code": "best_score <= 2", - "message": "Method majority_vote performs a lot better than baselines.\n Task id: label_projection\n Method id: majority_vote\n Metric id: f1\n Best score: -0.00876806656780211%\n" + "message": "Method scanvi_scarches performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: f1_weighted\n Best score: 0.9985%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score mlp_log_cp10k f1", - "value": 0.24709381340904898, + "name": "Worst score scgpt_zero_shot f1_weighted", + "value": 0, "severity": 0, - "severity_value": -0.24709381340904898, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method mlp_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: mlp_log_cp10k\n Metric id: f1\n Worst score: 0.24709381340904898%\n" + "message": "Method scgpt_zero_shot performs much worse than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: f1_weighted\n Worst score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score mlp_log_cp10k f1", - "value": 0.9846713012910284, + "name": "Best score scgpt_zero_shot f1_weighted", + "value": 0, "severity": 0, - "severity_value": 0.4923356506455142, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method mlp_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: mlp_log_cp10k\n Metric id: f1\n Best score: 0.9846713012910284%\n" + "message": "Method scgpt_zero_shot performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: f1_weighted\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score mlp_scran f1", - "value": 0.2789499346377783, + "name": "Worst score scimilarity f1_weighted", + "value": 0.0, "severity": 0, - "severity_value": -0.2789499346377783, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method mlp_scran performs much worse than baselines.\n Task id: label_projection\n Method id: mlp_scran\n Metric id: f1\n Worst score: 0.2789499346377783%\n" + "message": "Method scimilarity performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: f1_weighted\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score mlp_scran f1", - "value": 0.9806886504233998, + "name": "Best score scimilarity f1_weighted", + "value": 0.8833, "severity": 0, - "severity_value": 0.4903443252116999, + "severity_value": 0.44165, "code": "best_score <= 2", - "message": "Method mlp_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: mlp_scran\n Metric id: f1\n Best score: 0.9806886504233998%\n" + "message": "Method scimilarity performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: f1_weighted\n Best score: 0.8833%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score random_labels f1", + "name": "Worst score scimilarity_knn f1_weighted", "value": 0.0, "severity": 0, "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method random_labels performs much worse than baselines.\n Task id: label_projection\n Method id: random_labels\n Metric id: f1\n Worst score: 0.0%\n" + "message": "Method scimilarity_knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: f1_weighted\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score random_labels f1", - "value": 0.0, + "name": "Best score scimilarity_knn f1_weighted", + "value": 0.9375, + "severity": 0, + "severity_value": 0.46875, + "code": "best_score <= 2", + "message": "Method scimilarity_knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: f1_weighted\n Best score: 0.9375%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scprint f1_weighted", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scprint performs much worse than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: f1_weighted\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scprint f1_weighted", + "value": 0, "severity": 0, "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method random_labels performs a lot better than baselines.\n Task id: label_projection\n Method id: random_labels\n Metric id: f1\n Best score: 0.0%\n" + "message": "Method scprint performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: f1_weighted\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scanvi_all_genes f1", - "value": 0.2138243379016213, + "name": "Worst score seurat_transferdata f1_weighted", + "value": 0.0, "severity": 0, - "severity_value": -0.2138243379016213, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scanvi_all_genes performs much worse than baselines.\n Task id: label_projection\n Method id: scanvi_all_genes\n Metric id: f1\n Worst score: 0.2138243379016213%\n" + "message": "Method seurat_transferdata performs much worse than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: f1_weighted\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scanvi_all_genes f1", - "value": 0.9644662887296875, + "name": "Best score seurat_transferdata f1_weighted", + "value": 0.9933, "severity": 0, - "severity_value": 0.48223314436484377, + "severity_value": 0.49665, "code": "best_score <= 2", - "message": "Method scanvi_all_genes performs a lot better than baselines.\n Task id: label_projection\n Method id: scanvi_all_genes\n Metric id: f1\n Best score: 0.9644662887296875%\n" + "message": "Method seurat_transferdata performs a lot better than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: f1_weighted\n Best score: 0.9933%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scanvi_hvg f1", - "value": 0.19495198215872958, + "name": "Worst score singler f1_weighted", + "value": 0.0, "severity": 0, - "severity_value": -0.19495198215872958, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scanvi_hvg performs much worse than baselines.\n Task id: label_projection\n Method id: scanvi_hvg\n Metric id: f1\n Worst score: 0.19495198215872958%\n" + "message": "Method singler performs much worse than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: f1_weighted\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scanvi_hvg f1", - "value": 0.9689534392731689, + "name": "Best score singler f1_weighted", + "value": 0.9905, "severity": 0, - "severity_value": 0.48447671963658445, + "severity_value": 0.49525, "code": "best_score <= 2", - "message": "Method scanvi_hvg performs a lot better than baselines.\n Task id: label_projection\n Method id: scanvi_hvg\n Metric id: f1\n Best score: 0.9689534392731689%\n" + "message": "Method singler performs a lot better than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: f1_weighted\n Best score: 0.9905%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scarches_scanvi_all_genes f1", - "value": 0.1755299643837517, + "name": "Worst score uce f1_weighted", + "value": 0.0, "severity": 0, - "severity_value": -0.1755299643837517, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scarches_scanvi_all_genes performs much worse than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_all_genes\n Metric id: f1\n Worst score: 0.1755299643837517%\n" + "message": "Method uce performs much worse than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: f1_weighted\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scarches_scanvi_all_genes f1", - "value": 0.9357377852175203, + "name": "Best score uce f1_weighted", + "value": 0.1518, "severity": 0, - "severity_value": 0.46786889260876013, + "severity_value": 0.0759, "code": "best_score <= 2", - "message": "Method scarches_scanvi_all_genes performs a lot better than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_all_genes\n Metric id: f1\n Best score: 0.9357377852175203%\n" + "message": "Method uce performs a lot better than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: f1_weighted\n Best score: 0.1518%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scarches_scanvi_hvg f1", - "value": 0.013953154018990551, + "name": "Worst score xgboost f1_weighted", + "value": 0.2498, "severity": 0, - "severity_value": -0.013953154018990551, + "severity_value": -0.2498, "code": "worst_score >= -1", - "message": "Method scarches_scanvi_hvg performs much worse than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_hvg\n Metric id: f1\n Worst score: 0.013953154018990551%\n" + "message": "Method xgboost performs much worse than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: f1_weighted\n Worst score: 0.2498%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scarches_scanvi_hvg f1", - "value": 0.9385705708072999, + "name": "Best score xgboost f1_weighted", + "value": 0.9955, "severity": 0, - "severity_value": 0.46928528540364994, + "severity_value": 0.49775, "code": "best_score <= 2", - "message": "Method scarches_scanvi_hvg performs a lot better than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_hvg\n Metric id: f1\n Best score: 0.9385705708072999%\n" + "message": "Method xgboost performs a lot better than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: f1_weighted\n Best score: 0.9955%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score seurat f1", - "value": 0.3865924115203712, + "name": "Worst score majority_vote f1_macro", + "value": 0.0, "severity": 0, - "severity_value": -0.3865924115203712, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method seurat performs much worse than baselines.\n Task id: label_projection\n Method id: seurat\n Metric id: f1\n Worst score: 0.3865924115203712%\n" + "message": "Method majority_vote performs much worse than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: f1_macro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score seurat f1", - "value": 0.9764566346823227, + "name": "Best score majority_vote f1_macro", + "value": 0.0155, "severity": 0, - "severity_value": 0.48822831734116134, + "severity_value": 0.00775, "code": "best_score <= 2", - "message": "Method seurat performs a lot better than baselines.\n Task id: label_projection\n Method id: seurat\n Metric id: f1\n Best score: 0.9764566346823227%\n" + "message": "Method majority_vote performs a lot better than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: f1_macro\n Best score: 0.0155%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score true_labels f1", - "value": 1.0, + "name": "Worst score random_labels f1_macro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: f1_macro\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score random_labels f1_macro", + "value": 0.0482, + "severity": 0, + "severity_value": 0.0241, + "code": "best_score <= 2", + "message": "Method random_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: f1_macro\n Best score: 0.0482%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score true_labels f1_macro", + "value": 1, "severity": 0, "severity_value": -1.0, "code": "worst_score >= -1", - "message": "Method true_labels performs much worse than baselines.\n Task id: label_projection\n Method id: true_labels\n Metric id: f1\n Worst score: 1.0%\n" + "message": "Method true_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: f1_macro\n Worst score: 1%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score true_labels f1", - "value": 1.0, + "name": "Best score true_labels f1_macro", + "value": 1, "severity": 0, "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method true_labels performs a lot better than baselines.\n Task id: label_projection\n Method id: true_labels\n Metric id: f1\n Best score: 1.0%\n" + "message": "Method true_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: f1_macro\n Best score: 1%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score xgboost_log_cp10k f1", - "value": 0.2493315159411224, + "name": "Worst score geneformer f1_macro", + "value": 0, "severity": 0, - "severity_value": -0.2493315159411224, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method xgboost_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: xgboost_log_cp10k\n Metric id: f1\n Worst score: 0.2493315159411224%\n" + "message": "Method geneformer performs much worse than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: f1_macro\n Worst score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score xgboost_log_cp10k f1", - "value": 0.970651449032685, + "name": "Best score geneformer f1_macro", + "value": 0, "severity": 0, - "severity_value": 0.4853257245163425, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method xgboost_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: xgboost_log_cp10k\n Metric id: f1\n Best score: 0.970651449032685%\n" + "message": "Method geneformer performs a lot better than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: f1_macro\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score xgboost_scran f1", - "value": 0.26983439097881096, + "name": "Worst score knn f1_macro", + "value": 0.1876, "severity": 0, - "severity_value": -0.26983439097881096, + "severity_value": -0.1876, "code": "worst_score >= -1", - "message": "Method xgboost_scran performs much worse than baselines.\n Task id: label_projection\n Method id: xgboost_scran\n Metric id: f1\n Worst score: 0.26983439097881096%\n" + "message": "Method knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: f1_macro\n Worst score: 0.1876%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score xgboost_scran f1", - "value": 0.9601252037606909, + "name": "Best score knn f1_macro", + "value": 0.9997, "severity": 0, - "severity_value": 0.48006260188034544, + "severity_value": 0.49985, "code": "best_score <= 2", - "message": "Method xgboost_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: xgboost_scran\n Metric id: f1\n Best score: 0.9601252037606909%\n" + "message": "Method knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: f1_macro\n Best score: 0.9997%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score knn_classifier_log_cp10k f1_macro", - "value": 0.15159754888561494, + "name": "Worst score logistic_regression f1_macro", + "value": 0.1708, "severity": 0, - "severity_value": -0.15159754888561494, + "severity_value": -0.1708, "code": "worst_score >= -1", - "message": "Method knn_classifier_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: knn_classifier_log_cp10k\n Metric id: f1_macro\n Worst score: 0.15159754888561494%\n" + "message": "Method logistic_regression performs much worse than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: f1_macro\n Worst score: 0.1708%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score knn_classifier_log_cp10k f1_macro", - "value": 0.8876627930241076, + "name": "Best score logistic_regression f1_macro", + "value": 0.9997, "severity": 0, - "severity_value": 0.4438313965120538, + "severity_value": 0.49985, "code": "best_score <= 2", - "message": "Method knn_classifier_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: knn_classifier_log_cp10k\n Metric id: f1_macro\n Best score: 0.8876627930241076%\n" + "message": "Method logistic_regression performs a lot better than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: f1_macro\n Best score: 0.9997%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score knn_classifier_scran f1_macro", - "value": 0.18732004254704496, + "name": "Worst score mlp f1_macro", + "value": 0.2181, "severity": 0, - "severity_value": -0.18732004254704496, + "severity_value": -0.2181, "code": "worst_score >= -1", - "message": "Method knn_classifier_scran performs much worse than baselines.\n Task id: label_projection\n Method id: knn_classifier_scran\n Metric id: f1_macro\n Worst score: 0.18732004254704496%\n" + "message": "Method mlp performs much worse than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: f1_macro\n Worst score: 0.2181%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score knn_classifier_scran f1_macro", - "value": 0.8231022304862172, + "name": "Best score mlp f1_macro", + "value": 1.0, "severity": 0, - "severity_value": 0.4115511152431086, + "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method knn_classifier_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: knn_classifier_scran\n Metric id: f1_macro\n Best score: 0.8231022304862172%\n" + "message": "Method mlp performs a lot better than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: f1_macro\n Best score: 1.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score logistic_regression_log_cp10k f1_macro", - "value": 0.22097832109787452, + "name": "Worst score naive_bayes f1_macro", + "value": 0.1763, "severity": 0, - "severity_value": -0.22097832109787452, + "severity_value": -0.1763, "code": "worst_score >= -1", - "message": "Method logistic_regression_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: logistic_regression_log_cp10k\n Metric id: f1_macro\n Worst score: 0.22097832109787452%\n" + "message": "Method naive_bayes performs much worse than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: f1_macro\n Worst score: 0.1763%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score logistic_regression_log_cp10k f1_macro", - "value": 0.9700382220043122, + "name": "Best score naive_bayes f1_macro", + "value": 0.8912, "severity": 0, - "severity_value": 0.4850191110021561, + "severity_value": 0.4456, "code": "best_score <= 2", - "message": "Method logistic_regression_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: logistic_regression_log_cp10k\n Metric id: f1_macro\n Best score: 0.9700382220043122%\n" + "message": "Method naive_bayes performs a lot better than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: f1_macro\n Best score: 0.8912%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score logistic_regression_scran f1_macro", - "value": 0.21272783006514337, + "name": "Worst score scanvi f1_macro", + "value": 0.2035, "severity": 0, - "severity_value": -0.21272783006514337, + "severity_value": -0.2035, "code": "worst_score >= -1", - "message": "Method logistic_regression_scran performs much worse than baselines.\n Task id: label_projection\n Method id: logistic_regression_scran\n Metric id: f1_macro\n Worst score: 0.21272783006514337%\n" + "message": "Method scanvi performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: f1_macro\n Worst score: 0.2035%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score logistic_regression_scran f1_macro", - "value": 0.872600017009661, + "name": "Best score scanvi f1_macro", + "value": 0.999, "severity": 0, - "severity_value": 0.4363000085048305, + "severity_value": 0.4995, "code": "best_score <= 2", - "message": "Method logistic_regression_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: logistic_regression_scran\n Metric id: f1_macro\n Best score: 0.872600017009661%\n" + "message": "Method scanvi performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: f1_macro\n Best score: 0.999%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score majority_vote f1_macro", - "value": -0.04419766743032169, + "name": "Worst score scanvi_scarches f1_macro", + "value": 0.2324, "severity": 0, - "severity_value": 0.04419766743032169, + "severity_value": -0.2324, "code": "worst_score >= -1", - "message": "Method majority_vote performs much worse than baselines.\n Task id: label_projection\n Method id: majority_vote\n Metric id: f1_macro\n Worst score: -0.04419766743032169%\n" + "message": "Method scanvi_scarches performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: f1_macro\n Worst score: 0.2324%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score majority_vote f1_macro", - "value": -0.0016591611633974808, + "name": "Best score scanvi_scarches f1_macro", + "value": 0.9956, "severity": 0, - "severity_value": -0.0008295805816987404, + "severity_value": 0.4978, "code": "best_score <= 2", - "message": "Method majority_vote performs a lot better than baselines.\n Task id: label_projection\n Method id: majority_vote\n Metric id: f1_macro\n Best score: -0.0016591611633974808%\n" + "message": "Method scanvi_scarches performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: f1_macro\n Best score: 0.9956%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score mlp_log_cp10k f1_macro", - "value": 0.19155253850770848, + "name": "Worst score scgpt_zero_shot f1_macro", + "value": 0, "severity": 0, - "severity_value": -0.19155253850770848, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method mlp_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: mlp_log_cp10k\n Metric id: f1_macro\n Worst score: 0.19155253850770848%\n" + "message": "Method scgpt_zero_shot performs much worse than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: f1_macro\n Worst score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score mlp_log_cp10k f1_macro", - "value": 0.9685402219165037, + "name": "Best score scgpt_zero_shot f1_macro", + "value": 0, "severity": 0, - "severity_value": 0.48427011095825184, + "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method mlp_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: mlp_log_cp10k\n Metric id: f1_macro\n Best score: 0.9685402219165037%\n" + "message": "Method scgpt_zero_shot performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: f1_macro\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score mlp_scran f1_macro", - "value": 0.15550077289410666, + "name": "Worst score scimilarity f1_macro", + "value": 0.0, "severity": 0, - "severity_value": -0.15550077289410666, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method mlp_scran performs much worse than baselines.\n Task id: label_projection\n Method id: mlp_scran\n Metric id: f1_macro\n Worst score: 0.15550077289410666%\n" + "message": "Method scimilarity performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: f1_macro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score mlp_scran f1_macro", - "value": 0.8886750027095885, + "name": "Best score scimilarity f1_macro", + "value": 0.7501, "severity": 0, - "severity_value": 0.4443375013547943, + "severity_value": 0.37505, "code": "best_score <= 2", - "message": "Method mlp_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: mlp_scran\n Metric id: f1_macro\n Best score: 0.8886750027095885%\n" + "message": "Method scimilarity performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: f1_macro\n Best score: 0.7501%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score random_labels f1_macro", + "name": "Worst score scimilarity_knn f1_macro", "value": 0.0, "severity": 0, "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method random_labels performs much worse than baselines.\n Task id: label_projection\n Method id: random_labels\n Metric id: f1_macro\n Worst score: 0.0%\n" + "message": "Method scimilarity_knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: f1_macro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score random_labels f1_macro", - "value": 0.0, + "name": "Best score scimilarity_knn f1_macro", + "value": 0.839, + "severity": 0, + "severity_value": 0.4195, + "code": "best_score <= 2", + "message": "Method scimilarity_knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: f1_macro\n Best score: 0.839%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scprint f1_macro", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scprint performs much worse than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: f1_macro\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scprint f1_macro", + "value": 0, "severity": 0, "severity_value": 0.0, "code": "best_score <= 2", - "message": "Method random_labels performs a lot better than baselines.\n Task id: label_projection\n Method id: random_labels\n Metric id: f1_macro\n Best score: 0.0%\n" + "message": "Method scprint performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: f1_macro\n Best score: 0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scanvi_all_genes f1_macro", - "value": 0.19549993826984827, + "name": "Worst score seurat_transferdata f1_macro", + "value": 0.0, "severity": 0, - "severity_value": -0.19549993826984827, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scanvi_all_genes performs much worse than baselines.\n Task id: label_projection\n Method id: scanvi_all_genes\n Metric id: f1_macro\n Worst score: 0.19549993826984827%\n" + "message": "Method seurat_transferdata performs much worse than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: f1_macro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scanvi_all_genes f1_macro", - "value": 0.651766900954443, + "name": "Best score seurat_transferdata f1_macro", + "value": 0.9013, "severity": 0, - "severity_value": 0.3258834504772215, + "severity_value": 0.45065, "code": "best_score <= 2", - "message": "Method scanvi_all_genes performs a lot better than baselines.\n Task id: label_projection\n Method id: scanvi_all_genes\n Metric id: f1_macro\n Best score: 0.651766900954443%\n" + "message": "Method seurat_transferdata performs a lot better than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: f1_macro\n Best score: 0.9013%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scanvi_hvg f1_macro", - "value": 0.15940275123790326, + "name": "Worst score singler f1_macro", + "value": 0.0, "severity": 0, - "severity_value": -0.15940275123790326, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scanvi_hvg performs much worse than baselines.\n Task id: label_projection\n Method id: scanvi_hvg\n Metric id: f1_macro\n Worst score: 0.15940275123790326%\n" + "message": "Method singler performs much worse than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: f1_macro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scanvi_hvg f1_macro", - "value": 0.6639548374116637, + "name": "Best score singler f1_macro", + "value": 0.8229, "severity": 0, - "severity_value": 0.33197741870583186, + "severity_value": 0.41145, "code": "best_score <= 2", - "message": "Method scanvi_hvg performs a lot better than baselines.\n Task id: label_projection\n Method id: scanvi_hvg\n Metric id: f1_macro\n Best score: 0.6639548374116637%\n" + "message": "Method singler performs a lot better than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: f1_macro\n Best score: 0.8229%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scarches_scanvi_all_genes f1_macro", - "value": 0.07097550781577062, + "name": "Worst score uce f1_macro", + "value": 0.0, "severity": 0, - "severity_value": -0.07097550781577062, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method scarches_scanvi_all_genes performs much worse than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_all_genes\n Metric id: f1_macro\n Worst score: 0.07097550781577062%\n" + "message": "Method uce performs much worse than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: f1_macro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scarches_scanvi_all_genes f1_macro", - "value": 0.6184944871059823, + "name": "Best score uce f1_macro", + "value": 0.0501, "severity": 0, - "severity_value": 0.30924724355299116, + "severity_value": 0.02505, "code": "best_score <= 2", - "message": "Method scarches_scanvi_all_genes performs a lot better than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_all_genes\n Metric id: f1_macro\n Best score: 0.6184944871059823%\n" + "message": "Method uce performs a lot better than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: f1_macro\n Best score: 0.0501%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score scarches_scanvi_hvg f1_macro", - "value": 0.034737617054860816, + "name": "Worst score xgboost f1_macro", + "value": 0.2179, "severity": 0, - "severity_value": -0.034737617054860816, + "severity_value": -0.2179, "code": "worst_score >= -1", - "message": "Method scarches_scanvi_hvg performs much worse than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_hvg\n Metric id: f1_macro\n Worst score: 0.034737617054860816%\n" + "message": "Method xgboost performs much worse than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: f1_macro\n Worst score: 0.2179%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score scarches_scanvi_hvg f1_macro", - "value": 0.549779548817173, + "name": "Best score xgboost f1_macro", + "value": 0.9985, "severity": 0, - "severity_value": 0.2748897744085865, + "severity_value": 0.49925, "code": "best_score <= 2", - "message": "Method scarches_scanvi_hvg performs a lot better than baselines.\n Task id: label_projection\n Method id: scarches_scanvi_hvg\n Metric id: f1_macro\n Best score: 0.549779548817173%\n" + "message": "Method xgboost performs a lot better than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: f1_macro\n Best score: 0.9985%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score seurat f1_macro", - "value": 0.2725134501012934, + "name": "Worst score majority_vote f1_micro", + "value": 0.0, "severity": 0, - "severity_value": -0.2725134501012934, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method seurat performs much worse than baselines.\n Task id: label_projection\n Method id: seurat\n Metric id: f1_macro\n Worst score: 0.2725134501012934%\n" + "message": "Method majority_vote performs much worse than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: f1_micro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score seurat f1_macro", - "value": 0.8885645457392208, + "name": "Best score majority_vote f1_micro", + "value": 0.4546, "severity": 0, - "severity_value": 0.4442822728696104, + "severity_value": 0.2273, "code": "best_score <= 2", - "message": "Method seurat performs a lot better than baselines.\n Task id: label_projection\n Method id: seurat\n Metric id: f1_macro\n Best score: 0.8885645457392208%\n" + "message": "Method majority_vote performs a lot better than baselines.\n Task id: task_label_projection\n Method id: majority_vote\n Metric id: f1_micro\n Best score: 0.4546%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score true_labels f1_macro", - "value": 1.0, + "name": "Worst score random_labels f1_micro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method random_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: f1_micro\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score random_labels f1_micro", + "value": 0.0737, + "severity": 0, + "severity_value": 0.03685, + "code": "best_score <= 2", + "message": "Method random_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: random_labels\n Metric id: f1_micro\n Best score: 0.0737%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score true_labels f1_micro", + "value": 1, "severity": 0, "severity_value": -1.0, "code": "worst_score >= -1", - "message": "Method true_labels performs much worse than baselines.\n Task id: label_projection\n Method id: true_labels\n Metric id: f1_macro\n Worst score: 1.0%\n" + "message": "Method true_labels performs much worse than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: f1_micro\n Worst score: 1%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score true_labels f1_macro", + "name": "Best score true_labels f1_micro", + "value": 1, + "severity": 0, + "severity_value": 0.5, + "code": "best_score <= 2", + "message": "Method true_labels performs a lot better than baselines.\n Task id: task_label_projection\n Method id: true_labels\n Metric id: f1_micro\n Best score: 1%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score geneformer f1_micro", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method geneformer performs much worse than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: f1_micro\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score geneformer f1_micro", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method geneformer performs a lot better than baselines.\n Task id: task_label_projection\n Method id: geneformer\n Metric id: f1_micro\n Best score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score knn f1_micro", + "value": 0.2486, + "severity": 0, + "severity_value": -0.2486, + "code": "worst_score >= -1", + "message": "Method knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: f1_micro\n Worst score: 0.2486%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score knn f1_micro", + "value": 0.9992, + "severity": 0, + "severity_value": 0.4996, + "code": "best_score <= 2", + "message": "Method knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: knn\n Metric id: f1_micro\n Best score: 0.9992%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score logistic_regression f1_micro", + "value": -0.0107, + "severity": 0, + "severity_value": 0.0107, + "code": "worst_score >= -1", + "message": "Method logistic_regression performs much worse than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: f1_micro\n Worst score: -0.0107%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score logistic_regression f1_micro", + "value": 0.9992, + "severity": 0, + "severity_value": 0.4996, + "code": "best_score <= 2", + "message": "Method logistic_regression performs a lot better than baselines.\n Task id: task_label_projection\n Method id: logistic_regression\n Metric id: f1_micro\n Best score: 0.9992%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score mlp f1_micro", + "value": 0.2644, + "severity": 0, + "severity_value": -0.2644, + "code": "worst_score >= -1", + "message": "Method mlp performs much worse than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: f1_micro\n Worst score: 0.2644%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score mlp f1_micro", "value": 1.0, "severity": 0, "severity_value": 0.5, "code": "best_score <= 2", - "message": "Method true_labels performs a lot better than baselines.\n Task id: label_projection\n Method id: true_labels\n Metric id: f1_macro\n Best score: 1.0%\n" + "message": "Method mlp performs a lot better than baselines.\n Task id: task_label_projection\n Method id: mlp\n Metric id: f1_micro\n Best score: 1.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score naive_bayes f1_micro", + "value": 0.189, + "severity": 0, + "severity_value": -0.189, + "code": "worst_score >= -1", + "message": "Method naive_bayes performs much worse than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: f1_micro\n Worst score: 0.189%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score naive_bayes f1_micro", + "value": 0.9851, + "severity": 0, + "severity_value": 0.49255, + "code": "best_score <= 2", + "message": "Method naive_bayes performs a lot better than baselines.\n Task id: task_label_projection\n Method id: naive_bayes\n Metric id: f1_micro\n Best score: 0.9851%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scanvi f1_micro", + "value": 0.2797, + "severity": 0, + "severity_value": -0.2797, + "code": "worst_score >= -1", + "message": "Method scanvi performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: f1_micro\n Worst score: 0.2797%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scanvi f1_micro", + "value": 0.9967, + "severity": 0, + "severity_value": 0.49835, + "code": "best_score <= 2", + "message": "Method scanvi performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi\n Metric id: f1_micro\n Best score: 0.9967%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scanvi_scarches f1_micro", + "value": 0.28, + "severity": 0, + "severity_value": -0.28, + "code": "worst_score >= -1", + "message": "Method scanvi_scarches performs much worse than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: f1_micro\n Worst score: 0.28%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score xgboost_log_cp10k f1_macro", - "value": 0.2023839844503912, + "name": "Best score scanvi_scarches f1_micro", + "value": 0.9983, "severity": 0, - "severity_value": -0.2023839844503912, + "severity_value": 0.49915, + "code": "best_score <= 2", + "message": "Method scanvi_scarches performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scanvi_scarches\n Metric id: f1_micro\n Best score: 0.9983%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scgpt_zero_shot f1_micro", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scgpt_zero_shot performs much worse than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: f1_micro\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scgpt_zero_shot f1_micro", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method scgpt_zero_shot performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scgpt_zero_shot\n Metric id: f1_micro\n Best score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scimilarity f1_micro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scimilarity performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: f1_micro\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scimilarity f1_micro", + "value": 0.873, + "severity": 0, + "severity_value": 0.4365, + "code": "best_score <= 2", + "message": "Method scimilarity performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity\n Metric id: f1_micro\n Best score: 0.873%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scimilarity_knn f1_micro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scimilarity_knn performs much worse than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: f1_micro\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scimilarity_knn f1_micro", + "value": 0.9188, + "severity": 0, + "severity_value": 0.4594, + "code": "best_score <= 2", + "message": "Method scimilarity_knn performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scimilarity_knn\n Metric id: f1_micro\n Best score: 0.9188%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score scprint f1_micro", + "value": 0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method scprint performs much worse than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: f1_micro\n Worst score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score scprint f1_micro", + "value": 0, + "severity": 0, + "severity_value": 0.0, + "code": "best_score <= 2", + "message": "Method scprint performs a lot better than baselines.\n Task id: task_label_projection\n Method id: scprint\n Metric id: f1_micro\n Best score: 0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score seurat_transferdata f1_micro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method seurat_transferdata performs much worse than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: f1_micro\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score seurat_transferdata f1_micro", + "value": 0.9925, + "severity": 0, + "severity_value": 0.49625, + "code": "best_score <= 2", + "message": "Method seurat_transferdata performs a lot better than baselines.\n Task id: task_label_projection\n Method id: seurat_transferdata\n Metric id: f1_micro\n Best score: 0.9925%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score singler f1_micro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, + "code": "worst_score >= -1", + "message": "Method singler performs much worse than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: f1_micro\n Worst score: 0.0%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Best score singler f1_micro", + "value": 0.9892, + "severity": 0, + "severity_value": 0.4946, + "code": "best_score <= 2", + "message": "Method singler performs a lot better than baselines.\n Task id: task_label_projection\n Method id: singler\n Metric id: f1_micro\n Best score: 0.9892%\n" + }, + { + "task_id": "task_label_projection", + "category": "Scaling", + "name": "Worst score uce f1_micro", + "value": 0.0, + "severity": 0, + "severity_value": -0.0, "code": "worst_score >= -1", - "message": "Method xgboost_log_cp10k performs much worse than baselines.\n Task id: label_projection\n Method id: xgboost_log_cp10k\n Metric id: f1_macro\n Worst score: 0.2023839844503912%\n" + "message": "Method uce performs much worse than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: f1_micro\n Worst score: 0.0%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score xgboost_log_cp10k f1_macro", - "value": 0.9627584240440568, + "name": "Best score uce f1_micro", + "value": 0.0234, "severity": 0, - "severity_value": 0.4813792120220284, + "severity_value": 0.0117, "code": "best_score <= 2", - "message": "Method xgboost_log_cp10k performs a lot better than baselines.\n Task id: label_projection\n Method id: xgboost_log_cp10k\n Metric id: f1_macro\n Best score: 0.9627584240440568%\n" + "message": "Method uce performs a lot better than baselines.\n Task id: task_label_projection\n Method id: uce\n Metric id: f1_micro\n Best score: 0.0234%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Worst score xgboost_scran f1_macro", - "value": 0.18612566035812791, + "name": "Worst score xgboost f1_micro", + "value": 0.2561, "severity": 0, - "severity_value": -0.18612566035812791, + "severity_value": -0.2561, "code": "worst_score >= -1", - "message": "Method xgboost_scran performs much worse than baselines.\n Task id: label_projection\n Method id: xgboost_scran\n Metric id: f1_macro\n Worst score: 0.18612566035812791%\n" + "message": "Method xgboost performs much worse than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: f1_micro\n Worst score: 0.2561%\n" }, { - "task_id": "label_projection", + "task_id": "task_label_projection", "category": "Scaling", - "name": "Best score xgboost_scran f1_macro", - "value": 0.9118740057831154, + "name": "Best score xgboost f1_micro", + "value": 0.995, "severity": 0, - "severity_value": 0.4559370028915577, + "severity_value": 0.4975, "code": "best_score <= 2", - "message": "Method xgboost_scran performs a lot better than baselines.\n Task id: label_projection\n Method id: xgboost_scran\n Metric id: f1_macro\n Best score: 0.9118740057831154%\n" + "message": "Method xgboost performs a lot better than baselines.\n Task id: task_label_projection\n Method id: xgboost\n Metric id: f1_micro\n Best score: 0.995%\n" } ] \ No newline at end of file diff --git a/results/label_projection/data/results.json b/results/label_projection/data/results.json index 23bb2165..1b62ea6b 100644 --- a/results/label_projection/data/results.json +++ b/results/label_projection/data/results.json @@ -1,3330 +1,5618 @@ [ - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 17:53:22.005", - "code_version": "1.1.3", - "resources": { - "duration_sec": 410.0, - "cpu_pct": 134.5, - "peak_memory_mb": 2800.0, - "disk_read_mb": 768.5, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8492439964423362, - "f1": 0.8517360245741384, - "f1_macro": 0.7672490394127152 - }, - "scaled_scores": { - "accuracy": 0.8467065266542732, - "f1": 0.8492343712496994, - "f1_macro": 0.7660191091277334 - }, - "mean_score": 0.8206533356772353 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 17:54:52.087", - "code_version": "0.7.0", - "resources": { - "duration_sec": 379.0, - "cpu_pct": 24.7, - "peak_memory_mb": 989.8, - "disk_read_mb": 768.5, - "disk_write_mb": 722.9 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 17:55:02.568", - "code_version": "0.7.0", - "resources": { - "duration_sec": 379.0, - "cpu_pct": 21.2, - "peak_memory_mb": 992.6, - "disk_read_mb": 768.5, - "disk_write_mb": 722.9 - }, - "metric_values": { - "accuracy": 0.01655301907303093, - "f1": 0.016592995002742705, - "f1_macro": 0.005256541593617354 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 17:55:02.333", - "code_version": "0.7.0", - "resources": { - "duration_sec": 399.0, - "cpu_pct": 25.9, - "peak_memory_mb": 231.7, - "disk_read_mb": 768.5, - "disk_write_mb": 722.8 - }, - "metric_values": { - "accuracy": 0.048423757288269594, - "f1": 0.004473115481431257, - "f1_macro": 0.0005465940774858457 - }, - "scaled_scores": { - "accuracy": 0.03240717479776919, - "f1": -0.012324377861580568, - "f1_macro": -0.004734836380504603 - }, - "mean_score": 0.005115986851894672 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 17:55:02.509", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1079.0, - "cpu_pct": 492.5, - "peak_memory_mb": 1700.0, - "disk_read_mb": 768.6, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8575946239747011, - "f1": 0.8565399161183415, - "f1_macro": 0.7710990297473443 - }, - "scaled_scores": { - "accuracy": 0.8551977088881073, - "f1": 0.8541193187025766, - "f1_macro": 0.7698894440388039 - }, - "mean_score": 0.8264021572098293 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 17:53:31.989", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1343.0, - "cpu_pct": 280.7, - "peak_memory_mb": 1900.0, - "disk_read_mb": 768.5, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8904041901373654, - "f1": 0.8907696437309108, - "f1_macro": 0.8282477715392297 - }, - "scaled_scores": { - "accuracy": 0.8885595136411597, - "f1": 0.8889266034164626, - "f1_macro": 0.8273401780033578 - }, - "mean_score": 0.8682754316869934 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 18:33:05.132", - "code_version": "1.1.3", - "resources": { - "duration_sec": 2370.0, - "cpu_pct": 101.1, - "peak_memory_mb": 6800.0, - "disk_read_mb": 807.8, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.8380274730704615, - "f1": 0.8369482993272568, - "f1_macro": 0.7495378715122704 - }, - "scaled_scores": { - "accuracy": 0.8353012108727328, - "f1": 0.8341971331867847, - "f1_macro": 0.7482143497691559 - }, - "mean_score": 0.8059042312762245 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 18:34:53.628", - "code_version": "1.6.2", - "resources": { - "duration_sec": 2728.0, - "cpu_pct": 712.7, - "peak_memory_mb": 2000.0, - "disk_read_mb": 768.2, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.842474552821425, - "f1": 0.8449181894595059, - "f1_macro": 0.7863051512764242 - }, - "scaled_scores": { - "accuracy": 0.8398231422398633, - "f1": 0.8423014990208183, - "f1_macro": 0.7851759195622928 - }, - "mean_score": 0.8224335202743248 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 18:34:52.511", - "code_version": "1.1.3", - "resources": { - "duration_sec": 3188.0, - "cpu_pct": 138.5, - "peak_memory_mb": 6800.0, - "disk_read_mb": 807.8, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.8271568336792173, - "f1": 0.8404873765570953, - "f1_macro": 0.7906956090555932 - }, - "scaled_scores": { - "accuracy": 0.8242476008641914, - "f1": 0.8377959251537469, - "f1_macro": 0.7895895779202001 - }, - "mean_score": 0.817211034646046 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 18:34:53.797", - "code_version": "1.1.3", - "resources": { - "duration_sec": 3197.0, - "cpu_pct": 162.2, - "peak_memory_mb": 6800.0, - "disk_read_mb": 807.9, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.8760252989425833, - "f1": 0.8777868450203619, - "f1_macro": 0.8148727355232758 - }, - "scaled_scores": { - "accuracy": 0.8739386022207708, - "f1": 0.8757247463577108, - "f1_macro": 0.813894464032661 - }, - "mean_score": 0.8545192708703809 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 18:49:02.216", - "code_version": "4.3.0", - "resources": { - "duration_sec": 3809.0, - "cpu_pct": 105.6, - "peak_memory_mb": 873000.0, - "disk_read_mb": 4200.0, - "disk_write_mb": 4400.0 - }, - "metric_values": { - "accuracy": 0.8321474453997431, - "f1": 0.830794751253811, - "f1_macro": 0.7342634929447799 - }, - "scaled_scores": { - "accuracy": 0.829322212731749, - "f1": 0.8279397564931308, - "f1_macro": 0.7328592565153027 - }, - "mean_score": 0.7967070752467276 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 19:51:33.576", - "code_version": "1.6.2", - "resources": { - "duration_sec": 8268.0, - "cpu_pct": 752.9, - "peak_memory_mb": 6800.0, - "disk_read_mb": 813.9, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.8207332740389367, - "f1": 0.8261501329613843, - "f1_macro": 0.7658604484655689 - }, - "scaled_scores": { - "accuracy": 0.8177159222227806, - "f1": 0.8232167697045227, - "f1_macro": 0.7646231804232906 - }, - "mean_score": 0.801851957450198 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 19:43:53.714", - "code_version": "0.20.0", - "resources": { - "duration_sec": 8917.0, - "cpu_pct": 664.2, - "peak_memory_mb": 3700.0, - "disk_read_mb": 791.5, - "disk_write_mb": 723.0 - }, - "metric_values": { - "accuracy": 0.34884870046447275, - "f1": 0.21567196436308758, - "f1_macro": 0.04927592126738169 - }, - "scaled_scores": { - "accuracy": 0.3378887604883686, - "f1": 0.20243802245531098, - "f1_macro": 0.044251992110895685 - }, - "mean_score": 0.19485959168485842 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 19:43:52.760", - "code_version": "0.20.0", - "resources": { - "duration_sec": 10728.0, - "cpu_pct": 1184.6, - "peak_memory_mb": 6400.0, - "disk_read_mb": 791.5, - "disk_write_mb": 722.9 - }, - "metric_values": { - "accuracy": 0.6517936554995553, - "f1": 0.5691463256932892, - "f1_macro": 0.23546364663002034 - }, - "scaled_scores": { - "accuracy": 0.6459327739536753, - "f1": 0.5618765454005359, - "f1_macro": 0.2314235927775827 - }, - "mean_score": 0.4797443040439313 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 22:54:11.681", - "code_version": "0.20.0", - "resources": { - "duration_sec": 6500.0, - "cpu_pct": 2793.7, - "peak_memory_mb": 10100.0, - "disk_read_mb": 790.0, - "disk_write_mb": 723.2 - }, - "metric_values": { - "accuracy": 0.6659254867081728, - "f1": 0.5728069016471795, - "f1_macro": 0.24022628852988878 - }, - "scaled_scores": { - "accuracy": 0.6603024669647791, - "f1": 0.5655988861356424, - "f1_macro": 0.23621140199574875 - }, - "mean_score": 0.4873709183653901 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "cengen_random", - "submission_time": "2023-02-21 22:55:41.732", - "code_version": "0.20.0", - "resources": { - "duration_sec": 7540.0, - "cpu_pct": 2733.6, - "peak_memory_mb": 12900.0, - "disk_read_mb": 790.0, - "disk_write_mb": 723.2 - }, - "metric_values": { - "accuracy": 0.4243996442336199, - "f1": 0.28984918948885136, - "f1_macro": 0.07585896370042627 - }, - "scaled_scores": { - "accuracy": 0.4147113500477315, - "f1": 0.27786683753271696, - "f1_macro": 0.07097550781577062 - }, - "mean_score": 0.25451789846540634 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 17:56:22.253", - "code_version": "0.7.0", - "resources": { - "duration_sec": 349.0, - "cpu_pct": 11.5, - "peak_memory_mb": 990.3, - "disk_read_mb": 768.5, - "disk_write_mb": 722.9 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 17:56:22.487", - "code_version": "0.7.0", - "resources": { - "duration_sec": 349.0, - "cpu_pct": 12.5, - "peak_memory_mb": 992.0, - "disk_read_mb": 768.5, - "disk_write_mb": 722.8 - }, - "metric_values": { - "accuracy": 0.022168963451168363, - "f1": 0.0009616080277763535, - "f1_macro": 0.0005289794984702484 - }, - "scaled_scores": { - "accuracy": 0.00790273556231003, - "f1": -0.023216507030202435, - "f1_macro": -0.0016591611633974808 - }, - "mean_score": -0.005657644210429962 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 17:56:22.572", - "code_version": "0.7.0", - "resources": { - "duration_sec": 389.0, - "cpu_pct": 12.9, - "peak_memory_mb": 990.6, - "disk_read_mb": 768.5, - "disk_write_mb": 722.9 - }, - "metric_values": { - "accuracy": 0.014379868184541641, - "f1": 0.023629520137584254, - "f1_macro": 0.002184516197432137 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 17:56:21.767", - "code_version": "1.1.3", - "resources": { - "duration_sec": 490.0, - "cpu_pct": 108.2, - "peak_memory_mb": 2600.0, - "disk_read_mb": 768.5, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8124625524266027, - "f1": 0.831091398368635, - "f1_macro": 0.39834484361073036 - }, - "scaled_scores": { - "accuracy": 0.8097264437689969, - "f1": 0.8270035758811894, - "f1_macro": 0.39702764072529084 - }, - "mean_score": 0.6779192201251591 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 18:34:32.765", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1438.0, - "cpu_pct": 253.0, - "peak_memory_mb": 1800.0, - "disk_read_mb": 768.5, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8783702816057519, - "f1": 0.875140390576727, - "f1_macro": 0.5108683585871248 - }, - "scaled_scores": { - "accuracy": 0.876595744680851, - "f1": 0.8721186148101615, - "f1_macro": 0.5097975032930467 - }, - "mean_score": 0.7528372875946864 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 18:34:53.012", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1468.0, - "cpu_pct": 343.2, - "peak_memory_mb": 1600.0, - "disk_read_mb": 768.6, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8250449370880767, - "f1": 0.832031507991959, - "f1_macro": 0.402588128174668 - }, - "scaled_scores": { - "accuracy": 0.8224924012158055, - "f1": 0.8279664374616179, - "f1_macro": 0.4012802151068458 - }, - "mean_score": 0.6839130179280898 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 18:34:52.676", - "code_version": "1.1.3", - "resources": { - "duration_sec": 3068.0, - "cpu_pct": 103.5, - "peak_memory_mb": 6800.0, - "disk_read_mb": 807.8, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.7872977831036548, - "f1": 0.8046070625910249, - "f1_macro": 0.3507138447877583 - }, - "scaled_scores": { - "accuracy": 0.7841945288753799, - "f1": 0.7998782824358754, - "f1_macro": 0.3492923634158474 - }, - "mean_score": 0.6444550582423676 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 18:35:22.428", - "code_version": "1.1.3", - "resources": { - "duration_sec": 3159.0, - "cpu_pct": 145.7, - "peak_memory_mb": 6800.0, - "disk_read_mb": 807.8, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.7723187537447573, - "f1": 0.8036085106510055, - "f1_macro": 0.48353704258985897 - }, - "scaled_scores": { - "accuracy": 0.7689969604863222, - "f1": 0.7988555641536101, - "f1_macro": 0.48240635087966754 - }, - "mean_score": 0.6834196251732 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 18:35:24.021", - "code_version": "1.1.3", - "resources": { - "duration_sec": 3727.0, - "cpu_pct": 219.0, - "peak_memory_mb": 6800.0, - "disk_read_mb": 807.9, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.8568004793289394, - "f1": 0.8568529247812608, - "f1_macro": 0.48583701291538395 - }, - "scaled_scores": { - "accuracy": 0.8547112462006079, - "f1": 0.8533885669721287, - "f1_macro": 0.48471135652736513 - }, - "mean_score": 0.7309370565667006 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 18:50:21.245", - "code_version": "4.3.0", - "resources": { - "duration_sec": 3350.0, - "cpu_pct": 119.2, - "peak_memory_mb": 556200.0, - "disk_read_mb": 3800.0, - "disk_write_mb": 4000.0 - }, - "metric_values": { - "accuracy": 0.8478130617136009, - "f1": 0.8432431418068973, - "f1_macro": 0.4796671443590633 - }, - "scaled_scores": { - "accuracy": 0.8455927051671732, - "f1": 0.839449408368848, - "f1_macro": 0.4785279802854893 - }, - "mean_score": 0.7211900312738369 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 19:47:32.145", - "code_version": "1.6.2", - "resources": { - "duration_sec": 5270.0, - "cpu_pct": 3562.8, - "peak_memory_mb": 1900.0, - "disk_read_mb": 768.2, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8274415817855003, - "f1": 0.834051379104698, - "f1_macro": 0.4645690989104674 - }, - "scaled_scores": { - "accuracy": 0.8249240121580548, - "f1": 0.8300351922574651, - "f1_macro": 0.4633968807047743 - }, - "mean_score": 0.706118695040098 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 20:03:11.377", - "code_version": "1.6.2", - "resources": { - "duration_sec": 9350.0, - "cpu_pct": 741.9, - "peak_memory_mb": 6800.0, - "disk_read_mb": 813.9, - "disk_write_mb": 1400.0 - }, - "metric_values": { - "accuracy": 0.8304373876572798, - "f1": 0.84212350991216, - "f1_macro": 0.4723773954428312 - }, - "scaled_scores": { - "accuracy": 0.8279635258358663, - "f1": 0.8383026798289857, - "f1_macro": 0.4712222719310231 - }, - "mean_score": 0.7124961591986251 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 19:47:31.500", - "code_version": "0.20.0", - "resources": { - "duration_sec": 10520.0, - "cpu_pct": 1185.1, - "peak_memory_mb": 6600.0, - "disk_read_mb": 791.5, - "disk_write_mb": 722.9 - }, - "metric_values": { - "accuracy": 0.26243259436788496, - "f1": 0.22423389962996945, - "f1_macro": 0.16123904954334095 - }, - "scaled_scores": { - "accuracy": 0.25167173252279634, - "f1": 0.20545928377581954, - "f1_macro": 0.15940275123790326 - }, - "mean_score": 0.20551125584550636 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 23:00:52.616", - "code_version": "0.20.0", - "resources": { - "duration_sec": 5590.0, - "cpu_pct": 2312.4, - "peak_memory_mb": 17900.0, - "disk_read_mb": 791.6, - "disk_write_mb": 723.0 - }, - "metric_values": { - "accuracy": 0.08028759736369083, - "f1": 0.03725296782270025, - "f1_macro": 0.036846248365176415 - }, - "scaled_scores": { - "accuracy": 0.0668693009118541, - "f1": 0.013953154018990551, - "f1_macro": 0.034737617054860816 - }, - "mean_score": 0.03852002399523516 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 23:01:02.206", - "code_version": "0.20.0", - "resources": { - "duration_sec": 10970.0, - "cpu_pct": 2702.7, - "peak_memory_mb": 15800.0, - "disk_read_mb": 790.0, - "disk_write_mb": 723.2 - }, - "metric_values": { - "accuracy": 0.24565608148591972, - "f1": 0.19501179569318067, - "f1_macro": 0.09068453934420202 - }, - "scaled_scores": { - "accuracy": 0.23465045592705167, - "f1": 0.1755299643837517, - "f1_macro": 0.08869377613735335 - }, - "mean_score": 0.16629139881605223 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "cengen_batch", - "submission_time": "2023-02-21 23:03:31.738", - "code_version": "0.20.0", - "resources": { - "duration_sec": 14580.0, - "cpu_pct": 2792.0, - "peak_memory_mb": 11700.0, - "disk_read_mb": 790.0, - "disk_write_mb": 723.2 - }, - "metric_values": { - "accuracy": 0.6470940683043739, - "f1": 0.6168788176313944, - "f1_macro": 0.19725738168553295 - }, - "scaled_scores": { - "accuracy": 0.6419452887537994, - "f1": 0.6076067535116458, - "f1_macro": 0.19549993826984827 - }, - "mean_score": 0.48168399351176455 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:01.982", - "code_version": "1.1.3", - "resources": { - "duration_sec": 260.0, - "cpu_pct": 126.6, - "peak_memory_mb": 3300.0, - "disk_read_mb": 804.9, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.12981704943557804, - "f1": 0.14229308051699885, - "f1_macro": 0.16437714442609613 - }, - "scaled_scores": { - "accuracy": 0.1049049049049049, - "f1": 0.10780750046605095, - "f1_macro": 0.15159754888561494 - }, - "mean_score": 0.12143665141885691 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:01.995", - "code_version": "1.1.3", - "resources": { - "duration_sec": 260.0, - "cpu_pct": 110.7, - "peak_memory_mb": 1900.0, - "disk_read_mb": 804.9, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.24802127935642923, - "f1": 0.2875142098321384, - "f1_macro": 0.2327128251184122 - }, - "scaled_scores": { - "accuracy": 0.22649315982649318, - "f1": 0.2588674947434843, - "f1_macro": 0.22097832109787452 - }, - "mean_score": 0.23544632522261733 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.378", - "code_version": "0.7.0", - "resources": { - "duration_sec": 319.0, - "cpu_pct": 46.9, - "peak_memory_mb": 215.7, - "disk_read_mb": 804.8, - "disk_write_mb": 759.3 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.422", - "code_version": "0.7.0", - "resources": { - "duration_sec": 339.0, - "cpu_pct": 28.5, - "peak_memory_mb": 1022.1, - "disk_read_mb": 804.8, - "disk_write_mb": 759.3 - }, - "metric_values": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "scaled_scores": { - "accuracy": -0.028628628628628628, - "f1": -0.04020671778156426, - "f1_macro": -0.015293496887066585 - }, - "mean_score": -0.028042947765753157 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.627", - "code_version": "0.7.0", - "resources": { - "duration_sec": 339.0, - "cpu_pct": 32.8, - "peak_memory_mb": 1023.4, - "disk_read_mb": 804.8, - "disk_write_mb": 759.3 - }, - "metric_values": { - "accuracy": 0.027831841183339822, - "f1": 0.03865262268956753, - "f1_macro": 0.015063128971038524 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.473", - "code_version": "1.1.3", - "resources": { - "duration_sec": 439.0, - "cpu_pct": 141.8, - "peak_memory_mb": 1900.0, - "disk_read_mb": 804.9, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.23738160114181914, - "f1": 0.27619561215999017, - "f1_macro": 0.20373028688647557 - }, - "scaled_scores": { - "accuracy": 0.2155488822155489, - "f1": 0.24709381340904898, - "f1_macro": 0.19155253850770848 - }, - "mean_score": 0.21806507804410213 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.288", - "code_version": "1.6.2", - "resources": { - "duration_sec": 889.0, - "cpu_pct": 278.3, - "peak_memory_mb": 2200.0, - "disk_read_mb": 804.5, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.22440638380692876, - "f1": 0.27834682162039986, - "f1_macro": 0.21439857736198084 - }, - "scaled_scores": { - "accuracy": 0.2022022022022022, - "f1": 0.2493315159411224, - "f1_macro": 0.2023839844503912 - }, - "mean_score": 0.21797256753123861 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.452", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1093.0, - "cpu_pct": 113.0, - "peak_memory_mb": 5000.0, - "disk_read_mb": 844.2, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.23854937070195925, - "f1": 0.3068204107545123, - "f1_macro": 0.1682215736679451 - }, - "scaled_scores": { - "accuracy": 0.2167500834167501, - "f1": 0.2789499346377783, - "f1_macro": 0.15550077289410666 - }, - "mean_score": 0.2170669303162117 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.039", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1197.0, - "cpu_pct": 97.7, - "peak_memory_mb": 5000.0, - "disk_read_mb": 844.2, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.1913844556896328, - "f1": 0.2018575253249313, - "f1_macro": 0.1995615455583369 - }, - "scaled_scores": { - "accuracy": 0.16823490156823492, - "f1": 0.16976683609619148, - "f1_macro": 0.18732004254704496 - }, - "mean_score": 0.17510726007049046 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.159", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1499.0, - "cpu_pct": 115.1, - "peak_memory_mb": 5000.0, - "disk_read_mb": 844.1, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.24471259893603217, - "f1": 0.29034414714676077, - "f1_macro": 0.2245866122961815 - }, - "scaled_scores": { - "accuracy": 0.22308975642308976, - "f1": 0.26181121454905526, - "f1_macro": 0.21272783006514337 - }, - "mean_score": 0.23254293367909615 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 17:58:02.340", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1589.0, - "cpu_pct": 205.9, - "peak_memory_mb": 5900.0, - "disk_read_mb": 850.2, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.23588945114830673, - "f1": 0.2980572067652053, - "f1_macro": 0.19838515450237226 - }, - "scaled_scores": { - "accuracy": 0.21401401401401401, - "f1": 0.26983439097881096, - "f1_macro": 0.18612566035812791 - }, - "mean_score": 0.22332468845031764 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 18:33:35.173", - "code_version": "4.3.0", - "resources": { - "duration_sec": 1646.0, - "cpu_pct": 115.4, - "peak_memory_mb": 60000.0, - "disk_read_mb": 2400.0, - "disk_write_mb": 2800.0 - }, - "metric_values": { - "accuracy": 0.3442973919813157, - "f1": 0.41030222359279184, - "f1_macro": 0.28347167382711347 - }, - "scaled_scores": { - "accuracy": 0.32552552552552555, - "f1": 0.3865924115203712, - "f1_macro": 0.2725134501012934 - }, - "mean_score": 0.3282104623823967 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 18:35:21.411", - "code_version": "0.20.0", - "resources": { - "duration_sec": 3550.0, - "cpu_pct": 355.3, - "peak_memory_mb": 1700.0, - "disk_read_mb": 827.8, - "disk_write_mb": 759.5 - }, - "metric_values": { - "accuracy": 0.13280134942260283, - "f1": 0.1309776775288379, - "f1_macro": 0.21771900557535676 - }, - "scaled_scores": { - "accuracy": 0.10797464130797463, - "f1": 0.09603714226336035, - "f1_macro": 0.20575519362231212 - }, - "mean_score": 0.13658899239788236 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 23:00:53.863", - "code_version": "0.20.0", - "resources": { - "duration_sec": 5048.0, - "cpu_pct": 2616.4, - "peak_memory_mb": 5500.0, - "disk_read_mb": 826.4, - "disk_write_mb": 759.5 - }, - "metric_values": { - "accuracy": 0.22602828597379007, - "f1": 0.25597266236180566, - "f1_macro": 0.23106180199762813 - }, - "scaled_scores": { - "accuracy": 0.20387053720387055, - "f1": 0.22605776517561812, - "f1_macro": 0.21930204806013226 - }, - "mean_score": 0.21641011681320696 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 23:00:55.616", - "code_version": "0.20.0", - "resources": { - "duration_sec": 5627.0, - "cpu_pct": 2371.9, - "peak_memory_mb": 7100.0, - "disk_read_mb": 827.9, - "disk_write_mb": 759.5 - }, - "metric_values": { - "accuracy": 0.21006876865187493, - "f1": 0.22606919943933246, - "f1_macro": 0.2371223938182558 - }, - "scaled_scores": { - "accuracy": 0.18745412078745413, - "f1": 0.19495198215872958, - "f1_macro": 0.22545532752290248 - }, - "mean_score": 0.20262047682302872 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "zebrafish_labs", - "submission_time": "2023-02-21 23:00:54.531", - "code_version": "0.20.0", - "resources": { - "duration_sec": 6288.0, - "cpu_pct": 2718.5, - "peak_memory_mb": 4000.0, - "disk_read_mb": 826.4, - "disk_write_mb": 759.4 - }, - "metric_values": { - "accuracy": 0.21999480991306605, - "f1": 0.2442120891364309, - "f1_macro": 0.263874405982815 - }, - "scaled_scores": { - "accuracy": 0.19766433099766434, - "f1": 0.2138243379016213, - "f1_macro": 0.2526164715022231 - }, - "mean_score": 0.22136838013383628 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:02.896", - "code_version": "0.7.0", - "resources": { - "duration_sec": 309.0, - "cpu_pct": 57.8, - "peak_memory_mb": 1021.3, - "disk_read_mb": 804.8, - "disk_write_mb": 759.3 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:03.064", - "code_version": "0.7.0", - "resources": { - "duration_sec": 348.0, - "cpu_pct": 30.7, - "peak_memory_mb": 233.8, - "disk_read_mb": 804.8, - "disk_write_mb": 759.3 - }, - "metric_values": { - "accuracy": 0.1740139211136891, - "f1": 0.051585154480342614, - "f1_macro": 0.012351778656126482 - }, - "scaled_scores": { - "accuracy": 0.09491525423728814, - "f1": -0.03961726272153891, - "f1_macro": -0.03012033252188506 - }, - "mean_score": 0.008392552997954722 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:02.640", - "code_version": "1.1.3", - "resources": { - "duration_sec": 409.0, - "cpu_pct": 98.2, - "peak_memory_mb": 2500.0, - "disk_read_mb": 804.9, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8060711523588554, - "f1": 0.8014734891499417, - "f1_macro": 0.6240963193194412 - }, - "scaled_scores": { - "accuracy": 0.7875, - "f1": 0.7823825842007892, - "f1_macro": 0.6079312287810665 - }, - "mean_score": 0.7259379376606185 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:02.611", - "code_version": "1.1.3", - "resources": { - "duration_sec": 419.0, - "cpu_pct": 118.7, - "peak_memory_mb": 1600.0, - "disk_read_mb": 804.9, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8426140757927301, - "f1": 0.8410077225303916, - "f1_macro": 0.7110845534586984 - }, - "scaled_scores": { - "accuracy": 0.8275423728813559, - "f1": 0.8257185480829846, - "f1_macro": 0.6986602421488979 - }, - "mean_score": 0.7839737210377461 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:03.080", - "code_version": "1.1.3", - "resources": { - "duration_sec": 419.0, - "cpu_pct": 290.1, - "peak_memory_mb": 1600.0, - "disk_read_mb": 804.9, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8397138437741686, - "f1": 0.8403436701669355, - "f1_macro": 0.7096306108366662 - }, - "scaled_scores": { - "accuracy": 0.8243644067796609, - "f1": 0.8249906384518129, - "f1_macro": 0.6971437752278746 - }, - "mean_score": 0.7821662734864495 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:03.129", - "code_version": "0.7.0", - "resources": { - "duration_sec": 419.0, - "cpu_pct": 20.9, - "peak_memory_mb": 1022.3, - "disk_read_mb": 804.8, - "disk_write_mb": 759.3 - }, - "metric_values": { - "accuracy": 0.08739365815931942, - "f1": 0.08772691688778744, - "f1_macro": 0.0412302425620836 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:02.650", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1019.0, - "cpu_pct": 102.1, - "peak_memory_mb": 5000.0, - "disk_read_mb": 844.2, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.8022041763341067, - "f1": 0.8026197598424966, - "f1_macro": 0.6493289517704377 - }, - "scaled_scores": { - "accuracy": 0.7832627118644068, - "f1": 0.7836390837224504, - "f1_macro": 0.6342489471438408 - }, - "mean_score": 0.733716914243566 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:02.826", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1418.0, - "cpu_pct": 106.4, - "peak_memory_mb": 5000.0, - "disk_read_mb": 844.1, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.8431941221964424, - "f1": 0.8423899095360406, - "f1_macro": 0.7090452154929959 - }, - "scaled_scores": { - "accuracy": 0.8281779661016949, - "f1": 0.8272336503382587, - "f1_macro": 0.696533205965412 - }, - "mean_score": 0.7839816074684552 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:02.910", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1418.0, - "cpu_pct": 331.9, - "peak_memory_mb": 1900.0, - "disk_read_mb": 804.5, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8031709203402939, - "f1": 0.7995257960670158, - "f1_macro": 0.6814826306177239 - }, - "scaled_scores": { - "accuracy": 0.7843220338983051, - "f1": 0.7802475951070835, - "f1_macro": 0.6677853395861819 - }, - "mean_score": 0.7441183228638568 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 17:58:03.015", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1718.0, - "cpu_pct": 162.9, - "peak_memory_mb": 5000.0, - "disk_read_mb": 844.2, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.8343000773395205, - "f1": 0.8347189502833389, - "f1_macro": 0.6955124404833017 - }, - "scaled_scores": { - "accuracy": 0.8184322033898305, - "f1": 0.8188250286276055, - "f1_macro": 0.682418477267818 - }, - "mean_score": 0.7732252364284179 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 18:31:51.874", - "code_version": "4.3.0", - "resources": { - "duration_sec": 1589.0, - "cpu_pct": 129.3, - "peak_memory_mb": 376000.0, - "disk_read_mb": 2200.0, - "disk_write_mb": 2600.0 - }, - "metric_values": { - "accuracy": 0.8596287703016241, - "f1": 0.8577182706160404, - "f1_macro": 0.7417937504863691 - }, - "scaled_scores": { - "accuracy": 0.8461864406779661, - "f1": 0.8440360326114561, - "f1_macro": 0.7306900353181502 - }, - "mean_score": 0.8069708362025242 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 18:38:02.414", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1899.0, - "cpu_pct": 451.9, - "peak_memory_mb": 5500.0, - "disk_read_mb": 850.2, - "disk_write_mb": 1500.0 - }, - "metric_values": { - "accuracy": 0.7882830626450116, - "f1": 0.7842438906348819, - "f1_macro": 0.6613879616286572 - }, - "scaled_scores": { - "accuracy": 0.7680084745762712, - "f1": 0.7634961357962379, - "f1_macro": 0.6468265339572217 - }, - "mean_score": 0.7261103814432435 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 23:00:53.522", - "code_version": "0.20.0", - "resources": { - "duration_sec": 4959.0, - "cpu_pct": 2198.6, - "peak_memory_mb": 7800.0, - "disk_read_mb": 827.9, - "disk_write_mb": 759.5 - }, - "metric_values": { - "accuracy": 0.7790023201856149, - "f1": 0.7278565236325139, - "f1_macro": 0.43734666546002615 - }, - "scaled_scores": { - "accuracy": 0.7578389830508475, - "f1": 0.7016863903963155, - "f1_macro": 0.41315072761209043 - }, - "mean_score": 0.6242253670197512 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 23:00:55.143", - "code_version": "0.20.0", - "resources": { - "duration_sec": 5357.0, - "cpu_pct": 2746.9, - "peak_memory_mb": 6000.0, - "disk_read_mb": 826.4, - "disk_write_mb": 759.5 - }, - "metric_values": { - "accuracy": 0.6477184841453983, - "f1": 0.5550225618420579, - "f1_macro": 0.3165801404765449 - }, - "scaled_scores": { - "accuracy": 0.6139830508474576, - "f1": 0.5122321962630915, - "f1_macro": 0.2871908461633878 - }, - "mean_score": 0.47113536442464565 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 23:01:11.800", - "code_version": "0.20.0", - "resources": { - "duration_sec": 8230.0, - "cpu_pct": 2354.3, - "peak_memory_mb": 7700.0, - "disk_read_mb": 827.9, - "disk_write_mb": 759.5 - }, - "metric_values": { - "accuracy": 0.637277648878577, - "f1": 0.5346413116985615, - "f1_macro": 0.2832056442403825 - }, - "scaled_scores": { - "accuracy": 0.6025423728813559, - "f1": 0.48989102395318856, - "f1_macro": 0.2523811371824248 - }, - "mean_score": 0.44827151133898974 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "zebrafish_random", - "submission_time": "2023-02-21 23:01:02.773", - "code_version": "0.20.0", - "resources": { - "duration_sec": 9589.0, - "cpu_pct": 2729.8, - "peak_memory_mb": 5100.0, - "disk_read_mb": 826.4, - "disk_write_mb": 759.4 - }, - "metric_values": { - "accuracy": 0.7521268368136118, - "f1": 0.6923656409218792, - "f1_macro": 0.40005402866576983 - }, - "scaled_scores": { - "accuracy": 0.7283898305084746, - "f1": 0.6627825979161541, - "f1_macro": 0.37425438518477816 - }, - "mean_score": 0.588475604536469 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.547", - "code_version": "0.7.0", - "resources": { - "duration_sec": 350.0, - "cpu_pct": 23.8, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.34947254530700567, - "f1": 0.1810056237869919, - "f1_macro": 0.03699567620192996 - }, - "scaled_scores": { - "accuracy": 0.1729711141678129, - "f1": -0.042167518883946264, - "f1_macro": -0.036296851094743135 - }, - "mean_score": 0.03150224806304117 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.650", - "code_version": "1.1.3", - "resources": { - "duration_sec": 359.0, - "cpu_pct": 207.0, - "peak_memory_mb": 2300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9640248850419258, - "f1": 0.9645805182050861, - "f1_macro": 0.8790308464162443 - }, - "scaled_scores": { - "accuracy": 0.9542640990371389, - "f1": 0.9549288315836248, - "f1_macro": 0.8698241016779394 - }, - "mean_score": 0.9263390107662345 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.080", - "code_version": "1.1.3", - "resources": { - "duration_sec": 430.0, - "cpu_pct": 169.1, - "peak_memory_mb": 2300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9634839058696241, - "f1": 0.9639430099980332, - "f1_macro": 0.9390204193334389 - }, - "scaled_scores": { - "accuracy": 0.953576341127923, - "f1": 0.9541176045890214, - "f1_macro": 0.9343793731095585 - }, - "mean_score": 0.9473577729421677 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.340", - "code_version": "0.7.0", - "resources": { - "duration_sec": 460.0, - "cpu_pct": 12.4, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.067", - "code_version": "1.1.3", - "resources": { - "duration_sec": 520.0, - "cpu_pct": 87.0, - "peak_memory_mb": 2600.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.874492832025967, - "f1": 0.8731145520258626, - "f1_macro": 0.8089500380663799 - }, - "scaled_scores": { - "accuracy": 0.8404401650618981, - "f1": 0.8385387051018135, - "f1_macro": 0.7944095690321158 - }, - "mean_score": 0.8244628130652757 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.740", - "code_version": "0.7.0", - "resources": { - "duration_sec": 529.0, - "cpu_pct": 9.2, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.21341628347308628, - "f1": 0.21414325300594048, - "f1_macro": 0.07072541735434873 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.380", - "code_version": "1.6.2", - "resources": { - "duration_sec": 769.0, - "cpu_pct": 478.9, - "peak_memory_mb": 2700.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.93913984311604, - "f1": 0.9383958828902009, - "f1_macro": 0.9050628910586268 - }, - "scaled_scores": { - "accuracy": 0.922627235213205, - "f1": 0.921608973460573, - "f1_macro": 0.8978373984241703 - }, - "mean_score": 0.9140245356993161 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.239", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1369.0, - "cpu_pct": 99.9, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.8290505815526102, - "f1": 0.832422691059586, - "f1_macro": 0.5774678768528779 - }, - "scaled_scores": { - "accuracy": 0.782668500687758, - "f1": 0.7867584523751875, - "f1_macro": 0.5453097168070926 - }, - "mean_score": 0.7049122232900128 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.259", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1439.0, - "cpu_pct": 121.7, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9418447389775494, - "f1": 0.940413163463306, - "f1_macro": 0.6362141650537397 - }, - "scaled_scores": { - "accuracy": 0.9260660247592848, - "f1": 0.9241759560319147, - "f1_macro": 0.6085270793584396 - }, - "mean_score": 0.8195896867165464 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 17:58:22.452", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1668.0, - "cpu_pct": 236.9, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9342710305653232, - "f1": 0.9342714399948299, - "f1_macro": 0.7880000488209941 - }, - "scaled_scores": { - "accuracy": 0.9164374140302614, - "f1": 0.9163606341020992, - "f1_macro": 0.7718651137799977 - }, - "mean_score": 0.8682210539707862 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 18:37:31.536", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1480.0, - "cpu_pct": 145.9, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9537462807681905, - "f1": 0.9546716872929825, - "f1_macro": 0.7595492175376236 - }, - "scaled_scores": { - "accuracy": 0.9411966987620358, - "f1": 0.9423198784251703, - "f1_macro": 0.7412489408912797 - }, - "mean_score": 0.8749218393594953 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 18:44:41.862", - "code_version": "4.3.0", - "resources": { - "duration_sec": 1910.0, - "cpu_pct": 109.7, - "peak_memory_mb": 108200.0, - "disk_read_mb": 2800.0, - "disk_write_mb": 2400.0 - }, - "metric_values": { - "accuracy": 0.9588855829050582, - "f1": 0.9582796607282135, - "f1_macro": 0.8517420844591291 - }, - "scaled_scores": { - "accuracy": 0.9477303988995874, - "f1": 0.9469110121770045, - "f1_macro": 0.8404584411221284 - }, - "mean_score": 0.9116999507329068 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 18:36:19.798", - "code_version": "0.20.0", - "resources": { - "duration_sec": 2791.0, - "cpu_pct": 337.8, - "peak_memory_mb": 2200.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9596970516635109, - "f1": 0.9583537069413711, - "f1_macro": 0.6689106599819379 - }, - "scaled_scores": { - "accuracy": 0.9487620357634112, - "f1": 0.9470052357278499, - "f1_macro": 0.6437120457169414 - }, - "mean_score": 0.8464931057360676 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 18:36:51.529", - "code_version": "0.20.0", - "resources": { - "duration_sec": 2990.0, - "cpu_pct": 328.4, - "peak_memory_mb": 2300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9540167703543414, - "f1": 0.9517252686049228, - "f1_macro": 0.5816215781285415 - }, - "scaled_scores": { - "accuracy": 0.9415405777166438, - "f1": 0.9385705708072999, - "f1_macro": 0.549779548817173 - }, - "mean_score": 0.8099635657803722 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 19:52:42.299", - "code_version": "0.20.0", - "resources": { - "duration_sec": 8939.0, - "cpu_pct": 2151.9, - "peak_memory_mb": 5900.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9507708953205302, - "f1": 0.9494991049364069, - "f1_macro": 0.6454766237283966 - }, - "scaled_scores": { - "accuracy": 0.9374140302613481, - "f1": 0.9357377852175203, - "f1_macro": 0.6184944871059823 - }, - "mean_score": 0.8305487675282835 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "pancreas_batch", - "submission_time": "2023-02-21 23:00:59.251", - "code_version": "0.20.0", - "resources": { - "duration_sec": 5643.0, - "cpu_pct": 2762.1, - "peak_memory_mb": 4500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9575331349743035, - "f1": 0.9561489780630695, - "f1_macro": 0.6604518695765396 - }, - "scaled_scores": { - "accuracy": 0.9460110041265475, - "f1": 0.9441997258346858, - "f1_macro": 0.6346094719853796 - }, - "mean_score": 0.8416067339822044 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.196", - "code_version": "1.1.3", - "resources": { - "duration_sec": 368.0, - "cpu_pct": 127.5, - "peak_memory_mb": 2200.0, - "disk_read_mb": 629.0, - "disk_write_mb": 868.9 - }, - "metric_values": { - "accuracy": 0.8611835506519558, - "f1": 0.8585497668879454, - "f1_macro": 0.8714241831337218 - }, - "scaled_scores": { - "accuracy": 0.8475770925110132, - "f1": 0.8446930196084786, - "f1_macro": 0.868083461255843 - }, - "mean_score": 0.8534511911251116 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.233", - "code_version": "1.1.3", - "resources": { - "duration_sec": 418.0, - "cpu_pct": 148.3, - "peak_memory_mb": 1300.0, - "disk_read_mb": 629.0, - "disk_write_mb": 868.9 - }, - "metric_values": { - "accuracy": 0.925777331995988, - "f1": 0.9252414029903501, - "f1_macro": 0.9131036540313611 - }, - "scaled_scores": { - "accuracy": 0.9185022026431718, - "f1": 0.9179179015514406, - "f1_macro": 0.9108458692382295 - }, - "mean_score": 0.915755324477614 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.419", - "code_version": "0.7.0", - "resources": { - "duration_sec": 458.0, - "cpu_pct": 9.5, - "peak_memory_mb": 846.9, - "disk_read_mb": 629.0, - "disk_write_mb": 583.6 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.519", - "code_version": "0.7.0", - "resources": { - "duration_sec": 458.0, - "cpu_pct": 10.1, - "peak_memory_mb": 846.8, - "disk_read_mb": 629.0, - "disk_write_mb": 583.6 - }, - "metric_values": { - "accuracy": 0.22286860581745235, - "f1": 0.0812358992989467, - "f1_macro": 0.009346187495793795 - }, - "scaled_scores": { - "accuracy": 0.14669603524229075, - "f1": -0.00876806656780211, - "f1_macro": -0.0163934807054105 - }, - "mean_score": 0.04051149598969272 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.568", - "code_version": "0.7.0", - "resources": { - "duration_sec": 468.0, - "cpu_pct": 9.8, - "peak_memory_mb": 847.1, - "disk_read_mb": 629.0, - "disk_write_mb": 583.6 - }, - "metric_values": { - "accuracy": 0.08926780341023069, - "f1": 0.0892216643742255, - "f1_macro": 0.025324511313610666 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.569", - "code_version": "1.1.3", - "resources": { - "duration_sec": 568.0, - "cpu_pct": 384.9, - "peak_memory_mb": 1300.0, - "disk_read_mb": 629.1, - "disk_write_mb": 868.9 - }, - "metric_values": { - "accuracy": 0.931394182547643, - "f1": 0.9309033200595019, - "f1_macro": 0.9072989997095794 - }, - "scaled_scores": { - "accuracy": 0.9246696035242291, - "f1": 0.9241344713222418, - "f1_macro": 0.9048903954532009 - }, - "mean_score": 0.9178981567665573 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.200", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1206.0, - "cpu_pct": 100.2, - "peak_memory_mb": 5400.0, - "disk_read_mb": 668.3, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8686058174523571, - "f1": 0.8638651253107504, - "f1_macro": 0.8275820800516215 - }, - "scaled_scores": { - "accuracy": 0.8557268722466961, - "f1": 0.8505290811559385, - "f1_macro": 0.8231022304862172 - }, - "mean_score": 0.8431193946296173 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.447", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1507.0, - "cpu_pct": 290.0, - "peak_memory_mb": 1500.0, - "disk_read_mb": 628.7, - "disk_write_mb": 869.0 - }, - "metric_values": { - "accuracy": 0.8736208625877633, - "f1": 0.8711695852376224, - "f1_macro": 0.797480077558899 - }, - "scaled_scores": { - "accuracy": 0.8612334801762115, - "f1": 0.858549100562585, - "f1_macro": 0.7922181025460633 - }, - "mean_score": 0.8373335610949533 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.305", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1568.0, - "cpu_pct": 114.6, - "peak_memory_mb": 5300.0, - "disk_read_mb": 668.3, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.9251755265797392, - "f1": 0.9242577338820452, - "f1_macro": 0.8758263593202537 - }, - "scaled_scores": { - "accuracy": 0.91784140969163, - "f1": 0.9168378702531237, - "f1_macro": 0.872600017009661 - }, - "mean_score": 0.9024264323181382 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 17:58:13.434", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1739.0, - "cpu_pct": 330.1, - "peak_memory_mb": 5700.0, - "disk_read_mb": 674.4, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.8651955867602809, - "f1": 0.8639521670154929, - "f1_macro": 0.8174603029898384 - }, - "scaled_scores": { - "accuracy": 0.8519823788546256, - "f1": 0.8506246496398799, - "f1_macro": 0.8127174642955494 - }, - "mean_score": 0.8384414975966848 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 18:34:05.334", - "code_version": "4.3.0", - "resources": { - "duration_sec": 1486.0, - "cpu_pct": 122.6, - "peak_memory_mb": 568500.0, - "disk_read_mb": 1900.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9119358074222668, - "f1": 0.909781894158941, - "f1_macro": 0.8526389520351096 - }, - "scaled_scores": { - "accuracy": 0.9033039647577092, - "f1": 0.9009439483657983, - "f1_macro": 0.8488101427855799 - }, - "mean_score": 0.8843526853030292 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 18:37:01.573", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1630.0, - "cpu_pct": 165.7, - "peak_memory_mb": 5400.0, - "disk_read_mb": 668.4, - "disk_write_mb": 1100.0 - }, - "metric_values": { - "accuracy": 0.9285857572718155, - "f1": 0.9280688834551714, - "f1_macro": 0.8914942538629572 - }, - "scaled_scores": { - "accuracy": 0.9215859030837005, - "f1": 0.9210223676484285, - "f1_macro": 0.8886750027095885 - }, - "mean_score": 0.9104277578139058 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 18:36:51.552", - "code_version": "0.20.0", - "resources": { - "duration_sec": 3669.0, - "cpu_pct": 361.2, - "peak_memory_mb": 1600.0, - "disk_read_mb": 651.9, - "disk_write_mb": 583.7 - }, - "metric_values": { - "accuracy": 0.8409227683049147, - "f1": 0.8013792223646742, - "f1_macro": 0.39398162218102567 - }, - "scaled_scores": { - "accuracy": 0.8253303964757709, - "f1": 0.7819219343871875, - "f1_macro": 0.37823574630390006 - }, - "mean_score": 0.6618293590556195 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 23:00:54.927", - "code_version": "0.20.0", - "resources": { - "duration_sec": 5527.0, - "cpu_pct": 2559.0, - "peak_memory_mb": 5900.0, - "disk_read_mb": 650.5, - "disk_write_mb": 583.7 - }, - "metric_values": { - "accuracy": 0.7837512537612839, - "f1": 0.7239101602555007, - "f1_macro": 0.2807778194761088 - }, - "scaled_scores": { - "accuracy": 0.7625550660792951, - "f1": 0.6968638482657754, - "f1_macro": 0.26209062516467213 - }, - "mean_score": 0.5738365131699142 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 23:00:57.691", - "code_version": "0.20.0", - "resources": { - "duration_sec": 7265.0, - "cpu_pct": 2359.4, - "peak_memory_mb": 9300.0, - "disk_read_mb": 652.1, - "disk_write_mb": 583.8 - }, - "metric_values": { - "accuracy": 0.7223671013039117, - "f1": 0.6280996816595446, - "f1_macro": 0.23443165472180164 - }, - "scaled_scores": { - "accuracy": 0.6951541850220264, - "f1": 0.5916675838749157, - "f1_macro": 0.21454027092649408 - }, - "mean_score": 0.5004540132744787 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "tabula_muris_senis_lung_random", - "submission_time": "2023-02-21 23:00:58.998", - "code_version": "0.20.0", - "resources": { - "duration_sec": 8423.0, - "cpu_pct": 2801.4, - "peak_memory_mb": 3900.0, - "disk_read_mb": 650.5, - "disk_write_mb": 583.7 - }, - "metric_values": { - "accuracy": 0.8641925777331996, - "f1": 0.8323005307396097, - "f1_macro": 0.43557549460855116 - }, - "scaled_scores": { - "accuracy": 0.8508810572687224, - "f1": 0.8158723558733225, - "f1_macro": 0.4209103317534463 - }, - "mean_score": 0.6958879149651637 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.818", - "code_version": "0.7.0", - "resources": { - "duration_sec": 399.0, - "cpu_pct": 29.9, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:54.072", - "code_version": "0.7.0", - "resources": { - "duration_sec": 448.0, - "cpu_pct": 22.5, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.2209726443768997, - "f1": 0.2207732992070331, - "f1_macro": 0.07618531000986446 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:54.102", - "code_version": "0.7.0", - "resources": { - "duration_sec": 478.0, - "cpu_pct": 28.6, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.32887537993920973, - "f1": 0.16278278183633346, - "f1_macro": 0.03535485557443472 - }, - "scaled_scores": { - "accuracy": 0.13850955911041749, - "f1": -0.07442059841081748, - "f1_macro": -0.04419766743032169 - }, - "mean_score": 0.006630431089759438 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.335", - "code_version": "1.1.3", - "resources": { - "duration_sec": 519.0, - "cpu_pct": 97.1, - "peak_memory_mb": 2400.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9650455927051672, - "f1": 0.964346201217482, - "f1_macro": 0.8962212379632082 - }, - "scaled_scores": { - "accuracy": 0.9551307062036676, - "f1": 0.9542446392734802, - "f1_macro": 0.8876627930241076 - }, - "mean_score": 0.9323460461670852 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.294", - "code_version": "1.1.3", - "resources": { - "duration_sec": 529.0, - "cpu_pct": 121.4, - "peak_memory_mb": 2200.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9878419452887538, - "f1": 0.9877826521340638, - "f1_macro": 0.9723208693493605 - }, - "scaled_scores": { - "accuracy": 0.9843932891143191, - "f1": 0.9843211893874999, - "f1_macro": 0.9700382220043122 - }, - "mean_score": 0.9795842335020438 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.984", - "code_version": "1.1.3", - "resources": { - "duration_sec": 628.0, - "cpu_pct": 205.7, - "peak_memory_mb": 2200.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.988145896656535, - "f1": 0.9880554686775588, - "f1_macro": 0.9709369948626364 - }, - "scaled_scores": { - "accuracy": 0.9847834568864612, - "f1": 0.9846713012910284, - "f1_macro": 0.9685402219165037 - }, - "mean_score": 0.9793316600313311 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.867", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1056.0, - "cpu_pct": 302.4, - "peak_memory_mb": 2600.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9772036474164134, - "f1": 0.977130825456685, - "f1_macro": 0.9655956850535162 - }, - "scaled_scores": { - "accuracy": 0.9707374170893485, - "f1": 0.970651449032685, - "f1_macro": 0.9627584240440568 - }, - "mean_score": 0.9680490967220301 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.751", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1307.0, - "cpu_pct": 107.8, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9644376899696049, - "f1": 0.963539669184525, - "f1_macro": 0.7715105401247113 - }, - "scaled_scores": { - "accuracy": 0.9543503706593836, - "f1": 0.9532095976968298, - "f1_macro": 0.752667431735981 - }, - "mean_score": 0.8867424666973981 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.584", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1417.0, - "cpu_pct": 103.2, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9452887537993921, - "f1": 0.9377988609786475, - "f1_macro": 0.6317027390878511 - }, - "scaled_scores": { - "accuracy": 0.9297698010144363, - "f1": 0.9201758115346219, - "f1_macro": 0.6013299367256418 - }, - "mean_score": 0.8170918497582332 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.717", - "code_version": "0.20.0", - "resources": { - "duration_sec": 1948.0, - "cpu_pct": 320.4, - "peak_memory_mb": 2300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.956838905775076, - "f1": 0.9478276041369139, - "f1_macro": 0.529456773064677 - }, - "scaled_scores": { - "accuracy": 0.944596176355833, - "f1": 0.9330459341164852, - "f1_macro": 0.490651932650749 - }, - "mean_score": 0.7894313477076892 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:00:53.815", - "code_version": "1.6.2", - "resources": { - "duration_sec": 2028.0, - "cpu_pct": 218.1, - "peak_memory_mb": 8300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9693009118541034, - "f1": 0.9689284940816515, - "f1_macro": 0.9185879119724563 - }, - "scaled_scores": { - "accuracy": 0.9605930550136559, - "f1": 0.9601252037606909, - "f1_macro": 0.9118740057831154 - }, - "mean_score": 0.9441974215191541 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:39:13.606", - "code_version": "1.1.3", - "resources": { - "duration_sec": 2002.0, - "cpu_pct": 256.3, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9851063829787234, - "f1": 0.9849520807815663, - "f1_macro": 0.8786949882795664 - }, - "scaled_scores": { - "accuracy": 0.980881779165041, - "f1": 0.9806886504233998, - "f1_macro": 0.868691185543143 - }, - "mean_score": 0.9434205383771946 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:43:52.071", - "code_version": "4.3.0", - "resources": { - "duration_sec": 1899.0, - "cpu_pct": 110.2, - "peak_memory_mb": 45700.0, - "disk_read_mb": 2800.0, - "disk_write_mb": 2400.0 - }, - "metric_values": { - "accuracy": 0.982370820668693, - "f1": 0.9816543811179429, - "f1_macro": 0.8948274458832942 - }, - "scaled_scores": { - "accuracy": 0.9773702692157628, - "f1": 0.9764566346823227, - "f1_macro": 0.8861540574573145 - }, - "mean_score": 0.9466603204517999 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:38:22.399", - "code_version": "0.20.0", - "resources": { - "duration_sec": 2879.0, - "cpu_pct": 542.3, - "peak_memory_mb": 2300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.978419452887538, - "f1": 0.975807690913863, - "f1_macro": 0.6870492664816059 - }, - "scaled_scores": { - "accuracy": 0.9722980881779164, - "f1": 0.9689534392731689, - "f1_macro": 0.6612407911355731 - }, - "mean_score": 0.867497439528886 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 18:38:03.022", - "code_version": "0.20.0", - "resources": { - "duration_sec": 4448.0, - "cpu_pct": 630.8, - "peak_memory_mb": 3600.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9747720364741641, - "f1": 0.9723111833999045, - "f1_macro": 0.6753232347526958 - }, - "scaled_scores": { - "accuracy": 0.9676160749122122, - "f1": 0.9644662887296875, - "f1_macro": 0.6485477349891773 - }, - "mean_score": 0.8602100328770256 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "pancreas_random", - "submission_time": "2023-02-21 23:19:02.037", - "code_version": "0.20.0", - "resources": { - "duration_sec": 23550.0, - "cpu_pct": 2606.3, - "peak_memory_mb": 5900.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9556231003039514, - "f1": 0.9460786047979668, - "f1_macro": 0.5347595613474366 - }, - "scaled_scores": { - "accuracy": 0.9430355052672649, - "f1": 0.9308014020218237, - "f1_macro": 0.4963920321969213 - }, - "mean_score": 0.7900763131620033 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "majority_vote", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.447", - "code_version": "0.7.0", - "resources": { - "duration_sec": 389.0, - "cpu_pct": 25.4, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.3352850539291217, - "f1": 0.16837763151390925, - "f1_macro": 0.03587089116745244 - }, - "scaled_scores": { - "accuracy": 0.19574944071588368, - "f1": -0.024011749900275662, - "f1_macro": -0.02854216688504238 - }, - "mean_score": 0.047731841310188546 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "random_labels", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.563", - "code_version": "0.7.0", - "resources": { - "duration_sec": 389.0, - "cpu_pct": 27.1, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.17349768875192603, - "f1": 0.18787809947778522, - "f1_macro": 0.06262558806662334 - }, - "scaled_scores": { - "accuracy": 0.0, - "f1": 0.0, - "f1_macro": 0.0 - }, - "mean_score": 0.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_log_cp10k", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.062", - "code_version": "1.1.3", - "resources": { - "duration_sec": 419.0, - "cpu_pct": 113.8, - "peak_memory_mb": 2300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9463790446841295, - "f1": 0.9460810866713062, - "f1_macro": 0.8361237623993133 - }, - "scaled_scores": { - "accuracy": 0.9351230425055929, - "f1": 0.9336073644929135, - "f1_macro": 0.8251752602648021 - }, - "mean_score": 0.8979685557544362 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "true_labels", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.381", - "code_version": "0.7.0", - "resources": { - "duration_sec": 438.0, - "cpu_pct": 15.3, - "peak_memory_mb": 1500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "scaled_scores": { - "accuracy": 1.0, - "f1": 1.0, - "f1_macro": 1.0 - }, - "mean_score": 1.0 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_log_cp10k", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.624", - "code_version": "1.1.3", - "resources": { - "duration_sec": 548.0, - "cpu_pct": 447.1, - "peak_memory_mb": 2200.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.8080123266563944, - "f1": 0.8366856872231097, - "f1_macro": 0.5463249016394928 - }, - "scaled_scores": { - "accuracy": 0.767710663683818, - "f1": 0.7989041883098151, - "f1_macro": 0.516015060167066 - }, - "mean_score": 0.694209970720233 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_log_cp10k", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.070", - "code_version": "1.1.3", - "resources": { - "duration_sec": 579.0, - "cpu_pct": 121.8, - "peak_memory_mb": 2200.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9796610169491525, - "f1": 0.9791824244686764, - "f1_macro": 0.8964053737821305 - }, - "scaled_scores": { - "accuracy": 0.9753914988814317, - "f1": 0.9743664399175329, - "f1_macro": 0.889484260612362 - }, - "mean_score": 0.9464140664704422 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_log_cp10k", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.416", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1108.0, - "cpu_pct": 374.3, - "peak_memory_mb": 2600.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1700.0 - }, - "metric_values": { - "accuracy": 0.9639445300462249, - "f1": 0.9638063508972504, - "f1_macro": 0.8934197028901939 - }, - "scaled_scores": { - "accuracy": 0.9563758389261745, - "f1": 0.9554332310463785, - "f1_macro": 0.8862991183107084 - }, - "mean_score": 0.9327027294277538 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "knn_classifier_scran", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.124", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1368.0, - "cpu_pct": 95.9, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9411402157164869, - "f1": 0.9404360005933883, - "f1_macro": 0.7218714554626218 - }, - "scaled_scores": { - "accuracy": 0.9287844891871737, - "f1": 0.9266563315577248, - "f1_macro": 0.7032898050164121 - }, - "mean_score": 0.8529102085871035 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "mlp_scran", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.503", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1477.0, - "cpu_pct": 353.9, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9208012326656394, - "f1": 0.925397640256357, - "f1_macro": 0.6930725021224952 - }, - "scaled_scores": { - "accuracy": 0.9041759880686054, - "f1": 0.9081389632570285, - "f1_macro": 0.6725668057820641 - }, - "mean_score": 0.8282939190358993 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "logistic_regression_scran", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.327", - "code_version": "1.1.3", - "resources": { - "duration_sec": 1518.0, - "cpu_pct": 101.1, - "peak_memory_mb": 8500.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.8046224961479199, - "f1": 0.7759818120203118, - "f1_macro": 0.42839154368293453 - }, - "scaled_scores": { - "accuracy": 0.7636092468307234, - "f1": 0.7241569426515417, - "f1_macro": 0.3902026244368059 - }, - "mean_score": 0.6259896046396903 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "xgboost_scran", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:02:03.474", - "code_version": "1.6.2", - "resources": { - "duration_sec": 1729.0, - "cpu_pct": 290.2, - "peak_memory_mb": 8300.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 2100.0 - }, - "metric_values": { - "accuracy": 0.9531587057010786, - "f1": 0.9528870337485718, - "f1_macro": 0.7961599397904259 - }, - "scaled_scores": { - "accuracy": 0.9433258762117823, - "f1": 0.9419878146111644, - "f1_macro": 0.7825414715672205 - }, - "mean_score": 0.8892850541300558 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "seurat", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:45:41.465", - "code_version": "4.3.0", - "resources": { - "duration_sec": 1751.0, - "cpu_pct": 109.7, - "peak_memory_mb": 51500.0, - "disk_read_mb": 2800.0, - "disk_write_mb": 2400.0 - }, - "metric_values": { - "accuracy": 0.9756548536209553, - "f1": 0.975085587274387, - "f1_macro": 0.8955432565937734 - }, - "scaled_scores": { - "accuracy": 0.9705443698732289, - "f1": 0.969321831230518, - "f1_macro": 0.8885645457392208 - }, - "mean_score": 0.942810248947656 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_hvg", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:39:34.695", - "code_version": "0.20.0", - "resources": { - "duration_sec": 3756.0, - "cpu_pct": 521.2, - "peak_memory_mb": 2700.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9476117103235747, - "f1": 0.9372298848234738, - "f1_macro": 0.527328944140918 - }, - "scaled_scores": { - "accuracy": 0.9366144668158091, - "f1": 0.9227085057844598, - "f1_macro": 0.4957499907809764 - }, - "mean_score": 0.7850243211270818 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scarches_scanvi_all_genes", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:39:34.102", - "code_version": "0.20.0", - "resources": { - "duration_sec": 4127.0, - "cpu_pct": 677.5, - "peak_memory_mb": 4900.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9463790446841295, - "f1": 0.9363141652450575, - "f1_macro": 0.5241903797681743 - }, - "scaled_scores": { - "accuracy": 0.9351230425055929, - "f1": 0.9215809415877211, - "f1_macro": 0.4924017402497183 - }, - "mean_score": 0.7830352414476774 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_all_genes", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 18:39:41.726", - "code_version": "0.20.0", - "resources": { - "duration_sec": 4259.0, - "cpu_pct": 667.3, - "peak_memory_mb": 3700.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9704160246533128, - "f1": 0.9672436357663169, - "f1_macro": 0.6735752035664337 - }, - "scaled_scores": { - "accuracy": 0.9642058165548099, - "f1": 0.9596657050959715, - "f1_macro": 0.651766900954443 - }, - "mean_score": 0.8585461408684081 - }, - { - "task_id": "label_projection", - "commit_sha": "65efdc87e3f4048b94b98c6f9fbfe10dae8d5ab0", - "method_id": "scanvi_hvg", - "dataset_id": "pancreas_random_label_noise", - "submission_time": "2023-02-21 23:01:13.673", - "code_version": "0.20.0", - "resources": { - "duration_sec": 7679.0, - "cpu_pct": 1860.6, - "peak_memory_mb": 6000.0, - "disk_read_mb": 1300.0, - "disk_write_mb": 1200.0 - }, - "metric_values": { - "accuracy": 0.9673343605546996, - "f1": 0.963942105070359, - "f1_macro": 0.6849998633357023 - }, - "scaled_scores": { - "accuracy": 0.9604772557792692, - "f1": 0.9556003908939597, - "f1_macro": 0.6639548374116637 - }, - "mean_score": 0.8600108280282974 - } -] \ No newline at end of file + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "knn", + "metric_values": { + "accuracy": 0.9993, + "f1_macro": 0.9997, + "f1_micro": 0.9993, + "f1_weighted": 0.9993 + }, + "scaled_scores": { + "accuracy": 0.9992, + "f1_macro": 0.9997, + "f1_micro": 0.9992, + "f1_weighted": 0.9992 + }, + "mean_score": 0.9993, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 19.1, + "cpu_pct": 155.1, + "peak_memory_mb": 24064, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.9993, + "f1_macro": 0.9997, + "f1_micro": 0.9993, + "f1_weighted": 0.9993 + }, + "scaled_scores": { + "accuracy": 0.9992, + "f1_macro": 0.9997, + "f1_micro": 0.9992, + "f1_weighted": 0.9992 + }, + "mean_score": 0.9993, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 72, + "cpu_pct": 1120.6, + "peak_memory_mb": 24167, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.175, + "f1_macro": 0.0298, + "f1_micro": 0.175, + "f1_weighted": 0.0521 + }, + "scaled_scores": { + "accuracy": 0.0472, + "f1_macro": 0.0071, + "f1_micro": 0.0472, + "f1_weighted": 0 + }, + "mean_score": 0.0254, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 130, + "cpu_pct": 24.7, + "peak_memory_mb": 23962, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "mlp", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 403, + "cpu_pct": 628.5, + "peak_memory_mb": 21402, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.9871, + "f1_macro": 0.7359, + "f1_micro": 0.9871, + "f1_weighted": 0.9866 + }, + "scaled_scores": { + "accuracy": 0.9851, + "f1_macro": 0.7297, + "f1_micro": 0.9851, + "f1_weighted": 0.9858 + }, + "mean_score": 0.9214, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 143, + "cpu_pct": 22.7, + "peak_memory_mb": 20071, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.1341, + "f1_macro": 0.0229, + "f1_micro": 0.1341, + "f1_weighted": 0.1288 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0.0809 + }, + "mean_score": 0.0202, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 167, + "cpu_pct": 25.6, + "peak_memory_mb": 19968, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.9971, + "f1_macro": 0.999, + "f1_micro": 0.9971, + "f1_weighted": 0.9971 + }, + "scaled_scores": { + "accuracy": 0.9967, + "f1_macro": 0.999, + "f1_micro": 0.9967, + "f1_weighted": 0.997 + }, + "mean_score": 0.9973, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 839, + "cpu_pct": 100.3, + "peak_memory_mb": 67687, + "disk_read_mb": 17716, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.9986, + "f1_macro": 0.9957, + "f1_micro": 0.9986, + "f1_weighted": 0.9986 + }, + "scaled_scores": { + "accuracy": 0.9983, + "f1_macro": 0.9956, + "f1_micro": 0.9983, + "f1_weighted": 0.9985 + }, + "mean_score": 0.9977, + "resources": { + "submit": "2025-01-08 08:55:25", + "exit_code": 0, + "duration_sec": 2128, + "cpu_pct": 100.8, + "peak_memory_mb": 69940, + "disk_read_mb": 17716, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 550, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 420, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 500, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "singler", + "metric_values": { + "accuracy": 0.9907, + "f1_macro": 0.759, + "f1_micro": 0.9907, + "f1_weighted": 0.991 + }, + "scaled_scores": { + "accuracy": 0.9892, + "f1_macro": 0.7534, + "f1_micro": 0.9892, + "f1_weighted": 0.9905 + }, + "mean_score": 0.9306, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 3687, + "cpu_pct": 100.6, + "peak_memory_mb": 64410, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 18.6, + "cpu_pct": 114.6, + "peak_memory_mb": 7066, + "disk_read_mb": 636, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:13:45", + "exit_code": 1, + "duration_sec": 770, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.9957, + "f1_macro": 0.9985, + "f1_micro": 0.9957, + "f1_weighted": 0.9957 + }, + "scaled_scores": { + "accuracy": 0.995, + "f1_macro": 0.9985, + "f1_micro": 0.995, + "f1_weighted": 0.9955 + }, + "mean_score": 0.996, + "resources": { + "submit": "2025-01-08 08:18:15", + "exit_code": 0, + "duration_sec": 2144, + "cpu_pct": 2189.3, + "peak_memory_mb": 71373, + "disk_read_mb": 17613, + "disk_write_mb": 171 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "knn", + "metric_values": { + "accuracy": 0.9525, + "f1_macro": 0.881, + "f1_micro": 0.9525, + "f1_weighted": 0.9471 + }, + "scaled_scores": { + "accuracy": 0.9436, + "f1_macro": 0.8785, + "f1_micro": 0.9436, + "f1_weighted": 0.9447 + }, + "mean_score": 0.9276, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 0, + "duration_sec": 11.7, + "cpu_pct": 131.4, + "peak_memory_mb": 5632, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.9374, + "f1_macro": 0.8716, + "f1_micro": 0.9374, + "f1_weighted": 0.936 + }, + "scaled_scores": { + "accuracy": 0.9257, + "f1_macro": 0.8688, + "f1_micro": 0.9257, + "f1_weighted": 0.9331 + }, + "mean_score": 0.9134, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 15.4, + "cpu_pct": 1982, + "peak_memory_mb": 7066, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.1569, + "f1_macro": 0.0209, + "f1_micro": 0.1569, + "f1_weighted": 0.0426 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 0, + "duration_sec": 7.4, + "cpu_pct": 180.9, + "peak_memory_mb": 5530, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.9412, + "f1_macro": 0.8887, + "f1_micro": 0.9412, + "f1_weighted": 0.9365 + }, + "scaled_scores": { + "accuracy": 0.9303, + "f1_macro": 0.8864, + "f1_micro": 0.9303, + "f1_weighted": 0.9337 + }, + "mean_score": 0.9202, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 0, + "duration_sec": 27.2, + "cpu_pct": 479.4, + "peak_memory_mb": 6964, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.8941, + "f1_macro": 0.8339, + "f1_micro": 0.8941, + "f1_weighted": 0.9141 + }, + "scaled_scores": { + "accuracy": 0.8744, + "f1_macro": 0.8303, + "f1_micro": 0.8744, + "f1_weighted": 0.9102 + }, + "mean_score": 0.8723, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 10, + "cpu_pct": 130.8, + "peak_memory_mb": 6554, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.1705, + "f1_macro": 0.0681, + "f1_micro": 0.1705, + "f1_weighted": 0.1774 + }, + "scaled_scores": { + "accuracy": 0.016, + "f1_macro": 0.0482, + "f1_micro": 0.016, + "f1_weighted": 0.1408 + }, + "mean_score": 0.0553, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 7.9, + "cpu_pct": 167.9, + "peak_memory_mb": 6452, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.9354, + "f1_macro": 0.8745, + "f1_micro": 0.9354, + "f1_weighted": 0.9364 + }, + "scaled_scores": { + "accuracy": 0.9234, + "f1_macro": 0.8718, + "f1_micro": 0.9234, + "f1_weighted": 0.9336 + }, + "mean_score": 0.913, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 156, + "cpu_pct": 86, + "peak_memory_mb": 24167, + "disk_read_mb": 1434, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.9409, + "f1_macro": 0.8881, + "f1_micro": 0.9409, + "f1_weighted": 0.9428 + }, + "scaled_scores": { + "accuracy": 0.9298, + "f1_macro": 0.8857, + "f1_micro": 0.9298, + "f1_weighted": 0.9402 + }, + "mean_score": 0.9214, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 0, + "duration_sec": 1200, + "cpu_pct": 105.2, + "peak_memory_mb": 22631, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scimilarity", + "metric_values": { + "accuracy": 0.8929, + "f1_macro": 0.7093, + "f1_micro": 0.8929, + "f1_weighted": 0.8883 + }, + "scaled_scores": { + "accuracy": 0.873, + "f1_macro": 0.7031, + "f1_micro": 0.873, + "f1_weighted": 0.8833 + }, + "mean_score": 0.8331, + "resources": { + "submit": "2025-01-08 08:03:16", + "exit_code": 0, + "duration_sec": 534, + "cpu_pct": 57.2, + "peak_memory_mb": 31130, + "disk_read_mb": 40141, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": 0.9316, + "f1_macro": 0.8418, + "f1_micro": 0.9316, + "f1_weighted": 0.9401 + }, + "scaled_scores": { + "accuracy": 0.9188, + "f1_macro": 0.8385, + "f1_micro": 0.9188, + "f1_weighted": 0.9375 + }, + "mean_score": 0.9034, + "resources": { + "submit": "2025-01-08 08:03:20", + "exit_code": 0, + "duration_sec": 816, + "cpu_pct": 36.7, + "peak_memory_mb": 28365, + "disk_read_mb": 40141, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.9548, + "f1_macro": 0.8689, + "f1_micro": 0.9548, + "f1_weighted": 0.9488 + }, + "scaled_scores": { + "accuracy": 0.9464, + "f1_macro": 0.8661, + "f1_micro": 0.9464, + "f1_weighted": 0.9465 + }, + "mean_score": 0.9263, + "resources": { + "submit": "2025-01-08 08:03:16", + "exit_code": 0, + "duration_sec": 85, + "cpu_pct": 109.4, + "peak_memory_mb": 12288, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "singler", + "metric_values": { + "accuracy": 0.8918, + "f1_macro": 0.8266, + "f1_micro": 0.8918, + "f1_weighted": 0.8982 + }, + "scaled_scores": { + "accuracy": 0.8716, + "f1_macro": 0.8229, + "f1_micro": 0.8716, + "f1_weighted": 0.8937 + }, + "mean_score": 0.865, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 0, + "duration_sec": 281, + "cpu_pct": 104.8, + "peak_memory_mb": 10445, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 0, + "duration_sec": 6.3, + "cpu_pct": 193.8, + "peak_memory_mb": 5530, + "disk_read_mb": 183, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "uce", + "metric_values": { + "accuracy": 0.1767, + "f1_macro": 0.0699, + "f1_micro": 0.1767, + "f1_weighted": 0.188 + }, + "scaled_scores": { + "accuracy": 0.0234, + "f1_macro": 0.0501, + "f1_micro": 0.0234, + "f1_weighted": 0.1518 + }, + "mean_score": 0.0622, + "resources": { + "submit": "2025-01-08 08:03:20", + "exit_code": 0, + "duration_sec": 4844, + "cpu_pct": 289, + "peak_memory_mb": 476877, + "disk_read_mb": 42292, + "disk_write_mb": 22733 + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.9486, + "f1_macro": 0.8928, + "f1_micro": 0.9486, + "f1_weighted": 0.9465 + }, + "scaled_scores": { + "accuracy": 0.939, + "f1_macro": 0.8905, + "f1_micro": 0.939, + "f1_weighted": 0.9441 + }, + "mean_score": 0.9282, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 131, + "cpu_pct": 4215.5, + "peak_memory_mb": 25293, + "disk_read_mb": 1332, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "knn", + "metric_values": { + "accuracy": 0.8523, + "f1_macro": 0.3913, + "f1_micro": 0.8523, + "f1_weighted": 0.8582 + }, + "scaled_scores": { + "accuracy": 0.8476, + "f1_macro": 0.3871, + "f1_micro": 0.8476, + "f1_weighted": 0.8565 + }, + "mean_score": 0.7347, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 16.7, + "cpu_pct": 264.9, + "peak_memory_mb": 5632, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.8829, + "f1_macro": 0.4437, + "f1_micro": 0.8829, + "f1_weighted": 0.8851 + }, + "scaled_scores": { + "accuracy": 0.8791, + "f1_macro": 0.4398, + "f1_micro": 0.8791, + "f1_weighted": 0.8837 + }, + "mean_score": 0.7704, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 34.4, + "cpu_pct": 2301.9, + "peak_memory_mb": 8909, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.0799, + "f1_macro": 0.0082, + "f1_micro": 0.0799, + "f1_weighted": 0.0118 + }, + "scaled_scores": { + "accuracy": 0.0507, + "f1_macro": 0.0013, + "f1_micro": 0.0507, + "f1_weighted": 0 + }, + "mean_score": 0.0257, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 8.7, + "cpu_pct": 77, + "peak_memory_mb": 4506, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.8714, + "f1_macro": 0.3353, + "f1_micro": 0.8714, + "f1_weighted": 0.8748 + }, + "scaled_scores": { + "accuracy": 0.8673, + "f1_macro": 0.3307, + "f1_micro": 0.8673, + "f1_weighted": 0.8733 + }, + "mean_score": 0.7346, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 323, + "cpu_pct": 673.3, + "peak_memory_mb": 4711, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.7599, + "f1_macro": 0.3363, + "f1_micro": 0.7599, + "f1_weighted": 0.7937 + }, + "scaled_scores": { + "accuracy": 0.7523, + "f1_macro": 0.3316, + "f1_micro": 0.7523, + "f1_weighted": 0.7913 + }, + "mean_score": 0.6569, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 14.1, + "cpu_pct": 61.8, + "peak_memory_mb": 2253, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.0308, + "f1_macro": 0.0069, + "f1_micro": 0.0308, + "f1_weighted": 0.0381 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0.0266 + }, + "mean_score": 0.0066, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 7.4, + "cpu_pct": 175.3, + "peak_memory_mb": 6964, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.8854, + "f1_macro": 0.4032, + "f1_micro": 0.8854, + "f1_weighted": 0.8967 + }, + "scaled_scores": { + "accuracy": 0.8817, + "f1_macro": 0.399, + "f1_micro": 0.8817, + "f1_weighted": 0.8955 + }, + "mean_score": 0.7645, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 579, + "cpu_pct": 100.5, + "peak_memory_mb": 41677, + "disk_read_mb": 3072, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.8962, + "f1_macro": 0.4291, + "f1_micro": 0.8962, + "f1_weighted": 0.9054 + }, + "scaled_scores": { + "accuracy": 0.8929, + "f1_macro": 0.4251, + "f1_micro": 0.8929, + "f1_weighted": 0.9043 + }, + "mean_score": 0.7788, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 1536, + "cpu_pct": 101.5, + "peak_memory_mb": 41984, + "disk_read_mb": 3072, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scimilarity", + "metric_values": { + "accuracy": 0.6665, + "f1_macro": 0.3254, + "f1_micro": 0.6665, + "f1_weighted": 0.6504 + }, + "scaled_scores": { + "accuracy": 0.6559, + "f1_macro": 0.3207, + "f1_micro": 0.6559, + "f1_weighted": 0.6462 + }, + "mean_score": 0.5697, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 740, + "cpu_pct": 117.3, + "peak_memory_mb": 34714, + "disk_read_mb": 41882, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": 0.8253, + "f1_macro": 0.4563, + "f1_micro": 0.8253, + "f1_weighted": 0.8386 + }, + "scaled_scores": { + "accuracy": 0.8197, + "f1_macro": 0.4525, + "f1_micro": 0.8197, + "f1_weighted": 0.8367 + }, + "mean_score": 0.7321, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 508, + "cpu_pct": 128.1, + "peak_memory_mb": 33485, + "disk_read_mb": 41882, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.841, + "f1_macro": 0.4186, + "f1_micro": 0.841, + "f1_weighted": 0.8457 + }, + "scaled_scores": { + "accuracy": 0.8359, + "f1_macro": 0.4146, + "f1_micro": 0.8359, + "f1_weighted": 0.8438 + }, + "mean_score": 0.7326, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 230, + "cpu_pct": 104.4, + "peak_memory_mb": 23552, + "disk_read_mb": 3072, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "singler", + "metric_values": { + "accuracy": 0.7903, + "f1_macro": 0.3753, + "f1_micro": 0.7903, + "f1_weighted": 0.823 + }, + "scaled_scores": { + "accuracy": 0.7836, + "f1_macro": 0.3709, + "f1_micro": 0.7836, + "f1_weighted": 0.8209 + }, + "mean_score": 0.6898, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 3735, + "cpu_pct": 100.6, + "peak_memory_mb": 19149, + "disk_read_mb": 3072, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 5.9, + "cpu_pct": 225.6, + "peak_memory_mb": 5530, + "disk_read_mb": 411, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:21:45", + "exit_code": 143, + "duration_sec": 14400, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.8328, + "f1_macro": 0.3251, + "f1_micro": 0.8328, + "f1_weighted": 0.8408 + }, + "scaled_scores": { + "accuracy": 0.8275, + "f1_macro": 0.3204, + "f1_micro": 0.8275, + "f1_weighted": 0.8389 + }, + "mean_score": 0.7036, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 640, + "cpu_pct": 4597.1, + "peak_memory_mb": 34919, + "disk_read_mb": 2970, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "knn", + "metric_values": { + "accuracy": 0.9633, + "f1_macro": 0.7455, + "f1_micro": 0.9633, + "f1_weighted": 0.9606 + }, + "scaled_scores": { + "accuracy": 0.9633, + "f1_macro": 0.7455, + "f1_micro": 0.9633, + "f1_weighted": 0.9606 + }, + "mean_score": 0.9081, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 39.2, + "cpu_pct": 140.9, + "peak_memory_mb": 20173, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.9345, + "f1_macro": 0.6663, + "f1_micro": 0.9345, + "f1_weighted": 0.942 + }, + "scaled_scores": { + "accuracy": 0.9345, + "f1_macro": 0.6663, + "f1_micro": 0.9345, + "f1_weighted": 0.942 + }, + "mean_score": 0.8693, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 130, + "cpu_pct": 566.1, + "peak_memory_mb": 19968, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.0013, + "f1_macro": 0.0001, + "f1_micro": 0.0013, + "f1_weighted": 3.3113e-06 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 139, + "cpu_pct": 19.8, + "peak_memory_mb": 19149, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.9631, + "f1_macro": 0.6983, + "f1_micro": 0.9631, + "f1_weighted": 0.9633 + }, + "scaled_scores": { + "accuracy": 0.963, + "f1_macro": 0.6983, + "f1_micro": 0.963, + "f1_weighted": 0.9633 + }, + "mean_score": 0.8969, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 1116, + "cpu_pct": 688.4, + "peak_memory_mb": 19456, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.9215, + "f1_macro": 0.5996, + "f1_micro": 0.9215, + "f1_weighted": 0.9225 + }, + "scaled_scores": { + "accuracy": 0.9214, + "f1_macro": 0.5995, + "f1_micro": 0.9214, + "f1_weighted": 0.9225 + }, + "mean_score": 0.8412, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 23, + "cpu_pct": 82.1, + "peak_memory_mb": 19456, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.0749, + "f1_macro": 0.0102, + "f1_micro": 0.0749, + "f1_weighted": 0.1091 + }, + "scaled_scores": { + "accuracy": 0.0737, + "f1_macro": 0.0102, + "f1_micro": 0.0737, + "f1_weighted": 0.1091 + }, + "mean_score": 0.0667, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 142, + "cpu_pct": 20, + "peak_memory_mb": 18944, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.9609, + "f1_macro": 0.6825, + "f1_micro": 0.9609, + "f1_weighted": 0.9609 + }, + "scaled_scores": { + "accuracy": 0.9609, + "f1_macro": 0.6825, + "f1_micro": 0.9609, + "f1_weighted": 0.9609 + }, + "mean_score": 0.8913, + "resources": { + "submit": "2025-01-08 08:36:25", + "exit_code": 0, + "duration_sec": 851, + "cpu_pct": 100.6, + "peak_memory_mb": 93287, + "disk_read_mb": 17613, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.9642, + "f1_macro": 0.7126, + "f1_micro": 0.9642, + "f1_weighted": 0.9637 + }, + "scaled_scores": { + "accuracy": 0.9641, + "f1_macro": 0.7126, + "f1_micro": 0.9641, + "f1_weighted": 0.9637 + }, + "mean_score": 0.9011, + "resources": { + "submit": "2025-01-08 09:22:25", + "exit_code": 0, + "duration_sec": 1891, + "cpu_pct": 100.3, + "peak_memory_mb": 98202, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 1, + "duration_sec": 650, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 670, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 1040, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "singler", + "metric_values": { + "accuracy": 0.8704, + "f1_macro": 0.5036, + "f1_micro": 0.8704, + "f1_weighted": 0.8728 + }, + "scaled_scores": { + "accuracy": 0.8702, + "f1_macro": 0.5036, + "f1_micro": 0.8702, + "f1_weighted": 0.8728 + }, + "mean_score": 0.7792, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 10644, + "cpu_pct": 100.2, + "peak_memory_mb": 69632, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 3, + "cpu_pct": 245.6, + "peak_memory_mb": 2868, + "disk_read_mb": 395, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 1, + "duration_sec": 280, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.9386, + "f1_macro": 0.6541, + "f1_micro": 0.9386, + "f1_weighted": 0.9418 + }, + "scaled_scores": { + "accuracy": 0.9385, + "f1_macro": 0.6541, + "f1_micro": 0.9385, + "f1_weighted": 0.9418 + }, + "mean_score": 0.8682, + "resources": { + "submit": "2025-01-08 08:28:35", + "exit_code": 0, + "duration_sec": 1435, + "cpu_pct": 5044.8, + "peak_memory_mb": 82023, + "disk_read_mb": 17613, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "knn", + "metric_values": { + "accuracy": 0.9956, + "f1_macro": 0.8689, + "f1_micro": 0.9956, + "f1_weighted": 0.9962 + }, + "scaled_scores": { + "accuracy": 0.9925, + "f1_macro": 0.8566, + "f1_micro": 0.9925, + "f1_weighted": 0.9931 + }, + "mean_score": 0.9587, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 0, + "duration_sec": 15.5, + "cpu_pct": 139.3, + "peak_memory_mb": 22631, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.9971, + "f1_macro": 0.8541, + "f1_micro": 0.9971, + "f1_weighted": 0.9979 + }, + "scaled_scores": { + "accuracy": 0.995, + "f1_macro": 0.8404, + "f1_micro": 0.995, + "f1_weighted": 0.9962 + }, + "mean_score": 0.9566, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 35.3, + "cpu_pct": 1981.7, + "peak_memory_mb": 18740, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.6313, + "f1_macro": 0.0968, + "f1_micro": 0.6313, + "f1_weighted": 0.4887 + }, + "scaled_scores": { + "accuracy": 0.3682, + "f1_macro": 0.012, + "f1_micro": 0.3682, + "f1_weighted": 0.078 + }, + "mean_score": 0.2066, + "resources": { + "submit": "2025-01-08 08:03:20", + "exit_code": 0, + "duration_sec": 55.5, + "cpu_pct": 34.7, + "peak_memory_mb": 18228, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.9956, + "f1_macro": 0.8628, + "f1_micro": 0.9956, + "f1_weighted": 0.9963 + }, + "scaled_scores": { + "accuracy": 0.9925, + "f1_macro": 0.85, + "f1_micro": 0.9925, + "f1_weighted": 0.9934 + }, + "mean_score": 0.9571, + "resources": { + "submit": "2025-01-08 08:03:17", + "exit_code": 0, + "duration_sec": 188, + "cpu_pct": 649.3, + "peak_memory_mb": 18535, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.9913, + "f1_macro": 0.9005, + "f1_micro": 0.9913, + "f1_weighted": 0.9917 + }, + "scaled_scores": { + "accuracy": 0.9851, + "f1_macro": 0.8912, + "f1_micro": 0.9851, + "f1_weighted": 0.985 + }, + "mean_score": 0.9616, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 0, + "duration_sec": 13.7, + "cpu_pct": 120.5, + "peak_memory_mb": 15156, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.4165, + "f1_macro": 0.0858, + "f1_micro": 0.4165, + "f1_weighted": 0.4454 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 0, + "duration_sec": 56, + "cpu_pct": 34.7, + "peak_memory_mb": 18228, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.9956, + "f1_macro": 0.8724, + "f1_micro": 0.9956, + "f1_weighted": 0.9963 + }, + "scaled_scores": { + "accuracy": 0.9925, + "f1_macro": 0.8605, + "f1_micro": 0.9925, + "f1_weighted": 0.9933 + }, + "mean_score": 0.9597, + "resources": { + "submit": "2025-01-08 08:18:45", + "exit_code": 0, + "duration_sec": 912, + "cpu_pct": 100.9, + "peak_memory_mb": 87860, + "disk_read_mb": 12800, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.9971, + "f1_macro": 0.9569, + "f1_micro": 0.9971, + "f1_weighted": 0.9971 + }, + "scaled_scores": { + "accuracy": 0.995, + "f1_macro": 0.9529, + "f1_micro": 0.995, + "f1_weighted": 0.9948 + }, + "mean_score": 0.9844, + "resources": { + "submit": "2025-01-08 08:53:46", + "exit_code": 0, + "duration_sec": 2495, + "cpu_pct": 100.6, + "peak_memory_mb": 92468, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 1, + "duration_sec": 480, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 1, + "duration_sec": 400, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.9956, + "f1_macro": 0.8724, + "f1_micro": 0.9956, + "f1_weighted": 0.9963 + }, + "scaled_scores": { + "accuracy": 0.9925, + "f1_macro": 0.8605, + "f1_micro": 0.9925, + "f1_weighted": 0.9933 + }, + "mean_score": 0.9597, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 0, + "duration_sec": 511, + "cpu_pct": 102.6, + "peak_memory_mb": 69530, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "singler", + "metric_values": { + "accuracy": 0.9913, + "f1_macro": 0.8195, + "f1_micro": 0.9913, + "f1_weighted": 0.9921 + }, + "scaled_scores": { + "accuracy": 0.9851, + "f1_macro": 0.8026, + "f1_micro": 0.9851, + "f1_weighted": 0.9857 + }, + "mean_score": 0.9396, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 0, + "duration_sec": 2389, + "cpu_pct": 100.8, + "peak_memory_mb": 47616, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 0, + "duration_sec": 5.2, + "cpu_pct": 190.6, + "peak_memory_mb": 2765, + "disk_read_mb": 65, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:10:25", + "exit_code": 137, + "duration_sec": 610, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.9971, + "f1_macro": 0.9818, + "f1_micro": 0.9971, + "f1_weighted": 0.997 + }, + "scaled_scores": { + "accuracy": 0.995, + "f1_macro": 0.9801, + "f1_micro": 0.995, + "f1_weighted": 0.9946 + }, + "mean_score": 0.9912, + "resources": { + "submit": "2025-01-08 08:09:45", + "exit_code": 0, + "duration_sec": 320, + "cpu_pct": 3839.2, + "peak_memory_mb": 71373, + "disk_read_mb": 12800, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "knn", + "metric_values": { + "accuracy": 0.8739, + "f1_macro": 0.811, + "f1_micro": 0.8739, + "f1_weighted": 0.8704 + }, + "scaled_scores": { + "accuracy": 0.8663, + "f1_macro": 0.8098, + "f1_micro": 0.8663, + "f1_weighted": 0.8672 + }, + "mean_score": 0.8524, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 20.2, + "cpu_pct": 378.2, + "peak_memory_mb": 12493, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.902, + "f1_macro": 0.8277, + "f1_micro": 0.902, + "f1_weighted": 0.9004 + }, + "scaled_scores": { + "accuracy": 0.8961, + "f1_macro": 0.8266, + "f1_micro": 0.8961, + "f1_weighted": 0.898 + }, + "mean_score": 0.8792, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 68, + "cpu_pct": 718.3, + "peak_memory_mb": 11879, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.1159, + "f1_macro": 0.0063, + "f1_micro": 0.1159, + "f1_weighted": 0.0241 + }, + "scaled_scores": { + "accuracy": 0.0624, + "f1_macro": 0, + "f1_micro": 0.0624, + "f1_weighted": 0 + }, + "mean_score": 0.0312, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 8.9, + "cpu_pct": 129.9, + "peak_memory_mb": 11367, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.8959, + "f1_macro": 0.8458, + "f1_micro": 0.8959, + "f1_weighted": 0.8954 + }, + "scaled_scores": { + "accuracy": 0.8896, + "f1_macro": 0.8448, + "f1_micro": 0.8896, + "f1_weighted": 0.8928 + }, + "mean_score": 0.8792, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 362, + "cpu_pct": 682.7, + "peak_memory_mb": 15668, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.806, + "f1_macro": 0.72, + "f1_micro": 0.806, + "f1_weighted": 0.8074 + }, + "scaled_scores": { + "accuracy": 0.7943, + "f1_macro": 0.7183, + "f1_micro": 0.7943, + "f1_weighted": 0.8027 + }, + "mean_score": 0.7774, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 17.5, + "cpu_pct": 78.3, + "peak_memory_mb": 11674, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.0571, + "f1_macro": 0.0244, + "f1_micro": 0.0571, + "f1_weighted": 0.0655 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0.0182, + "f1_micro": 0, + "f1_weighted": 0.0424 + }, + "mean_score": 0.0152, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 12.1, + "cpu_pct": 91, + "peak_memory_mb": 10957, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.908, + "f1_macro": 0.8569, + "f1_micro": 0.908, + "f1_weighted": 0.9068 + }, + "scaled_scores": { + "accuracy": 0.9024, + "f1_macro": 0.856, + "f1_micro": 0.9024, + "f1_weighted": 0.9045 + }, + "mean_score": 0.8913, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 721, + "cpu_pct": 100.8, + "peak_memory_mb": 61850, + "disk_read_mb": 9933, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.91, + "f1_macro": 0.8661, + "f1_micro": 0.91, + "f1_weighted": 0.908 + }, + "scaled_scores": { + "accuracy": 0.9045, + "f1_macro": 0.8653, + "f1_micro": 0.9045, + "f1_weighted": 0.9058 + }, + "mean_score": 0.895, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 1808, + "cpu_pct": 100.7, + "peak_memory_mb": 63591, + "disk_read_mb": 9933, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scimilarity", + "metric_values": { + "accuracy": 0.843, + "f1_macro": 0.7517, + "f1_micro": 0.843, + "f1_weighted": 0.8424 + }, + "scaled_scores": { + "accuracy": 0.8334, + "f1_macro": 0.7501, + "f1_micro": 0.8334, + "f1_weighted": 0.8385 + }, + "mean_score": 0.8139, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 1259, + "cpu_pct": 147.7, + "peak_memory_mb": 49255, + "disk_read_mb": 48640, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": 0.8867, + "f1_macro": 0.84, + "f1_micro": 0.8867, + "f1_weighted": 0.8843 + }, + "scaled_scores": { + "accuracy": 0.8799, + "f1_macro": 0.839, + "f1_micro": 0.8799, + "f1_weighted": 0.8815 + }, + "mean_score": 0.87, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 750, + "cpu_pct": 256.3, + "peak_memory_mb": 51200, + "disk_read_mb": 48640, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.8359, + "f1_macro": 0.7821, + "f1_micro": 0.8359, + "f1_weighted": 0.8226 + }, + "scaled_scores": { + "accuracy": 0.826, + "f1_macro": 0.7807, + "f1_micro": 0.826, + "f1_weighted": 0.8182 + }, + "mean_score": 0.8127, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 797, + "cpu_pct": 101.1, + "peak_memory_mb": 50996, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "singler", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 143, + "duration_sec": 14401, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 0, + "duration_sec": 1.9, + "cpu_pct": 478.5, + "peak_memory_mb": 2765, + "disk_read_mb": 668, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:08:35", + "exit_code": 143, + "duration_sec": 14411, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.8562, + "f1_macro": 0.7775, + "f1_micro": 0.8562, + "f1_weighted": 0.8544 + }, + "scaled_scores": { + "accuracy": 0.8475, + "f1_macro": 0.7761, + "f1_micro": 0.8475, + "f1_weighted": 0.8509 + }, + "mean_score": 0.8305, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 0, + "duration_sec": 781, + "cpu_pct": 4787.4, + "peak_memory_mb": 56628, + "disk_read_mb": 9831, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "knn", + "metric_values": { + "accuracy": 0.964, + "f1_macro": 0.8905, + "f1_micro": 0.964, + "f1_weighted": 0.9632 + }, + "scaled_scores": { + "accuracy": 0.9542, + "f1_macro": 0.8835, + "f1_micro": 0.9542, + "f1_weighted": 0.9502 + }, + "mean_score": 0.9355, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 14.3, + "cpu_pct": 217.2, + "peak_memory_mb": 14951, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.9606, + "f1_macro": 0.8765, + "f1_micro": 0.9606, + "f1_weighted": 0.9608 + }, + "scaled_scores": { + "accuracy": 0.95, + "f1_macro": 0.8686, + "f1_micro": 0.95, + "f1_weighted": 0.9469 + }, + "mean_score": 0.9289, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 23.7, + "cpu_pct": 2958.6, + "peak_memory_mb": 19252, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.5704, + "f1_macro": 0.0605, + "f1_micro": 0.5704, + "f1_weighted": 0.4144 + }, + "scaled_scores": { + "accuracy": 0.4546, + "f1_macro": 0, + "f1_micro": 0.4546, + "f1_weighted": 0.2066 + }, + "mean_score": 0.279, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 56.6, + "cpu_pct": 31.1, + "peak_memory_mb": 14848, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.9576, + "f1_macro": 0.8296, + "f1_micro": 0.9576, + "f1_weighted": 0.9577 + }, + "scaled_scores": { + "accuracy": 0.9462, + "f1_macro": 0.8186, + "f1_micro": 0.9462, + "f1_weighted": 0.9427 + }, + "mean_score": 0.9134, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 201, + "cpu_pct": 681.1, + "peak_memory_mb": 19047, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.867, + "f1_macro": 0.7004, + "f1_micro": 0.867, + "f1_weighted": 0.8885 + }, + "scaled_scores": { + "accuracy": 0.8312, + "f1_macro": 0.681, + "f1_micro": 0.8312, + "f1_weighted": 0.8489 + }, + "mean_score": 0.7981, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 8.7, + "cpu_pct": 205, + "peak_memory_mb": 19047, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.2124, + "f1_macro": 0.0683, + "f1_micro": 0.2124, + "f1_weighted": 0.2619 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0.0083, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0.0021, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 56.4, + "cpu_pct": 32.3, + "peak_memory_mb": 14848, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.9615, + "f1_macro": 0.8799, + "f1_micro": 0.9615, + "f1_weighted": 0.9617 + }, + "scaled_scores": { + "accuracy": 0.9512, + "f1_macro": 0.8722, + "f1_micro": 0.9512, + "f1_weighted": 0.9481 + }, + "mean_score": 0.9306, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 681, + "cpu_pct": 101.5, + "peak_memory_mb": 58368, + "disk_read_mb": 13415, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.9618, + "f1_macro": 0.8733, + "f1_micro": 0.9618, + "f1_weighted": 0.9608 + }, + "scaled_scores": { + "accuracy": 0.9515, + "f1_macro": 0.8651, + "f1_micro": 0.9515, + "f1_weighted": 0.9468 + }, + "mean_score": 0.9288, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 1725, + "cpu_pct": 101, + "peak_memory_mb": 59597, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 1, + "duration_sec": 410, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 1, + "duration_sec": 340, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.947, + "f1_macro": 0.8217, + "f1_micro": 0.947, + "f1_weighted": 0.9467 + }, + "scaled_scores": { + "accuracy": 0.9327, + "f1_macro": 0.8103, + "f1_micro": 0.9327, + "f1_weighted": 0.9277 + }, + "mean_score": 0.9008, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 433, + "cpu_pct": 102.9, + "peak_memory_mb": 71885, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "singler", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": "NA", + "duration_sec": 3660, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 1.7, + "cpu_pct": 475.7, + "peak_memory_mb": 2765, + "disk_read_mb": 273, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:15:15", + "exit_code": 143, + "duration_sec": 14400, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.9618, + "f1_macro": 0.866, + "f1_micro": 0.9618, + "f1_weighted": 0.9604 + }, + "scaled_scores": { + "accuracy": 0.9515, + "f1_macro": 0.8573, + "f1_micro": 0.9515, + "f1_weighted": 0.9463 + }, + "mean_score": 0.9267, + "resources": { + "submit": "2025-01-08 08:39:55", + "exit_code": 0, + "duration_sec": 332, + "cpu_pct": 4778.9, + "peak_memory_mb": 67072, + "disk_read_mb": 13312, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "knn", + "metric_values": { + "accuracy": 0.7721, + "f1_macro": 0.2696, + "f1_micro": 0.7721, + "f1_weighted": 0.7757 + }, + "scaled_scores": { + "accuracy": 0.7654, + "f1_macro": 0.2675, + "f1_micro": 0.7654, + "f1_weighted": 0.7724 + }, + "mean_score": 0.6427, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 14.9, + "cpu_pct": 907.2, + "peak_memory_mb": 29184, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.7768, + "f1_macro": 0.2977, + "f1_micro": 0.7768, + "f1_weighted": 0.7793 + }, + "scaled_scores": { + "accuracy": 0.7703, + "f1_macro": 0.2957, + "f1_micro": 0.7703, + "f1_weighted": 0.7761 + }, + "mean_score": 0.6531, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 295, + "cpu_pct": 615.3, + "peak_memory_mb": 22528, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.0892, + "f1_macro": 0.0029, + "f1_micro": 0.0892, + "f1_weighted": 0.0146 + }, + "scaled_scores": { + "accuracy": 0.0625, + "f1_macro": 0, + "f1_micro": 0.0625, + "f1_weighted": 0 + }, + "mean_score": 0.0312, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 293, + "cpu_pct": 15, + "peak_memory_mb": 20992, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.7634, + "f1_macro": 0.2442, + "f1_micro": 0.7634, + "f1_weighted": 0.7614 + }, + "scaled_scores": { + "accuracy": 0.7565, + "f1_macro": 0.242, + "f1_micro": 0.7565, + "f1_weighted": 0.7579 + }, + "mean_score": 0.6282, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 11010, + "cpu_pct": 843.4, + "peak_memory_mb": 21709, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.7496, + "f1_macro": 0.2954, + "f1_micro": 0.7496, + "f1_weighted": 0.7566 + }, + "scaled_scores": { + "accuracy": 0.7422, + "f1_macro": 0.2934, + "f1_micro": 0.7422, + "f1_weighted": 0.753 + }, + "mean_score": 0.6327, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 50.5, + "cpu_pct": 90.6, + "peak_memory_mb": 22631, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.0285, + "f1_macro": 0.0032, + "f1_micro": 0.0285, + "f1_weighted": 0.0408 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0.0003, + "f1_micro": 0, + "f1_weighted": 0.0265 + }, + "mean_score": 0.0067, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 192, + "cpu_pct": 15.1, + "peak_memory_mb": 25088, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.8121, + "f1_macro": 0.299, + "f1_micro": 0.8121, + "f1_weighted": 0.8111 + }, + "scaled_scores": { + "accuracy": 0.8065, + "f1_macro": 0.297, + "f1_micro": 0.8065, + "f1_weighted": 0.8083 + }, + "mean_score": 0.6796, + "resources": { + "submit": "2025-01-08 09:17:45", + "exit_code": 0, + "duration_sec": 1393, + "cpu_pct": 100.2, + "peak_memory_mb": 127693, + "disk_read_mb": 19559, + "disk_write_mb": 3 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.8264, + "f1_macro": 0.3288, + "f1_micro": 0.8264, + "f1_weighted": 0.8227 + }, + "scaled_scores": { + "accuracy": 0.8213, + "f1_macro": 0.3269, + "f1_micro": 0.8213, + "f1_weighted": 0.82 + }, + "mean_score": 0.6974, + "resources": { + "submit": "2025-01-08 10:16:05", + "exit_code": 0, + "duration_sec": 3377, + "cpu_pct": 100.3, + "peak_memory_mb": 134554, + "disk_read_mb": 19559, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scimilarity", + "metric_values": { + "accuracy": 0.8548, + "f1_macro": 0.3666, + "f1_micro": 0.8548, + "f1_weighted": 0.8564 + }, + "scaled_scores": { + "accuracy": 0.8505, + "f1_macro": 0.3648, + "f1_micro": 0.8505, + "f1_weighted": 0.8543 + }, + "mean_score": 0.73, + "resources": { + "submit": "2025-01-08 09:10:55", + "exit_code": 0, + "duration_sec": 2312, + "cpu_pct": 164.7, + "peak_memory_mb": 79565, + "disk_read_mb": 58368, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": 0.8138, + "f1_macro": 0.3342, + "f1_micro": 0.8138, + "f1_weighted": 0.8184 + }, + "scaled_scores": { + "accuracy": 0.8083, + "f1_macro": 0.3323, + "f1_micro": 0.8083, + "f1_weighted": 0.8157 + }, + "mean_score": 0.6911, + "resources": { + "submit": "2025-01-08 09:05:15", + "exit_code": 0, + "duration_sec": 532, + "cpu_pct": 215.7, + "peak_memory_mb": 83456, + "disk_read_mb": 58368, + "disk_write_mb": 41882 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.8534, + "f1_macro": 0.4562, + "f1_micro": 0.8534, + "f1_weighted": 0.8527 + }, + "scaled_scores": { + "accuracy": 0.8491, + "f1_macro": 0.4546, + "f1_micro": 0.8491, + "f1_weighted": 0.8505 + }, + "mean_score": 0.7508, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 676, + "cpu_pct": 102.6, + "peak_memory_mb": 96564, + "disk_read_mb": 19559, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "singler", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 143, + "duration_sec": 14400, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 26.8, + "cpu_pct": 26.5, + "peak_memory_mb": 4813, + "disk_read_mb": 3277, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:29:55", + "exit_code": 137, + "duration_sec": 700, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.7757, + "f1_macro": 0.2713, + "f1_micro": 0.7757, + "f1_weighted": 0.8019 + }, + "scaled_scores": { + "accuracy": 0.7691, + "f1_macro": 0.2692, + "f1_micro": 0.7691, + "f1_weighted": 0.799 + }, + "mean_score": 0.6516, + "resources": { + "submit": "2025-01-08 08:56:05", + "exit_code": 0, + "duration_sec": 4600, + "cpu_pct": 4782.9, + "peak_memory_mb": 97690, + "disk_read_mb": 19456, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "knn", + "metric_values": { + "accuracy": 0.8179, + "f1_macro": 0.3601, + "f1_micro": 0.8179, + "f1_weighted": 0.8226 + }, + "scaled_scores": { + "accuracy": 0.8159, + "f1_macro": 0.3597, + "f1_micro": 0.8159, + "f1_weighted": 0.8224 + }, + "mean_score": 0.7035, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 18.4, + "cpu_pct": 97.5, + "peak_memory_mb": 3687, + "disk_read_mb": 769, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.879, + "f1_macro": 0.4803, + "f1_micro": 0.879, + "f1_weighted": 0.8692 + }, + "scaled_scores": { + "accuracy": 0.8776, + "f1_macro": 0.48, + "f1_micro": 0.8776, + "f1_weighted": 0.8691 + }, + "mean_score": 0.7761, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 32.1, + "cpu_pct": 2647.1, + "peak_memory_mb": 6759, + "disk_read_mb": 768, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.0222, + "f1_macro": 0.0005, + "f1_micro": 0.0222, + "f1_weighted": 0.001 + }, + "scaled_scores": { + "accuracy": 0.0115, + "f1_macro": 0, + "f1_micro": 0.0115, + "f1_weighted": 0 + }, + "mean_score": 0.0058, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 11, + "cpu_pct": 89.5, + "peak_memory_mb": 3584, + "disk_read_mb": 756, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.8352, + "f1_macro": 0.3762, + "f1_micro": 0.8352, + "f1_weighted": 0.8383 + }, + "scaled_scores": { + "accuracy": 0.8334, + "f1_macro": 0.3759, + "f1_micro": 0.8334, + "f1_weighted": 0.8382 + }, + "mean_score": 0.7202, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 9650, + "cpu_pct": 794.9, + "peak_memory_mb": 2458, + "disk_read_mb": 767, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.7831, + "f1_macro": 0.4153, + "f1_micro": 0.7831, + "f1_weighted": 0.8108 + }, + "scaled_scores": { + "accuracy": 0.7807, + "f1_macro": 0.415, + "f1_micro": 0.7807, + "f1_weighted": 0.8106 + }, + "mean_score": 0.6968, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 9.3, + "cpu_pct": 130.3, + "peak_memory_mb": 5735, + "disk_read_mb": 766, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.0108, + "f1_macro": 0.0023, + "f1_micro": 0.0108, + "f1_weighted": 0.0176 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0.0017, + "f1_micro": 0, + "f1_weighted": 0.0166 + }, + "mean_score": 0.0046, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 7.9, + "cpu_pct": 48.6, + "peak_memory_mb": 2253, + "disk_read_mb": 755, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.8922, + "f1_macro": 0.5552, + "f1_micro": 0.8922, + "f1_weighted": 0.8881 + }, + "scaled_scores": { + "accuracy": 0.891, + "f1_macro": 0.555, + "f1_micro": 0.891, + "f1_weighted": 0.888 + }, + "mean_score": 0.8062, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 258, + "cpu_pct": 102.8, + "peak_memory_mb": 28058, + "disk_read_mb": 827, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.9077, + "f1_macro": 0.5656, + "f1_micro": 0.9077, + "f1_weighted": 0.8931 + }, + "scaled_scores": { + "accuracy": 0.9067, + "f1_macro": 0.5653, + "f1_micro": 0.9067, + "f1_weighted": 0.893 + }, + "mean_score": 0.8179, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 1122, + "cpu_pct": 105.2, + "peak_memory_mb": 27034, + "disk_read_mb": 824, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 231, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 510, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.8586, + "f1_macro": 0.4654, + "f1_micro": 0.8586, + "f1_weighted": 0.8464 + }, + "scaled_scores": { + "accuracy": 0.8571, + "f1_macro": 0.4651, + "f1_micro": 0.8571, + "f1_weighted": 0.8462 + }, + "mean_score": 0.7564, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 102, + "cpu_pct": 113.4, + "peak_memory_mb": 11776, + "disk_read_mb": 793, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "singler", + "metric_values": { + "accuracy": 0.8544, + "f1_macro": 0.4786, + "f1_micro": 0.8544, + "f1_weighted": 0.8471 + }, + "scaled_scores": { + "accuracy": 0.8528, + "f1_macro": 0.4783, + "f1_micro": 0.8528, + "f1_weighted": 0.8469 + }, + "mean_score": 0.7577, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 439, + "cpu_pct": 104.1, + "peak_memory_mb": 11060, + "disk_read_mb": 785, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 2.4, + "cpu_pct": 455.3, + "peak_memory_mb": 5530, + "disk_read_mb": 34, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 131, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.8382, + "f1_macro": 0.5006, + "f1_micro": 0.8382, + "f1_weighted": 0.842 + }, + "scaled_scores": { + "accuracy": 0.8365, + "f1_macro": 0.5003, + "f1_micro": 0.8365, + "f1_weighted": 0.8419 + }, + "mean_score": 0.7538, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 1400, + "cpu_pct": 2058.1, + "peak_memory_mb": 13415, + "disk_read_mb": 767, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "knn", + "metric_values": { + "accuracy": 0.8627, + "f1_macro": 0.763, + "f1_micro": 0.8627, + "f1_weighted": 0.859 + }, + "scaled_scores": { + "accuracy": 0.8299, + "f1_macro": 0.7498, + "f1_micro": 0.8299, + "f1_weighted": 0.8209 + }, + "mean_score": 0.8076, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 7.4, + "cpu_pct": 156.7, + "peak_memory_mb": 5325, + "disk_read_mb": 827, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.9022, + "f1_macro": 0.8118, + "f1_micro": 0.9022, + "f1_weighted": 0.9016 + }, + "scaled_scores": { + "accuracy": 0.8789, + "f1_macro": 0.8013, + "f1_micro": 0.8789, + "f1_weighted": 0.875 + }, + "mean_score": 0.8585, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 15.7, + "cpu_pct": 2772.5, + "peak_memory_mb": 6554, + "disk_read_mb": 827, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.4378, + "f1_macro": 0.0677, + "f1_micro": 0.4378, + "f1_weighted": 0.2666 + }, + "scaled_scores": { + "accuracy": 0.3036, + "f1_macro": 0.0155, + "f1_micro": 0.3036, + "f1_weighted": 0.0683 + }, + "mean_score": 0.1728, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 6.9, + "cpu_pct": 166.2, + "peak_memory_mb": 5530, + "disk_read_mb": 814, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.8881, + "f1_macro": 0.7177, + "f1_micro": 0.8881, + "f1_weighted": 0.8887 + }, + "scaled_scores": { + "accuracy": 0.8613, + "f1_macro": 0.7019, + "f1_micro": 0.8613, + "f1_weighted": 0.8586 + }, + "mean_score": 0.8208, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 19.8, + "cpu_pct": 682.5, + "peak_memory_mb": 6452, + "disk_read_mb": 826, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.8747, + "f1_macro": 0.791, + "f1_micro": 0.8747, + "f1_weighted": 0.8765 + }, + "scaled_scores": { + "accuracy": 0.8448, + "f1_macro": 0.7793, + "f1_micro": 0.8448, + "f1_weighted": 0.8431 + }, + "mean_score": 0.828, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 7.1, + "cpu_pct": 138.4, + "peak_memory_mb": 3277, + "disk_read_mb": 825, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.1927, + "f1_macro": 0.053, + "f1_micro": 0.1927, + "f1_weighted": 0.2128 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 2.9, + "cpu_pct": 359.9, + "peak_memory_mb": 5530, + "disk_read_mb": 814, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.8889, + "f1_macro": 0.8155, + "f1_micro": 0.8889, + "f1_weighted": 0.8885 + }, + "scaled_scores": { + "accuracy": 0.8624, + "f1_macro": 0.8052, + "f1_micro": 0.8624, + "f1_weighted": 0.8584 + }, + "mean_score": 0.8471, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 89, + "cpu_pct": 99.2, + "peak_memory_mb": 21095, + "disk_read_mb": 886, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.891, + "f1_macro": 0.8116, + "f1_micro": 0.891, + "f1_weighted": 0.8928 + }, + "scaled_scores": { + "accuracy": 0.8649, + "f1_macro": 0.8011, + "f1_micro": 0.8649, + "f1_weighted": 0.8639 + }, + "mean_score": 0.8487, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 517, + "cpu_pct": 98.7, + "peak_memory_mb": 18944, + "disk_read_mb": 882, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 1, + "duration_sec": 260, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 1, + "duration_sec": 240, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.903, + "f1_macro": 0.9065, + "f1_micro": 0.903, + "f1_weighted": 0.9035 + }, + "scaled_scores": { + "accuracy": 0.8799, + "f1_macro": 0.9013, + "f1_micro": 0.8799, + "f1_weighted": 0.8774 + }, + "mean_score": 0.8846, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 57.4, + "cpu_pct": 112.1, + "peak_memory_mb": 9933, + "disk_read_mb": 851, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "singler", + "metric_values": { + "accuracy": 0.8539, + "f1_macro": 0.7819, + "f1_micro": 0.8539, + "f1_weighted": 0.8511 + }, + "scaled_scores": { + "accuracy": 0.8191, + "f1_macro": 0.7697, + "f1_micro": 0.8191, + "f1_weighted": 0.8108 + }, + "mean_score": 0.8047, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 499, + "cpu_pct": 102.6, + "peak_memory_mb": 9114, + "disk_read_mb": 844, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 0, + "duration_sec": 6.7, + "cpu_pct": 74.1, + "peak_memory_mb": 1536, + "disk_read_mb": 92, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 1, + "duration_sec": 140, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.891, + "f1_macro": 0.8061, + "f1_micro": 0.891, + "f1_weighted": 0.8923 + }, + "scaled_scores": { + "accuracy": 0.8649, + "f1_macro": 0.7953, + "f1_micro": 0.8649, + "f1_weighted": 0.8632 + }, + "mean_score": 0.8471, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 0, + "duration_sec": 187, + "cpu_pct": 1695.5, + "peak_memory_mb": 12596, + "disk_read_mb": 826, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "knn", + "metric_values": { + "accuracy": 0.9483, + "f1_macro": 0.6335, + "f1_micro": 0.9483, + "f1_weighted": 0.9379 + }, + "scaled_scores": { + "accuracy": 0.9296, + "f1_macro": 0.6174, + "f1_micro": 0.9296, + "f1_weighted": 0.922 + }, + "mean_score": 0.8497, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 15.4, + "cpu_pct": 71, + "peak_memory_mb": 2868, + "disk_read_mb": 887, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.2571, + "f1_macro": 0.3054, + "f1_micro": 0.2571, + "f1_weighted": 0.3379 + }, + "scaled_scores": { + "accuracy": -0.0107, + "f1_macro": 0.275, + "f1_micro": -0.0107, + "f1_weighted": 0.1681 + }, + "mean_score": 0.1108, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 15.6, + "cpu_pct": 738.3, + "peak_memory_mb": 6554, + "disk_read_mb": 886, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.3746, + "f1_macro": 0.0419, + "f1_micro": 0.3746, + "f1_weighted": 0.2042 + }, + "scaled_scores": { + "accuracy": 0.1493, + "f1_macro": 0, + "f1_micro": 0.1493, + "f1_weighted": 0 + }, + "mean_score": 0.0746, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 10.8, + "cpu_pct": 100.5, + "peak_memory_mb": 6349, + "disk_read_mb": 874, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.9561, + "f1_macro": 0.668, + "f1_micro": 0.9561, + "f1_weighted": 0.9508 + }, + "scaled_scores": { + "accuracy": 0.9403, + "f1_macro": 0.6535, + "f1_micro": 0.9403, + "f1_weighted": 0.9381 + }, + "mean_score": 0.8681, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 11.6, + "cpu_pct": 394.6, + "peak_memory_mb": 6554, + "disk_read_mb": 885, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.6238, + "f1_macro": 0.5442, + "f1_micro": 0.6238, + "f1_weighted": 0.5887 + }, + "scaled_scores": { + "accuracy": 0.4883, + "f1_macro": 0.5243, + "f1_micro": 0.4883, + "f1_weighted": 0.4832 + }, + "mean_score": 0.496, + "resources": { + "submit": "2025-01-08 08:03:57", + "exit_code": 0, + "duration_sec": 12.1, + "cpu_pct": 90.2, + "peak_memory_mb": 3789, + "disk_read_mb": 884, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.2649, + "f1_macro": 0.0831, + "f1_micro": 0.2649, + "f1_weighted": 0.2847 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0.043, + "f1_micro": 0, + "f1_weighted": 0.1011 + }, + "mean_score": 0.036, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 8.5, + "cpu_pct": 115.3, + "peak_memory_mb": 5530, + "disk_read_mb": 874, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.9514, + "f1_macro": 0.6666, + "f1_micro": 0.9514, + "f1_weighted": 0.9485 + }, + "scaled_scores": { + "accuracy": 0.9339, + "f1_macro": 0.652, + "f1_micro": 0.9339, + "f1_weighted": 0.9353 + }, + "mean_score": 0.8638, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 62, + "cpu_pct": 97.2, + "peak_memory_mb": 21300, + "disk_read_mb": 946, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.8041, + "f1_macro": 0.6407, + "f1_micro": 0.8041, + "f1_weighted": 0.8422 + }, + "scaled_scores": { + "accuracy": 0.7335, + "f1_macro": 0.625, + "f1_micro": 0.7335, + "f1_weighted": 0.8018 + }, + "mean_score": 0.7234, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 324, + "cpu_pct": 98.9, + "peak_memory_mb": 19047, + "disk_read_mb": 942, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 260, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:57", + "exit_code": 1, + "duration_sec": 410, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.9765, + "f1_macro": 0.804, + "f1_micro": 0.9765, + "f1_weighted": 0.9746 + }, + "scaled_scores": { + "accuracy": 0.968, + "f1_macro": 0.7954, + "f1_micro": 0.968, + "f1_weighted": 0.968 + }, + "mean_score": 0.9249, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 36.9, + "cpu_pct": 127.2, + "peak_memory_mb": 10138, + "disk_read_mb": 911, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "singler", + "metric_values": { + "accuracy": 0.9624, + "f1_macro": 0.8058, + "f1_micro": 0.9624, + "f1_weighted": 0.958 + }, + "scaled_scores": { + "accuracy": 0.9488, + "f1_macro": 0.7973, + "f1_micro": 0.9488, + "f1_weighted": 0.9472 + }, + "mean_score": 0.9105, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 112, + "cpu_pct": 108.6, + "peak_memory_mb": 6247, + "disk_read_mb": 904, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 4.6, + "cpu_pct": 217.5, + "peak_memory_mb": 5530, + "disk_read_mb": 158, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:57", + "exit_code": 1, + "duration_sec": 290, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.9561, + "f1_macro": 0.8257, + "f1_micro": 0.9561, + "f1_weighted": 0.952 + }, + "scaled_scores": { + "accuracy": 0.9403, + "f1_macro": 0.8181, + "f1_micro": 0.9403, + "f1_weighted": 0.9397 + }, + "mean_score": 0.9096, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 0, + "duration_sec": 97, + "cpu_pct": 4902.3, + "peak_memory_mb": 21402, + "disk_read_mb": 886, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "knn", + "metric_values": { + "accuracy": 0.2718, + "f1_macro": 0.1904, + "f1_micro": 0.2718, + "f1_weighted": 0.2486 + }, + "scaled_scores": { + "accuracy": 0.2486, + "f1_macro": 0.1876, + "f1_micro": 0.2486, + "f1_weighted": 0.2459 + }, + "mean_score": 0.2327, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 9.9, + "cpu_pct": 133.8, + "peak_memory_mb": 5530, + "disk_read_mb": 795, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "logistic_regression", + "metric_values": { + "accuracy": 0.2821, + "f1_macro": 0.1736, + "f1_micro": 0.2821, + "f1_weighted": 0.229 + }, + "scaled_scores": { + "accuracy": 0.2592, + "f1_macro": 0.1708, + "f1_micro": 0.2592, + "f1_weighted": 0.2262 + }, + "mean_score": 0.2289, + "resources": { + "submit": "2025-01-08 08:05:35", + "exit_code": 0, + "duration_sec": 30.1, + "cpu_pct": 872.7, + "peak_memory_mb": 3789, + "disk_read_mb": 794, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "majority_vote", + "metric_values": { + "accuracy": 0.0432, + "f1_macro": 0.0034, + "f1_micro": 0.0432, + "f1_weighted": 0.0036 + }, + "scaled_scores": { + "accuracy": 0.0127, + "f1_macro": 0, + "f1_micro": 0.0127, + "f1_weighted": 0 + }, + "mean_score": 0.0064, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 3.8, + "cpu_pct": 73.5, + "peak_memory_mb": 2253, + "disk_read_mb": 782, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "mlp", + "metric_values": { + "accuracy": 0.287, + "f1_macro": 0.2208, + "f1_micro": 0.287, + "f1_weighted": 0.2621 + }, + "scaled_scores": { + "accuracy": 0.2644, + "f1_macro": 0.2181, + "f1_micro": 0.2644, + "f1_weighted": 0.2594 + }, + "mean_score": 0.2516, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 16.2, + "cpu_pct": 428.7, + "peak_memory_mb": 6452, + "disk_read_mb": 793, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "naive_bayes", + "metric_values": { + "accuracy": 0.214, + "f1_macro": 0.1791, + "f1_micro": 0.214, + "f1_weighted": 0.2182 + }, + "scaled_scores": { + "accuracy": 0.189, + "f1_macro": 0.1763, + "f1_micro": 0.189, + "f1_weighted": 0.2154 + }, + "mean_score": 0.1924, + "resources": { + "submit": "2025-01-08 08:05:35", + "exit_code": 0, + "duration_sec": 2.7, + "cpu_pct": 389.3, + "peak_memory_mb": 5632, + "disk_read_mb": 792, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "random_labels", + "metric_values": { + "accuracy": 0.0308, + "f1_macro": 0.0165, + "f1_micro": 0.0308, + "f1_weighted": 0.0203 + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0.0131, + "f1_micro": 0, + "f1_weighted": 0.0167 + }, + "mean_score": 0.0075, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 10.9, + "cpu_pct": 90.3, + "peak_memory_mb": 5530, + "disk_read_mb": 782, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scanvi", + "metric_values": { + "accuracy": 0.3019, + "f1_macro": 0.2063, + "f1_micro": 0.3019, + "f1_weighted": 0.2552 + }, + "scaled_scores": { + "accuracy": 0.2797, + "f1_macro": 0.2035, + "f1_micro": 0.2797, + "f1_weighted": 0.2525 + }, + "mean_score": 0.2539, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 96, + "cpu_pct": 89.8, + "peak_memory_mb": 21607, + "disk_read_mb": 853, + "disk_write_mb": 2 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scanvi_scarches", + "metric_values": { + "accuracy": 0.3022, + "f1_macro": 0.235, + "f1_micro": 0.3022, + "f1_weighted": 0.2665 + }, + "scaled_scores": { + "accuracy": 0.28, + "f1_macro": 0.2324, + "f1_micro": 0.28, + "f1_weighted": 0.2639 + }, + "mean_score": 0.2641, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 505, + "cpu_pct": 95.5, + "peak_memory_mb": 19456, + "disk_read_mb": 850, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scimilarity", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 1, + "duration_sec": 520, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scimilarity_knn", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 1, + "duration_sec": 510, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "seurat_transferdata", + "metric_values": { + "accuracy": 0.3436, + "f1_macro": 0.2637, + "f1_micro": 0.3436, + "f1_weighted": 0.3013 + }, + "scaled_scores": { + "accuracy": 0.3227, + "f1_macro": 0.2611, + "f1_micro": 0.3227, + "f1_weighted": 0.2988 + }, + "mean_score": 0.3013, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 44.8, + "cpu_pct": 130.2, + "peak_memory_mb": 9114, + "disk_read_mb": 819, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "singler", + "metric_values": { + "accuracy": 0.3462, + "f1_macro": 0.2639, + "f1_micro": 0.3462, + "f1_weighted": 0.3091 + }, + "scaled_scores": { + "accuracy": 0.3254, + "f1_macro": 0.2613, + "f1_micro": 0.3254, + "f1_weighted": 0.3066 + }, + "mean_score": 0.3047, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 2749, + "cpu_pct": 100.3, + "peak_memory_mb": 8807, + "disk_read_mb": 811, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "true_labels", + "metric_values": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "scaled_scores": { + "accuracy": 1, + "f1_macro": 1, + "f1_micro": 1, + "f1_weighted": 1 + }, + "mean_score": 1, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 5.1, + "cpu_pct": 218.7, + "peak_memory_mb": 2868, + "disk_read_mb": 575, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "uce", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:05:35", + "exit_code": 1, + "duration_sec": 140, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "xgboost", + "metric_values": { + "accuracy": 0.279, + "f1_macro": 0.2206, + "f1_micro": 0.279, + "f1_weighted": 0.2525 + }, + "scaled_scores": { + "accuracy": 0.2561, + "f1_macro": 0.2179, + "f1_micro": 0.2561, + "f1_weighted": 0.2498 + }, + "mean_score": 0.245, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 0, + "duration_sec": 116, + "cpu_pct": 4664.1, + "peak_memory_mb": 20890, + "disk_read_mb": 793, + "disk_write_mb": 1 + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 170, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 170, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 170, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 1, + "duration_sec": 530, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 1, + "duration_sec": 240, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/dkd", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 1, + "duration_sec": 60, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:25", + "exit_code": 1, + "duration_sec": 1380, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 270, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/gtex_v9", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 60, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 1, + "duration_sec": 160, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 160, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hcla", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 1, + "duration_sec": 180, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:20", + "exit_code": 1, + "duration_sec": 120, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:19", + "exit_code": 1, + "duration_sec": 120, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/hypomap", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:18", + "exit_code": 1, + "duration_sec": 130, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:26", + "exit_code": 1, + "duration_sec": 1730, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 1, + "duration_sec": 160, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/immune_cell_atlas", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:27", + "exit_code": 1, + "duration_sec": 110, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 1, + "duration_sec": 130, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 1, + "duration_sec": 130, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/mouse_pancreas_atlas", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 1, + "duration_sec": 140, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:18:25", + "exit_code": 143, + "duration_sec": 14412, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 09:08:45", + "exit_code": 1, + "duration_sec": 3650, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "cellxgene_census/tabula_sapiens", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 180, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 30.1, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 30, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/cengen", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 40.6, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:46", + "exit_code": 1, + "duration_sec": 30.5, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 1, + "duration_sec": 20.2, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/immune_cells", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:47", + "exit_code": 1, + "duration_sec": 40, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 30.2, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:57", + "exit_code": 1, + "duration_sec": 29.9, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/pancreas", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:03:56", + "exit_code": 1, + "duration_sec": 40.1, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "geneformer", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 1, + "duration_sec": 30.1, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scgpt_zero_shot", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 1, + "duration_sec": 30.1, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + }, + { + "dataset_id": "openproblems_v1/zebrafish", + "method_id": "scprint", + "metric_values": { + "accuracy": "NA", + "f1_macro": "NA", + "f1_micro": "NA", + "f1_weighted": "NA" + }, + "scaled_scores": { + "accuracy": 0, + "f1_macro": 0, + "f1_micro": 0, + "f1_weighted": 0 + }, + "mean_score": 0, + "resources": { + "submit": "2025-01-08 08:05:36", + "exit_code": 1, + "duration_sec": 40.1, + "cpu_pct": "NA", + "peak_memory_mb": "NA", + "disk_read_mb": "NA", + "disk_write_mb": "NA" + } + } +] diff --git a/results/label_projection/data/state.yaml b/results/label_projection/data/state.yaml new file mode 100644 index 00000000..abbb0fc1 --- /dev/null +++ b/results/label_projection/data/state.yaml @@ -0,0 +1,9 @@ +id: process +output_scores: !file results.json +output_method_info: !file method_info.json +output_metric_info: !file metric_info.json +output_dataset_info: !file dataset_info.json +output_task_info: !file task_info.json +output_qc: !file quality_control.json +output_metric_execution_info: !file metric_execution_info.json + diff --git a/results/label_projection/data/task_info.json b/results/label_projection/data/task_info.json index cd2fd192..51b972df 100644 --- a/results/label_projection/data/task_info.json +++ b/results/label_projection/data/task_info.json @@ -1,10 +1,11 @@ { - "task_id": "label_projection", - "commit_sha": "c97decf07adb2e3050561d6fa9ae46132be07bef", - "task_name": "Label Projection", + "task_id": "task_label_projection", + "commit_sha": null, + "task_name": "Label projection", "task_summary": "Automated cell type annotation from rich, labeled reference data", - "task_description": "\nA major challenge for integrating single cell datasets is creating matching cell type\nannotations for each cell. One of the most common strategies for annotating cell types\nis referred to as\n[\"cluster-then-annotate\"](https://openproblems.bio/bibliography#kiselev2019challenges) whereby\ncells are aggregated into clusters based on feature similarity and then manually\ncharacterized based on differential gene expression or previously identified marker\ngenes. Recently, methods have emerged to build on this strategy and annotate cells\nusing [known marker genes](https://openproblems.bio/bibliography#pliner2019supervised). However,\nthese strategies pose a difficulty for integrating atlas-scale datasets as the\nparticular annotations may not match.\n\nTo ensure that the cell type labels in newly generated datasets match existing reference\ndatasets, some methods align cells to a previously annotated [reference\ndataset](https://openproblems.bio/bibliography#hou2019scmatch) and then\n_project_ labels from the reference to the new dataset.\n\nHere, we compare methods for annotation based on a reference dataset. The datasets\nconsist of two or more samples of single cell profiles that have been manually annotated\nwith matching labels. These datasets are then split into training and test batches, and\nthe task of each method is to train a cell type classifer on the training set and\nproject those labels onto the test set.\n\n", - "repo": "https://github.com/openproblems-bio/openproblems/tree/v1.0.0/openproblems/tasks/label_projection", + "task_description": "A major challenge for integrating single cell datasets is creating matching\ncell type annotations for each cell. One of the most common strategies for\nannotating cell types is referred to as\n[\"cluster-then-annotate\"](https://www.nature.com/articles/s41576-018-0088-9)\nwhereby cells are aggregated into clusters based on feature similarity and\nthen manually characterized based on differential gene expression or previously\nidentified marker genes. Recently, methods have emerged to build on this\nstrategy and annotate cells using\n[known marker genes](https://www.nature.com/articles/s41592-019-0535-3).\nHowever, these strategies pose a difficulty for integrating atlas-scale\ndatasets as the particular annotations may not match.\n\nTo ensure that the cell type labels in newly generated datasets match\nexisting reference datasets, some methods align cells to a previously\nannotated [reference dataset](https://academic.oup.com/bioinformatics/article/35/22/4688/54802990)\nand then _project_ labels from the reference to the new dataset.\n\nHere, we compare methods for annotation based on a reference dataset.\nThe datasets consist of two or more samples of single cell profiles that\nhave been manually annotated with matching labels. These datasets are then\nsplit into training and test batches, and the task of each method is to\ntrain a cell type classifer on the training set and project those labels\nonto the test set.\n", + "repo": "https://github.com/openproblems-bio/task_label_projection", + "issue_tracker": "https://github.com/openproblems-bio/task_label_projection/issues", "authors": [ { "name": "Nikolay Markov", @@ -30,6 +31,6 @@ } } ], - "version": "v1.0.0", + "version": "build_main", "license": "MIT" }