qc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import sys\n",
    "\n",
    "import pandas as pd\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import wraps\n",
    "\n",
    "def validate_checklist_values(func):\n",
    "    @wraps(func)\n",
    "    def wrapper(checklist, *args, **kwargs):\n",
    "        # Ensure the first argument is a dictionary\n",
    "        if isinstance(checklist, dict) and all(value in (True, None) for value in checklist.values()):\n",
    "            func(checklist, *args, **kwargs)\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_if_dataset(basedir, dataset):\n",
    "    dataset_path = os.path.join(basedir, dataset)\n",
    "    is_dataset_name = lambda dataset_name: (\"GSE\" in dataset_name) or ('E-MTAB' in dataset_name) or ('PRJNA' in dataset_name) or ('SDY' in dataset_name)\n",
    "    return is_dataset_name(dataset) and os.path.isdir(dataset_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check if metafiles exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_full_path(basedir, dataset, filename):\n",
    "    return os.path.join(basedir, dataset, filename)\n",
    "\n",
    "def exist_nonempty(path, type='file'):\n",
    "    if type == 'file':\n",
    "        return os.path.isfile(path) and os.path.getsize(path) > 0\n",
    "    elif type == 'dir':\n",
    "        return os.path.isdir(path) and os.listdir(path)\n",
    "    else:\n",
    "        raise ValueError\n",
    "\n",
    "@validate_checklist_values\n",
    "def check_metafiles_exist(checklist, basedir, datasetname, metafile_suffixes):\n",
    "    # draft output dict \n",
    "    checklist.update({\"meta_exist\": True, \"meta_lost\": None})\n",
    "    # make paths from suffixes\n",
    "    meta_paths = [make_full_path(basedir, datasetname, f'{datasetname}.{meta_suffix}') for meta_suffix in metafile_suffixes]\n",
    "    # find lost files\n",
    "    lost_files = [file for file in meta_paths if not exist_nonempty(file, type='file')]\n",
    "    if lost_files:\n",
    "        checklist['meta_exist'] = False\n",
    "        checklist['meta_lost'] = ','.join(lost_files)\n",
    "\n",
    "\n",
    "def check_db_meta_exist(checklist, basedir, datasetname, db_meta_suffixes):\n",
    "    # make paths from suffixes\n",
    "    db_meta_paths = [make_full_path(basedir, datasetname, f'{datasetname}{meta_suffix}') for meta_suffix in db_meta_suffixes]\n",
    "    # check if exist\n",
    "    exist_list = [exist_nonempty(file, type='file') for file in db_meta_paths]\n",
    "    checklist['db_meta_exist'] = any(exist_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check if metafiles contain all samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_sample_x_run(filepath):\n",
    "    process_line = lambda line: line.rstrip().split('\\t', 1)\n",
    "    with open(filepath, 'r') as file:\n",
    "        splited_lines = list(map(process_line, file.readlines()))\n",
    "        sample_to_run = {line[0]:(line[1].split(',') if len(line) == 2 else None) for line in splited_lines}\n",
    "    return sample_to_run\n",
    "\n",
    "\n",
    "def get_first_column(filepath):\n",
    "    first_col = lambda line: line.rstrip().split('\\t', 1)[0]\n",
    "    with open(filepath, 'r') as file:\n",
    "        first_column_list = [first_col(line) for line in file.readlines()]\n",
    "    return first_column_list\n",
    "\n",
    "\n",
    "def validate_file(basedir, dataset, filename, correct_list):\n",
    "    filepath = os.path.join(basedir, dataset, filename)\n",
    "    list_from_file = get_first_column(filepath)\n",
    "    return set(list_from_file) == set(correct_list)\n",
    "\n",
    "def check_sample_x_run_file(checklist, filepath):\n",
    "    # read sample x run file\n",
    "    sample_to_run = read_sample_x_run(filepath)\n",
    "    # check if all samples have runs\n",
    "    samples_with_lost_runs = [key for key, value in sample_to_run.items() if value is None]\n",
    "    checklist['all samples have runs'] = not samples_with_lost_runs\n",
    "    checklist['missing_runs_smaples'] = ','.join(samples_with_lost_runs) if samples_with_lost_runs else None\n",
    "    return sample_to_run\n",
    "    \n",
    "        \n",
    "@validate_checklist_values\n",
    "def check_metafiles(checklist, basedir, dataset, sample_to_run):\n",
    "    # get sample to run dict\n",
    "    samples = list(sample_to_run.keys())\n",
    "    runs = list(chain.from_iterable(sample_to_run.values()))\n",
    "    \n",
    "    # validate files\n",
    "    checklist['all runs in run.list'] = validate_file(basedir, dataset, f'{dataset}.run.list', runs)\n",
    "    checklist['all samples in sample.list'] = validate_file(basedir, dataset, f'{dataset}.sample.list', samples)\n",
    "    checklist['all runs in parsed.list'] = validate_file(basedir, dataset, f'{dataset}.parsed.tsv', runs)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check if all fastqs are there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "@validate_checklist_values\n",
    "def validate_fastqs(checklist, basedir, dataset, sample_to_runs):\n",
    "    samples = set(sample_to_runs.keys())\n",
    "    # get fastqdir path\n",
    "    fastqdir_path = os.path.join(basedir, dataset, 'fastqs')\n",
    "    # check if dir exists\n",
    "    if exist_nonempty(fastqdir_path, type='dir'):\n",
    "        checklist['fastqdir_nonemptyexist'] = True\n",
    "        lost_samples = samples.difference(os.listdir(fastqdir_path))\n",
    "        # check if all files are present \n",
    "        if not lost_samples:\n",
    "            checklist['all_fastq_samples'] = True\n",
    "            # check if all runs are present\n",
    "            fastqnum_persample = {sample: len(os.listdir(os.path.join(fastqdir_path, sample))) for sample in samples}\n",
    "            lostrun_samples_list = [sample for sample in samples if fastqnum_persample[sample] / 2 != len(sample_to_runs[sample])]\n",
    "            if not lostrun_samples_list:\n",
    "                checklist['all_fastq_runs_exist'] = True\n",
    "            else:\n",
    "                # write missing sample with missing runs\n",
    "                checklist['all_fastq_runs_exist'] = False\n",
    "                checklist['missing_runs_fastq_samples'] = lostrun_samples_list\n",
    "        else:\n",
    "            # write missing samples to checklist\n",
    "            checklist['all_fastq_samples'] = False\n",
    "            checklist['missing_fastq_samples'] = ','.join(lost_samples)\n",
    "            \n",
    "    else:\n",
    "        checklist['fastqdir_nonemptyexist'] = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validate starsolo output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_starsolo(checklist, basedir, dataset, sample_to_runs):\n",
    "    samples = set(sample_to_runs.keys())\n",
    "    # check if all starsolo sample dirs exists and not empty\n",
    "    not_ok_dirs = [sample for sample in samples if not exist_nonempty(os.path.join(basedir, dataset, sample), type='dir')]\n",
    "    if not not_ok_dirs:\n",
    "        checklist['starsolo_allnonemptyexist'] = True\n",
    "        # check there is output dir and no tmp dir\n",
    "        no_output_dirs = [sample for sample in samples if not exist_nonempty(os.path.join(basedir, dataset, sample, 'output'), type='dir')]\n",
    "        no_final_log_file = [sample for sample in samples if not exist_nonempty(os.path.join(basedir, dataset, sample, 'Log.final.out'), type='file')]\n",
    "        tmp_exists = [sample for sample in samples if exist_nonempty(os.path.join(basedir, dataset, sample, '_STARtmp'), type='dir')]\n",
    "        \n",
    "        checklist['starsolo_existOutput'] = not bool(no_output_dirs)\n",
    "        checklist['starsolo_emptyOutput_samples'] = ','.join(no_output_dirs) if no_output_dirs else None\n",
    "        checklist['starsolo_existFinalLog'] = not bool(no_final_log_file)\n",
    "        checklist['starsolo_noFinalLog_samples'] = ','.join(no_final_log_file) if no_final_log_file else None\n",
    "        checklist['starsolo_noTmp'] = not bool(tmp_exists)\n",
    "        checklist['starsolo_existTmp_samples'] = ','.join(tmp_exists) if tmp_exists else None\n",
    "    else:\n",
    "        checklist['starsolo_allnonemptyexist'] = False\n",
    "        checklist['missing_starsolo_samples'] = ','.join(not_ok_dirs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validate starsolo qc file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_solo_qc(checklist, basedir, dataset, sample_to_runs):\n",
    "    samples = set(sample_to_runs.keys())\n",
    "    solo_qc_path = os.path.join(basedir, dataset, f'{dataset}.solo_qc.tsv')\n",
    "    # check if file exists\n",
    "    checklist['solo_qc_exists'] = exist_nonempty(solo_qc_path, type='file')\n",
    "    if checklist['solo_qc_exists']:\n",
    "        # check if file is empty\n",
    "        with open(solo_qc_path, 'r') as file:\n",
    "            lines = file.readlines()\n",
    "            checklist['solo_qc_nonempty'] = bool(len(lines) - 1)\n",
    "        # check if all samples are in the file\n",
    "        if checklist['solo_qc_nonempty']:\n",
    "            samples_in_file = get_first_column(solo_qc_path)[1:]\n",
    "            lost_samples = samples.difference(samples_in_file)\n",
    "            checklist['solo_qc_all_samples'] = not bool(lost_samples)\n",
    "            checklist['missing_solo_qc_samples'] = ','.join(lost_samples) if lost_samples else None\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_basedir(basedir, checklist_columns, metafile_suffixes, db_metafile_suffixes):\n",
    "    # get valid dataset names\n",
    "    datasets = [dataset for dataset in os.listdir(basedir) if check_if_dataset(basedir, dataset)]\n",
    "\n",
    "    checklist_dict = dict()\n",
    "    for dataset in datasets:\n",
    "        checklist = {col:None for col in checklist_columns}\n",
    "\n",
    "        # check that all metadata files exist\n",
    "        check_metafiles_exist(checklist, basedir, dataset, metafile_suffixes)\n",
    "\n",
    "        # read sample_to_run file\n",
    "        if isinstance(checklist, dict) and all(value in (True, None) for value in checklist.values()):\n",
    "            sample_x_run_path = os.path.join(basedir, dataset, f'{dataset}.sample_x_run.tsv')\n",
    "            sample_to_run = check_sample_x_run_file(checklist, sample_x_run_path)\n",
    "\n",
    "            # check that all metadata is consistent and all directories and files exist\n",
    "            check_metafiles(checklist, basedir, dataset, sample_to_run)\n",
    "\n",
    "            # validate directories\n",
    "            validate_fastqs(checklist, basedir, dataset, sample_to_run)\n",
    "\n",
    "            # validate starsolo output\n",
    "            validate_starsolo(checklist, basedir, dataset, sample_to_run)\n",
    "\n",
    "            # validate solo_qc.tsv\n",
    "            validate_solo_qc(checklist, basedir, dataset, sample_to_run)\n",
    "            \n",
    "            # check that there exists db metadata files\n",
    "            check_db_meta_exist(checklist, basedir, dataset, db_metafile_suffixes)\n",
    "            \n",
    "            checklist_dict[dataset] = checklist\n",
    "    return pd.DataFrame(checklist_dict).T.sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check all directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "ok_datasets = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "metafile_suffixes = ['run.list', 'sample.list', 'sample_x_run.tsv', 'parsed.tsv']\n",
    "db_metafile_suffixes = ['.idf.txt', '.sdrf.txt', '_family.soft']\n",
    "\n",
    "informative_columns = [\n",
    "    'meta_exist',\n",
    "    'db_meta_exist',\n",
    "    'all samples have runs',\n",
    "    'all runs in run.list',\n",
    "    'all samples in sample.list',\n",
    "    'all runs in parsed.list',\n",
    "    'fastqdir_nonemptyexist',\n",
    "    'all_fastq_samples',\n",
    "    'all_fastq_runs_exist',\n",
    "    'starsolo_allnonemptyexist',\n",
    "    'starsolo_existOutput',\n",
    "    'starsolo_existFinalLog',\n",
    "    'starsolo_noTmp',\n",
    "    'solo_qc_exists',\n",
    "    'solo_qc_nonempty',\n",
    "    'solo_qc_all_samples'\n",
    "]\n",
    "\n",
    "additional_columns = [\n",
    "    'meta_lost',\n",
    "    'missing_runs_fastq_samples',\n",
    "    'missing_fastq_samples',\n",
    "    'missing_starsolo_samples',\n",
    "    'starsolo_emptyOutput_samples',\n",
    "    'starsolo_noFinalLog_samples',\n",
    "    'starsolo_existTmp_samples',\n",
    "    'missing_solo_qc_samples'\n",
    "]\n",
    "\n",
    "must_be_true_columns = [\n",
    "    'meta_exist',\n",
    "    'missing_runs_smaples',\n",
    "    'all runs in run.list',\n",
    "    'all samples in sample.list',\n",
    "    'all runs in parsed.list',\n",
    "    'starsolo_allnonemptyexist',\n",
    "    'starsolo_existOutput',\n",
    "    'starsolo_existFinalLog',\n",
    "    'starsolo_noTmp',\n",
    "    'solo_qc_exists',\n",
    "    'solo_qc_nonempty',\n",
    "    'solo_qc_all_samples'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "checklist_columns = informative_columns + additional_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0_Current\n",
    "Let's start with `/lustre/scratch127/cellgen/cellgeni/aljes/reprocess/reprocessing-datasets-project/0_Current`. We need to chech that:\n",
    "- All files in metadata actually exist\n",
    "- All metadata files exist: `run.list.tsv`, `sample_x_run.tsv`, `solo_qc.tsv`, `parsed.tsv`\n",
    "- QC are not empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "basedir = \"/lustre/scratch127/cellgen/cellgeni/aljes/reprocess/reprocessing-datasets-project/0_Current\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_exist</th>\n",
       "      <th>db_meta_exist</th>\n",
       "      <th>all samples have runs</th>\n",
       "      <th>all runs in run.list</th>\n",
       "      <th>all samples in sample.list</th>\n",
       "      <th>all runs in parsed.list</th>\n",
       "      <th>fastqdir_nonemptyexist</th>\n",
       "      <th>all_fastq_samples</th>\n",
       "      <th>all_fastq_runs_exist</th>\n",
       "      <th>starsolo_allnonemptyexist</th>\n",
       "      <th>...</th>\n",
       "      <th>solo_qc_all_samples</th>\n",
       "      <th>meta_lost</th>\n",
       "      <th>missing_runs_fastq_samples</th>\n",
       "      <th>missing_fastq_samples</th>\n",
       "      <th>missing_starsolo_samples</th>\n",
       "      <th>starsolo_emptyOutput_samples</th>\n",
       "      <th>starsolo_noFinalLog_samples</th>\n",
       "      <th>starsolo_existTmp_samples</th>\n",
       "      <th>missing_solo_qc_samples</th>\n",
       "      <th>missing_runs_smaples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GSE112903</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GSM3090973,GSM3090974</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE114802</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE115982</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE116113</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE117211</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          meta_exist db_meta_exist all samples have runs all runs in run.list  \\\n",
       "GSE112903       True          True                  True                 True   \n",
       "GSE114802       True          True                  True                 True   \n",
       "GSE115982       True          True                  True                 True   \n",
       "GSE116113       True          True                  True                 True   \n",
       "GSE117211       True          True                  True                 True   \n",
       "\n",
       "          all samples in sample.list all runs in parsed.list  \\\n",
       "GSE112903                       True                    True   \n",
       "GSE114802                       True                    True   \n",
       "GSE115982                       True                    True   \n",
       "GSE116113                       True                    True   \n",
       "GSE117211                       True                    True   \n",
       "\n",
       "          fastqdir_nonemptyexist all_fastq_samples all_fastq_runs_exist  \\\n",
       "GSE112903                  False              None                 None   \n",
       "GSE114802                   True              True                 True   \n",
       "GSE115982                   True              True                 True   \n",
       "GSE116113                   True              True                 True   \n",
       "GSE117211                   True              True                 True   \n",
       "\n",
       "          starsolo_allnonemptyexist  ... solo_qc_all_samples meta_lost  \\\n",
       "GSE112903                     False  ...                None      None   \n",
       "GSE114802                      True  ...                True      None   \n",
       "GSE115982                      True  ...                True      None   \n",
       "GSE116113                      True  ...                True      None   \n",
       "GSE117211                      True  ...                True      None   \n",
       "\n",
       "          missing_runs_fastq_samples missing_fastq_samples  \\\n",
       "GSE112903                       None                  None   \n",
       "GSE114802                       None                  None   \n",
       "GSE115982                       None                  None   \n",
       "GSE116113                       None                  None   \n",
       "GSE117211                       None                  None   \n",
       "\n",
       "          missing_starsolo_samples starsolo_emptyOutput_samples  \\\n",
       "GSE112903    GSM3090973,GSM3090974                         None   \n",
       "GSE114802                     None                         None   \n",
       "GSE115982                     None                         None   \n",
       "GSE116113                     None                         None   \n",
       "GSE117211                     None                         None   \n",
       "\n",
       "          starsolo_noFinalLog_samples starsolo_existTmp_samples  \\\n",
       "GSE112903                        None                      None   \n",
       "GSE114802                        None                      None   \n",
       "GSE115982                        None                      None   \n",
       "GSE116113                        None                      None   \n",
       "GSE117211                        None                      None   \n",
       "\n",
       "          missing_solo_qc_samples missing_runs_smaples  \n",
       "GSE112903                    None                 None  \n",
       "GSE114802                    None                 None  \n",
       "GSE115982                    None                 None  \n",
       "GSE116113                    None                 None  \n",
       "GSE117211                    None                 None  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklist_df = validate_basedir(basedir, checklist_columns, metafile_suffixes, db_metafile_suffixes)\n",
    "checklist_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "checklist_df.to_csv('0_current_validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ok_datasets=234, all_datasets=325\n"
     ]
    }
   ],
   "source": [
    "ok_datasets['0_current'] = checklist_df[checklist_df[must_be_true_columns].all(axis=1)].index.tolist()\n",
    "print(f\"ok_datasets={len(ok_datasets['0_current'])}, all_datasets={checklist_df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.int64(234)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((checklist_df[must_be_true_columns].all(axis=1)) & (checklist_df.db_meta_exist)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('0_current_validation.txt', 'w') as f:\n",
    "    for line in ok_datasets['0_current']:\n",
    "        f.write(f\"{line}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0_Current/0_SUCCESS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "basedir = \"/lustre/scratch127/cellgen/cellgeni/aljes/reprocess/reprocessing-datasets-project/0_Current/0_SUCCESS\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_exist</th>\n",
       "      <th>db_meta_exist</th>\n",
       "      <th>all samples have runs</th>\n",
       "      <th>all runs in run.list</th>\n",
       "      <th>all samples in sample.list</th>\n",
       "      <th>all runs in parsed.list</th>\n",
       "      <th>fastqdir_nonemptyexist</th>\n",
       "      <th>all_fastq_samples</th>\n",
       "      <th>all_fastq_runs_exist</th>\n",
       "      <th>starsolo_allnonemptyexist</th>\n",
       "      <th>...</th>\n",
       "      <th>solo_qc_all_samples</th>\n",
       "      <th>meta_lost</th>\n",
       "      <th>missing_runs_fastq_samples</th>\n",
       "      <th>missing_fastq_samples</th>\n",
       "      <th>missing_starsolo_samples</th>\n",
       "      <th>starsolo_emptyOutput_samples</th>\n",
       "      <th>starsolo_noFinalLog_samples</th>\n",
       "      <th>starsolo_existTmp_samples</th>\n",
       "      <th>missing_solo_qc_samples</th>\n",
       "      <th>missing_runs_smaples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GSE103544</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE103574</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE104556</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE106543</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSE108067</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          meta_exist db_meta_exist all samples have runs all runs in run.list  \\\n",
       "GSE103544       True          True                  True                 True   \n",
       "GSE103574       True          True                  True                 True   \n",
       "GSE104556       True          True                  True                 True   \n",
       "GSE106543       True          True                  True                 True   \n",
       "GSE108067       True          True                  True                 True   \n",
       "\n",
       "          all samples in sample.list all runs in parsed.list  \\\n",
       "GSE103544                       True                    True   \n",
       "GSE103574                       True                    True   \n",
       "GSE104556                       True                    True   \n",
       "GSE106543                       True                    True   \n",
       "GSE108067                       True                    True   \n",
       "\n",
       "          fastqdir_nonemptyexist all_fastq_samples all_fastq_runs_exist  \\\n",
       "GSE103544                  False              None                 None   \n",
       "GSE103574                  False              None                 None   \n",
       "GSE104556                  False              None                 None   \n",
       "GSE106543                  False              None                 None   \n",
       "GSE108067                  False              None                 None   \n",
       "\n",
       "          starsolo_allnonemptyexist  ... solo_qc_all_samples meta_lost  \\\n",
       "GSE103544                      True  ...                True      None   \n",
       "GSE103574                      True  ...                True      None   \n",
       "GSE104556                      True  ...                True      None   \n",
       "GSE106543                      True  ...                True      None   \n",
       "GSE108067                      True  ...                True      None   \n",
       "\n",
       "          missing_runs_fastq_samples missing_fastq_samples  \\\n",
       "GSE103544                       None                  None   \n",
       "GSE103574                       None                  None   \n",
       "GSE104556                       None                  None   \n",
       "GSE106543                       None                  None   \n",
       "GSE108067                       None                  None   \n",
       "\n",
       "          missing_starsolo_samples starsolo_emptyOutput_samples  \\\n",
       "GSE103544                     None                         None   \n",
       "GSE103574                     None                         None   \n",
       "GSE104556                     None                         None   \n",
       "GSE106543                     None                         None   \n",
       "GSE108067                     None                         None   \n",
       "\n",
       "          starsolo_noFinalLog_samples starsolo_existTmp_samples  \\\n",
       "GSE103544                        None                      None   \n",
       "GSE103574                        None                      None   \n",
       "GSE104556                        None                      None   \n",
       "GSE106543                        None                      None   \n",
       "GSE108067                        None                      None   \n",
       "\n",
       "          missing_solo_qc_samples missing_runs_smaples  \n",
       "GSE103544                    None                 None  \n",
       "GSE103574                    None                 None  \n",
       "GSE104556                    None                 None  \n",
       "GSE106543                    None                 None  \n",
       "GSE108067                    None                 None  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklist_df = validate_basedir(basedir, checklist_columns, metafile_suffixes, db_metafile_suffixes)\n",
    "checklist_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "checklist_df.to_csv('0_success_validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ok_datasets=858, all_datasets=859\n"
     ]
    }
   ],
   "source": [
    "ok_datasets['0_success'] = checklist_df[checklist_df[must_be_true_columns].all(axis=1)].index.tolist()\n",
    "print(f\"ok_datasets={len(ok_datasets['0_success'])}, all_datasets={checklist_df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.int64(858)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((checklist_df[must_be_true_columns].all(axis=1)) & (checklist_df.db_meta_exist)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('0_success_validation.txt', 'w') as f:\n",
    "    for line in ok_datasets['0_success']:\n",
    "        f.write(f\"{line}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "basedir = \"/lustre/scratch127/cellgen/cellgeni/aljes/reprocess/reprocessing-datasets-project\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_exist</th>\n",
       "      <th>db_meta_exist</th>\n",
       "      <th>all samples have runs</th>\n",
       "      <th>all runs in run.list</th>\n",
       "      <th>all samples in sample.list</th>\n",
       "      <th>all runs in parsed.list</th>\n",
       "      <th>fastqdir_nonemptyexist</th>\n",
       "      <th>all_fastq_samples</th>\n",
       "      <th>all_fastq_runs_exist</th>\n",
       "      <th>starsolo_allnonemptyexist</th>\n",
       "      <th>...</th>\n",
       "      <th>solo_qc_all_samples</th>\n",
       "      <th>meta_lost</th>\n",
       "      <th>missing_runs_fastq_samples</th>\n",
       "      <th>missing_fastq_samples</th>\n",
       "      <th>missing_starsolo_samples</th>\n",
       "      <th>starsolo_emptyOutput_samples</th>\n",
       "      <th>starsolo_noFinalLog_samples</th>\n",
       "      <th>starsolo_existTmp_samples</th>\n",
       "      <th>missing_solo_qc_samples</th>\n",
       "      <th>missing_runs_smaples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>E-MTAB-10018</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E-MTAB-10042</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E-MTAB-10060</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E-MTAB-10143</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ERS11457215,ERS11457212,ERS11457195,ERS1145721...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ERS11457215,ERS11457210,ERS11457192,ERS1145719...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E-MTAB-10169</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ERS5913312,ERS5913318,ERS5913311,ERS5913322,ER...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ERS5913312,ERS5913318,ERS5913324,ERS5913311,ER...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             meta_exist db_meta_exist all samples have runs  \\\n",
       "E-MTAB-10018       True          True                  True   \n",
       "E-MTAB-10042       True          True                  True   \n",
       "E-MTAB-10060       True          True                  True   \n",
       "E-MTAB-10143       True          True                  True   \n",
       "E-MTAB-10169       True          True                  True   \n",
       "\n",
       "             all runs in run.list all samples in sample.list  \\\n",
       "E-MTAB-10018                 True                       True   \n",
       "E-MTAB-10042                 True                       True   \n",
       "E-MTAB-10060                 True                       True   \n",
       "E-MTAB-10143                 True                       True   \n",
       "E-MTAB-10169                 True                       True   \n",
       "\n",
       "             all runs in parsed.list fastqdir_nonemptyexist all_fastq_samples  \\\n",
       "E-MTAB-10018                    True                  False              None   \n",
       "E-MTAB-10042                    True                  False              None   \n",
       "E-MTAB-10060                    True                  False              None   \n",
       "E-MTAB-10143                    True                  False              None   \n",
       "E-MTAB-10169                    True                  False              None   \n",
       "\n",
       "             all_fastq_runs_exist starsolo_allnonemptyexist  ...  \\\n",
       "E-MTAB-10018                 None                      True  ...   \n",
       "E-MTAB-10042                 None                      True  ...   \n",
       "E-MTAB-10060                 None                      True  ...   \n",
       "E-MTAB-10143                 None                     False  ...   \n",
       "E-MTAB-10169                 None                     False  ...   \n",
       "\n",
       "             solo_qc_all_samples meta_lost missing_runs_fastq_samples  \\\n",
       "E-MTAB-10018                True      None                       None   \n",
       "E-MTAB-10042                True      None                       None   \n",
       "E-MTAB-10060                True      None                       None   \n",
       "E-MTAB-10143               False      None                       None   \n",
       "E-MTAB-10169               False      None                       None   \n",
       "\n",
       "             missing_fastq_samples  \\\n",
       "E-MTAB-10018                  None   \n",
       "E-MTAB-10042                  None   \n",
       "E-MTAB-10060                  None   \n",
       "E-MTAB-10143                  None   \n",
       "E-MTAB-10169                  None   \n",
       "\n",
       "                                       missing_starsolo_samples  \\\n",
       "E-MTAB-10018                                               None   \n",
       "E-MTAB-10042                                               None   \n",
       "E-MTAB-10060                                               None   \n",
       "E-MTAB-10143  ERS11457215,ERS11457212,ERS11457195,ERS1145721...   \n",
       "E-MTAB-10169  ERS5913312,ERS5913318,ERS5913311,ERS5913322,ER...   \n",
       "\n",
       "             starsolo_emptyOutput_samples starsolo_noFinalLog_samples  \\\n",
       "E-MTAB-10018                         None                        None   \n",
       "E-MTAB-10042                         None                        None   \n",
       "E-MTAB-10060                         None                        None   \n",
       "E-MTAB-10143                         None                        None   \n",
       "E-MTAB-10169                         None                        None   \n",
       "\n",
       "             starsolo_existTmp_samples  \\\n",
       "E-MTAB-10018                      None   \n",
       "E-MTAB-10042                      None   \n",
       "E-MTAB-10060                      None   \n",
       "E-MTAB-10143                      None   \n",
       "E-MTAB-10169                      None   \n",
       "\n",
       "                                        missing_solo_qc_samples  \\\n",
       "E-MTAB-10018                                               None   \n",
       "E-MTAB-10042                                               None   \n",
       "E-MTAB-10060                                               None   \n",
       "E-MTAB-10143  ERS11457215,ERS11457210,ERS11457192,ERS1145719...   \n",
       "E-MTAB-10169  ERS5913312,ERS5913318,ERS5913324,ERS5913311,ER...   \n",
       "\n",
       "             missing_runs_smaples  \n",
       "E-MTAB-10018                 None  \n",
       "E-MTAB-10042                 None  \n",
       "E-MTAB-10060                 None  \n",
       "E-MTAB-10143                 None  \n",
       "E-MTAB-10169                 None  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklist_df = validate_basedir(basedir, checklist_columns, metafile_suffixes, db_metafile_suffixes)\n",
    "checklist_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "checklist_df.to_csv('main_dir_validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ok_datasets=201, all_datasets=278\n"
     ]
    }
   ],
   "source": [
    "ok_datasets['main_dir'] = checklist_df[checklist_df[must_be_true_columns].all(axis=1)].index.tolist()\n",
    "print(f\"ok_datasets={len(ok_datasets['main_dir'])}, all_datasets={checklist_df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_exist</th>\n",
       "      <th>db_meta_exist</th>\n",
       "      <th>all samples have runs</th>\n",
       "      <th>all runs in run.list</th>\n",
       "      <th>all samples in sample.list</th>\n",
       "      <th>all runs in parsed.list</th>\n",
       "      <th>fastqdir_nonemptyexist</th>\n",
       "      <th>all_fastq_samples</th>\n",
       "      <th>all_fastq_runs_exist</th>\n",
       "      <th>starsolo_allnonemptyexist</th>\n",
       "      <th>...</th>\n",
       "      <th>solo_qc_all_samples</th>\n",
       "      <th>meta_lost</th>\n",
       "      <th>missing_runs_fastq_samples</th>\n",
       "      <th>missing_fastq_samples</th>\n",
       "      <th>missing_starsolo_samples</th>\n",
       "      <th>starsolo_emptyOutput_samples</th>\n",
       "      <th>starsolo_noFinalLog_samples</th>\n",
       "      <th>starsolo_existTmp_samples</th>\n",
       "      <th>missing_solo_qc_samples</th>\n",
       "      <th>missing_runs_smaples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PRJNA826269</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            meta_exist db_meta_exist all samples have runs  \\\n",
       "PRJNA826269       True         False                  True   \n",
       "\n",
       "            all runs in run.list all samples in sample.list  \\\n",
       "PRJNA826269                 True                       True   \n",
       "\n",
       "            all runs in parsed.list fastqdir_nonemptyexist all_fastq_samples  \\\n",
       "PRJNA826269                    True                  False              None   \n",
       "\n",
       "            all_fastq_runs_exist starsolo_allnonemptyexist  ...  \\\n",
       "PRJNA826269                 None                      True  ...   \n",
       "\n",
       "            solo_qc_all_samples meta_lost missing_runs_fastq_samples  \\\n",
       "PRJNA826269                True      None                       None   \n",
       "\n",
       "            missing_fastq_samples missing_starsolo_samples  \\\n",
       "PRJNA826269                  None                     None   \n",
       "\n",
       "            starsolo_emptyOutput_samples starsolo_noFinalLog_samples  \\\n",
       "PRJNA826269                         None                        None   \n",
       "\n",
       "            starsolo_existTmp_samples missing_solo_qc_samples  \\\n",
       "PRJNA826269                      None                    None   \n",
       "\n",
       "            missing_runs_smaples  \n",
       "PRJNA826269                 None  \n",
       "\n",
       "[1 rows x 25 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklist_df[((checklist_df[must_be_true_columns].all(axis=1)) & (checklist_df.db_meta_exist != True))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'GSE115982'}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(ok_datasets['0_current']).intersection(ok_datasets['0_success'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['0_current', '0_success', 'main_dir'])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ok_datasets.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('main_dir_validation.txt', 'w') as f:\n",
    "    for line in ok_datasets['main_dir']:\n",
    "        f.write(f\"{line}\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}