CSV2MISSING.txt

#!/bin/bash
#SBATCH --time=01:00:00
#SBATCH --job-name=processCmds

# Set file paths
DATAMISSING=/scratch/groups/jzeitzer/UKBB/Data/MissingData/Batch_rest
DATACSV=/scratch/groups/jzeitzer/UKBB/Data/Outputs/Batch_rest/timeSeries
DATACOMM=/scratch/groups/jzeitzer/UKBB/Data/Commands/Batch_rest
CODEUKBB=/scratch/groups/jzeitzer/UKBB/Code/ukbb

# Export environment variables
export DATAMISSING
export DATACSV
export DATACOMM
export CODEUKBB

# Make sure directories exist
mkdir -p $DATACSV
mkdir -p $DATACOMM
mkdir -p $DATAMISSING

# Load modules
ml R
ml python/3.6.1
ml py-numpy/1.18.1_py36
ml py-scipy/1.4.1_py36
ml py-scikit-learn/0.24.2_py36
ml py-pandas/1.0.3_py36

# Generate input files 
cd $DATACOMM
python3 /scratch/groups/jzeitzer/UKBB/Code/ukbb/chunkMissing.py

# Submit jobs  
cd $UKBB
for FILE in $DATACOMM/inputMissing_*.txt; do
    echo ${FILE}
    export FILE
    sbatch -n 1 -t 1-00:00:00 --job-name=CSV2MISSING --wrap="bash /scratch/groups/jzeitzer/UKBB/Code/ukbb/runMissingDataLength.txt"
#    sbatch -n 1 -t 1-00:00:00 --job-name=CSV2MISSING --wrap="bash /scratch/groups/jzeitzer/UKBB/Code/ukbb/runMissingDataCheck.txt"
done