upload2NCI.sge

#!/bin/bash

# SGE OPTIONS
#$ -V
#$ -cwd
#$ -b n
#$ -l mem_requested=8G,fast_dm=10
#$ -q short.q
#$ -S /bin/bash
#$ -N nci_backup
#$ -l tmp_requested=1000G,tmpfree=1000G
#$ -M a.senabouth@garvan.org.au
#$ -m ae

# Remember to change this to your NCI username
LOGIN=as8574@gadi-dm.nci.org.au

# INPUT DIRECTORIES
# So for the command... qsub upload2NCI.sge /path/to/backup backup_filename.tar.gz /path/on/nci/to/move/to
INPUT_DIR=$1
TAR_FILENAME=$2
MD5_FILE=${TAR_FILENAME}.md5
NCI_DIR=$3

## CLEAN UP NCI_DIR so it does not end with a forward slash
[[ "${NCI_DIR}" == */ ]] && NCI_DIR="${NCI_DIR: : -1}"

# Check if the input variable is actually a directory
# 1. Go to parent directory of the directory we are backing up
# 2. Create a TAR file on SCRATCH
# 3. Create an MD5 file of the TAR file
# 4. Transfer to NCI
# 5. Run remote checksum to ensure it is fine

if [[ -d $INPUT_DIR ]]
then
    # Get paths from input directory
    PARENT_DIR="$(dirname "$INPUT_DIR")"
    ARCHIVE_DIR="$(basename "$INPUT_DIR")"

    # Navigate to parent directory
    cd $PARENT_DIR

    # Archive the file on scratch
    tar -zchvf ${TMPDIR}/${TAR_FILENAME} ${ARCHIVE_DIR}/

    # Move to scratch disk to run checksums
    cd $TMPDIR
    md5sum $TAR_FILENAME > $MD5_FILE
    md5sum -c $MD5_FILE

    # Create directory on NCI if it doesn't exist.
    ssh ${LOGIN} "mkdir -p ${NCI_DIR}"
    
    fi
    # Once it is complete, transfer to NCI
    # If you are using my script, remember to change to your username and ensure your SSH keys are properly set
    rsync -az ${TAR_FILENAME} ${LOGIN}:${NCI_DIR}/${TAR_FILENAME}
    rsync -az ${MD5_FILE} ${LOGIN}:${NCI_DIR}/${MD5_FILE}

    # Run SSH remotely to check the transfer went fine.
    ssh ${LOGIN} "cd ${NCI_DIR}; md5sum -c ${MD5_FILE}"
else
    echo "$INPUT_DIR is not a directory."
fi