diff --git a/bioinfoconda/docker/Dockerfile b/bioinfoconda/docker/Dockerfile deleted file mode 100644 index 74eb346..0000000 --- a/bioinfoconda/docker/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -FROM ubuntu:18.04 - -# Set the default shell -SHELL ["/bin/bash", "-c"] - -# Add the docker:bioinfo user -RUN addgroup --gid 5000 bioinfo \ - && yes | adduser --quiet --disabled-password \ - --no-create-home --gid 5000 docker - -# Install base system utils -RUN apt-get update && apt-get install -y \ - gawk \ - git \ - locales \ - python \ - wget \ - && rm -rf /var/lib/apt/lists/* - -# Set the locale -RUN locale-gen en_US.UTF-8 \ - && update-locale LANG="en_US.UTF-8" \ - && update-locale LANGUAGE="en_US.UTF-8" - -# Replicate xavier's directory structure in the image -RUN mkdir -p /bioinfo/bioinfotree \ - /bioinfo/data \ - /bioinfo/prj - -# Add bioinfotree tools -COPY bioinfotree /bioinfo/bioinfotree - -# Install miniconda3 -RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /bioinfo/Miniconda3-latest-Linux-x86_64.sh \ - && bash /bioinfo/Miniconda3-latest-Linux-x86_64.sh -b -p /bioinfo/miniconda3 \ - && rm /bioinfo/Miniconda3-latest-Linux-x86_64.sh - -# Set permissions -RUN chown -R docker:bioinfo /bioinfo \ - && chmod -R 2775 /bioinfo - -# Add bioinfotree's and miniconda's executables to $PATH -ENV PATH="/bioinfo/bioinfotree/local/bin:/bioinfo/miniconda3/bin:$PATH" \ - PYTOHNPATH="/bioinfo/bioinfotree/local/lib/python" \ - PERL5LIB="/bioinfo/bioinfotree/local/lib/perl" \ - LC_ALL="en_US.UTF8" \ - BIOINFO_ROOT="/bioinfo" - -# NOTES ON OCCAM -# -# On occam, when we run an image the user is emariell:emariell. When whe -# build the image, the user is root (or the one specified with the -# "USER" directive in the dockerfile). The umask cannot be set (unless -# we set it with a script), so the new files created under /bioinfo will -# not be writable by the group. Given all this, I think that the most -# economic option is to chmod 777 everything. -# -# Setting the workdir does not work on occam, either. We have to -# manually cd into the required directory. -# -# PYTHONPATH and PERL5LIB must be reset in the project dockerfile to -# include the bioinfotree libraries. Appending the old variables does -# not work, but maybe there is a right way to append variables for -# dockerfiles. I should investigate about that. diff --git a/bioinfoconda/docker/dockerignore b/bioinfoconda/docker/dockerignore deleted file mode 100644 index 8b37f4d..0000000 --- a/bioinfoconda/docker/dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -data -docker -miniconda3 -prj diff --git a/bioinfoconda/local/bin/X-conda b/bioinfoconda/local/bin/X-conda deleted file mode 120000 index c7dc556..0000000 --- a/bioinfoconda/local/bin/X-conda +++ /dev/null @@ -1 +0,0 @@ -../src/X-conda.sh \ No newline at end of file diff --git a/bioinfoconda/local/bin/X-conda-export b/bioinfoconda/local/bin/X-conda-export deleted file mode 120000 index 7c990ab..0000000 --- a/bioinfoconda/local/bin/X-conda-export +++ /dev/null @@ -1 +0,0 @@ -../src/X-conda-export.sh \ No newline at end of file diff --git a/bioinfoconda/local/bin/X-conda-get-channels b/bioinfoconda/local/bin/X-conda-get-channels deleted file mode 120000 index a215fec..0000000 --- a/bioinfoconda/local/bin/X-conda-get-channels +++ /dev/null @@ -1 +0,0 @@ -../src/X-conda-get-channels.pl \ No newline at end of file diff --git a/bioinfoconda/local/bin/X-conda-get-specs b/bioinfoconda/local/bin/X-conda-get-specs deleted file mode 120000 index c9ec4c8..0000000 --- a/bioinfoconda/local/bin/X-conda-get-specs +++ /dev/null @@ -1 +0,0 @@ -../src/X-conda-get-specs.pl \ No newline at end of file diff --git a/bioinfoconda/local/bin/X-conda-install b/bioinfoconda/local/bin/X-conda-install deleted file mode 120000 index e1746d6..0000000 --- a/bioinfoconda/local/bin/X-conda-install +++ /dev/null @@ -1 +0,0 @@ -../src/X-conda-install.sh \ No newline at end of file diff --git a/bioinfoconda/local/bin/X-getdata b/bioinfoconda/local/bin/X-getdata deleted file mode 120000 index ffe9e1a..0000000 --- a/bioinfoconda/local/bin/X-getdata +++ /dev/null @@ -1 +0,0 @@ -../src/X-getdata.pl \ No newline at end of file diff --git a/bioinfoconda/local/bin/X-mkprj b/bioinfoconda/local/bin/X-mkprj deleted file mode 120000 index d2389ba..0000000 --- a/bioinfoconda/local/bin/X-mkprj +++ /dev/null @@ -1 +0,0 @@ -../src/X-mkprj.sh \ No newline at end of file diff --git a/bioinfoconda/local/lib/bash/bash_completion b/bioinfoconda/local/lib/bash/bash_completion deleted file mode 100644 index 0d0e734..0000000 --- a/bioinfoconda/local/lib/bash/bash_completion +++ /dev/null @@ -1,33 +0,0 @@ -####### -# cdp # -####### - -# FIXME: prjhome does not change; we should not define it locally inside -# the _cdp function. - -function _cdp -{ - local cur=${COMP_WORDS[COMP_CWORD]} - local prjhome="$BIOINFO_ROOT/prj" - local IFS=$' \t\n' - - IFS=$'\n' - COMPREPLY=( $(cd $prjhome && compgen -d -- "$cur") ) - COMPREPLY=( "${COMPREPLY[@]/%//}" ) - IFS=$' \t\n' - - # variable names if appropriate shell option set and no - # completions - if shopt -q cdable_vars && [[ ${#COMPREPLY[@]} -eq 0 ]]; then - COMPREPLY=( $(compgen -v -- "$cur") ) - fi - - return 0 -} -complete -o filenames -o nospace -o bashdefault -F _cdp cdp - -############# -# snakemake # -############# - -complete -o filenames -o nospace -o bashdefault -C snakemake-bash-completion snakemake diff --git a/bioinfoconda/local/lib/bash/bash_functions b/bioinfoconda/local/lib/bash/bash_functions deleted file mode 100644 index 193ca32..0000000 --- a/bioinfoconda/local/lib/bash/bash_functions +++ /dev/null @@ -1,85 +0,0 @@ -##################### -# Logging Functions # -##################### - -# FIXME: perhaps warn and info should not print to stderr. - -# Input: error string -function bug() -{ - >&2 echo "BUG: $1" - exit 99 -} - -# Input: error string, exit code -function error() -{ - >&2 echo "ERROR: $1" - - re='^[0-9]+$' - if ! [[ $2 =~ $re ]] ; then - bug "argument to error() is not a number." - else - exit $2 - fi -} - -# Input: warning string -function warn() -{ - >&2 echo "WARNING: $1" -} - -# Input: info string -function info() -{ - >&2 echo "INFO: $1" -} - -############################### -# Path Manipulation Functions # -############################### - -# NOTE: By "Output" we mean what is sent to the standard output; the -# return value is 0 if no error occurred, different from 0 otherwise. - -# Input: an absolute path -# Output: the project contained in that path, if any -function extract_prj -{ - if [[ $1 != "${BIOINFO_ROOT}/prj"* ]]; then - return 1 - fi - - if [ "$1" == "$BIOINFO_ROOT/prj" ]; then - echo $BIOINFO_ROOT/prj - else - echo $1 | cut -d '/' -f 4 - fi - - return 0 -} - -# Input: a relative path -# Output: the absolute path -function absolutise -{ - if [[ $1 == "/"* ]]; then - echo $1 - return 0 - fi - - abs_path=$PWD - rel_path=`echo $1 | sed 's#^./##'` - - for d in `echo $rel_path | cut -d '/' -f 1- --output-delimiter=' '`; do - if [ "$d" == ".." ]; then - abs_path=`dirname $abs_path` - else - abs_path=$abs_path"/"$d - fi - done - - echo $abs_path - return 0 -} diff --git a/bioinfoconda/local/lib/bash/bash_utils b/bioinfoconda/local/lib/bash/bash_utils deleted file mode 100644 index bf20230..0000000 --- a/bioinfoconda/local/lib/bash/bash_utils +++ /dev/null @@ -1,73 +0,0 @@ -source $BIOINFO_ROOT/bioinfoconda/local/lib/bash/bash_functions - -####### -# cdp # -####### - -export BIOINFOCONDA_OLDPWD="${BIOINFO_ROOT}/prj" - -# Update BIOINFOCONDA_OLWPWD -function cd -{ - # Find current project, if any - cur_prj="" - if extract_prj $PWD > /dev/null; then - cur_prj=`extract_prj $PWD` - fi - - # Find new project (exclude cd options), if any - new_prj="" - for p in $*; do - if [[ $p != "-"* ]] && extract_prj $(absolutise $p) > /dev/null; then - dest=$p - new_prj=`extract_prj $(absolutise $dest)` - fi - done - - # If we are entering another project... - if [ "$new_prj" != "" ]; then - # ...From another project - if [ "$cur_prj" != "" ] && [ $cur_prj != $new_prj ]; then - BIOINFOCONDA_OLDPWD=$PWD - fi - # Elif we go out to the outside world... - elif [ "$new_prj" == "" ]; then - # ...From a project - if [ "$cur_prj" != "" ]; then - BIOINFOCONDA_OLDPWD=$PWD - fi - fi - - builtin cd "$@" - return $? -} - -# Note that in cdp we use the cd function we defined above, therefore we -# need not worry about updating BIOINFOCONDA_OLDPWD -function cdp -{ - prj_root=${BIOINFO_ROOT}/prj - - if [ "$#" == "0" ]; then - cd $prj_root - elif [ "$#" == "1" ]; then - if [ $1 == "-" ]; then - cd $BIOINFOCONDA_OLDPWD - elif [ $1 == "." ]; then - if extract_prj $PWD > /dev/null; then - cur_prj=`extract_prj $PWD` - cd $prj_root/$cur_prj/dataset - else - cd $prj_root - fi - else - cd $prj_root/$1 - fi - else - echo "ERROR: Too many arguments." - echo "Usage: cdp [prjname]" - return 1 - fi - - return $? -} diff --git a/bioinfoconda/local/src/X-conda-export.sh b/bioinfoconda/local/src/X-conda-export.sh deleted file mode 100755 index 2a6c4fe..0000000 --- a/bioinfoconda/local/src/X-conda-export.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -source $BIOINFO_ROOT/bioinfoconda/local/lib/bash/bash_functions - -# Usage string -read -r -d '' usage << END -Usage: - X-conda export - -Options: - -h Prints this help - -Notes: - The environment is exported in the local/condafiles of the project - and the files are ordered by name and exportation date. - -Reporting bugs: - federicomarotta AT mail DOT com -END - -# Validate environment and define defaults -if [[ -z "${CONDA_PREFIX}" ]]; then - error 'You must be inside a conda environment' 3 -fi - -options=h -longoptions=help - -date=$(date +%Y-%m-%d) -prjname=$(basename ${CONDA_PREFIX}) -prjpath=${BIOINFO_ROOT}/prj/$prjname -env_file=$prjpath/local/condafiles/${prjname}_${date}.yml - -# Parse the options -PARSER=$(getopt --options=$options --longoptions=$longoptions --name "$0" -- "$@") -eval set -- "$PARSER" - -while true; do - case "$1" in - -h|--help ) - echo "$usage" - exit 0 - ;; - -- ) - shift - break - ;; - * ) - error "Unexpected arguments. $usage" 3 - shift - ;; - esac -done - -# Start the real script -info "Exporting environment..." -conda env export -n $prjname -f $env_file - -if [[ $? == 0 ]]; then - info "Done. The environment file is $env_file." -else - error "Something went wrong" $? -fi diff --git a/bioinfoconda/local/src/X-conda-get-channels.pl b/bioinfoconda/local/src/X-conda-get-channels.pl deleted file mode 100755 index b843a0e..0000000 --- a/bioinfoconda/local/src/X-conda-get-channels.pl +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/perl - -# NOTE: this program is for internal use only. -# Input: a conda environment file (as recent as possible) -# Output: the list of channels used in that environment - -use strict; -use warnings; -use File::Basename; -use Getopt::Long; -use YAML::Tiny; - -my $NAME = basename($0); -my $USAGE =< \$env_file, - "help|h" => \$help -); - -# Print help message -if ($help) -{ - print $USAGE; - exit 0; -} - -# Validate arguments -if ($env_file eq '') -{ - die "Error: $USAGE"; -} -elsif (! -f $env_file) -{ - die "Error: $env_file is not a file.\n"; -} - -# Open the environment file and parse it -my $yaml = YAML::Tiny->new; -$yaml = YAML::Tiny->read($env_file); -my $channels = $yaml->[0]->{channels}; - -print "@$channels"; - -exit 0; diff --git a/bioinfoconda/local/src/X-conda-get-specs.pl b/bioinfoconda/local/src/X-conda-get-specs.pl deleted file mode 100755 index 4a3e233..0000000 --- a/bioinfoconda/local/src/X-conda-get-specs.pl +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/perl - -# NOTE: this program is for internal use only. -# Input: a conda history file -# Output: the list of manually installed packages, excluding manually -# removed packages - -use strict; -use warnings; -use File::Basename; -use Getopt::Long; - -my $NAME = basename($0); -my $USAGE =< \$history_file, - "help|h" => \$help -); - -# Print help message -if ($help) -{ - print $USAGE; - exit 0; -} - -# Validate arguments -if ($history_file eq '') -{ - die "Error: $USAGE"; -} -elsif (! -f $history_file) -{ - die "Error: $history_file is not a file.\n"; -} - -# Get the specs from history -open HISTORY, "<", $history_file - or die "Error: could not open $history_file: $!"; -my @update_specs; -while () -{ - if (m/^# update specs: \[(.*)\]/) - { - my @specs = split(/, /, $1); - foreach my $spec (@specs) - { - push(@update_specs, substr($spec, 1, length($spec) - 2)); - } - } - elsif (m/^# remove specs: \[(.*)\]/) - { - my @specs = split(/, /, $1); - foreach my $spec (@specs) - { - $spec = substr($spec, 1, length($spec) - 2); - for (my $i = 0; $i <= $#update_specs; $i++) - { - if ($update_specs[$i] eq $spec) - { - splice(@update_specs, $i, 1); - } - } - } - } -} -close(HISTORY); -@update_specs = uniq(@update_specs); - -print "@update_specs"; - -exit 0; - -# https://stackoverflow.com/questions/7651/how-do-i-remove-duplicate-items-from-an-array-in-perl -sub uniq -{ - my %seen; - grep !$seen{$_}++, @_; -} diff --git a/bioinfoconda/local/src/X-conda-install.sh b/bioinfoconda/local/src/X-conda-install.sh deleted file mode 100755 index e68a2fb..0000000 --- a/bioinfoconda/local/src/X-conda-install.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash - -# TODO: we should parse all the options with optparse and validate them. - -# validate the environment -if [[ ! -z "${BIOINFO_ROOT}" ]]; then - source $BIOINFO_ROOT/bioinfoconda/local/lib/bash/bash_functions -else - >&2 echo 'ERROR: ${BIOINFO_ROOT} is not defined' - exit 3 -fi -if [[ -z "${CONDA_PREFIX}" ]]; then - error 'You must be inside a conda environment' 3 -fi - -options=yfmCc:n:p:qkh -longoptions=help - -date=$(date +%Y-%m-%d) -minicondapath=$(conda info --base) -prjname=$(basename ${CONDA_PREFIX}) -prjpath=${BIOINFO_ROOT}/prj/$prjname -env_file=$prjpath/local/condafiles/${prjname}_${date}.yml -history_file=${CONDA_PREFIX}/conda-meta/history -argv=$@ - -# Usage string -read -r -d '' usage << END -Usage: - X-conda install -c CHANNEL PACKAGE1 PACKAGE2 ... - -More in detail: - `basename $0` install [-y] [--dry-run] [-f] [--no-deps] [-m] - [-C] [--use-local] [--offline] [--no-pin] [-c CHANNEL] - [--override-channels] [-n ENVIRONMENT | -p PATH] [-q] - [--copy] [-k] [--update-dependencies] - [--no-update-dependencies] [--channel-priority] - [--no-channel-priority] [--clobber] - [--show-channel-urls] [--no-show-channel-urls] - [--download-only] [--json] [--debug] [--verbose] - [PACKAGE1 [PACKAGE2 [...]]] - -Options: - For an explanation of the options, see the usage of \`conda - create'. - -Notes: - This program removes and recreate the current conda environment. - It is the heavy artillery to be deployed when "conda install" - fails. As specified in the documentation of conda, the - installation of all the packages at once reduces the chances - of conflicts. - - If you do not provide any option, the environment will be - simply regenerated; this might be useful to solve some - conda-related problems. - -Reporting bugs: - federicomarotta AT mail DOT com -END - -PARSER=$(getopt --options=$options --longoptions=$longoptions --name "$0" -- "$@") -eval set -- "$PARSER" - -while true; do - case "$1" in - -h|--help ) - echo "$usage" - exit 0 - ;; - -- ) - shift - break - ;; - * ) - # For now we do not exclude any argument - shift - ;; - esac -done - -# Export the environment as it is now -X-conda-export - -# Get a list of channels and update_specs -info "Getting the channels and packages of the environment..." -channels="-c $(X-conda-get-channels -f $env_file | sed 's/ / -c /g')" -specs=$(X-conda-get-specs -f $history_file) - -# Recreate the environment -info "Deactivating environment..." -source "$minicondapath/etc/profile.d/conda.sh" -conda deactivate - -info "Running: conda env remove -n $prjname" -conda env remove -y -q -n $prjname - -info "Running: conda create --name $prjname $channels $argv $specs" -conda create -y --name $prjname $channels $argv $specs - -if [[ $? -eq 0 ]]; then - # info "Reactivating environment..." - info "All done." -else - error "Sorry, that did not work. You can regenerate the environment as it was through $env_file." 9 -fi - -# It's not necessary to reactivate the environment, because it was only -# deactivated for the shell opened by this script; activating the -# environment here would be pointless. - -# source activate $prjname -# info "All done." diff --git a/bioinfoconda/local/src/X-conda.sh b/bioinfoconda/local/src/X-conda.sh deleted file mode 100755 index 2a4ec7c..0000000 --- a/bioinfoconda/local/src/X-conda.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# TODO: allow to run outside of an environment (parse -p and -n options) - -# validate the environment -if [[ ! -z "${BIOINFO_ROOT}" ]]; then - source $BIOINFO_ROOT/bioinfoconda/local/lib/bash/bash_functions -else - >&2 echo 'ERROR: ${BIOINFO_ROOT} is not defined' - exit 3 -fi -if [[ -z "${CONDA_PREFIX}" ]]; then - error 'You must be inside a conda environment' 3 -fi - -# Usage string -read -r -d '' usage << END -Usage: - `basename $0` [-h] command ... - -Commands: - The command can be one of the following: - - * install: brute-force installation of packages - * export: save the environment to a file - -Reporting bugs: - federicomarotta AT mail DOT com -END - -# Check syntax and acquire options and arguments -if [[ $# -eq 0 ]]; then - error "$usage" 1 -elif [ $1 == "-h" ]; then - echo "$usage" -elif [ $1 == "install" ]; then - X-conda-install "${@:2}" -elif [ $1 == "export" ]; then - X-conda-export "${@:2}" -else - error "$usage" 1 -fi diff --git a/bioinfoconda/local/src/X-getdata.pl b/bioinfoconda/local/src/X-getdata.pl deleted file mode 100755 index d7f6f2e..0000000 --- a/bioinfoconda/local/src/X-getdata.pl +++ /dev/null @@ -1,828 +0,0 @@ -#!/usr/bin/perl - -use warnings; -use strict; -#use Digest::SHA; -use Digest::MD5; -use File::Basename; -use File::Copy; -use File::Find; -use File::Path qw(make_path); -use Fcntl qw(:flock SEEK_END); -use Getopt::Long; - -# TODO: implement -R option. - -# TODO: accept patterns for http too. - -# TODO: accept patterns in the middle of the path. - -# TODO: check if new dest is overriding something. - -# TODO: pass @ARGV as @params for rsync, e.g. to specify port or -# something else in the command line. - -my $NAME = basename($0); -my $USAGE =< \$reject, - "fix-index|f" => \$fix_index, - "help|h" => \$help -); - -# Print help message -if ($help) -{ - print $USAGE; - exit 0; -} - -# Fix index -if ($fix_index) -{ - # Check syntax - if (scalar(@ARGV) != 0) - { - print "You cannot specify an URL when fixing the index.\n"; - print $USAGE; - exit 1; - } - if ($reject ne '') - { - print "-f is not compatible with -R.\n"; - print $USAGE; - exit 1; - } - - # Try to fix the index - fix_index($index_file); - - exit 0; -} - -# By default, download -# Parse arguments -if (scalar(@ARGV) != 1) -{ - print $USAGE; - exit 1; -} -my $source = $ARGV[0]; -my @output = (); - -# Recognise the protocol -if ($source =~ m#^https?://# or $source =~ m#^ftps?://#) -{ - # Parse the URL - my ($protocol, $domain, $port, $path, @params) = parse_url($source); - - # Validate the URL - my $v_source = validate_url($protocol, $domain, $port, $path, \@params); - - # Find an appropriate destination path - my $dest = find_dest_url($domain, $path, \@params); - - # Ensure that dest ends with a / for recursive downloads - if (is_recursive($v_source) and $dest !~ m/\/$/) { - $dest .= "/"; - } - print "\nSaving files to $data_root$dest.\n\n"; - - # Finally download the files - if (is_recursive($v_source)) - { - @output = wget_recursive($v_source, $data_root . $dest); - } - else - { - @output = wget_simple($v_source, $data_root . $dest); - } -} -elsif ($source =~ m#^rsync://# or $source =~ m/@/) -{ - # Parse the hostname - my ($protocol, $user, $host, $path) = parse_host($source); - - # Validate the hostname - my $v_source = validate_host($protocol, $user, $host, $path); - - # Get an appropriate destination path - my $dest = find_dest_host($host, $path); - - # Ensure that dest ends with a / for recursive downloads - if (is_recursive($v_source) and $dest !~ m/\/$/) { - $dest .= "/"; - } - print "\nSaving files to $data_root$dest.\n\n"; - - # Finally download the files - if (is_recursive($v_source)) - { - @output = rsync_recursive($v_source, $data_root . $dest); - } - else - { - @output = rsync_simple($v_source, $data_root . $dest); - } -} -else -{ - die("ERROR: unrecognised source. Please specify the protocol or the username\@host.\n"); -} - -if (@output) -{ - # Add an entry to the index file - print "Updating index...\n"; - update_index(\@output); - - # Remove write permissions to everybody - for (my $i = 0; $i < scalar(@output); $i += 4) - { - chmod(0444, $output[$i+3]); - } -} - -print "Done."; -exit 0; - -# Subroutines - -sub parse_url -{ - my $url = $_[0]; - - my @parsed_url = ($url =~ m#^((.*)://)?(.*?)(:(\d+))?(/.*?)(\?(.+))?$#); - my @path = split(//, $parsed_url[5]); - - for (my $i = 0; $i < scalar(@parsed_url); $i++) - { - if (! $parsed_url[$i]) - { - $parsed_url[$i] = ''; - } - } - if (! $parsed_url[1]) - { - $parsed_url[1] = "http"; - } - - # If url has a trailing slash, download recursively - if ($path[-1] eq '/') - { - $parsed_url[5] .= '*'; - } - my @params = split(/&/, $parsed_url[6]); - - # The fields are as follows: protocol, domain name, port, path, query parameters - return ($parsed_url[1], $parsed_url[2], $parsed_url[4], $parsed_url[5], @params); -} - -sub parse_host -{ - my $host = $_[0]; - - my @parsed_host = ($host =~ m#^((.*)://)?((.*)@)?(.*?)(:(.*))$#); - my @path = split(//, $parsed_host[6]); - - for (my $i = 0; $i < scalar(@parsed_host); $i++) - { - if (! $parsed_host[$i]) - { - $parsed_host[$i] = ''; - } - } - - # If the path has a trailing slash, download recursively - if ($path[-1] eq '/') - { - $parsed_host[6] .= '*'; - } - - # The fields are as follows: username, hostname, path - return ($parsed_host[1], $parsed_host[3], $parsed_host[4], $parsed_host[6]); -} - -sub validate_url -{ - my $protocol = $_[0]; - my $domain = $_[1]; - my $port = $_[2]; - my $path = $_[3]; - my @params = @{$_[4]}; - my $v_url = ''; - - # TODO: other validations - - if ($protocol =~ m/^http/) - { - # Path cannot contain patterns - if ($path =~ m/[\*\?\[\]\{\}]/) - { - die("ERROR: path cannot contain patterns if the protocol is http.\n"); - } - } - elsif ($protocol =~ m/^ftp/) - { - # Only the last item can contain patterns - my @path = split(/\//, $path); - for (my $i = 0; $i < scalar(@path); $i++) - { - if ($path[$i] =~ m/[\*\?\[\]\{\}]/ and $i != $#path) - { - die("ERROR: metacharacters are allowed only for file names, not for directories.\n"); - } - } - } - else - { - die("ERROR: protocol $protocol is not supported.\n"); - } - - # Recompose the URL - $v_url = $protocol . "://" . $domain; - if ($port) - { - $v_url .= ":" . $port; - } - $v_url .= $path; - if (@params) - { - $v_url .= "?" . join('&', @params); - } - - return $v_url; -} - -sub validate_host -{ - my $protocol = $_[0]; - my $username = $_[1]; - my $hostname = $_[2]; - my $path = $_[3]; - my $v_host = ''; - - if ($username eq '') - { - $username = $ENV{USERNAME}; - } - if ($path eq '') - { - die("ERROR: please provide the path of the file.\n"); - } - - # Only the last item can contain patterns - my @path = split(/\//, $path); - for (my $i = 0; $i < scalar(@path); $i++) - { - if ($path[$i] =~ m/[\*\?\[\]\{\}]/ and $i != $#path) - { - die("ERROR: metacharacters are allowed only for file names, not for directories.\n"); - } - } - - if ($protocol) - { - $v_host .= $protocol . "://"; - } - - $v_host .= $username . "@" . $hostname . ":" . $path; - return $v_host; -} - -sub find_dest_url -{ - my $domain = $_[0]; - my $path = $_[1]; - my @params = @{$_[2]}; - - # Extract the first meaningful level domain - my @levels = split(/\./, $domain); - my $fld = ''; - for (my $i = 0; $i < scalar(@levels); $i++) - { - if (is_prefix($levels[$i]) || is_pseudo($levels[$i])) - { - next; - } - else - { - $fld = $levels[$i]; - last; - } - } - - # Extract a path - my @dirs = split(/\//, $path); - my $localdirs = ''; - for (my $i = 1; $i < scalar(@dirs); $i++) - { - if (is_pseudo($dirs[$i])) - { - next; - } - elsif ($dirs[$i] =~ m/[\*\?\[\]\{\}]/) - { - $localdirs .= '/'; - last; - } - else - { - $localdirs .= '/' . $dirs[$i]; - } - } - - # Add params - if (@params) - { - for (my $i = 0; $i < scalar(@params); $i++) - { - my @value = split(/=/, $params[$i]); - $localdirs .= '/' . $value[1]; - } - } - - # Recompose the destination path - my $tmp_dest = $fld . $localdirs; - $tmp_dest =~ s#^/##; - - # Manually override the proposed destination - my $dest = edit_dest($tmp_dest); - - return $dest; -} - -sub find_dest_host -{ - my $host = $_[0]; - my $path = $_[1]; - - # Extract a path - my @dirs = split(/\//, $path); - my $localdirs = '/'; - for (my $i = 1; $i < scalar(@dirs); $i++) - { - if ($dirs[$i] =~ m/[\*\?\[\]\{\}]/) - { - $localdirs .= '/'; - last; - } - else - { - $localdirs .= '/' . $dirs[$i]; - } - } - - my $tmp_dest = $host . $localdirs; - my $dest = edit_dest($tmp_dest); - - return $dest; -} - -sub edit_dest -{ - my $dest = $_[0]; - my $tmp_dest = ''; - - print "\nThe suggested path for the files is $dest\n"; - - print "\nIn order to maintain a systematic ordering of the directories, the \ -destination path should be a subpath of the url (for instance, if the url is \ -https://www.foo.com/bar/baz/bin/a.txt, a suitable destination could be \ -foo/bin/a.txt, if bar/baz is not relevant.\n"; - - print "\nEnter an empty line if the suggested path is good, otherwise type a path \ -(relative to $data_root):\n"; - - while (my $tmp_dest = ) - { - chomp $tmp_dest; - - if ($tmp_dest ne '') - { - if (is_valid_dest($tmp_dest)) - { - system("echo \"$ENV{USER}: Overridding suggested path $dest with new path $tmp_dest.\" | systemd-cat -t $NAME -p notice"); - $dest = $tmp_dest; - last; - } - else - { - print "The path must be relative to $data_root, and cannot contain '..'.\nTry again:\n"; - next; - } - } - else - { - last; - } - } - - return $dest; -} - -sub wget_simple -{ - my $url = $_[0]; - my $dest = $_[1]; - my $dirname = dirname($dest); - my @output = (0); - - unless(-d $dirname or make_path($dirname)) - { - die("ERROR: cannot make path $dirname.\n"); - } - - open(WGETOUTPUT, "wget -O $dest $url 2>&1 |"); - while (my $line = ) - { - # update the progress line... - if ($line =~ m/^\s*\d+[KMGT]\s+/) - { - chomp $line; - print "\r$line"; - # put back the "\n" to the last line - if ($line =~ m/100%/) - { - print "\n"; - } - } - # ...and append the other lines - else - { - print $line; - # extract the information to be logged - if ($line =~ m/ saved \[.*\]/) - { - # We want @output to contain: date, - # hour, url, path. But wget simple - # returns only date, hour, path, so we - # add the url. - @output = ($line =~ m/(.*) (.*) \(.*\) - '(.*)' saved \[.*\]/); - splice(@output, 2, 0, $url); - } - } - } - - return @output; -} - -sub wget_recursive -{ - my $url = $_[0]; - my $dest = $_[1]; - # my $dirname = dirname($dest); - my $dirname = $dest; - my @output = (); - my @urldirs = split("/", $url); - my $cutdirs; - - unless(-d $dirname or make_path($dirname)) - { - die("ERROR: cannot make path $dirname.\n"); - } - - $cutdirs = $#urldirs-2; - # we subtract the false dirs generated by :// - open(WGETOUTPUT, "wget -P $dirname \\ - -N --no-verbose --show-progress \\ - -r -l inf --no-parent \\ - -nH --cut-dirs=$cutdirs \\ - $url 2>&1 |"); - while (my $line = ) - { - # update the progress line... - if ($line =~ m/^\s*\d+[KMGT]\s+/ and $line =~ m/%/) - { - chomp $line; - print "\r$line"; - # put back the "\n" to the last line - if ($line =~ m/100%/) - { - print "\n"; - # extract the information to be logged - if ($line !~ m/.listing/) - { - push @output, ($line =~ m/100% .*(....-..-..) (.*) URL: (.*) \[\d+\] -> "(.*)" \[\d+\]$/); - } - } - } - # ...and append the other lines - else - { - print $line; - } - } - - return @output; -} - -sub rsync_simple -{ - my $source = $_[0]; - my $dest = $_[1]; - my $dirname = dirname($dest); - my @output = (); - - unless(-d $dirname or make_path($dirname)) - { - die("ERROR: cannot make path $dirname.\n"); - } - - open(RSYNCOUTPUT, "rsync -rtvz --out-format=\"%t %f\" $source $dest 2>&1 |"); - while (my $line = ) - { - if ($line =~ m/^\d{4}\/\d{2}\/\d{2}/) - { - chomp($line); - $line .= " " . $dest . "\n"; - my @line = split(" ", $line); - $line[2] = $source; - push @output, @line; - } - print $line; - } - - return @output; -} - -sub rsync_recursive -{ - my $source = $_[0]; - my $dest = $_[1]; - my $dirname = dirname($dest); - my @output = (); - - unless(-d $dirname or make_path($dirname)) - { - die("ERROR: cannot make path $dirname.\n"); - } - - open(RSYNCOUTPUT, "rsync -rtvz --out-format=\"%t %f\" $source $dest 2>&1 |"); - while (my $line = ) - { - if ($line =~ m/^\d{4}\/\d{2}\/\d{2}/) - { - chomp($line); - $line .= " " . $dest . "\n"; - my @line = split(" ", $line); - $line[3] = $line[3] . $line[2]; - $line[2] = dirname($source) . "/" . $line[2]; - push @output, @line; - } - print $line; - } - - return @output; -} - -sub update_index -{ - my @log = @{$_[0]}; - #my $ctx = Digest::SHA->new(1); - my $ctx = Digest::MD5->new; - - open(INDEXFILE, ">>", $index_file) - or die "ERROR: cannot open $index_file."; - - for (my $i = 0; $i < scalar(@log); $i += 4) - { - # @log contains, for each file, date, hour, url, path. - # Logs for different files are stored sequentially, in a - # flat fashion. - - # Compute checksum for each downloaded file - my $digest = checksum($ctx, $log[$i+3]); - - # Append line to the index - print INDEXFILE "$log[$i] $log[$i+1]\t$ENV{USER}\t$log[$i+2]\t$log[$i+3]\t$digest\n"; - } - - close(INDEXFILE); -} - -sub fix_index -{ - my $index_file = $_[0]; - #my $ctx = Digest::SHA->new(1); - my $ctx = Digest::MD5->new; - - # Backup the previous index - copy($index_file, $index_file.".old") or die "Failed to backup index file: $!"; - - # Actual files under $data_root - my %tree; - my %tree_bypath; - find(sub { - return unless -f; # Must be a file - return if /^\./; # Must not be hidden - - my $date = substr(`stat -c '%y' $File::Find::name`, 0, 19); - my $user = (getpwuid((stat $File::Find::name)[4]))[0]; - my $digest = checksum($ctx, $File::Find::name); - - $tree{$digest} = [$date, $user, $File::Find::name, $File::Find::name]; - $tree_bypath{$File::Find::name} = [$date, $user, $File::Find::name, $digest]; - - }, $data_root); - - # Files in the index - my %index; - my %index_bypath; - open(INDEXFILE, "+<", $index_file) or die "ERROR: cannot open $index_file: $!"; - flock(INDEXFILE, LOCK_EX) or die "Cannot lock $index_file: $!"; - while () - { - chomp; - @_ = split /\t/; - $index{$_[4]} = [$_[0], $_[1], $_[2], $_[3]]; - $index_bypath{$_[3]} = [$_[0], $_[1], $_[2], $_[4]]; - } - - # Find checksums in common and check if they have the same path - foreach my $checksum (keys(%tree)) - { - if (! exists $index{$checksum}) - { - # We have to add this file to the index - $index{$checksum} = $tree{$checksum}; - } - elsif ($tree{$checksum}[3] ne $index{$checksum}[3]) - { - # We have to rename this file in the index - $index{$checksum}[3] = $tree{$checksum}[3]; - $index{$checksum}[0] = $tree{$checksum}[0]; - } - } - - # Find checksums only in the index - foreach my $checksum (keys(%index)) - { - if (! exists($tree{$checksum})) - { - # We have to delete the files from the index - delete($index{$checksum}); - } - } - - # Now check if the files with the same path have actually the - # same checksum - foreach my $path (keys(%tree_bypath)) - { - if (exists($index_bypath{$path}) and $tree_bypath{$path}[3] ne $index_bypath{$path}[3]) - { - # We have a problem - print "Attention! The file $path may be corrupted.\n"; - } - } - - # Print the new healthy index - seek(INDEXFILE, 0, 0); - truncate(INDEXFILE, 0); - foreach my $checksum (keys(%index)) - { - print INDEXFILE "$index{$checksum}[0]\t$index{$checksum}[1]\t$index{$checksum}[2]\t$index{$checksum}[3]\t$checksum\n"; - } - close(INDEXFILE); - - return 0; -} - -sub checksum -{ - my $ctx = $_[0]; - my $path = $_[1]; - - # MD5 algorithm - open(my $fh, "<", $path) - or die "Could not open $path."; - $ctx->addfile($fh); - my $digest = $ctx->hexdigest; - - close($fh); - return $digest; - - # SHA algorithm - #$ctx->addfile($path); - #my $digest = $ctx->hexdigest; - - #return $digest; -} - -sub is_valid_dest -{ - my $dest = $_[0]; - - if ($dest =~ m#^/#) - { - return 0; - } - if ($dest =~ m#\.\./#) - { - return 0; - } - - return 1; -} - -sub is_prefix -{ - my $level = $_[0]; - - # list of prefixes - # soe: ucsc - # cse: ucsc - my @prefixes = ("www", "ftp", "soe", "cse", "com"); - - for (my $i = 0; $i < scalar(@prefixes); $i++) - { - if ($level eq $prefixes[$i]) - { - return 1; - } - } - - return 0; -} - -sub is_pseudo -{ - my $dir = $_[0]; - my @pseudodirs = ("pub", ".*?download.*?", "tmp", "files", - "goldenPath", "ftp", "databases", "storage", "googleapis"); - - for (my $i = 0; $i < scalar(@pseudodirs); $i++) - { - if ($dir =~ m/$pseudodirs[$i]/i) - { - return 1; - } - } - - return 0; -} - -sub is_recursive -{ - my $path = $_[0]; - - return ($path =~ m/[\*\?\[\]\{\}]/); -} diff --git a/bioinfoconda/local/src/X-mkprj.sh b/bioinfoconda/local/src/X-mkprj.sh deleted file mode 100755 index 5b1b081..0000000 --- a/bioinfoconda/local/src/X-mkprj.sh +++ /dev/null @@ -1,545 +0,0 @@ -#!/bin/bash - -# TODO: save templates as files and default environment as yml. - -# TODO: improve documentation of -l. - -# TODO: use relative paths inside a project. - -# validate the environment -if [[ -z "${BIOINFO_ROOT}" ]]; then - error '${BIOINFO_ROOT} is not defined' 3 -fi - -source $BIOINFO_ROOT/bioinfoconda/local/lib/bash/bash_functions - -if which conda > /dev/null; then - minicondapath=$(conda info --base) -fi -bioinfotreepath="$BIOINFO_ROOT/bioinfotree" -gitlab_config_file="$BIOINFO_ROOT/bioinfoconda/local/etc/gitlab" - -options=hTGlCir -longoptions=help,no-templates,no-git,gitlab,no-conda,interactive-conda,remove - -# Usage string -read -r -d '' usage << END -Usage: `basename $0` [options] prjname - -Options: - -T|--no-templates do not create template Snakefile and - Dockerfile. - -G|--no-git do not initialise local git repository. - -l|--gitlab create remote gitlab repository. - -C|--no-conda do not create the conda environment. - -i|--interactive-conda enable the creation of a customised conda - environment. - -r|--remove remove project directory and conda - environment. - -h|--help print this message. - -Notes: - This program does the following things: - 1) Creating a directory structure for the project. - 2) Adding default Dockerfile and Snakefile. - 3) Initialising a git repository. - 4) Creating a conda environment. - 5) Setting up direnv for project environment and libraries. - - With the \`-i' option, you will be prompted to choose the channels - and packages that will be added initially to the conda environment; - otherwise the environment will contain R, perl, cpanm and snakemake. - Use this option only if you know what you are doing. - - The \`-l' option is used to create a remote repository on gitlab - and associate it to the local repository. (If you pass this - option together with -r, the gitlab repository is deleted; see - below.) - - To remove a project, use the \`-r' option and provide the - project name. The entire directory of the project, as well as - the related conda environment, if it exists, are eliminated. If - you also pass the -l option, the remote gitlab repository is - deleted too, if it exists. - -Reporting bugs: - federicomarotta AT mail DOT com -END - -# Functions definition -function clean_prj() -{ - prjpath=$1 - prjname=$(basename $prjpath) - gitlab_username=$( awk '(NR == 2) {print $1}' $gitlab_config_file ) - gitlab_group_id=$( awk '(NR == 2) {print $2}' $gitlab_config_file ) - gitlab_token=$( awk '(NR == 2) {print $3}' $gitlab_config_file ) - - # Remove local directory - if [ -d $prjpath ]; then - rm -rf $prjpath - fi - - # Remove conda environment - if conda info --envs | grep "^$prjname[[:space:]]" > /dev/null; then - conda env remove -y -n $prjname - fi - - # Remove remote gitlab repository - if ((gitlab)); then - # Check for invalid gitlab configuration - if [ -z "$gitlab_username" ] || [ -z "$gitlab_group_id" ] || [ -z "$gitlab_token" ]; then - echo "Invalid gitlab configuration. This feature requires - you to provide some configuration parameters; check the - documentation for help." - return 4 - fi - curl -s --header "Private-Token: $gitlab_token" \ - -X DELETE "https://gitlab.com/api/v4/projects/$gitlab_username%2F$prjname" - fi - - #response=$( curl -s --header "Private-Token: $gitlab_token" \ - #-X DELETE "https://gitlab.com/api/v4/projects/" \ - #-d "name=$prjname" | jq -r ".message" ) - - #if [ "$?" != "0" ]; then - #echo "curl had some problems to send your request." - #return $? - #fi - #if [ "$response" != "null" ]; then - #echo "The GitLab API complained about your request:" - #echo $response - #return 8 - #fi - - return 0 -} -function is_valid_name() -{ - re='^[0-9a-z_-]+$' - if [[ $1 =~ $re ]] ; then - return 0 - else - return 1 - fi -} -function make_dirs() -{ - prjpath=$1 - - mkdir -p $prjpath/dataset \ - && mkdir -p $prjpath/local/{R,benchmark,bin,builds,config,data,doc,dockerfiles,lib,log,python,snakefiles,src,tmp,condafiles} \ - && mkdir -p $prjpath/local/lib/{perl5,python,R} \ - || return $? - - configure_direnv_local $prjpath - - return 0 -} -function create_templates() -{ - prjpath=$1 - prjname=$(basename $prjpath) - - cat <<- END > $prjpath/local/dockerfiles/Dockerfile - # Use the bioinfoconda parent image - # NOTE: you should set a version tag! - FROM cbuatmbc/bioinfoconda - - # Copy the project directory into the container - # NOTE: remember to add the Snakefiles to the .dockerignore - COPY . $prjpath - - # Create the conda environment from the yml file - # NOTE: remember to export the environment before building the - # image, and to edit the following path with the new file. - RUN conda env create -f $prjpath/local/condafiles/$prjname.yml - - RUN chown -R root:bioinfo /bioinfo \ - && chmod -R 2777 /bioinfo - - # Set up the environment - ENV PATH="$minicondapath/envs/$prjname/bin:$prjpath/local/bin:\$PATH" \\ - PERL5LIB="$prjpath/local/lib/perl:$bioinfotreepath/local/lib/perl" \\ - PERL_CPANM_HOME="$prjpath/local/builds/perl5" \\ - PERL_CPANM_OPT="-l $prjpath/local --no-man-pages --save-dists=$prjpath/local/src/perl5" \\ - PYTHONPATH="$prjpath/local/lib/python:$bioinfotreepath/local/lib/python" \\ - R_PROFILE_USER="$prjpath/.Rprofile" \\ - CONDA_DEFAULT_ENV="$prjname" \\ - CONDA_PREFIX="$minicondapath/envs/$prjname" - - # Establish the entry point - ENTRYPOINT ["$prjpath/local/dockerfiles/docker-entrypoint.sh"] - CMD ["snakemake --dag -Tsvg > dag.svg"] - END - - cat <<- END > $prjpath/local/dockerfiles/docker-entrypoint.sh - #!/usr/bin/env bash - cd $prjpath/dataset - eval "\$@" - END - - cat <<- END > $prjpath/local/dockerfiles/dockerignore - # Exclude some hidden files - .gitignore - .Rproj.user - .Rhistory - .envrc - # Exclude dataset and local/data (this speeds up the build time) - dataset/* - local/data/* - END - - cat <<- END > $prjpath/local/snakefiles/Snakefile - # This is a template Snakefile, change it at will - - # Set the config file - configfile: "../local/config/snakemake_config.yml" - - # Define the final targets: all the others depend on them - ALL = ["foo.bed", "bar.png"] - - # Run the entire pipeline - rule all: - input: ALL - END - - cat <<- END > $prjpath/local/config/snakemake_config.yml - BIOINFOCONDA: - ROOT: - "$BIOINFO_ROOT/" - DATA: - "$BIOINFO_ROOT/data/" - - PRJ: - ROOT: - "$prjpath/" - DATA: - "$prjpath/local/data/" - END - - ln -s ../local/snakefiles/Snakefile $prjpath/dataset/Snakefile - #ln -s local/dockerfiles/Dockerfile $prjpath/Dockerfile - ln -s local/dockerfiles/dockerignore $prjpath/.dockerignore - - return 0 -} -function initialise_repo() -{ - prjpath=$1 - - git init --shared=group $prjpath > /dev/null || return $? - echo .RData > $prjpath/.gitignore - echo .Rhistory >> $prjpath/.gitignore - echo .Rproj.user >> $prjpath/.gitignore - echo dataset/* >> $prjpath/.gitignore - echo !dataset/Snakefile >> $prjpath/.gitignore - echo local/benchmark/* >> $prjpath/.gitignore - echo local/data/* >> $prjpath/.gitignore - echo local/log/* >> $prjpath/.gitignore - echo local/tmp/* >> $prjpath/.gitignore - - return 0 -} -function initialise_gitlab_repo() -{ - prjpath=$1 - prjname=$(basename $prjpath) - gitlab_username=$( awk '(NR == 2) {print $1}' $gitlab_config_file ) - gitlab_group_id=$( awk '(NR == 2) {print $2}' $gitlab_config_file ) - gitlab_token=$( awk '(NR == 2) {print $3}' $gitlab_config_file ) - - # Check for invalid configuration - if [ -z "$gitlab_username" ] || [ -z "$gitlab_group_id" ] || [ -z "$gitlab_token" ]; then - echo "Invalid gitlab configuration. This feature requires you to - provide some configuration parameters; check the documentation - for help." - return 4 - fi - - # Create repository - response=$( curl -s --header "Private-Token: $gitlab_token" \ - -X POST "https://gitlab.com/api/v4/projects" \ - -d "name=$prjname" | jq -r ".message" ) - - if [ "$?" != "0" ]; then - echo "curl had some problems to send your request." - return $? - fi - if [ "$response" != "null" ]; then - echo "The GitLab API complained about your request:" - echo $response - return 8 - fi - - # Share the project with the group - response=$( curl -s --header "Private-Token: $gitlab_token" \ - -X POST "https://gitlab.com/api/v4/projects/$gitlab_username%2F$prjname/share" \ - -d "group_id=$gitlab_group_id" \ - -d "group_access=40" | jq -r ".message" ) - - if [ "$?" != "0" ]; then - echo "curl had some problems to send your request." - return $? - fi - if [ "$response" != "null" ]; then - echo "The GitLab API complained about your request:" - echo $response - return 8 - fi - - # Add the remote origins - cd $prjpath - git remote add origin "https://gitlab.com/$gitlab_username/$prjname.git" - git remote add ssh-origin "git@gitlab.com:$gitlab_username/$prjname.git" - cd $OLDPWD - - return 0 -} -function create_default_conda() -{ - prjpath=$1 - prjname=$(basename $prjpath) - date=$(date +%Y-%m-%d) - - conda create -y --name $prjname -c r -c conda-forge -c bioconda r-base r-essentials perl perl-app-cpanminus snakemake \ - && conda config --file $minicondapath/envs/$prjname/.condarc --add channels r --add channels conda-forge --add channels bioconda \ - && conda env export -n $prjname -f $prjpath/local/condafiles/${prjname}_${date}.yml \ - || return $? - - configure_direnv_conda $prjpath - - return 0 -} -function create_custom_conda() -{ - prjpath=$1 - prjname=$(basename $prjpath) - date=$(date +%Y-%m-%d) - - echo "Complete the command, then press enter to run it:" - while read -p "conda create -y --name $prjname " args; do - if ! conda create -y --name $prjname $args; then - echo "That did not work. Please try again:" - else - conda env export -n $prjname -f $prjpath/local/condafiles/${prjname}_${date}.yml - echo "Success!" - break - fi - done - - configure_direnv_conda $prjpath - - return 0 -} -function configure_direnv_conda() -{ - prjpath=$1 - prjname=$(basename $prjpath) - - echo "source activate $prjname" >> "$prjpath/.envrc" - # It is important that the miniconda libraries come last. - echo ".libPaths( c(.libPaths(), \"$minicondapath/envs/$prjname/lib/R/library\") )" >> $prjpath/.Rprofile -} -function configure_direnv_local() -{ - prjpath=$1 - - echo "export PATH=$prjpath/local/bin:\$PATH" > "$prjpath/.envrc" - echo "export PERL5LIB=$prjpath/local/lib/perl5" >> "$prjpath/.envrc" - echo "export PERL_CPANM_HOME=$prjpath/local/builds/perl5" >> "$prjpath/.envrc" - echo "export PERL_CPANM_OPT=\"-l $prjpath/local --no-man-pages --save-dists=$prjpath/local/src/perl5\"" >> "$prjpath/.envrc" - echo "export PYTHONPATH=$prjpath/local/lib/python" >> "$prjpath/.envrc" - echo "export R_PROFILE_USER=$prjpath/.Rprofile" >> "$prjpath/.envrc" - echo ".libPaths( c(\"$prjpath/local/lib/R\", .libPaths()) )" > $prjpath/.Rprofile -} - -# check syntax and acquire options and arguments -PARSER=$(getopt --options=$options --longoptions=$longoptions --name "$0" -- "$@") -eval set -- "$PARSER" - -templates=1 -git=1 -gitlab=0 -conda=1 -interactive=0 -remove=0 -while true; do - case "$1" in - -h|--help ) - echo "$usage" - exit 0 - ;; - -T|--no-templates ) - templates=0 - shift - ;; - -G|--no-git ) - git=0 - shift - ;; - -l|--gitlab ) - gitlab=1 - shift - ;; - -C|--no-conda ) - conda=0 - shift - ;; - -i|--interactive-conda ) - interactive=1 - shift - ;; - -r|--remove ) - remove=1 - shift - ;; - -- ) - shift - break - ;; - * ) - bug "Unexpected arguments problem." - ;; - esac -done - -# validate the options -if ((remove)); then - if [[ $templates == 0 || $conda == 0 || $git == 0 || $interactive == 1 ]]; then - error "-r is not compatible with any other option (except -l)." 5 - fi -fi -if ! ((git)) && (($gitlab)); then - error "-G is not compatible with -l." 5 -fi - -# parse and validate the arguments -if [[ $# -eq 1 ]]; then - if ! is_valid_name $1; then - error "Project name is not valid. Only lowercase letters, numbers, dashes and underscores are allowed." 5 - fi - if ((remove)); then - if ! [ -e ${BIOINFO_ROOT}/prj/$1 ]; then - error "This project does not exist." 7 - fi - else - if [ -e ${BIOINFO_ROOT}/prj/$1 ]; then - error "A project with this name already exists." 7 - fi - fi - prjpath=${BIOINFO_ROOT}/prj/$1 -else - error "$usage" 1 -fi - -# start the script - -# Remove a project -if ((remove)); then - prjname=$(basename $prjpath) - - echo "Are you sure you want to remove project \"$prjname\"? Choose y/n." - echo "y) Yes, get rid of it! n) No, wait! Let me reconsider..." - - while read answer; do - case ${answer:0:1} in - 'y'|'Y' ) - echo - info "Removing project $prjpath..." - echo - clean_prj $prjpath - info "All done. The project no longer exists." - break - ;; - 'n'|'N'|'' ) - echo - info "OK, the project is safe. Bye" - break - ;; - * ) - echo -n "Please answer yes or no. Try again: [yes/no] " - ;; - esac - done - - exit -fi - -# Otherwise create a project -info "Creating project at $prjpath..." - -# make directories -if ! make_dirs $prjpath; then - clean_prj $prjpath - error "Could not create directories." 9 -fi - -# add links to dockerfiles and snakefiles -if ((templates)); then - info "Adding templates for Dockerfile and Snakefile..." - if ! create_templates $prjpath; then - clean_prj $prjpath - error "Could not create links to templates." 9 - fi -fi - -# initialise a git repository -if ((git)); then - info "Initialising local git repository..." - if ! initialise_repo $prjpath; then - clean_prj $prjpath - error "Could not initialise git repository." 9 - fi -fi - -# create remote gitlab repository -if ((gitlab)); then - info "Creating remote repository on GitLab..." - if ! initialise_gitlab_repo $prjpath; then - warn "Could not create remote repository." - fi -fi - -# create a conda environment -if ((conda)); then - info "Starting the creation of a conda environment..." - - if ! ((interactive)); then - if ! create_default_conda $prjpath; then - clean_prj $prjpath - error "Could not create conda environment." 9 - fi - else - echo "Do you want to create a custom conda environment? Choose y/n." - echo "y) create custom environment n) accept default" - - while read answer; do - case ${answer:0:1} in - 'y'|'Y' ) - if ! create_custom_conda $prjpath; then - clean_prj $prjpath - error "Could not create conda environment." 9 - fi - break - ;; - 'n'|'N'|'' ) - if ! create_default_conda $prjpath; then - clean_prj $prjpath - error "Could not create conda environment." 9 - fi - break - ;; - * ) - echo -n "Please answer yes or no. Try again: [yes/no] " - ;; - esac - done - fi -fi - -# whitelist directory for direnv -direnv allow "$prjpath" - -info "All done. New project is at $prjpath" -exit