Merge pull request #425 from bgruening/18.01

Galaxy Docker 18.01 release

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "galaxy/roles/galaxyproject.galaxyextras"]
 	path = galaxy/roles/galaxyprojectdotorg.galaxyextras
 	url = https://github.com/galaxyproject/ansible-galaxy-extras
+        branch = 18.01
 [submodule "galaxy/roles/galaxy-postgresql"]
 	path = galaxy/roles/galaxy-postgresql
 	url = https://github.com/galaxyproject/ansible-postgresql

.travis.yml

@@ -267,23 +267,30 @@ script:
   # Run a ton of BioBlend test against our servers.
   - cd $TRAVIS_BUILD_DIR/test/bioblend/ && . ./test.sh && cd $WORKING_DIR/
 
-  # Test the 'old' tool installation script
-  - docker_exec_run bash -c "install-repository $INSTALL_REPO_ARG '--url https://toolshed.g2.bx.psu.edu -o iuc --name samtools_sort --panel-section-name SAMtools'"
-  - |
-    if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
-    then
-        # Test without install-repository wrapper
-        sleep 10
-        docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools'
-    fi
+  # not working anymore in 18.01
+  # executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid  --log-file=galaxy_install.log --pid-file=galaxy_install.pid
+  # [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml
+  # /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log'
+  # getopt_long() error
+  # cat: galaxy_install.pid: No such file or directory
+  # tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory
+  #- |
+  #  if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
+  #  then
+  #      # Test without install-repository wrapper
+  #      sleep 10
+  #      docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools'
+  #  fi
+
+
   # Test the 'new' tool installation script
 #  - docker_exec bash -c "install-tools $GALAXY_HOME/ephemeris/sample_tool_list.yaml"
   - |
     if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
     then
         # Compose uses the online installer (uses the running instance)
         sleep 10
-        docker_exec_run shed-install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS"
+        docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS"
     else
       docker_exec_run install-tools "$SAMPLE_TOOLS"
     fi

README.md

@@ -8,7 +8,7 @@
 Galaxy Docker Image
 ===================
 
-The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features.
+The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features. 
 
 One of the main goals is to make the access to entire tool suites as easy as possible. Usually,
 this includes the setup of a public available web-service that needs to be maintained, or that the Tool-user needs to either setup a Galaxy Server by its own or to have Admin access to a local Galaxy server.
@@ -18,7 +18,6 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and
 
 ![Docker hierarchy](https://raw.githubusercontent.com/bgruening/docker-galaxy-stable/master/chart.png)
 
-
 # Table of Contents <a name="toc" />
 
 - [Usage](#Usage)
@@ -54,6 +53,7 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and
 
 
 # Usage <a name="Usage" /> [[toc]](#toc)
+This chapter explains how to launch the container manually. An [ansible](https://www.ansible.com/) playbook that launches the container automatically is available [here](https://github.com/LUMC/galaxy-launcher).
 
 At first you need to install docker. Please follow the [very good instructions](https://docs.docker.com/installation/) from the Docker project.
 

compose/.env_htcondor_docker

@@ -6,6 +6,8 @@
 # Galaxy deployments.
 # ln -sf .env_htcondor_docker .env
 
+TAG=18.01
+
 GALAXY_DESTINATIONS_DEFAULT=condor_cluster
 
 GALAXY_RUNNERS_ENABLE_CONDOR=True

compose/.env_k8_native

@@ -3,3 +3,5 @@ GALAXY_DESTINATIONS_DEFAULT=docker_dispatch
 GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8_default
 GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container
 GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml
+
+TAG=18.01

compose/.env_slurm

@@ -6,6 +6,7 @@
 # Galaxy deployments.
 # ln -sf .env_slurm .env
 
+TAG=18.01
 
 # default destination is our SLURM cluster
 GALAXY_DESTINATIONS_DEFAULT=slurm_cluster
@@ -17,4 +18,3 @@ GALAXY_DOCKER_ENABLED=false
 
 GALAXY_CONFIG_CLEANUP_JOB=onsuccess
 GALAXY_PORT=8080
-

compose/.env_slurm_singularity

@@ -6,6 +6,8 @@
 # Galaxy deployments.
 # ln -sf .env_slurm .env
 
+TAG=18.01
+
 # default destination is our SLURM cluster with Singularity enabled
 GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity
 GALAXY_RUNNERS_ENABLE_SLURM=True

compose/README.md

@@ -24,6 +24,11 @@ Galaxy Docker Compose
 
 <p align="right"><a href="#toc">&#x25B2; back to top</a></p>
 
+# Architectural Overview
+
+![Architectural Overview](./galaxy-compose.svg)
+
+
 # Usage <a name="Usage" />
 
 At first you need to install docker with [compose](https://docs.docker.com/compose).

compose/buildlocal.sh

@@ -2,31 +2,36 @@
 set -x -e
 
 ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras
-ANSIBLE_RELEASE=1d90fdc073d8a94278366290d20f17caae059fbb
+ANSIBLE_RELEASE=18.01
 
-GALAXY_RELEASE=release_17.09
+GALAXY_RELEASE=release_18.01
 GALAXY_REPO=galaxyproject/galaxy
 
-DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache"
+DOCKER_ADDITIONAL_BUILD_ARGS=""
+#"--no-cache"
+
+# For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch.
+TAG=":18.01"
 
 docker pull postgres
 
-docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/bgruening/galaxy-base ./galaxy-base/
-docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg GALAXY_REPO=$GALAXY_REPO --build-arg GALAXY_RELEASE=$GALAXY_RELEASE -t quay.io/bgruening/galaxy-init ./galaxy-init/
+docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/bgruening/galaxy-base$TAG ./galaxy-base/
+docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg GALAXY_REPO=$GALAXY_REPO --build-arg GALAXY_RELEASE=$GALAXY_RELEASE -t quay.io/bgruening/galaxy-init$TAG ./galaxy-init/
 
 # Build the Galaxy web-application container
-docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/bgruening/galaxy-web ./galaxy-web/
+docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/bgruening/galaxy-web$TAG ./galaxy-web/
 
-docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/galaxy/proftpd ./galaxy-proftpd
+docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/galaxy/proftpd$TAG ./galaxy-proftpd
 
 # Build the postgres container
-docker build -t quay.io/galaxy/postgres ./galaxy-postgres
+docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/galaxy/postgres$TAG ./galaxy-postgres
 
 # The SLURM cluster
-docker build -t quay.io/galaxy/slurm ./galaxy-slurm
+docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/galaxy/slurm$TAG ./galaxy-slurm
 
 # we build a common HTCondor and derive from that laster
-docker build -t quay.io/bgruening/galaxy-htcondor-base ./galaxy-htcondor-base
-docker build -t quay.io/bgruening/galaxy-htcondor ./galaxy-htcondor
-docker build -t quay.io/bgruening/galaxy-htcondor-executor ./galaxy-htcondor-executor
+docker build -t quay.io/bgruening/galaxy-htcondor-base$TAG ./galaxy-htcondor-base
+docker build -t quay.io/bgruening/galaxy-htcondor$TAG ./galaxy-htcondor
+docker build -t quay.io/bgruening/galaxy-htcondor-executor$TAG ./galaxy-htcondor-executor
 
+docker build -t quay.io/bgruening/galaxy-grafana$TAG ./galaxy-grafana

compose/docker-compose.yml

@@ -34,7 +34,7 @@ services:
       replicas: 1
       placement:
         constraints: [node.role == manager]
-    labels: 
+    labels:
         kompose.service.type: nodeport
 
   galaxy-postgres:
@@ -75,10 +75,12 @@ services:
     restart: unless-stopped
     networks:
       - galaxy
+    depends_on:
+      - galaxy-postgres
     deploy:
       placement:
         constraints: [node.role == manager]
-    labels: 
+    labels:
         kompose.service.type: nodeport
 
   # SLURM container - this is your compute cluster :)
@@ -157,7 +159,7 @@ services:
 
 
   galaxy-htcondor-executor-big:
-    image: quay.io/bgruening/galaxy-htcondor-executor
+    image: quay.io/bgruening/galaxy-htcondor-executor:${TAG:-latest}
     # privileged is needed to run Docker jobs, moreover HTCondor wants
     # a RW cgroups mount. A RO cgroups mount throws an error in ProcD for running jobs.
     privileged: True
@@ -199,14 +201,46 @@ services:
         constraints: [node.role == worker]
 
 
+  # Grafana and InfluxDB for statistics
+  influxdb:
+    image: influxdb:1.4-alpine
+    environment:
+        - RABBITMQ_DEFAULT_USER=galaxy
+        - RABBITMQ_DEFAULT_PASS=vaiJa3ieghai2ief0jao
+        - RABBITMQ_DEFAULT_VHOST=galaxy
+    volumes:
+        - ${EXPORT_DIR:-/export}/influxdb:/var/lib/influxdb
+    networks:
+      - galaxy
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == worker]
+
+  grafana:
+    image: quay.io/bgruening/galaxy-grafana:${TAG:-latest}
+    volumes:
+        - ${EXPORT_DIR:-/export}/grafana:/var/lib/grafana
+    networks:
+      - galaxy
+    ports:
+        - "3000:3000"
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == worker]
+
   # This container provides the galaxy uwsgi webhandlers, job handlers, nginx
   galaxy-web:
     # build: galaxy-web
     image: quay.io/bgruening/galaxy-web:${TAG:-latest}
     environment:
         # In case the galaxy-init container is not used, you need to include "sleeplock" to NONUSE
         - NONUSE=proftpd,postgres,slurmd,slurmctld
-        - [email protected]
+        - GALAXY_DEFAULT_ADMIN_USER=admin
+        - [email protected]
         - GALAXY_DEFAULT_ADMIN_PASSWORD=admin
         - GALAXY_DEFAULT_ADMIN_KEY=admin
         - GALAXY_HANDLER_NUMPROCS=2
@@ -217,11 +251,11 @@ services:
         - GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True
         - GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:chaopagoosaequuashie@galaxy-postgres/galaxy?client_encoding=utf8
         - GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml
-        # Configurate admin and master api key. Can be overridden in galaxy.ini
+        # Configurate admin and master api key. Can be overridden in galaxy.yml
         - [email protected]
         - GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3
         - GALAXY_CONFIG_DATABASE_AUTO_MIGRATE=${GALAXY_CONFIG_DATABASE_AUTO_MIGRATE:-false}
-        - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy 
+        - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy
         # Condor
         - GALAXY_DESTINATIONS_DEFAULT=${GALAXY_DESTINATIONS_DEFAULT:-slurm_cluster}
         - GALAXY_RUNNERS_ENABLE_CONDOR=${GALAXY_RUNNERS_ENABLE_CONDOR:-False}
@@ -259,6 +293,7 @@ services:
     volumes:
         # This is the directory where all your files from Galaxy will be stored
         # on your host system
+        - ${EXPORT_DIR:-/export}/galaxy-central/config/galaxy.yml.sample:/etc/galaxy/galaxy.yml
         - ${EXPORT_DIR:-/export}/:/export/
         - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
@@ -268,10 +303,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-    labels: 
+    labels:
         kompose.service.type: nodeport
 
 networks:
     galaxy:
-
-

compose/dumpsql.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+TAG=18.01
+
 # Sets the image of postgres to use
 POSTGRES=postgres:9.6.5
 
@@ -22,9 +24,9 @@ pg_end=`date +%s`
 init_start=`date +%s`
 
 docker run -i --rm --name "dumpsql_galaxy_installdb" \
-    -e "GALAXY_CONFIG_FILE=/galaxy-export/config/galaxy.ini.sample" \
+    -e "GALAXY_CONFIG_FILE=/etc/galaxy/galaxy.yml" \
     -e "GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB?client_encoding=utf8" \
-    --link "dumpsql_postgres:db" quay.io/bgruening/galaxy-init install_db.sh
+    --link "dumpsql_postgres:db" quay.io/bgruening/galaxy-init:$TAG install_db.sh
 
 init_end=`date +%s`
 dump_start=`date +%s`

compose/galaxy-base/Dockerfile

@@ -7,7 +7,7 @@ FROM ubuntu:14.04
 MAINTAINER Björn A. Grüning, [email protected]
 
 ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras
-ARG ANSIBLE_RELEASE=1d90fdc073d8a94278366290d20f17caae059fbb
+ARG ANSIBLE_RELEASE=18.01
 
 ENV DEBIAN_FRONTEND=noninteractive \
 GALAXY_USER=galaxy \
@@ -17,18 +17,20 @@ GALAXY_HOME=/home/galaxy \
 GALAXY_LOGS_DIR=/home/galaxy/logs \
 GALAXY_ROOT=/export/galaxy-central \
 GALAXY_VIRTUAL_ENV=/export/venv \
+GALAXY_CONFIG_DIR=/etc/galaxy \
 EXPORT_DIR=/export \
 # Setting a standard encoding. This can get important for things like the unix sort tool.
 LC_ALL=en_US.UTF-8 \
 LANG=en_US.UTF-8
 
 ENV \
-GALAXY_CONFIG_FILE=/export/config/galaxy.ini \
+GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \
 GALAXY_CONFIG_STATIC_ENABLED=False \
 GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://localhost/galaxy?client_encoding=utf8 \
 GALAXY_CONFIG_TEMPLATE_CACHE_PATH=/export/galaxy-central/database/compiled_templates \
 GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=/export/galaxy-central/database/citations/data \
 GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=/export/galaxy-central/database/pbs \
+GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \
 GALAXY_CONFIG_FTP_UPLOAD_DIR=/export/ftp \
 GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \
 GALAXY_CONFIG_USE_PBKDF2=False \
@@ -72,14 +74,14 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
     apt-get update -qq && apt-get upgrade -y && \
     apt-get install --no-install-recommends -y sudo python-virtualenv \
     supervisor linux-image-extra-virtual munge \
-    nodejs-legacy npm ansible nano python-pip \
+    ansible nano python-pip \
     unattended-upgrades \
     gridengine-drmaa1.0 && \
     wget -q --no-check-certificate https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \
     dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \
     rm condor_8.6.3_linux_all.deb && \
     pip install --upgrade pip && \
-    pip install "ephemeris<0.8" && \
+    pip install ephemeris && \
     apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     mkdir -p /tmp/download && \
     wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \
@@ -121,5 +123,17 @@ RUN ln -s -f /export/shed_tools /shed_tools && \
     ln -s -f /export/venv /galaxy_venv && \
     mkdir /etc/galaxy && ln -s -f /export/welcome /etc/galaxy/web
 
+RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.tar.gz && \
+    cd / && tar xvfz telegraf-1.5.0_linux_amd64.tar.gz && \
+    cp -Rv telegraf/* / && \
+    rm -rf telegraf && \
+    rm telegraf-1.5.0_linux_amd64.tar.gz
+
+ADD telegraf.conf /etc/telegraf/telegraf.conf
+
+ENV GALAXY_CONFIG_STATSD_HOST=localhost \
+    GALAXY_CONFIG_STATSD_PORT=8125 \
+    GALAXY_CONFIG_STATSD_PREFIX=galaxy
+
 # Autostart script that is invoked during container start
 CMD ["/usr/bin/startup"]