From 945085072e1d147f8dfe6aa7c99aa3012afac55c Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 4 Nov 2017 15:20:09 +0100 Subject: [PATCH 01/60] track dev branch --- compose/buildlocal.sh | 2 +- compose/galaxy-init/Dockerfile | 2 +- galaxy/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index c43fab466..46f94676d 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -4,7 +4,7 @@ set -x -e ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras ANSIBLE_RELEASE=86a127ae3aaaea125c8faa0271471106f2a4f889 -GALAXY_RELEASE=release_17.09 +GALAXY_RELEASE=dev GALAXY_REPO=galaxyproject/galaxy DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache" diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 61a15c0f3..077a51557 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -6,7 +6,7 @@ FROM quay.io/bgruening/galaxy-base MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com -ARG GALAXY_RELEASE=release_17.09 +ARG GALAXY_RELEASE=dev ARG GALAXY_REPO=galaxyproject/galaxy # Create these folders and link to target directory for installation diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 68ca45352..5da54fdf6 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -9,7 +9,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_17.09} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-dev} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ From ee4c3346f834e5a501f3e1d9774d8af00ba6ebbb Mon Sep 17 00:00:00 2001 From: rhpvorderman Date: Fri, 10 Nov 2017 13:51:26 +0100 Subject: [PATCH 02/60] add galaxy-launcher reference --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cddc223ba..f22e47920 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and ![Docker hierarchy](https://raw.githubusercontent.com/bgruening/docker-galaxy-stable/master/chart.png) +An [ansible](https://www.ansible.com/) playbook that launches a galaxy production server using this image is available [here](https://github.com/LUMC/galaxy-launcher). # Table of Contents From b641a3118e7f28edafb364ebaa18c2e8c4b7588a Mon Sep 17 00:00:00 2001 From: rhpvorderman Date: Fri, 10 Nov 2017 14:05:08 +0100 Subject: [PATCH 03/60] add galaxy-launcher reference --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index f22e47920..dbcf38701 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Galaxy Docker Image =================== -The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features. +The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features. An [ansible](https://www.ansible.com/) playbook that launches a galaxy production server using this image is available [here](https://github.com/LUMC/galaxy-launcher). One of the main goals is to make the access to entire tool suites as easy as possible. Usually, this includes the setup of a public available web-service that needs to be maintained, or that the Tool-user needs to either setup a Galaxy Server by its own or to have Admin access to a local Galaxy server. @@ -18,8 +18,6 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and ![Docker hierarchy](https://raw.githubusercontent.com/bgruening/docker-galaxy-stable/master/chart.png) -An [ansible](https://www.ansible.com/) playbook that launches a galaxy production server using this image is available [here](https://github.com/LUMC/galaxy-launcher). - # Table of Contents - [Usage](#Usage) From 7b64f7832d6cebd6e11101b9b9f77a297cbe0dca Mon Sep 17 00:00:00 2001 From: rhpvorderman Date: Mon, 13 Nov 2017 13:48:52 +0100 Subject: [PATCH 04/60] Transfer line to Usage chapter. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dbcf38701..cba176537 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Galaxy Docker Image =================== -The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features. An [ansible](https://www.ansible.com/) playbook that launches a galaxy production server using this image is available [here](https://github.com/LUMC/galaxy-launcher). +The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features. One of the main goals is to make the access to entire tool suites as easy as possible. Usually, this includes the setup of a public available web-service that needs to be maintained, or that the Tool-user needs to either setup a Galaxy Server by its own or to have Admin access to a local Galaxy server. @@ -53,6 +53,7 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and # Usage [[toc]](#toc) +This chapter explains how to launch the container manually. An [ansible](https://www.ansible.com/) playbook that launches the container automatically is available [here](https://github.com/LUMC/galaxy-launcher). At first you need to install docker. Please follow the [very good instructions](https://docs.docker.com/installation/) from the Docker project. From d794f86987173530a495df358ff66bad4a177e19 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Thu, 16 Nov 2017 15:10:30 +0000 Subject: [PATCH 05/60] Makes ansible tags configurable on build time. --- compose/galaxy-web/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 91c67a2d4..9398970c9 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -6,6 +6,8 @@ FROM quay.io/bgruening/galaxy-base MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com +ARG GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx,cvmfs + ENV GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \ # The following 2 ENV vars can be used to set the number of uwsgi processes and threads UWSGI_PROCESSES=2 \ @@ -66,7 +68,7 @@ RUN rm -f /usr/bin/startup && \ # Used for detecting privileged mode --extra-vars host_docker_legacy=False \ --extra-vars galaxy_extras_docker_legacy=False \ - --tags=supervisor,startup,scripts,nginx,cvmfs -c local && \ + --tags=$GALAXY_ANSIBLE_TAGS -c local && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN chmod +x /usr/bin/startup From 24c0320d007aeaeafc4d56cc0dc0a0761a8b8333 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 17 Nov 2017 14:37:25 +0100 Subject: [PATCH 06/60] Add -v option for install-tools command --- galaxy/install_tools_wrapper.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index 4ee5992fc..fd91fc256 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -13,7 +13,7 @@ else export PORT=8080 service postgresql start install_log='galaxy_install.log' - + # wait for database to finish starting up STATUS=$(psql 2>&1) while [[ ${STATUS} =~ "starting up" ]] @@ -22,12 +22,12 @@ else STATUS=$(psql 2>&1) sleep 1 done - + echo "starting Galaxy" sudo -E -u galaxy ./run.sh --daemon --log-file=$install_log --pid-file=galaxy_install.pid - + galaxy_install_pid=`cat galaxy_install.pid` - + while : ; do tail -n 2 $install_log | grep -E -q "Removing PID file galaxy_install.pid|Daemon is already running" if [ $? -eq 0 ] ; then @@ -52,6 +52,10 @@ shed-install -g "http://localhost:$PORT" -a admin -t "$1" exit_code=$? if [ $exit_code != 0 ] ; then + if [ "$2" == "-v" ] ; then + echo "Installation failed, Galaxy server log:" + cat $install_log + fi exit $exit_code fi From 3246327446df76979ff0efe207a44c8a121fee31 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 17 Nov 2017 22:20:26 +0100 Subject: [PATCH 07/60] add overview image --- compose/galaxy-compose.svg | 1872 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1872 insertions(+) create mode 100644 compose/galaxy-compose.svg diff --git a/compose/galaxy-compose.svg b/compose/galaxy-compose.svg new file mode 100644 index 000000000..83f6bdd6a --- /dev/null +++ b/compose/galaxy-compose.svg @@ -0,0 +1,1872 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + Cluster + + + + + + + + + + + + + + + + quai.io/galaxy/ + github.com/bgruening/compose/ + galaxy-slurm + galaxy-slurm + + + + + + quai.io/bgruening/ + github.com/bgruening/compose/ + galaxy-htcondor + galaxy-htcondor + + + + + + quai.io/bgruening/ + github.com/bgruening/compose/ + galaxy-htcondor-executor + galaxy-htcondor-executor + + + + + + + quai.io/bgruening/ + github.com/bgruening/compose/ + galaxy-htcondor-base + galaxy-htcondor-base + + + + + + build base + + + + + + library/ + github.com/docker-library/ + rabbitmq + rabbitmq + + + + + + quai.io/galaxy/ + github.com/bgruening/ + proftpd + galaxy-proftpd + + + + + + + Database + + + quai.io/galaxy/, library/ + github.com/bgruening/compose/ + postgres + galaxy-postgres + + + + + + thajeztah/ + github.com/thaJeztah/ + pgadmin4 + pgadmin4-docker + + + + + + + + + + + + + /export/ftp/ + + + + + /export/rabbitmq/ + + + network + + + /export/postgres/ + + + + + /export/* + + /export/slurm.conf + /export/munge.key + + + + + + quai.io/bgruening/ + github.com/bgruening/compose/ + galaxy-web + galaxy-web + + + + + + quai.io/bgruening/ + + + github.com/bgruening/compose/ + galaxy-init + galaxy-init + + + + quai.io/bgruening/ + github.com/bgruening/compose/ + galaxy-base + galaxy-base + + + + + build base + + + + + + + + + + + /export/* + /export/.initdone + + + + /export/* + /export/.initdone + + + + lock file + port open + + + + + + + Galaxy + + + + /export/* + + + + + /export/* + + + + + + + + + From b4e1bede61698c06680e4ca792a3780638f0f051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 18 Nov 2017 11:10:23 +0100 Subject: [PATCH 08/60] add overview image --- compose/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compose/README.md b/compose/README.md index 045be1b12..30dc025b8 100644 --- a/compose/README.md +++ b/compose/README.md @@ -24,6 +24,11 @@ Galaxy Docker Compose

▲ back to top

+# Architectural Overview + +![Architectural Overview](./galaxy-compose.svg) + + # Usage At first you need to install docker with [compose](https://docs.docker.com/compose). From dfda84de8b8ed8e6e828d2a37443eefa43078cc3 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 20 Nov 2017 16:40:55 +0000 Subject: [PATCH 09/60] Replaces k8 with k8s for ansible parts --- compose/galaxy-init/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 077a51557..676e3ea6e 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -40,7 +40,7 @@ RUN ansible-playbook /ansible/provision.yml \ --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \ --extra-vars galaxy_extras_config_condor=True \ --extra-vars galaxy_extras_config_condor_docker=True \ - --extra-vars galaxy_extras_config_k8_jobs=True \ + --extra-vars galaxy_extras_config_k8s_jobs=True \ --extra-vars galaxy_minimum_version=17.09 \ --extra-vars galaxy_extras_config_rabbitmq=False \ --extra-vars nginx_upload_store_path=/export/nginx_upload_store \ @@ -50,7 +50,7 @@ RUN ansible-playbook /ansible/provision.yml \ #--extra-vars container_resolution_explicit=True \ #--extra-vars container_resolution_cached_mulled=False \ #--extra-vars container_resolution_build_mulled=False \ - --tags=ie,pbs,slurm,uwsgi,metrics,k8 -c local && \ + --tags=ie,pbs,slurm,uwsgi,metrics,k8s -c local && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 2370b8a22eae7103ac0b4b3ded5b87cab4b90d39 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 20 Nov 2017 17:04:58 +0000 Subject: [PATCH 10/60] Updates the sha for ansible-galaxy-extras to reflect the var changes k8 to k8s. --- compose/galaxy-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 150e2b2c1..77fae9330 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -7,7 +7,7 @@ FROM ubuntu:14.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ARG ANSIBLE_RELEASE=1e07cc8a56cd18821a9f20129c400a242d25d1a0 +ARG ANSIBLE_RELEASE=5d9c8be9b496a6890cf04a858ed650f0211baf89 ENV DEBIAN_FRONTEND=noninteractive \ GALAXY_USER=galaxy \ From 7ba8abd7caade74e3b8c14889b39103f7d1a47b4 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 20 Nov 2017 17:07:09 +0000 Subject: [PATCH 11/60] =?UTF-8?q?Updates=20buildlocal.sh=20ansible=20sha?= =?UTF-8?q?=20as=20suggested=20by=20Bj=C3=B6rn.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- compose/buildlocal.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 46f94676d..6d371bbe3 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -2,7 +2,7 @@ set -x -e ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ANSIBLE_RELEASE=86a127ae3aaaea125c8faa0271471106f2a4f889 +ANSIBLE_RELEASE=5d9c8be9b496a6890cf04a858ed650f0211baf89 GALAXY_RELEASE=dev GALAXY_REPO=galaxyproject/galaxy From b6429db868486e65827adb4a761acc3a8aab8d27 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 21 Nov 2017 10:00:54 +0000 Subject: [PATCH 12/60] Uses :dev tag for FROMs in all relevant containers. --- compose/buildlocal.sh | 17 ++++++++++------- compose/galaxy-htcondor-executor/Dockerfile | 2 +- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-init/Dockerfile | 2 +- compose/galaxy-web/Dockerfile | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 6d371bbe3..967d17272 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -9,15 +9,18 @@ GALAXY_REPO=galaxyproject/galaxy DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache" +# For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. +TAG=":dev" + docker pull postgres -docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/bgruening/galaxy-base ./galaxy-base/ -docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg GALAXY_REPO=$GALAXY_REPO --build-arg GALAXY_RELEASE=$GALAXY_RELEASE -t quay.io/bgruening/galaxy-init ./galaxy-init/ +docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/bgruening/galaxy-base$TAG ./galaxy-base/ +docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg GALAXY_REPO=$GALAXY_REPO --build-arg GALAXY_RELEASE=$GALAXY_RELEASE -t quay.io/bgruening/galaxy-init$TAG ./galaxy-init/ # Build the Galaxy web-application container -docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/bgruening/galaxy-web ./galaxy-web/ +docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/bgruening/galaxy-web$TAG ./galaxy-web/ -docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/galaxy/proftpd ./galaxy-proftpd +docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/galaxy/proftpd$TAG ./galaxy-proftpd # Build the postgres container docker build -t quay.io/galaxy/postgres ./galaxy-postgres @@ -26,7 +29,7 @@ docker build -t quay.io/galaxy/postgres ./galaxy-postgres docker build -t quay.io/galaxy/slurm ./galaxy-slurm # we build a common HTCondor and derive from that laster -docker build -t quay.io/bgruening/galaxy-htcondor-base ./galaxy-htcondor-base -docker build -t quay.io/bgruening/galaxy-htcondor ./galaxy-htcondor -docker build -t quay.io/bgruening/galaxy-htcondor-executor ./galaxy-htcondor-executor +docker build -t quay.io/bgruening/galaxy-htcondor-base$TAG ./galaxy-htcondor-base +docker build -t quay.io/bgruening/galaxy-htcondor$TAG ./galaxy-htcondor +docker build -t quay.io/bgruening/galaxy-htcondor-executor$TAG ./galaxy-htcondor-executor diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index 7b2288ae0..3ae0255d8 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base +FROM quay.io/bgruening/galaxy-htcondor-base:dev ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 1e5fd7bec..af80dbaa3 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base +FROM quay.io/bgruening/galaxy-htcondor-base:dev ADD condor_config.local /etc/condor/condor_config.local ADD supervisord.conf /etc/supervisord.conf diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 676e3ea6e..7d829c48a 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base +FROM quay.io/bgruening/galaxy-base:dev MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 9398970c9..a2d3350aa 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base +FROM quay.io/bgruening/galaxy-base:dev MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com From 7f121f5f17de79ddec6afeab381dbcfb298ca257 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 21 Nov 2017 11:00:42 +0000 Subject: [PATCH 13/60] Sets tag on buildlocal to latest --- compose/buildlocal.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 967d17272..2f125a6bf 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -10,7 +10,7 @@ GALAXY_REPO=galaxyproject/galaxy DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -TAG=":dev" +TAG=":latest" docker pull postgres From 00611012f1b2f04b385cc2a17214115a5ee0f942 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 14 Dec 2017 14:25:18 +0100 Subject: [PATCH 14/60] upgrade DB --- compose/galaxy-postgres/init-galaxy-db.sql.in | 600 +++++++++++++++++- 1 file changed, 593 insertions(+), 7 deletions(-) diff --git a/compose/galaxy-postgres/init-galaxy-db.sql.in b/compose/galaxy-postgres/init-galaxy-db.sql.in index 2a3da8c0a..4af8ee621 100644 --- a/compose/galaxy-postgres/init-galaxy-db.sql.in +++ b/compose/galaxy-postgres/init-galaxy-db.sql.in @@ -467,7 +467,8 @@ CREATE TABLE dataset_collection ( create_time timestamp without time zone, update_time timestamp without time zone, populated_state character varying(64) DEFAULT 'ok'::character varying NOT NULL, - populated_state_message text + populated_state_message text, + element_count integer ); @@ -1553,7 +1554,9 @@ CREATE TABLE history_dataset_collection_association ( deleted boolean, visible boolean, copied_from_history_dataset_collection_association_id integer, - implicit_output_name character varying(255) + implicit_output_name character varying(255), + implicit_collection_jobs_id integer, + job_id integer ); @@ -1755,6 +1758,66 @@ CREATE SEQUENCE history_user_share_association_id_seq ALTER SEQUENCE history_user_share_association_id_seq OWNED BY history_user_share_association.id; +-- +-- Name: implicit_collection_jobs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE implicit_collection_jobs ( + id integer NOT NULL, + populated_state character varying(64) NOT NULL +); + + +-- +-- Name: implicit_collection_jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE implicit_collection_jobs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: implicit_collection_jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE implicit_collection_jobs_id_seq OWNED BY implicit_collection_jobs.id; + + +-- +-- Name: implicit_collection_jobs_job_association; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE implicit_collection_jobs_job_association ( + implicit_collection_jobs_id integer, + id integer NOT NULL, + job_id integer, + order_index integer NOT NULL +); + + +-- +-- Name: implicit_collection_jobs_job_association_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE implicit_collection_jobs_job_association_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: implicit_collection_jobs_job_association_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE implicit_collection_jobs_job_association_id_seq OWNED BY implicit_collection_jobs_job_association.id; + + -- -- Name: implicitly_converted_dataset_association; Type: TABLE; Schema: public; Owner: - -- @@ -4795,6 +4858,70 @@ CREATE SEQUENCE workflow_invocation_id_seq ALTER SEQUENCE workflow_invocation_id_seq OWNED BY workflow_invocation.id; +-- +-- Name: workflow_invocation_output_dataset_association; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE workflow_invocation_output_dataset_association ( + id integer NOT NULL, + workflow_invocation_id integer, + workflow_step_id integer, + dataset_id integer, + workflow_output_id integer +); + + +-- +-- Name: workflow_invocation_output_dataset_association_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE workflow_invocation_output_dataset_association_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: workflow_invocation_output_dataset_association_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE workflow_invocation_output_dataset_association_id_seq OWNED BY workflow_invocation_output_dataset_association.id; + + +-- +-- Name: workflow_invocation_output_dataset_collection_association; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE workflow_invocation_output_dataset_collection_association ( + id integer NOT NULL, + workflow_invocation_id integer, + workflow_step_id integer, + dataset_collection_id integer, + workflow_output_id integer +); + + +-- +-- Name: workflow_invocation_output_dataset_collection_associatio_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE workflow_invocation_output_dataset_collection_associatio_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: workflow_invocation_output_dataset_collection_associatio_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE workflow_invocation_output_dataset_collection_associatio_id_seq OWNED BY workflow_invocation_output_dataset_collection_association.id; + + -- -- Name: workflow_invocation_step; Type: TABLE; Schema: public; Owner: - -- @@ -4806,7 +4933,9 @@ CREATE TABLE workflow_invocation_step ( workflow_invocation_id integer NOT NULL, workflow_step_id integer NOT NULL, job_id integer, - action bytea + action bytea, + implicit_collection_jobs_id integer, + state character varying(64) ); @@ -4829,6 +4958,69 @@ CREATE SEQUENCE workflow_invocation_step_id_seq ALTER SEQUENCE workflow_invocation_step_id_seq OWNED BY workflow_invocation_step.id; +-- +-- Name: workflow_invocation_step_output_dataset_association; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE workflow_invocation_step_output_dataset_association ( + id integer NOT NULL, + workflow_invocation_step_id integer, + dataset_id integer, + output_name character varying(255) +); + + +-- +-- Name: workflow_invocation_step_output_dataset_association_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE workflow_invocation_step_output_dataset_association_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: workflow_invocation_step_output_dataset_association_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE workflow_invocation_step_output_dataset_association_id_seq OWNED BY workflow_invocation_step_output_dataset_association.id; + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_association; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE workflow_invocation_step_output_dataset_collection_association ( + id integer NOT NULL, + workflow_invocation_step_id integer, + workflow_step_id integer, + dataset_collection_id integer, + output_name character varying(255) +); + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_assoc_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE workflow_invocation_step_output_dataset_collection_assoc_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_assoc_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE workflow_invocation_step_output_dataset_collection_assoc_id_seq OWNED BY workflow_invocation_step_output_dataset_collection_association.id; + + -- -- Name: workflow_invocation_to_subworkflow_invocation_association; Type: TABLE; Schema: public; Owner: - -- @@ -5588,6 +5780,20 @@ ALTER TABLE ONLY history_tag_association ALTER COLUMN id SET DEFAULT nextval('hi ALTER TABLE ONLY history_user_share_association ALTER COLUMN id SET DEFAULT nextval('history_user_share_association_id_seq'::regclass); +-- +-- Name: implicit_collection_jobs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY implicit_collection_jobs ALTER COLUMN id SET DEFAULT nextval('implicit_collection_jobs_id_seq'::regclass); + + +-- +-- Name: implicit_collection_jobs_job_association id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY implicit_collection_jobs_job_association ALTER COLUMN id SET DEFAULT nextval('implicit_collection_jobs_job_association_id_seq'::regclass); + + -- -- Name: implicitly_converted_dataset_association id; Type: DEFAULT; Schema: public; Owner: - -- @@ -6211,6 +6417,20 @@ ALTER TABLE ONLY workflow ALTER COLUMN id SET DEFAULT nextval('workflow_id_seq': ALTER TABLE ONLY workflow_invocation ALTER COLUMN id SET DEFAULT nextval('workflow_invocation_id_seq'::regclass); +-- +-- Name: workflow_invocation_output_dataset_association id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_association ALTER COLUMN id SET DEFAULT nextval('workflow_invocation_output_dataset_association_id_seq'::regclass); + + +-- +-- Name: workflow_invocation_output_dataset_collection_association id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_collection_association ALTER COLUMN id SET DEFAULT nextval('workflow_invocation_output_dataset_collection_associatio_id_seq'::regclass); + + -- -- Name: workflow_invocation_step id; Type: DEFAULT; Schema: public; Owner: - -- @@ -6218,6 +6438,20 @@ ALTER TABLE ONLY workflow_invocation ALTER COLUMN id SET DEFAULT nextval('workfl ALTER TABLE ONLY workflow_invocation_step ALTER COLUMN id SET DEFAULT nextval('workflow_invocation_step_id_seq'::regclass); +-- +-- Name: workflow_invocation_step_output_dataset_association id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_association ALTER COLUMN id SET DEFAULT nextval('workflow_invocation_step_output_dataset_association_id_seq'::regclass); + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_association id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_collection_association ALTER COLUMN id SET DEFAULT nextval('workflow_invocation_step_output_dataset_collection_assoc_id_seq'::regclass); + + -- -- Name: workflow_invocation_to_subworkflow_invocation_association id; Type: DEFAULT; Schema: public; Owner: - -- @@ -6509,7 +6743,7 @@ COPY dataset (id, create_time, update_time, state, deleted, purged, purgable, ex -- Data for Name: dataset_collection; Type: TABLE DATA; Schema: public; Owner: - -- -COPY dataset_collection (id, collection_type, create_time, update_time, populated_state, populated_state_message) FROM stdin; +COPY dataset_collection (id, collection_type, create_time, update_time, populated_state, populated_state_message, element_count) FROM stdin; \. @@ -6989,7 +7223,7 @@ SELECT pg_catalog.setval('history_dataset_collection_annotation_association_id_s -- Data for Name: history_dataset_collection_association; Type: TABLE DATA; Schema: public; Owner: - -- -COPY history_dataset_collection_association (id, collection_id, history_id, hid, name, deleted, visible, copied_from_history_dataset_collection_association_id, implicit_output_name) FROM stdin; +COPY history_dataset_collection_association (id, collection_id, history_id, hid, name, deleted, visible, copied_from_history_dataset_collection_association_id, implicit_output_name, implicit_collection_jobs_id, job_id) FROM stdin; \. @@ -7082,6 +7316,36 @@ COPY history_user_share_association (id, history_id, user_id) FROM stdin; SELECT pg_catalog.setval('history_user_share_association_id_seq', 1, false); +-- +-- Data for Name: implicit_collection_jobs; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY implicit_collection_jobs (id, populated_state) FROM stdin; +\. + + +-- +-- Name: implicit_collection_jobs_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('implicit_collection_jobs_id_seq', 1, false); + + +-- +-- Data for Name: implicit_collection_jobs_job_association; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY implicit_collection_jobs_job_association (implicit_collection_jobs_id, id, job_id, order_index) FROM stdin; +\. + + +-- +-- Name: implicit_collection_jobs_job_association_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('implicit_collection_jobs_job_association_id_seq', 1, false); + + -- -- Data for Name: implicitly_converted_dataset_association; Type: TABLE DATA; Schema: public; Owner: - -- @@ -7606,7 +7870,7 @@ GalaxyTools lib/tool_shed/galaxy_install/migrate 1 -- COPY migrate_version (repository_id, repository_path, version) FROM stdin; -Galaxy lib/galaxy/model/migrate 135 +Galaxy lib/galaxy/model/migrate 136 \. @@ -8443,11 +8707,41 @@ COPY workflow_invocation (id, create_time, update_time, workflow_id, history_id, SELECT pg_catalog.setval('workflow_invocation_id_seq', 1, false); +-- +-- Data for Name: workflow_invocation_output_dataset_association; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY workflow_invocation_output_dataset_association (id, workflow_invocation_id, workflow_step_id, dataset_id, workflow_output_id) FROM stdin; +\. + + +-- +-- Name: workflow_invocation_output_dataset_association_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('workflow_invocation_output_dataset_association_id_seq', 1, false); + + +-- +-- Name: workflow_invocation_output_dataset_collection_associatio_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('workflow_invocation_output_dataset_collection_associatio_id_seq', 1, false); + + +-- +-- Data for Name: workflow_invocation_output_dataset_collection_association; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY workflow_invocation_output_dataset_collection_association (id, workflow_invocation_id, workflow_step_id, dataset_collection_id, workflow_output_id) FROM stdin; +\. + + -- -- Data for Name: workflow_invocation_step; Type: TABLE DATA; Schema: public; Owner: - -- -COPY workflow_invocation_step (id, create_time, update_time, workflow_invocation_id, workflow_step_id, job_id, action) FROM stdin; +COPY workflow_invocation_step (id, create_time, update_time, workflow_invocation_id, workflow_step_id, job_id, action, implicit_collection_jobs_id, state) FROM stdin; \. @@ -8458,6 +8752,36 @@ COPY workflow_invocation_step (id, create_time, update_time, workflow_invocation SELECT pg_catalog.setval('workflow_invocation_step_id_seq', 1, false); +-- +-- Data for Name: workflow_invocation_step_output_dataset_association; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY workflow_invocation_step_output_dataset_association (id, workflow_invocation_step_id, dataset_id, output_name) FROM stdin; +\. + + +-- +-- Name: workflow_invocation_step_output_dataset_association_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('workflow_invocation_step_output_dataset_association_id_seq', 1, false); + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_assoc_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('workflow_invocation_step_output_dataset_collection_assoc_id_seq', 1, false); + + +-- +-- Data for Name: workflow_invocation_step_output_dataset_collection_association; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY workflow_invocation_step_output_dataset_collection_association (id, workflow_invocation_step_id, workflow_step_id, dataset_collection_id, output_name) FROM stdin; +\. + + -- -- Name: workflow_invocation_to_subworkflow_invocation_associatio_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - -- @@ -9046,6 +9370,22 @@ ALTER TABLE ONLY history_user_share_association ADD CONSTRAINT history_user_share_association_pkey PRIMARY KEY (id); +-- +-- Name: implicit_collection_jobs_job_association implicit_collection_jobs_job_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY implicit_collection_jobs_job_association + ADD CONSTRAINT implicit_collection_jobs_job_association_pkey PRIMARY KEY (id); + + +-- +-- Name: implicit_collection_jobs implicit_collection_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY implicit_collection_jobs + ADD CONSTRAINT implicit_collection_jobs_pkey PRIMARY KEY (id); + + -- -- Name: implicitly_converted_dataset_association implicitly_converted_dataset_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -9774,6 +10114,22 @@ ALTER TABLE ONLY visualization_user_share_association ADD CONSTRAINT visualization_user_share_association_pkey PRIMARY KEY (id); +-- +-- Name: workflow_invocation_output_dataset_association workflow_invocation_output_dataset_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_association + ADD CONSTRAINT workflow_invocation_output_dataset_association_pkey PRIMARY KEY (id); + + +-- +-- Name: workflow_invocation_output_dataset_collection_association workflow_invocation_output_dataset_collection_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_output_dataset_collection_association_pkey PRIMARY KEY (id); + + -- -- Name: workflow_invocation workflow_invocation_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -9782,6 +10138,22 @@ ALTER TABLE ONLY workflow_invocation ADD CONSTRAINT workflow_invocation_pkey PRIMARY KEY (id); +-- +-- Name: workflow_invocation_step_output_dataset_association workflow_invocation_step_output_dataset_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_association + ADD CONSTRAINT workflow_invocation_step_output_dataset_association_pkey PRIMARY KEY (id); + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_association workflow_invocation_step_output_dataset_collection_associa_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_step_output_dataset_collection_associa_pkey PRIMARY KEY (id); + + -- -- Name: workflow_invocation_step workflow_invocation_step_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -10930,6 +11302,20 @@ CREATE INDEX ix_icda_ldda_id ON implicitly_converted_dataset_association USING b CREATE INDEX ix_icda_ldda_parent_id ON implicitly_converted_dataset_association USING btree (ldda_parent_id); +-- +-- Name: ix_implicit_collection_jobs_job_association_implicit_co_ea04; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_implicit_collection_jobs_job_association_implicit_co_ea04 ON implicit_collection_jobs_job_association USING btree (implicit_collection_jobs_id); + + +-- +-- Name: ix_implicit_collection_jobs_job_association_job_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_implicit_collection_jobs_job_association_job_id ON implicit_collection_jobs_job_association USING btree (job_id); + + -- -- Name: ix_implicitly_converted_dataset_assoc_ldda_parent_id; Type: INDEX; Schema: public; Owner: - -- @@ -12848,6 +13234,34 @@ CREATE INDEX ix_wfinv_swfinv_wfi ON workflow_invocation_to_subworkflow_invocatio CREATE INDEX ix_wfreq_inputstep_wfi ON workflow_request_input_step_parameter USING btree (workflow_invocation_id); +-- +-- Name: ix_workflow_invocation_output_dataset_association_dataset_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_output_dataset_association_dataset_id ON workflow_invocation_output_dataset_association USING btree (dataset_id); + + +-- +-- Name: ix_workflow_invocation_output_dataset_association_workf_5924; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_output_dataset_association_workf_5924 ON workflow_invocation_output_dataset_association USING btree (workflow_invocation_id); + + +-- +-- Name: ix_workflow_invocation_output_dataset_collection_associ_ab6c; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_output_dataset_collection_associ_ab6c ON workflow_invocation_output_dataset_collection_association USING btree (workflow_invocation_id); + + +-- +-- Name: ix_workflow_invocation_output_dataset_collection_associ_ec97; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_output_dataset_collection_associ_ec97 ON workflow_invocation_output_dataset_collection_association USING btree (dataset_collection_id); + + -- -- Name: ix_workflow_invocation_step_job_id; Type: INDEX; Schema: public; Owner: - -- @@ -12855,6 +13269,34 @@ CREATE INDEX ix_wfreq_inputstep_wfi ON workflow_request_input_step_parameter USI CREATE INDEX ix_workflow_invocation_step_job_id ON workflow_invocation_step USING btree (job_id); +-- +-- Name: ix_workflow_invocation_step_output_dataset_association__66f5; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_step_output_dataset_association__66f5 ON workflow_invocation_step_output_dataset_association USING btree (dataset_id); + + +-- +-- Name: ix_workflow_invocation_step_output_dataset_association__bcc0; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_step_output_dataset_association__bcc0 ON workflow_invocation_step_output_dataset_association USING btree (workflow_invocation_step_id); + + +-- +-- Name: ix_workflow_invocation_step_output_dataset_collection_a_b73b; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_step_output_dataset_collection_a_b73b ON workflow_invocation_step_output_dataset_collection_association USING btree (dataset_collection_id); + + +-- +-- Name: ix_workflow_invocation_step_output_dataset_collection_a_db49; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_workflow_invocation_step_output_dataset_collection_a_db49 ON workflow_invocation_step_output_dataset_collection_association USING btree (workflow_invocation_step_id); + + -- -- Name: ix_workflow_invocation_step_workflow_invocation_id; Type: INDEX; Schema: public; Owner: - -- @@ -13732,6 +14174,14 @@ ALTER TABLE ONLY history_dataset_collection_association ADD CONSTRAINT history_dataset_collection_as_copied_from_history_dataset__fkey FOREIGN KEY (copied_from_history_dataset_collection_association_id) REFERENCES history_dataset_collection_association(id); +-- +-- Name: history_dataset_collection_association history_dataset_collection_ass_implicit_collection_jobs_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_dataset_collection_association + ADD CONSTRAINT history_dataset_collection_ass_implicit_collection_jobs_id_fkey FOREIGN KEY (implicit_collection_jobs_id) REFERENCES implicit_collection_jobs(id); + + -- -- Name: history_dataset_collection_association history_dataset_collection_association_collection_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -13748,6 +14198,14 @@ ALTER TABLE ONLY history_dataset_collection_association ADD CONSTRAINT history_dataset_collection_association_history_id_fkey FOREIGN KEY (history_id) REFERENCES history(id); +-- +-- Name: history_dataset_collection_association history_dataset_collection_association_job_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_dataset_collection_association + ADD CONSTRAINT history_dataset_collection_association_job_id_fkey FOREIGN KEY (job_id) REFERENCES job(id); + + -- -- Name: history_dataset_collection_rating_association history_dataset_collection_ra_history_dataset_collection_i_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -13852,6 +14310,22 @@ ALTER TABLE ONLY history_user_share_association ADD CONSTRAINT history_user_share_association_user_id_fkey FOREIGN KEY (user_id) REFERENCES galaxy_user(id); +-- +-- Name: implicit_collection_jobs_job_association implicit_collection_jobs_job_a_implicit_collection_jobs_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY implicit_collection_jobs_job_association + ADD CONSTRAINT implicit_collection_jobs_job_a_implicit_collection_jobs_id_fkey FOREIGN KEY (implicit_collection_jobs_id) REFERENCES implicit_collection_jobs(id); + + +-- +-- Name: implicit_collection_jobs_job_association implicit_collection_jobs_job_association_job_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY implicit_collection_jobs_job_association + ADD CONSTRAINT implicit_collection_jobs_job_association_job_id_fkey FOREIGN KEY (job_id) REFERENCES job(id); + + -- -- Name: implicitly_converted_dataset_association implicitly_converted_dataset_association_hda_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -15212,6 +15686,78 @@ ALTER TABLE ONLY workflow_invocation ADD CONSTRAINT workflow_invocation_history_id_fkey FOREIGN KEY (history_id) REFERENCES history(id); +-- +-- Name: workflow_invocation_output_dataset_association workflow_invocation_output_dataset__workflow_invocation_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_association + ADD CONSTRAINT workflow_invocation_output_dataset__workflow_invocation_id_fkey FOREIGN KEY (workflow_invocation_id) REFERENCES workflow_invocation(id); + + +-- +-- Name: workflow_invocation_output_dataset_association workflow_invocation_output_dataset_asso_workflow_output_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_association + ADD CONSTRAINT workflow_invocation_output_dataset_asso_workflow_output_id_fkey FOREIGN KEY (workflow_output_id) REFERENCES workflow_output(id); + + +-- +-- Name: workflow_invocation_output_dataset_association workflow_invocation_output_dataset_associ_workflow_step_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_association + ADD CONSTRAINT workflow_invocation_output_dataset_associ_workflow_step_id_fkey FOREIGN KEY (workflow_step_id) REFERENCES workflow_step(id); + + +-- +-- Name: workflow_invocation_output_dataset_association workflow_invocation_output_dataset_association_dataset_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_association + ADD CONSTRAINT workflow_invocation_output_dataset_association_dataset_id_fkey FOREIGN KEY (dataset_id) REFERENCES history_dataset_association(id); + + +-- +-- Name: workflow_invocation_output_dataset_collection_association workflow_invocation_output_dataset_c_dataset_collection_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_output_dataset_c_dataset_collection_id_fkey FOREIGN KEY (dataset_collection_id) REFERENCES history_dataset_collection_association(id); + + +-- +-- Name: workflow_invocation_output_dataset_collection_association workflow_invocation_output_dataset_coll_workflow_output_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_output_dataset_coll_workflow_output_id_fkey FOREIGN KEY (workflow_output_id) REFERENCES workflow_output(id); + + +-- +-- Name: workflow_invocation_output_dataset_collection_association workflow_invocation_output_dataset_collec_workflow_step_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_output_dataset_collec_workflow_step_id_fkey FOREIGN KEY (workflow_step_id) REFERENCES workflow_step(id); + + +-- +-- Name: workflow_invocation_output_dataset_collection_association workflow_invocation_output_dataset_workflow_invocation_id_fkey1; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_output_dataset_workflow_invocation_id_fkey1 FOREIGN KEY (workflow_invocation_id) REFERENCES workflow_invocation(id); + + +-- +-- Name: workflow_invocation_step workflow_invocation_step_implicit_collection_jobs_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step + ADD CONSTRAINT workflow_invocation_step_implicit_collection_jobs_id_fkey FOREIGN KEY (implicit_collection_jobs_id) REFERENCES implicit_collection_jobs(id); + + -- -- Name: workflow_invocation_step workflow_invocation_step_job_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -15220,6 +15766,46 @@ ALTER TABLE ONLY workflow_invocation_step ADD CONSTRAINT workflow_invocation_step_job_id_fkey FOREIGN KEY (job_id) REFERENCES job(id); +-- +-- Name: workflow_invocation_step_output_dataset_collection_association workflow_invocation_step_outp_workflow_invocation_step_id_fkey1; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_step_outp_workflow_invocation_step_id_fkey1 FOREIGN KEY (workflow_invocation_step_id) REFERENCES workflow_invocation_step(id); + + +-- +-- Name: workflow_invocation_step_output_dataset_association workflow_invocation_step_outpu_workflow_invocation_step_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_association + ADD CONSTRAINT workflow_invocation_step_outpu_workflow_invocation_step_id_fkey FOREIGN KEY (workflow_invocation_step_id) REFERENCES workflow_invocation_step(id); + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_association workflow_invocation_step_output_data_dataset_collection_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_step_output_data_dataset_collection_id_fkey FOREIGN KEY (dataset_collection_id) REFERENCES history_dataset_collection_association(id); + + +-- +-- Name: workflow_invocation_step_output_dataset_association workflow_invocation_step_output_dataset_associa_dataset_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_association + ADD CONSTRAINT workflow_invocation_step_output_dataset_associa_dataset_id_fkey FOREIGN KEY (dataset_id) REFERENCES history_dataset_association(id); + + +-- +-- Name: workflow_invocation_step_output_dataset_collection_association workflow_invocation_step_output_dataset_c_workflow_step_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY workflow_invocation_step_output_dataset_collection_association + ADD CONSTRAINT workflow_invocation_step_output_dataset_c_workflow_step_id_fkey FOREIGN KEY (workflow_step_id) REFERENCES workflow_step(id); + + -- -- Name: workflow_invocation_step workflow_invocation_step_workflow_invocation_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- From e6e8078f89bb3c6e1136d0f8e109bb3ba256cd25 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 18 Dec 2017 14:34:10 +0100 Subject: [PATCH 15/60] Implement grafana support --- compose/buildlocal.sh | 1 + compose/docker-compose.yml | 35 +++++- compose/galaxy-base/Dockerfile | 12 ++ compose/galaxy-base/telegraf.conf | 105 +++++++++++++++++ compose/galaxy-grafana/Dockerfile | 2 + compose/galaxy-grafana/configure_slurm.py | 114 +++++++++++++++++++ compose/galaxy-grafana/grafana.db | Bin 0 -> 211685 bytes compose/galaxy-grafana/munge.conf | 13 +++ compose/galaxy-grafana/requirements.txt | 71 ++++++++++++ compose/galaxy-grafana/startup.sh | 26 +++++ compose/galaxy-grafana/supervisor_slurm.conf | 18 +++ compose/galaxy-htcondor-base/Dockerfile | 12 +- compose/galaxy-htcondor-base/telegraf.conf | 94 +++++++++++++++ compose/galaxy-htcondor-executor/startup.sh | 1 + compose/galaxy-htcondor/supervisord.conf | 12 ++ 15 files changed, 512 insertions(+), 4 deletions(-) create mode 100644 compose/galaxy-base/telegraf.conf create mode 100644 compose/galaxy-grafana/Dockerfile create mode 100644 compose/galaxy-grafana/configure_slurm.py create mode 100644 compose/galaxy-grafana/grafana.db create mode 100644 compose/galaxy-grafana/munge.conf create mode 100644 compose/galaxy-grafana/requirements.txt create mode 100644 compose/galaxy-grafana/startup.sh create mode 100644 compose/galaxy-grafana/supervisor_slurm.conf create mode 100644 compose/galaxy-htcondor-base/telegraf.conf diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 2f125a6bf..e776d1f58 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -33,3 +33,4 @@ docker build -t quay.io/bgruening/galaxy-htcondor-base$TAG ./galaxy-htcondor-bas docker build -t quay.io/bgruening/galaxy-htcondor$TAG ./galaxy-htcondor docker build -t quay.io/bgruening/galaxy-htcondor-executor$TAG ./galaxy-htcondor-executor +docker build -t quay.io/bgruening/galaxy-grafana$TAG ./galaxy-grafana diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 7c7c5be11..12fe3e2a6 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -78,7 +78,7 @@ services: deploy: placement: constraints: [node.role == manager] - labels: + labels: kompose.service.type: nodeport # SLURM container - this is your compute cluster :) @@ -199,6 +199,37 @@ services: constraints: [node.role == worker] + # Grafana and InfluxDB for statistics + influxdb: + image: influxdb:1.4-alpine + environment: + - RABBITMQ_DEFAULT_USER=galaxy + - RABBITMQ_DEFAULT_PASS=vaiJa3ieghai2ief0jao + - RABBITMQ_DEFAULT_VHOST=galaxy + volumes: + - ${EXPORT_DIR:-/export}/influxdb:/var/lib/influxdb + networks: + - galaxy + deploy: + mode: replicated + replicas: 1 + placement: + constraints: [node.role == worker] + + grafana: + image: quay.io/bgruening/galaxy-grafana:${TAG:-latest} + volumes: + - ${EXPORT_DIR:-/export}/grafana:/var/lib/grafana + networks: + - galaxy + ports: + - "3000:3000" + deploy: + mode: replicated + replicas: 1 + placement: + constraints: [node.role == worker] + # This container provides the galaxy uwsgi webhandlers, job handlers, nginx galaxy-web: # build: galaxy-web @@ -268,7 +299,7 @@ services: deploy: placement: constraints: [node.role == manager] - labels: + labels: kompose.service.type: nodeport networks: diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 77fae9330..843c87492 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -121,5 +121,17 @@ RUN ln -s -f /export/shed_tools /shed_tools && \ ln -s -f /export/venv /galaxy_venv && \ mkdir /etc/galaxy && ln -s -f /export/welcome /etc/galaxy/web +RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.tar.gz && \ + cd / && tar xvfz telegraf-1.5.0_linux_amd64.tar.gz && \ + cp -Rv telegraf/* / && \ + rm -rf telegraf && \ + rm telegraf-1.5.0_linux_amd64.tar.gz + +ADD telegraf.conf /etc/telegraf/telegraf.conf + +ENV GALAXY_CONFIG_STATSD_HOST=localhost \ + GALAXY_CONFIG_STATSD_PORT=8125 \ + GALAXY_CONFIG_STATSD_PREFIX=galaxy + # Autostart script that is invoked during container start CMD ["/usr/bin/startup"] diff --git a/compose/galaxy-base/telegraf.conf b/compose/galaxy-base/telegraf.conf new file mode 100644 index 000000000..0e2f13605 --- /dev/null +++ b/compose/galaxy-base/telegraf.conf @@ -0,0 +1,105 @@ +# Global tags can be specified here in key="value" format. +[global_tags] + dc = "docker" # will tag all metrics with dc=us-east-1 + +# Configuration for telegraf agent +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logfile = "" + hostname = "" + omit_hostname = false + + +# Configuration for influxdb server to send metrics to +[[outputs.influxdb]] + urls = ["http://influxdb:8086"] # required + database = "default" # required + retention_policy = "" + write_consistency = "any" + timeout = "5s" + + +# Read metrics about cpu usage +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false + +# Read metrics about disk usage by mount point +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs"] + +# Read metrics about disk IO by device +[[inputs.diskio]] + +# Get kernel statistics from /proc/stat +[[inputs.kernel]] + # no configuration + + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + + +# Get the number of processes and group them by status +[[inputs.processes]] + # no configuration + + +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration + + +# Read metrics about system load & uptime +[[inputs.system]] + # no configuration + + +# # Collect statistics about itself +[[inputs.internal]] + collect_memstats = true + +# Get kernel statistics from /proc/vmstat +[[inputs.kernel_vmstat]] + # no configuration + + +# Provides Linux sysctl fs metrics +[[inputs.linux_sysctl_fs]] + # no configuration + +# Read metrics about network interface usage +[[inputs.net]] + ## By default, telegraf gathers stats from any up interface (excluding loopback) + ## Setting interfaces will tell it to gather these explicit interfaces, + ## regardless of status. + ## + # interfaces = ["eth0"] + +# Read TCP metrics such as established, time wait and sockets counts. +[[inputs.netstat]] + # no configuration + + +[[inputs.statsd]] + protocol = "udp" + ## Address and port to host UDP listener on + service_address = ":8125" + delete_gauges = true + delete_counters = true + delete_sets = true + delete_timings = true + percentiles = [90] + metric_separator = "." diff --git a/compose/galaxy-grafana/Dockerfile b/compose/galaxy-grafana/Dockerfile new file mode 100644 index 000000000..6ca89d6c0 --- /dev/null +++ b/compose/galaxy-grafana/Dockerfile @@ -0,0 +1,2 @@ +FROM grafana/grafana:4.6.3 +ADD grafana.db /var/lib/grafana/grafana.db diff --git a/compose/galaxy-grafana/configure_slurm.py b/compose/galaxy-grafana/configure_slurm.py new file mode 100644 index 000000000..9e389496d --- /dev/null +++ b/compose/galaxy-grafana/configure_slurm.py @@ -0,0 +1,114 @@ +from socket import gethostname +from string import Template +from os import environ +import psutil + + +SLURM_CONFIG_TEMPLATE = ''' +# slurm.conf file generated by configurator.html. +# Put this file on all nodes of your cluster. +# See the slurm.conf man page for more information. +# +ControlMachine=$control_machine +#ControlAddr= +#BackupController= +#BackupAddr= +# +AuthType=auth/munge +CacheGroups=0 +#CheckpointType=checkpoint/none +CryptoType=crypto/munge +MpiDefault=none +#PluginDir= +#PlugStackConfig= +#PrivateData=jobs +ProctrackType=proctrack/pgid +#Prolog= +#PrologSlurmctld= +#PropagatePrioProcess=0 +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +ReturnToService=1 +#SallocDefaultCommand= +SlurmctldPidFile=/var/run/slurmctld.pid +SlurmctldPort=6817 +SlurmdPidFile=/var/run/slurmd.pid +SlurmdPort=6818 +SlurmdSpoolDir=/tmp/slurmd +SlurmUser=$user +#SlurmdUser=root +#SrunEpilog= +#SrunProlog= +StateSaveLocation=/tmp/slurm +SwitchType=switch/none +#TaskEpilog= +TaskPlugin=task/none +#TaskPluginParam= +#TaskProlog= +InactiveLimit=0 +KillWait=30 +MinJobAge=300 +#OverTimeLimit=0 +SlurmctldTimeout=120 +SlurmdTimeout=300 +#UnkillableStepTimeout=60 +#VSizeFactor=0 +Waittime=0 +FastSchedule=1 +SchedulerType=sched/backfill +SchedulerPort=7321 +SelectType=select/cons_res +SelectTypeParameters=CR_Core_Memory +AccountingStorageType=accounting_storage/none +#AccountingStorageUser= +AccountingStoreJobComment=YES +ClusterName=$cluster_name +#DebugFlags= +#JobCompHost= +#JobCompLoc= +#JobCompPass= +#JobCompPort= +JobCompType=jobcomp/none +#JobCompUser= +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/none +SlurmctldDebug=3 +#SlurmctldLogFile= +SlurmdDebug=3 +#SlurmdLogFile= +NodeName=$node_name NodeAddr=$hostname NodeHostname=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN +PartitionName=$partition_name Nodes=$nodes Default=YES MaxTime=INFINITE State=UP Shared=YES +''' + +try: + # psutil version 0.2 + cpus = psutil.NUM_CPUS + mem = psutil.TOTAL_PHYMEM +except: + # psutil version 0.3 + cpus = psutil.cpu_count() + mem = psutil.virtual_memory().total + +def main(): + hostname = gethostname() + node_name = environ.get('SLURM_NODE_NAME', hostname) + template_params = { + "hostname": hostname, + "node_name": node_name, + "nodes": ",".join(environ.get('SLURM_NODES', node_name).split(',')), + "cluster_name": environ.get('SLURM_CLUSTER_NAME', 'Cluster'), + "control_machine": environ.get('SLURM_CONTROL_MACHINE', hostname), + "user": environ.get('SLURM_USER_NAME', '{{ galaxy_user_name }}'), + "cpus": environ.get("SLURM_CPUS", cpus), + "partition_name": environ.get('SLURM_PARTITION_NAME', 'debug'), + "memory": environ.get("SLURM_MEMORY", int(mem / (1024 * 1024))) + } + config_contents = Template(SLURM_CONFIG_TEMPLATE).substitute(template_params) + control_addr = environ.get('SLURM_CONTROL_ADDR', None) + if control_addr: + config_contents = config_contents.replace("#ControlAddr=", "ControlAddr=%s" % control_addr) + # TODO: NodeAddr should probably galaxy-slurm in the Kubernetes case. + open("/etc/slurm-llnl/slurm.conf", "w").write(config_contents) + +if __name__ == "__main__": + main() diff --git a/compose/galaxy-grafana/grafana.db b/compose/galaxy-grafana/grafana.db new file mode 100644 index 0000000000000000000000000000000000000000..07c3a31d80bcf8c31c6839f78d19e8fec68050ab GIT binary patch literal 211685 zcmeIb3vgUldLDSY8)O5LAdN<&!Duv+u3FTPkN~i|0T2(3Mgx){i4k8SKv5iu)4b@u zXxw4H)DM6Zht@VIjcjL~j3=3OyiT&~>?Za&e#N_!U3-(*>tvmH<8@N&s+3c)Yqz#4 zai!L-c*>7#?Rcy9`_DP|-gCRpZFDypPhQNZ*?sRlum7I&pa1{<$60*qyj3;zm0~$> zRP}MKPYZ>#FX_6bX+MGgejESUf4A|a&;EgbL(+G@uRo!U{?Y$_2zMWk{0co|M!p~U z^T_uie;E0L$p01j{mAb|ekbzVk>8B`)5x##2mC5(dHwCa!$ZSQhwfX2jCtqkTl1+} z#Vn_C#jI7JZ--7@nV(yl*RNh&c~d zU7x$UbZOxtN?n-0xTIgag#WIdKR@`$h<^ChEjxAC%(P8Vy1YDylqr~6hV zWn}V}-{{h1)2N#0)@i(+rG*RgUeeS`8QEJqC^~fXXz0Nlt21w9%SIKih9;8dN7z%}Ezdp7p6=#>TMJRM z`Gup~%FDGh#)}t5(+-Q|)*L>*@;W}QPUGX^1U}9t@$scme9R2vBl$8uUU>l@FMSRl z&ko_^Q=h=cz@zwx9-`&?Y-A0en$>E#M!p;Q#mL`_{N2b;M*c?R$0FZ~{7B>nBO8&g z^P*H-(+Yi2?bHVYgX2afqt^-+-jB{!sZQ%?jLl*}O`p{%Cc3;j`K964lJUgUXd*e9 znAQ`C+2o1Y_*5a%vOTt(TDK=woZKG2q-^g&JlwAx|Hji8A)m}_TsoUTo>{VRhRyBy zcNP|x76X>CD=b4bVK$x*^=oqvQtk_M7b4#K=5NyTcn?=U7SBq30H4csT999@hw=H& zJ$x*#;`yb80J@xI7Q1CXeURHaxer%m*E@R5Gj z`d}pTf?HiU8jd_YKpXVoXyhqsc96y=%l=)CHz*cRAfUkhqX4b{KJB~~{y#z^+CRjX zE&K&4fzEM*KqcS^$Znzt`${G7q9g490)#_(02#AfE&}0dh8L<<-mFxOd`St$D0N?A z3Zcl$8vY;r1r!JSy*DEs(Qu9!A*tHnxn z#Y|7FOl3|?k55idPfnSWE5?i&pP8DPNKP%EFi%V*SK=pDGMTBFnU%D8Vti^UJw9$G zCZ|>=CQihc4Rd+ioH;R*o?4zbkz78Jh-b#9CYMhnGv>;)nMtH);--1-%!TZw<@0Gm18|$))kx@x<)JBo_)UXyLyI|BUu`vwtsJ3tlO=QZ& zk}#?@FQCNfE0-=enY{VBQB``VMWxOHQCG^>g`!f<5Z=%tKOOovp-Ko=g`d{m(5AdB zLGkPGV~5$*?4!}4$rnQPDB}_fr{76c&3uU)`JuB^YZd4BlY(Ce8^7vVZs~VQkhz~% za%=e3;3J2J5-)^4P&AY-W=yBhCt5W_PbwN=mxr?h{$t_O-f85CR*l$8gzS_~2^x#I zF`Vg-4vm9o5ye>8>vu{$-l`pm!#V}18Ge2E!TD%(=!F+T-{8`f^N91M!BxV!-nFI7 z9=IcHwC_wI#?q|(s(dEO#axpg1%I>6<;DC_sMS_s4UP$!)bd?ET|J_f&F)ygAgUA~ zrznTzK9C|$5sxgxoJh)Mz6k#Z7}*`=-mZKL;@0rnhgCv3>kYB8N5#bH*mBS8ilNAV z((wP_FQ7m`frok%kybMS?f(J4J$%+>M*y~;I1IORG3fdD4wPM_9KT{?gD z0;GMpWP>cIE0nL3k4LpPUPY@{nK!NEpQxu-jdI1Tj*c=L(UtkPt}a}eKdq>vet587 zyIOx%^j&mUg5vD1b3J#H0Z8SV{I=}`&3*0FV3HzKeYtZ zKsZjb`#aJ*$%fmahx(Z-moBsfHHa_tX+Nxm9}WGbP!wNw?eE@G{o3_~QfmWY7L1YHk(vR}JdA86e=pAv1fZ|lsyV1cR+Kk#I~cB6i%br;xd zZlec$*0UH383M3j1XcSF%Ae@hPJJwOfJ(~zYc1iB-C6Wx?Y0-b+){Uz9h-1Ut=wLP zEp&F3r9A)}sg7kUm$M4llmP6!ov~qU$pb|mgAF^YZmZy(>H70S{hCpK11|{U^yY`a zV9I0UWdir)lp7*zWqh4ZsD$lh<0-+NvFJ8~?s=WJMsCTq-comP?$b!`urcM%AgYB= zp;q%pwACHxCusOv^>}xBc1E%1DEKw@e%aMjF13~I_Rn1Ok^3}$UYt&dg`grN0T-=ar@IvMC_V=f!L%J zo8Wp89Xk7TsQ$Y1CP}zZ$~ko7Y2SD2TdY0tsQ&C@ERJsD8TS6EHu~==?+lpAunPL6a(1lQ zCsmcBfC&Jh$e(EVfAAMjAfUj(r@&*87X}|R(7Skd>8m@)@Q|KBzh<{G|8ZJ10 zXY{CcRsupv;U%@BQyg^eg{bzrx1LU`c;|VfDkj@RgD<=7N=@BTJG9GBC?dnBFFvRC zk{9J#R;sG6{@myKwQOBN!I>_q+pq%D+lZ1t>V*!+uwP$Q)TTNECEP*|>cDCt=z&)X zs9sd7M(Mx)Y`=C*?Wx?*lH4sRM7-fC-+rjz=;@7TKHIOo{(yr0g{Qb(otP1unwsq4 zc4saSG*UxLHr>>Bw*z=mxN=YDpMm*d-6fEB`VhV(znJE5Vj3bhoRJ4Rez~xesyVI) z{FWB^Eoa>aUjhmQ6xfFpNMa8k7#JEy&>T{xb)*OUg%P#dWE$C2Po}m0T_B-as&@uQcmE z)li(tkumr7S|SFS)_n+aEwyrMxE}vBJ9%c~1!h}uOCYbi`0?P;A(m+8Bh&kc^K8s( zh;goWZH;MdkYgQJB)q9wRX1;rY|Nz4THA?9zTQ%N-GT)ac2(hzgycJtFAe;ka7$FI zx|J=M4JR7$A|<>ktH5l$j#pGWZw`6=@fu(Bc|CNm>sREwWPi&uLdo+@WFM~geKI<9 z>bX#T(pd!b9u$h4efe{~)8Brg3uP|-Byv=t7jBNKhRW=ppB1flqq7NA@cUZi`!){< z{tYM)P~ah;KwsYgqyaj`C`{SL{vyaX_7J@EV2A+)hys0){!lm)=@;t2uW8|53MWF} z)P8Nhk*9v)2+pY6c-;k1UBM6HBZrwi@@2!(BA&;0UfNV)wDsp-@^$@pyX)#oOds== zL48YbDBrnFxUpNOqkpM7_(7~+Tc|$+np0>3N35NC#@tGpB~$0Al>1cceyHsVZ@$>C z&3rhgc}?8yF0Q0SAEXp!4C# zA83(3#YgZLP#~bdL8QQ=hx)Y8GvTa}Gw$3Se59{W8+eBQpySj-+!*llN}E9a7b2l2 zw1Gro#ZsxV`CiRJAb)K3#=V%8 ziOt5cWn;xC7_pHUc`)I^S?5CaZiz0qPY9h0*CxhJj3s0DN8CzEtDY*@PfEt#q+)s+cR5LGp5T**J!S!;*;vJ|-te^u(~+)j78l z`n$DwJb6N&%bDeB#qGW$gU0UPypJxKm2}xEQNK|;bt7Z(@r#adc{es&sO54aGz&_r zOT}WYYL#NMao*2U#X=QMdKmYLk*lEdG^P0S$kK3|rWpuSHdip^R-K2+#dSUbA=2lJ zQU${m#aB%$y9(+hPn;NuF>><y%E{%4M8<;{47Vciit5NGDpUtQOauA5}~U z)GcnU8M&I%A9Nn1p`ZM9(~os4!}>mgmH%_0!O-WBd;D8ks`>3O^tI^zL%N zdi2@o&;kzY8D}^X}FidWJI3%F#*onx((*2aA`cY zTDKKrmWSCXLI{R^XZV9>qC;<>tx?t%d3Za|*>lR{8S{D3UbEA#vRuP>sc8qS;Jz@~ z&omy8)mV}9`{h<+71cHmRooaRJ>Zb`O)YXU{4c}rgr3CLUH$uDm=caFQ+_)m7O#N3 zrR|duTdaP6Vk?o6jnl`FF6qM=05igo-D%ftPMI#rd2X+x&Gw|&5u~|xG^(ZZpr+>x zK!;Xzx`6XTc^aT|_m(al3Fur^pJ_0C|MO9;Ld`au3+R?>WI{MjubgKd_>G4JZc;`W zEOPwie(iSs8amLF)5%la7=A4m4Y)@E&7>Z*!AX=oZ2VI1WO;uSI@8@*j4kD?y_H1p*3e4+S3W8+azf@w@P+`vwvN%~&3( z0#W1@jjV*Pha;iCABt4HB&L z0NZN$s(Gi1bG*c{SaJQ@x%n&edR*o1vosRbUZZGiCN(ph&Lf?xgQlH^ePr>C!AD;0 z*Is+TT(|fEW5N?beY}`cA1TI6A7(i}c~FH&|A{aP_RV&TVL$BDtL%q;;+N2N*frao z$2}x$%hgM+JDw)QDPPW5d}vc2dWF0-=q(C$lR*wFs@yHh$H2V5WUaSnaD!?t3_cS1 zbC?Lfr$xRO`G2>EDFw9#6bLA=XDC3bKp%S3cPJ7Hg%5Fg;J6m~M&z~dCqmy29n=1r zcHFnO>4m6v-Pt=h7b6VcC|Rl7=3U7RvW0o;*ixQ?` zBCgYz*F}*RPEldlHQazyA3=LCa!3nnKdxy%9{DTy`lG%aV5e>zevwt8OMeV2B!w5KYxIG)yj2qZNB(qFohh1i`6cpsXx{oS_ zg(EZlnpr<9Hp^PUdaq^*nL(>xeRf&5TEG~KUVsK4mh)1|7U{(k-BF4bA0C}T3a^iA?*1-yV)H#-dbn%ODvv;Eoea3^tzK}~Eih5#)FD+N z6oD7PH?`57k^+Ku0ty5aP*b1~`V{hoJ@?$uH;Y ztJP|0_Bi~zxQFcQbbR`R5Y6u0motqeXOoH9$rBBilCu-j_wLjB@6+bA$PY!}NAR2Y z*6CmU&==`QB~omKa_J%n(#|5J6Bz{cxc)dBUOQrpN}2!gi?2h^`oZVien_ngoz3w& z@^J%lI??q$7-$%d3bL;>4CUr{{|iy=Hc3?t9#GCS;XY8#PaiKR=YE49lylw3!ccn+ z&6{Xm61llYqg&FIyO=cFBzJG}yh`{cmLSe28`l1Pr|e+AJYSFMNFnu5DuEwQ|t zP~fq?f#(Knz=6XARV{KWWP~)Wx)c4c51b^!={OW1GMsWdsorP@YR_=mSgMuvNaZn@ znu9r*VnR-rizU67%LqV05|~N(dnxJXXXk2cYYCOo_2BDY>et@*n4uTk(atI1=W6FX zKqdlahHSLqG2xY6l&p3hTO7Zc;U0N5M9(hL$N*TI{FVy3W{;6ElYb#ICHp2I5cZu} zP1!dns$>sk-yPc``{mXxYAT^^6LEvOaD3cLgLCz5-r21T&~Y?$-lPu@P=D?X-veyq z8OSN`00DN0oyytJZBi=K%5-R(ELG6kVCYW~ikQ&C|DX292t!OnKHh{s+&mH;diB{* zJ;b0w!YFJn7r1?huP1yBuF|tjt}x8~=BqCq9!fkL`oOWO+9mi{#jo*Zo1FK#rpm?4 z9^MGWqC?L<8+ve^DbS*n_-9g7E8G^u)$IWk{hSYW(0K;tqk*9pUkp9S%Cx{lwrRPR z81qmh=H5U$`L?}^HikbR9ZDoZ5AF-l!cK`+DIi6_YO(72+}c;X?ZSbiu}If1#gFcL z2UJ>rIXd(jR@lo;D@?&_DL z+MDzK&4~rz+XTHF*|#8p^h~qyX~~Ruz&KnmoPYuW1vaCAtO#uOY;ZrIKtO>5MuC6< z9IzJ}3@4yKClnwJ;1GO@BTt8aHGC(Oz}Nlxw{kVAEv9S@iQ&(P-E2gYQ)?6YOI%BG z+>7VmSy)_Jl+JfZ+2FK7Wt!PIz0|M0_2IKt^=suqpx?U zk~5LNu33h0=bUyN$=QVg%qi3J7ItH=>-R*`I( z#(QP{t*Z-H=1(^(vNKCpqS_mm@JJKU8T`*xO@wNlRy6SOm3eTiv`q7jEDc_|+-Q znJtISP=P+}ffk+%eHWn)543wb#u5&{B2wc%K8)1333X~*d%f_pWUY)dpPhyKx>$G; z7}xxeU7Q1R&G!i#WT(|qe%$)=$D%`*(4hs#SkqhrQMWm>YEnXm=9^#ewcs>AD^=90 zcW!g&;K0%7&>T9b%Bt&7Oxd&7I_VkBQE$Rqhw2WP`rr%P4)~3i_OKn04Wv1Ffkacq z(bFxFQG2qegVT{G$}ATQ#FZ;!FYVhYpU`t>lI$?k=3Yh9TKJ{V4{E=wO}EPm>d#&8 z*YfP3EU{0rg`@L$Nx^lO#+o9;f&`^SSZn)>HX3%QHU zHV@}DIjf?JwXE##XRcAW^(LvT?gwQs0S5p<|vi$s&~zuKcx zk;A)HE9~K0_^LM@zyum0tb)+0RuRIFEtoaK!J8C5HNstDHS{yP~ZSkfEf%z z$YcI9k$1wQp%3si_-l&-r@k81<|L^5y&6uK>lEm&zk0J@E2?1&R-s7EVuPDFsivGY ztIXxi7v^hGkYj!90fP608a(0vqd4@2HK0stI$707=T_j5iz98gI-ItLIUhI zd*s4sr(bLa>FhgVAPUqkXXP!wREjn>ud;Y-k}*+f5V?E>EJW4O|yQ?d#LgvhFI`exv5krbDihPSZaU^^1Sv7?CH zG{(dA*l2WUaX3`J%H9>3LA$KP1SG-!~2j&)NmSvk@JaHqeE{F z$yV4FvJ1O4=i9FqCEJ&JHndlwLqkKM2N1My05q$UeZ1kFr|eqqr{`XhW&&v7Kr;GZ zPyq$nqd=e%v`44lsel3pkpc`F)Xr$(k3;`5bW;0MeC^_|{%RTr;cOgo9dm@Xz;O?c zSk>TCD<0%xmh0bj`O3nDxhvQ8H|MW2Khldf;zM~^Zs{k_Upnc%&U|3u=GH9MDKtqEaEblmDr=nTjXLlJ}nXfEsU2wq668=nBW$ z`sA&CZJiyp?Q!F3LqBMZIeL4zC9Oc^sj{(_LSm|q2HjcYZE*9+( zai3|H%4HGg16^>xph!T0gF=BoA2=v)G8jidfzBxq2mqbaEGQ9B;J{FT0|Id^Qj44k z|48VYT3kD(8r1c95Y_o|) zzUx@hny~_y+g6Kt6NkXs(L|){DW9uadGq~Z!8t2bZlStr=9M?LU9X%R) zFfI(P#@(D%sit7B&6jf0k1sb+TfWydz1za|PDTz_FsyXG2VmU)+vo@1` zxPBrY9eVD$(8e1c8rYp%P>;OpLHh-1TLr`gVq7X1z(49}xI;^Oy?>0Q!>#iaumUL) z|IB-bx&ts8aRzuLX_21>XIKrr8H#BC06%;6clB!s2Y(CL56jLbu?PXit@-2tm!4F` zgTHA-wYNM#pR`gs#re*xMzynw$~vuQJ}EH?LxV4H*#Jm*PwY zb^d$LgR0s8b_NsnP5&17Olw`as@?!5=oQ-w>Io>Y|0ocw|NZyQf*uDH=oJNM{YOH- zhh*g8qxdKIJ3tgz$m2{VS;=v6oR0OJ4{~JDe?gMrn<`tg&x>)AyHVaSJHeZ*-Sx~- zT!v~6BvhnQHm=^@M86*QQ3KpJ7H9$oXqg8?2`JD53Iv)!4~Q645l~>CQh?TfIP|y{ zD&ymqwTs<35Fei6)bZ7>#Vo$JuHg;-H&Ma3w_BeyCToKr>lyHz;_dLl%!q|1d zz|ui=lpS<&^6=2BL!nATOO+}@-dT~i{F-2oG}O(n4KGhbhsM$WXhW^sD3r1?tmeyp zH9LH3x>lvQTC-d(qJicbOxPDOo5|aq-?Tm8?HeG&>A9u(rG*Rgy5k;S zFxOLz6japngsk-NCy=SRkfpTI&4yNteJiGF1`lWjE%asWcks_P|D{V&ZK?(x)4*}Y zY~kr_?(1xrerruk74;x>oGh=IaBBR`^WTKd)OE-LJ8>7Pd-j`)Is? zY#cvKfd#?}#)l?MU@fz81Pp*hUk40;vxuE!0II1RxE5|R9}KDwD6p3)5HNtf{C0xA z1r*q33Iq&bn+Y6LA5dT~Q-E-RsP?=T`6&FQ&@Tfp{7r2gKgHj*b(|~ZIS|Sy6p(L| za1z@kc0z5WbRSgNC@-&h^D{EVIEAXM6)P=Q-}c#6r)u83pt4flb|v*=)qc&WFE(Qe z-o7~i1HobfVQ4&fQXC!N7(W&98=F9PgLE}BoCTgvGa7!7Tn z7yKJg;NhbHt^dBzSuONSp|jzi3rF$u-}+JE!8FyvkqCYGrfYZgh!D-aT56b>%$6<7o!c`( z_lVR1BVv1^-f#d6_;J7BdO(4PfC7O+@DRM@V2A+)d?>&a0xhkD!=ZH~Ax--{ww25E z$M5xPZ`A3;Tggwz(I)hy?nI)DT5yeg<8!9i1Gy+K`@o z%Cs_4=%Lg4Y`;L+ug5o~N)>07s|w`JYS~KLcWk8+QCDl+jnKX$H{$phs&2|2i*g~e zR7|fn~jimf#7*9l!?@|f3H0s`m8*l`uywBq2i074av{m z*)8miT1MPbHolf(e8}A^ZYt~>ozx^8^_?Of5kT(2t&v-HAMllP!{M*L79E;IkNgO&oAXR;(mMBkyX_Ww-5JNAaP|gOW_8MsI_Zv z#WPhS>x4-_6d11e&qjw9Fh9pz%uidab0^L9cIK#EtmOi79tsdbnf(Rd0}4EB6bKN) zhwY6A;|(a_QXoJCTv`O*0t!5I6d(a0sui`!o8jLGe?9acLw`B+41R9cU;Xp-aKE-t z&jOG>RW{*+qq`ao4vx**pmfVDwB91z+e5nM(Z*!76!m*mHRq-hE-lwXQEhP@U6s6Z zNt>fH$K3XKm%YBa4&LDP(QQx#RnE9yt>b;nSZI``ua)c_JP4b4YPW4lQF!~}RJ9NK zwVU;4z2kB;*{%oY)F9!}(05&3ST)>pKM>X4l+vO#l2vct?>l^-tS-YCmeT4n(Wcc! zKdvUq>f48Uos#*`SZN*$ElrYCzdqs`hd@B{rt&bRYz57rP7U4Jfb= zD8S7E|4NH|5dO39Y3*NW|2+K1`=HxRT|9hJoSyRDTy*G#7eXH;m``cd#CaZML3O^Q zy&j~_^{yR9xxqJK1KYO6mtRgcUS?-^q?Rp6BL&!xokTHqF@8T&tS!??9#`h)&bz5- z_`Q`}(LhPnEp?T2t_ru(NbNb|524ib0A;sH(Kk5>PSaEDQ6zQAmz0kU*T=pT9l9|b zs++Fkv9kq|ke70IdB|hz3UWAk9x=olQbc5Vo>NxUxJ6P6WI~dffQ&do$UMn|!}aJJ z(V<~!Kx!aC4kZcDXe8faUt3B-XW#u8iz zDDd!5fW(1FXjF@QDg00HPw>}@0uMe}Kh&?yd_)n}!o|)JCpa?T*)bZ98*&~EB#&N5 z9xCnj{d!+iv);q&xmqImMr5&bckT+XoQ#ebVV)5X*bYv9tKPkOneqV2IM3qz5`W}% z{peBs+PV2FxXpr(kfZL4dRzsze4!rc*J=#!A`q!ou#g(jPN(OQWX9YcxkAxSN0tb}>hcK|d$RE=1emnC0$e%~P$A9!-y!D@72tB3^^!E>p4-9Vn zo*{G39VYMq<(sW@DqHx*)PXI*RJe0&8csTCSNRu~jPrs`Kwwx>z_>%oWQZc{#gm z9E*?W$wUGllanKQV%Y8KoU{8?^ml9Vc=CikN1JKI?Y@I%$L`;}k1m;&blECV;u_RW z-N;yc{G#z}!K^M|iWF+O+z8E+(&|#Nn5$Z)*le8l^Hi}=Ef;eb_ll9Lpz}1P`13id zaGRzX2!z8qFy&UAhswouJ^|CkT+S#}FkDf5)wHs!pkDICiIEs1H-FH`Sw@8s0>AH} z^UFpV-$W@I*m<*%t)htpdPcu#iaWQR=`$OXX4i_9%dwY3oyI%LnOU=dulHicTGlC@ zu9b0AF#05F%Nuvx?-ocWu2)uz>&}mAv5GbQFKDa(*$6)VSr{L`5u*M7koJZa{wVYx zLL=JW#@9~%j5E=p<4=X^gANcxkjriJZi>NJ_;K4;!tp%G5glA27zgD6-mn@7yP1y; zJ%u)S-fldC|9{ORa}&hhm7N)`g_iZok^S$+yp3_dknID z(Y+>~TJ?YQOINv)KN zd7o$OSK~s&G8nr_>LPJFwd|LNy+z)4;{Hw3ewesHpSGsqoZ^3`{Q14);3FNlW5#J@Ygrd3K^{keLSJOI2wo6Vheuc#7Mxp&3jx?9+)4nFu) z{cyi_>7&J2d)!J){%(;|Nn`M;qS{4u@;e*pyo z3LGQ~9EyZO;X{3qflwd&8eyN|NCf^(k6@sOBEPQz75I0NKaKn$z6O5*1p*2j1PVm^ zLRk5cLx=kY@NXag8IBz83k`(A;l92BFaV6RPy3!0u7;lpW%1>p|5fUP{n~5wCxlMG zRAdOD*)a#s@RjcIlnZ+JLm4VTt3FnLtX~`5kd7)sZEYA9cMS4S_4=Lqqfsq;oJf_c znq`GTL<%NpZh2sMwLIm~fGEE>e}4WHB`M}wT3l82gMK(LLpuw7NE%_qSTl9=4ggHx zrfxI%hU%??d&O4P9JOKL;^O?3B~q*}H9s}@$T2kHY7bUs1iyuzLtnibK3(ys^K?O| z-_loT;`B=h;PjQciCi6nkMNPRD*_uD<83;9uvJK2%zjAf(k4&VnDLX1iwbWRwIFQ9-w1p@a$f7%3h0t!47 z6o4(08&&f~vJJ<`W!y3S@;lu8$gRfAnuanr$jp1!xFgK{E5&ji#_`$M$r4$GbH=g> z`=_wh7PBV}GQ`3T$}F6sJu+}MHb7Q?r!KTxCWCwTwcWhWOvFeagud~%$|n6^fN4J4Ak7+30E&#{CRT`*4ilMVs#2Z*UxtjMv;t0LE^skj~XI<{a6w zg(+IRi`cm$JQ46darTMj+!S6OtNdP(+5g#G8HvG{u6!3ATs(K_n!Yr5R)2f${MGqI z{o2CPIsMJ~>-y{ZyD=*GF1ZUhR((tv1S^^_i__%_%?FxBpdBMgC)lH`KyreW z&7kZQeT>IZgXrA*#R5$)T(u|aSvIxwi&)w2WEF}C93k$5!LcGVuTZRH!VRVX>Y*C( zx=Te2uxvo{!vH7o2>r%aaT~_d46J$NLV`FKnh+J72yIp_p^-9tO~lKul#H|~dr{y5 z7XLV{2_t>mUSE}}St^;C^A>eVv`{t5S(93XPf@0_#9yW4*pn?6Yo(KzS-1~X&&boG zu_Gzx#V`W%V(c{&hjAY)5Y1@H@F$qt1})#bX;f-uGtay;V(C&1Lo1ds){}Rah(XTA z<`z#8$%;8EP4ilu1B6SMyTn6cbL@Ks3*?N=9H|2-P9fZ^m^m|z`Fz9E^%@=Uoyyi= zbM|!1%w?EAlGAs$*XV4)doR{lo;kS?V^Xwb;fU`oG}TbNWlP0%!`jNQti}pjwpM;` zE5q`XZq-h~g7cXBHdd_8UR6W!RxMSED^<(WuTC5D)`0b2q`BO(&8lhSwlF*_YESu= zZQaGw`K>h8SV61S=&X`0wAE1DZiyxW-`VfQfH#7=&UQ#52XnW3{Rgo8?Tz{PSC7yW zM~;*09F<-rhqhuaBbX;KjWcXx;@dy{UR@fmp{_dy@H*P_wS?4`WY=1f_B@ZB{BE?oJ5?%ef*JAr}~JX{EEBrnv^2X?ewhjA?A8nTF!(HcCd8 z^qlUu&87;}?UZYU!qysUu0h>WC1;wYtu>{n!l$ijktF3UkPjY>r4rWvd!AbC)^=|b zyUf>>zQ}R_Y`&i*gZ&KJiV{kJUbqwc8Q#Di+0ORtRwi`W-s!i%6vGFrJY^AGg*3n5{LApEV=3V)HfcK8LJk5VBoDZckrc z8BIuS03}^*cv+3zusgeZO!zb=g?^QQh|L-B9#SaStOqyRGvM>~@?QdIDY^(ST9*tM z>%U_2riKusu62rCVZvgcgR)&=2*4PU?qm{}yods08uv@fMrLJY0#{@u$k`yh1t;Z# ziIu78G*u-PO!^c|cuK~NiSenlw)OhV{AJE zit^XlJ*RFgZ?jnMen3x6O1!$${h+bDe=C?MTO5pBf!$8W8wS>YwP*0tc~B1%V*lVz zoo*MgIEF_?L5`;o59q0>a(j0RSj}j+=zg=XInln}D6m9)j8QOd?GIyA9te4l_%7Ce ztqp^e2O{W{DFP;!aSEGPc{%TLZDj>^MESu^+fh`ZZVU0^TQS!(6j!&AHemkW%4nJ@ zu-oaF$M=lCUoi7HgneIe_>7rezDxRM2TH*XaC!D}m!P#5YmO0z392Vy3@3v&WeO1N zmR*0F`*6sY0P5f1X&aIgNY^kq!T*bo%~0Y7md_zMK7~9E$(a#-0;wJ1W7F=fNnA)w zC9(e3a2MrKZfclh67n4u`7pc*4Qv}W5i@Jl$VcEFx$}s=CE`#=fJ2F&_`!Ed(Pzh~ zb)J+n+du(n!8UeXfKv>dW$dk$Yc{lJ3I7V)^aLr~)-du$iSSnL6M#ph-$b2zcG3}M z6Cn?F@?A&o*>C7gw1Eqcm9kODn$3?SnJa*C=QN1IY$K?b4Ybi%0&klFZ24I7NOb~3 z$`Z9CjAbPl_f{<7!6JT~L2e7-FEkl{8HI4*r27o}-t-9T;{~L&U}LjgRk-H{8yOsv zj+4^?cUqwvLc@To1wcCNd*$1t#L$Z)M+au*g66{}%QzB$Bofp3n8t*Ok0qvQ9<5L< z_Ee;@Qki3{NRnpK%s4$Pz3}_!1?3&Vpz2H!`XmxN{6cTd@9enp#6cadC;IAq% z12}Xw*lgM}up!CWj`1<3=pYAQdQIy1CQ3Bc|J^3nQSx7Q1r}cuTSH5>e%hedllBFV zSL8TztK6hh-F9eGI~%$otK@c%sbjn2uy+y}h`2u%x)8YSbr~_T-EIz0Who$n^mW^< z-BF1MQ3cc5b#6{R7_MU6sGZVp@S$e+eysoZxl8}{yZE=s(fO*Jjor9`MTzs2s;f6| z>c{x66-3aK%$qk!+1!R$0BTHPQ_W1Dx8L%S_V$Gg?CU$WG3-kLw-XP9-FE@=6>-~h zao%A8H_bp9f|0QP?;(hcEAppAGExU1i#d8bwn%?m&_S{wrW2IKd=k723BH;phr756 zfaI{q&T6|Q6K-@4t~eA%B)h8LBkMg2_o8&N4l*dk4k4b7qhU7JASO-B0m={+_~9&ezrmoCKfvaw+NaNWduM&MVYK;SBYoQjbd52MW;>1OW7rAo1L#z}*{Z|3^2yo^lLZz-ff)~g?X%f?iWVjo!M5OtQ|-!iBpuGvITSYgq6=R! z%ZMH2?5y3i&Ibi)WUaH9#bg9#pnkw*bXKO>EhP*YXTe3+xwdou^p|!jMka{baGEdZcPvG>Z4on^sj7wP+XVl`6G-Z<_Y0OWist1XKX)|3B+N z2khz;RjJ#rWqA9F0-a9m7FGGSokI*F8-H)wZ&jV$husqnk*rLg=W7aFv9*Pcb`?CBySo0ikUVM(VjWJNNzKm@@&B3%-6OXB_8X4 z1MoRRY40-^0Z%5JO3GyRb22Xq8+@4OAM{y&6Hcwj zpl2^cp#{kUxe;2y7=z@29t(X3TS^zn1F`-;>bkk66ok94Kn2MI#VHJ1-yL_z07`zM z$nQ@9@9^nb!@&o_WytnDwSV^n@7hOLm%Ekq+=t4|0EPAc4dD{#Wi&Vfp`E<=rFOtHsisK9;W1hSpOeyDS@EPoAz~1s33V@+vloy zn3lp!(%!bQ!$~+AlmB&UVs0#N%eh`Rm&W9O-L#@g>et*#?~(^%{r{j0;n9Y*JE5>a z@<3R6y5D&gK3o`wq*jr zAbDWL6q(69P>8TK9vsW6n;qL8=U$a?gXDoC?<~PH%5x&!5^1#4i#tyQCLk!Od6C@8LJsvO;j*o5g&}ppyA92pi?k0g?v#WTJJP_%= z@e+z)P}_RiZ;(9DPHeQ7fm3j&J)Arc>;EtF^MJkiy7oxCPsctG+%gj+4}?F#ih(_V zZ3Zg~aIWcaV{kTwAbB9^$GejsyFDkatt1b``u{7jTrqpnKyIK}c7z7bso;{fI;fHv znvkRmr#g^xc91-f%qOs#avz`|d0>z{a3?wKR+9%}{eSZ~j_Y8lA-hQ+7$grYtmJBU zPM-u1pd)n{6Qu;GLGnP#<{Dec6-fx;`6lh`*(|EZ_n4Ax4D#;rF(vyyc_7yRk9yM( zs(qV$69{(T?;0czq-?=NNmb&%PEGu(I{e>VQ|`Y`n`^4TzpW-0z|Mkn8;bk4qsaW% z>8KP{v~I5<^ItdZHB@1@w>zG9SpVN5{if>#f|1}7uB4eU58KJrdEi(Rf)Bzz8`;0~1c8iREownVgD&MwS zW&7)*{Z`f4eb^o0kOadZc_4_;9uB`LYR=x~ww_qgaq>W{|8J`u*o2qY0m==sjkuC0 zKb|5gZZC*>-H3U;@d#-vS>FzxCfQbh9**RJSpVAWRuTvT$`yRC*o%;hlQ`UdE8ZNv zPWg(Bx98^{VM|>vmT&8)%_?$wlkHGG1!;mEo?c0>nt9`GvrI=iz^1@>7Y;kaW|T3? zbabEy(~M<}oN?zaOLJ}JOF5k5R6q`mdpHdeUPxr0HFCMP4Zt+^i*oX6(3weohq{r= zN{l831hZT%W6qIfLu?5mNHKPf`#I&|W?(rt3;e+MRoe%uivA0@|Gk=7#tBKW#dDXg z=}U8G^|$BFU!7mnuPrQ{)8CxGuD`Co8>52nlE-=3T!9n!g9PDQJjODxKjjfl`1b|@B+ZKYhKGqMWB_0jQFsuE{7 z(J$cK?3t34zD=ORJ-dTY7?C?12{>t#u}PY~bbjtgY@B@}UV0=pNgoOJIUZ-4#k_#m0sT1r`RvCQKtza^&CeJua~HOTCX7Z_yr#cc6SLK1AwwvBw1m{Kv95j>4#y)%qu@LkZ?<$lLA z`?j|+WBvbiwTMpvEMO~?1Q=D>tiQ)8fx*X1MKDmbzyoH=NCah;Tcf&ewvL&2byXK} z02Ve!aZo2_@=KG`iHZ0rTyc!@VkEpBDYr3v4DCLMQNC}X6WDLY$t%Jq-a7$*$<9uy z=eqoC?8x}ok%_S*ljvLfr`~ywWLIby4OTNYYU)0~yh^QX<|&gjg?VvtVeFM7$I*VV z3}TvZjyWTPL}X&lFBY;@vt z*;p5Qx_D8L5w_ym$be4JsQnDCH|c4fAHy~AfOO3hv4dBem(D2Z`ppWur&8HmaoNbF zQtiE7lK*8VQF3ExKYGA12h7A%(lYFxq*St74l6{RD+I^tn9`7g*?Fd|54_k!_n~7` zaf~8cJz~;{*|Iwxq4~|#BXS3g(}}>$e=|nkuw$)r4oh}PCxWUMuom7}&lM!`CXee) z?^0Hja9k`J@0p@x%g_x<0F%7;qY|)OF0RAt3UA-53*dqy|0hZQ|9c91Phgyr&xPm| z)F(4yJLX0ot6Hbc`BY=u4!q`6JghvCtBXdJ_4u?ZoyaAlZymQP)g$eiEgId=#&7*Z?!AH(v4$QFzw2oQR6H6jiitufemn zoAw&2u-hY=>`q$DZFbDX`u_>B*>g2+0e}o_NMv;4G627PrMON=3%^9_^=QprhZ;U%MBG8>A3_qNqf zA_&fD4Y879T{6<tV$C=&$d!v1~p zrv9ZMICnn<=k7pYF4q6w>~<>%l5^x31P#$uGO^K#DBwYDn%1^x!3vUdkbJp2rkM6% zt*s>I*nz|xSpRiI z_jvd8wjepzesG(I-noa9bL~K4F0B9mY72h5L2^!P^^S)TX-7zt8ysh+0uF-YoRCyH zOU~)I*YqX~LzS2l>;JztF94Nes&)~1fMc&?N>|N0z98a_${S%#HQ`lafOi%>tvA#cID7!+EPLzyc^;9C8x8$;40mAXv|V@g0x7 z^K^0|IEn2jXM6HpK*S0{4jhz^p@?w(wIe5NyMlx&=8aMb4tz^YoM&kXo!^{Qi8vF+ ziu7txXB^AnRS9B8;*sERj`84dj(vGJ2W`6Bd@u*r|KD~EzMX3C1V3$#6Vz5#;LQ4H(jHpzxq|9_`fCq=B6ttxm4^^^;Ooeth}w!E(`kToNy0813!u%cge8#%|c1-KN*shA4L${t5ck3hll7N!(}X;#Op^{S`w3jE$jW*erSh zc-x3v6XK2a|94r=6-$SPj=iBNhO|G8Ey;Fc;6JppJ5(|l$WSf2L z-vEo9du;sTIWJJ$H2wCt=m{?EqfzX6A&%V8tRk)ImN_1<{(ox6)G(K53N?&hsv6}i z3qXVos(rb)$2O`G=TnSDwI*M$M5pCk5rLOmf=HWcutg28T$F)av<0!Kp}4vYxP(&@ zvhIbirV7;Ulxu~;)*5QALETa%XPRV>+QOI=Rrs`3EtX1KnhTYt8tj&u#7?aLpYHxW z0Y|}J?3TBi@lLrK37ZHA!_#uAlTNjRrV3iM2FpZQ>R6|3H59j7Y9il`iB6|3U)@*Y z`qm|!eJL*`Ah+);feGvXJKF-ew%qG&b!0DW_UTe>Qz@xe+dSul*_w&eoG#bwlsPq) zx4Dyw>slG4MeDw)hT<(-Dy|#WR)%FYR?xDw@_SnumZx;9b_!N{EA(q9-m0ZaaiwZ` z`qk;YXs)4Udu;E<`u{K7>7l?qTjcrosXbFdN>t@Z0-X*hNY2ku1Yfgkker{4JF65? zx)o?}hrpDMlk@LDV*WuefKVu}g(!=&6IM$TF2#>*1Ys*?&P>A{_=d+Xlp%9l;K9CN zzbD5c@h01ikR1Q$H#*l=%GKJn#j^`~*+lxvO0}XN*9n-~P0ah-eUV4C&_I7^d|>c9 zSHt0CJTWzzNRB3^^+a+ud15vm2g+6`045Weprth4%QYm+T%cUw4OzQKNA&YRcJV<5+w|PbT0Tn4Fv((G!$nQ}|5Ixt-A8 zt;OTX6Z%}vEF&|NgS%#o%Ib2_C}$`$E?Ui4Rk__P!tWQHj1df^U9x?PAS6?_P9eM; zvEsH<0Y|*O!R1a8@EFUZO)2e|donDz>QoX_6R2RjQ%N*Ny-)~y*<#tdL0OQd|lsf&i z6Ky|2w*MdF>;DT{=nMFz`m6s!B=m$fkVtI&o-r^!J}}Tf$Wx7u7U0q`nqfKYMZ!iY z%lN%m3VG(eo{5Qz3j*SIpKy{C#wNy2j3rrwqFqUeQlPt4 zu%Ds=_U>`lE$)_-eBDLi1^4h?QC>=jFISQni;ux)qzSi2 zn|->*o92117QNSL$XWD4P27CdCr=={XuDUvv#4nnQjhn$VFb(lXZZR*uZ7O57XR*C ztN-KvFj&qr&uo1ESr5zMY-Z20mT$iteq0MZ^UQaaM6V;98`$06r>nkKi#*ubZJBJ_ zclP!&`eSwf7XJH1h$dda@Ywjwh(3w`AQzHQm_ssc$p0)1$li7NxguAf8WcI#lO&gK64V{ixL9Tdn4ep;6cFqflDs}MN_VfOWv){M%Au#noL3m-06MFjj|#sRNA#X;-}baRr05h zImyB=iJ!8tY2&#{`#wDg1%Z4sAqw(hvZE%uWoMHj8t(yz;ZD(O4u#R#4j}Qa*{zR% zwURZ*PA(+Juo!FR*ehQvE>~I~^nmsM_vuWgOs^pBj$nbD0U-Q-&lLW}v-A3wF=XNm zwyt>(_z?2t?!utQEoeBJQnG?8D3PenSQz9hoz_p@-dzgZS2PvGPvmYt;MwQ}3W`v30_r0cdaYS0tHH!(3~E;oem zwHf?3nOvT12pb&t3mXI@OWW~=CnlET>1mcjX;b-5>`|MAs@ekH_VCL#bNX=C&dq$y zd!=IX_)nRGn>kn^Aq_ZLi`<(EmzKuzoP^i{JoasRYO9faa(Vw&5LxQ>w^JXPtpyB{ z4e+}{7lsU!F2s{FV>rdJR5K! z+^%i|`OUUE-XTpFwf+#AryazkI7rSi|l*`BN z4d=uNc-^xxHX`P88>IJT=5_AJEuEE?9vHt%DC%!XatUH;M zrX^Dq`;0$V1yEeI3d~Qi$IMl#S>llak0oHfO&)896>d>%7EJaE4|8mj%fii~1Jf4P z|NpqR4*xuWoYw0s7I4}9!(~^E0s^SYJ?5~Y)|T9LhvZYN|6krlzyB^})ZX&XbNnUp zq&Ek~_$IuO4!rrzoAP}(nWxZw4~pu~)PAi0Uuo%ZK&SmdqfFnnt(YF&e^W7aH3nXmZCo=DgLt-B%brcN@M;1!(0v~i57CC!bKD_w&w;Z8B&&f zO**hlC1!TOz1~GsIbE=pHw*rpjK@n9fJwd?dny979V?aKjlK5RCmSLq1_!FfIc+-` zzFnt|nRs(5YV-{f9=Np0JzJ~SxkC18iH>}fpJVv9WSfsMC4xPeW>bE(IMx9FOBM zCgnGx>sbH)7<1UEuouZxwoy1Zu3u?~ z_I6!+x$WaY4p35Z0I$P1GFdEI?~@xjsQogCO>QT|QthY?dzT{-U;>j2Taq2}hGdr} zFDwMDG?HTE>;e=Xc@8-{>5z6AfMUJp$v~Wx4)iWjVe)f1c*D$1k*L9wgCr-A6l8LO{}&&dVceIph9t+QkSzqEj5R4Q84etwWK^*wP!Q+^6M+)s zquIm;_^K<^^2?yiNDK~ZbYLapld>nKC)+t2;2s~bI|17qGAOrfL5yM!dQw6?uM>`y zQQI)E{(lwZ+v3rW%{naJI(F7dt!(BQvy2tlLKY~J4c8-)}8@+i6sxq|(<5@cRBC3c@7 zVg3Kw=4`lQro_dF?pV@pu_Ly*Q||GD&yr~VH5o#?U{7F*J%@ojJ2p%4ASA2Raho6d z_$5R5kfl4e0iN2<(7%S1-_JxSiQ3ck|GlN%$EUI zZ66wtTkryF@(4vqhh60EL6-CwbeSY3t_dplhnmWdDh9*NVq%6UV~(|0k4lYy0vij1$_m-%}-S^AQQ%aICQ- zC&rFUVGXn|H|+-Bx)C3pxjB{w>NCc_ns?GBOO?AxW~zh9z}(UA4eo{Y|4#+3Z14Vl zg4fpNq;2A1+swbb-uvyjRc#`51rpbh$^FW|*zvR6hw1=uFs%Ror2~zT4PIvsWDL^Y zH{#dB%DlWPqsMz*2pueRUVOK0owjXT3Z55FxH+xFh1N-}#6xO7BX)DYshOR7|Fkck zd2pZZ)k&c-N|M(r@4h=`pM&9r!La@{GF#6rlE+l3n6uKmEqLt;6lPOr9|B_KQXa3a zZ7i(O1EkkN8Ia4K+XUE2E+jBFID4S23euS5jkl4;#45sLAi<)l;rc-Ehf%&wg^*R6 zoEo_28v|_2d?^QuD{OQC4syuGDBwo6=5UF!RHK4IaeedzGxQ>dAN>Nn!%Vj&3#YZh zA>$r~3b>}SiW9Hku>c90wKPe4Bl+!(#KzetJG=x=;ke2^$K&ifRh^`d1or_4sFNyk z-H19Y6-dxitSbptfZLywlvHPmKE~syL3HkYc*>!>%XDl4^NnCs!xiokg{PgfF%tJ# z{(tR;M*r{eU;XD7LXT+!{rv+t?gpoaG@N!bF*}jmv#I=RcQVT$R4+~mL7cupF4v;2 zMLIdez9t}k=8`18ml(ql$yW%ommC3{;{31}%tCG)P?f}@*rz8AG`Uju>gWLutV!++ zPUdzYpgfdEj9|Uj`TGAZvViq?^>1f&^?#HNKc%uY@paMtp$umwmb zeomLFbI!zaZamL|+!XQu)@NDFvok5cYHe5dwobB8`&oLAh-v41gu+`>MHX*mjtRVodAL;FfPYSCljh&v3nK;CU&M*I`JN`H^dv`+}*vx zoM-~&-ZpYIh-6$xbrAS_Hg1JMir6Inibq$g{0IW{jhSo#5;Wd{Tl1TzdFCD zUt3r@r@uLWU4I>+p;V9^_~z)|VE%}FFmbFHC$gVlFYbkBx{OOsv54I0#qWhoByjf-q2!>?}WlPdalKHyvazd0zpy$(P3I zyy=N?e*Wj07Fr`t5d7^71?oRL)(2>mp9H?~yTcv;L%bRA2KNwd38%gn+v6vJZySo| zbvt#~h+tk1C4>ia1Lt(zIO}4r=keeFmaCY&cp%vuWI}Q?o{6c|wkPqf=64VhU?ADU zbas%kI*9k$sbt?Csz>`>*YB(HGypMe=_4mqD58+FKl^r3!-4oWQCRqLuC&kOQ1OYel@FtCi aGFCAxW7V{>tCW}o8^xZ>{)V~!oBuyY&6s}x literal 0 HcmV?d00001 diff --git a/compose/galaxy-grafana/munge.conf b/compose/galaxy-grafana/munge.conf new file mode 100644 index 000000000..8d0046893 --- /dev/null +++ b/compose/galaxy-grafana/munge.conf @@ -0,0 +1,13 @@ +############################################################################### +# $Id: munge.sysconfig 507 2006-05-11 20:28:55Z dun $ +############################################################################### + +## +# Pass additional command-line options to the daemon. +## +OPTIONS="--force --key-file /etc/munge/munge.key --num-threads 1" + +## +# Adjust the scheduling priority of the daemon. +## +# NICE= diff --git a/compose/galaxy-grafana/requirements.txt b/compose/galaxy-grafana/requirements.txt new file mode 100644 index 000000000..3f79978d5 --- /dev/null +++ b/compose/galaxy-grafana/requirements.txt @@ -0,0 +1,71 @@ +# packages with C extensions +bx-python==0.7.3 +MarkupSafe==0.23 +PyYAML==3.11 +SQLAlchemy==1.0.15 +mercurial==3.7.3 +numpy==1.9.2 +pycrypto==2.6.1 + +# Install python_lzo if you want to support indexed access to lzo-compressed +# locally cached maf files via bx-python +#python_lzo==1.8 + +# pure Python packages +Paste==2.0.2 +PasteDeploy==1.5.2 +docutils==0.12 +wchartype==0.1 +repoze.lru==0.6 +Routes==2.2 +WebOb==1.4.1 +WebHelpers==1.3 +Mako==1.0.2 +pytz==2015.4 +Babel==2.0 +Beaker==1.7.0 +dictobj==0.3.1 +nose==1.3.7 +Parsley==1.3 +six==1.9.0 +Whoosh==2.7.4 +testfixtures==4.10.0 + +# Cheetah and dependencies +Cheetah==2.4.4 +Markdown==2.6.3 + +# BioBlend and dependencies +bioblend==0.7.0 +boto==2.38.0 +requests==2.8.1 +requests-toolbelt==0.4.0 + +# kombu and dependencies +kombu==3.0.30 +amqp==1.4.8 +anyjson==0.3.3 + +# Pulsar requirements +psutil==4.1.0 +pulsar-galaxy-lib==0.7.0.dev5 + +# sqlalchemy-migrate and dependencies +sqlalchemy-migrate==0.10.0 +decorator==4.0.2 +Tempita==0.5.3dev +sqlparse==0.1.16 +pbr==1.8.0 + +# svgwrite and dependencies +svgwrite==1.1.6 +pyparsing==2.1.1 + +# Fabric and dependencies +Fabric==1.10.2 +paramiko==1.15.2 +ecdsa==0.13 + +# Flexible BAM index naming +pysam==0.8.4+gx5 + diff --git a/compose/galaxy-grafana/startup.sh b/compose/galaxy-grafana/startup.sh new file mode 100644 index 000000000..e6ebf8994 --- /dev/null +++ b/compose/galaxy-grafana/startup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Setup the galaxy user UID/GID and pass control on to supervisor +if id "$SLURM_USER_NAME" >/dev/null 2>&1; then + echo "user exists" +else + echo "user does not exist, creating" + useradd -m -d /var/"$SLURM_USER_NAME" "$SLURM_USER_NAME" +fi +usermod -u $SLURM_UID $SLURM_USER_NAME +groupmod -g $SLURM_GID $SLURM_USER_NAME +if [ ! -f "$MUNGE_KEY_PATH" ] + then + cp /etc/munge/munge.key "$MUNGE_KEY_PATH" +fi + +if [ ! -f "$SLURM_CONF_PATH" ] + then + python /usr/local/bin/configure_slurm.py + cp /etc/slurm-llnl/slurm.conf "$SLURM_CONF_PATH" +fi +mkdir -p /tmp/slurm +chown $SLURM_USER_NAME /tmp/slurm +ln -sf "$GALAXY_DIR" "$SYMLINK_TARGET" +ln -sf "$SLURM_CONF_PATH" /etc/slurm-llnl/slurm.conf +exec /usr/local/bin/supervisord -n -c /etc/supervisor/supervisord.conf diff --git a/compose/galaxy-grafana/supervisor_slurm.conf b/compose/galaxy-grafana/supervisor_slurm.conf new file mode 100644 index 000000000..06482a1c5 --- /dev/null +++ b/compose/galaxy-grafana/supervisor_slurm.conf @@ -0,0 +1,18 @@ +[program:munge] +user=root +command=/usr/sbin/munged --key-file=%(ENV_MUNGE_KEY_PATH)s -F --force + +[program:slurmctld] +user=root +command=/usr/sbin/slurmctld -D -L /var/log/slurm-llnl/slurmctld.log -f %(ENV_SLURM_CONF_PATH)s +autostart = %(ENV_SLURMCTLD_AUTOSTART)s +autorestart = true +priority = 200 + +[program:slurmd] +user=root +command=/usr/sbin/slurmd -f %(ENV_SLURM_CONF_PATH)s -D -L /var/log/slurm-llnl/slurmd.log +autostart = %(ENV_SLURMD_AUTOSTART)s +autorestart = true +priority = 300 + diff --git a/compose/galaxy-htcondor-base/Dockerfile b/compose/galaxy-htcondor-base/Dockerfile index c99a9a4c3..2469c4434 100644 --- a/compose/galaxy-htcondor-base/Dockerfile +++ b/compose/galaxy-htcondor-base/Dockerfile @@ -7,10 +7,18 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/99AcquireRetries && \ locale-gen en_US.UTF-8 && dpkg-reconfigure locales && \ apt-get update -qq && apt-get install -y --no-install-recommends \ - apt-transport-https wget supervisor unattended-upgrades && \ + apt-transport-https wget supervisor unattended-upgrades ca-certificates && \ # Condor mirror is really unreliable and times out too often, using a copy of the deb from cargo port #echo "deb [arch=amd64] http://research.cs.wisc.edu/htcondor/ubuntu/stable/ trusty contrib" > /etc/apt/sources.list.d/htcondor.list - wget -q --no-check-certificate https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \ + wget -q https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \ dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \ rm condor_8.6.3_linux_all.deb && \ rm -rf /var/lib/apt/lists/* + +RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.tar.gz && \ + cd / && tar xvfz telegraf-1.5.0_linux_amd64.tar.gz && \ + cp -Rv telegraf/* / && \ + rm -rf telegraf && \ + rm telegraf-1.5.0_linux_amd64.tar.gz + +ADD telegraf.conf /etc/telegraf/telegraf.conf diff --git a/compose/galaxy-htcondor-base/telegraf.conf b/compose/galaxy-htcondor-base/telegraf.conf new file mode 100644 index 000000000..70db77900 --- /dev/null +++ b/compose/galaxy-htcondor-base/telegraf.conf @@ -0,0 +1,94 @@ +# Global tags can be specified here in key="value" format. +[global_tags] + dc = "docker" # will tag all metrics with dc=us-east-1 + +# Configuration for telegraf agent +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logfile = "" + hostname = "" + omit_hostname = false + + +# Configuration for influxdb server to send metrics to +[[outputs.influxdb]] + urls = ["http://influxdb:8086"] # required + database = "default" # required + retention_policy = "" + write_consistency = "any" + timeout = "5s" + + +# Read metrics about cpu usage +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false + +# Read metrics about disk usage by mount point +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs"] + +# Read metrics about disk IO by device +[[inputs.diskio]] + +# Get kernel statistics from /proc/stat +[[inputs.kernel]] + # no configuration + + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + + +# Get the number of processes and group them by status +[[inputs.processes]] + # no configuration + + +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration + + +# Read metrics about system load & uptime +[[inputs.system]] + # no configuration + + +# # Collect statistics about itself +[[inputs.internal]] + collect_memstats = true + +# Get kernel statistics from /proc/vmstat +[[inputs.kernel_vmstat]] + # no configuration + + +# Provides Linux sysctl fs metrics +[[inputs.linux_sysctl_fs]] + # no configuration + +# Read metrics about network interface usage +[[inputs.net]] + ## By default, telegraf gathers stats from any up interface (excluding loopback) + ## Setting interfaces will tell it to gather these explicit interfaces, + ## regardless of status. + ## + # interfaces = ["eth0"] + +# Read TCP metrics such as established, time wait and sockets counts. +[[inputs.netstat]] + # no configuration + diff --git a/compose/galaxy-htcondor-executor/startup.sh b/compose/galaxy-htcondor-executor/startup.sh index cf8bb9645..cf7f728ab 100644 --- a/compose/galaxy-htcondor-executor/startup.sh +++ b/compose/galaxy-htcondor-executor/startup.sh @@ -29,6 +29,7 @@ UID_DOMAIN = galaxy SCHED_NAME = $CONDOR_HOST " > /etc/condor/condor_config.local +/usr/bin/telegraf --config /etc/telegraf/telegraf.conf & sudo -u condor touch /var/log/condor/StartLog sudo -u condor touch /var/log/condor/StarterLog tail -f -n 1000 /var/log/condor/StartLog /var/log/condor/StarterLog & diff --git a/compose/galaxy-htcondor/supervisord.conf b/compose/galaxy-htcondor/supervisord.conf index d63b615c7..b4c2d8265 100644 --- a/compose/galaxy-htcondor/supervisord.conf +++ b/compose/galaxy-htcondor/supervisord.conf @@ -42,6 +42,18 @@ autostart=true autorestart=false user=condor +[program:telegraf] +command=/usr/bin/telegraf --config /etc/telegraf/telegraf.conf +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +stopwaitsecs=1 +startretries=5 +autostart=true +autorestart=false +user=root + [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface From 8b7ca79621c300b28dca81f6bacad38fe5a9a533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 24 Dec 2017 00:02:36 +0100 Subject: [PATCH 16/60] skip one more test for the time being test_tool_dependency_install -> samtools is not needed anymore --- test/bioblend/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index 05ad6209a..3696ddd79 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -12,7 +12,7 @@ then export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; cd /home/galaxy/bioblend-master ; - tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params"' + tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install"' else docker build -t bioblend_test . From 2710b810385bb39072a8dce4f45a8adfbd1ba34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 24 Dec 2017 00:59:42 +0100 Subject: [PATCH 17/60] skip test also for the mono-container --- test/bioblend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 8fbc235da..90748868f 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -14,7 +14,7 @@ ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ BIOBLEND_GALAXY_URL=http://galaxy -CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs'" +CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install'" # library tests, needs share /tmp filesystem # * test_upload_from_galaxy_filesystem From 27c33b841dc6ffe5c719078f2936ea4950a08a6d Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 1 Jan 2018 23:43:39 +0100 Subject: [PATCH 18/60] use new ephemeris syntax --- .travis.yml | 2 +- galaxy/install_tools_wrapper.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6f18f5bae..3c5da495c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -283,7 +283,7 @@ script: then # Compose uses the online installer (uses the running instance) sleep 10 - docker_exec_run shed-install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" + docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" else docker_exec_run install-tools "$SAMPLE_TOOLS" fi diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index fd91fc256..83c2e4220 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -47,7 +47,7 @@ else done fi -shed-install -g "http://localhost:$PORT" -a admin -t "$1" +shed-tools install -g "http://localhost:$PORT" -a admin -t "$1" exit_code=$? From fbff10ec1952c5635c9be5e156bc76f811845882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 13 Feb 2018 17:45:30 +0100 Subject: [PATCH 19/60] track 18.01 --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 5da54fdf6..dd9568759 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -9,7 +9,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-dev} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_18.01} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ @@ -131,7 +131,7 @@ RUN ansible-playbook /ansible/postgresql_provision.yml && \ --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \ --extra-vars proftpd_use_sftp=True \ --extra-vars galaxy_extras_docker_legacy=False \ - --extra-vars galaxy_minimum_version=17.09 \ + --extra-vars galaxy_minimum_version=18.01 \ --extra-vars nginx_upload_store_path=$EXPORT_DIR/nginx_upload_store \ --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \ --extra-vars supervisor_postgres_autostart=false \ From 0876061596609f9743304c2c65747933e4a32467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 13 Feb 2018 18:07:33 +0100 Subject: [PATCH 20/60] use yml instead of ini --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index dd9568759..200a08834 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -105,7 +105,7 @@ RUN mkdir $GALAXY_ROOT && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install --extra-index-url https://wheels.galaxyproject.org/ uwsgi -v --pre && \ rm -rf ~/.cache/ -RUN su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.ini.sample $GALAXY_CONFIG_FILE" +RUN su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" ADD ./reports_wsgi.ini.sample $GALAXY_CONFIG_DIR/reports_wsgi.ini ADD sample_tool_list.yaml $GALAXY_HOME/ephemeris/sample_tool_list.yaml From 7fa457218693d3c7633bf35f33ecd8569837f6cf Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 22 Feb 2018 17:44:45 +0100 Subject: [PATCH 21/60] fix some unavailable resources --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 200a08834..9b52be794 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -16,7 +16,7 @@ GALAXY_CONFIG_DIR=/etc/galaxy \ EXPORT_DIR=/export \ DEBIAN_FRONTEND=noninteractive -ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.ini \ +ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \ GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \ @@ -63,7 +63,6 @@ RUN groupadd -r postgres -g $GALAXY_POSTGRES_GID && \ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache && \ apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https software-properties-common wget && \ - apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \ sh -c "wget -qO - https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -" && \ sh -c "echo deb http://research.cs.wisc.edu/htcondor/ubuntu/stable/ trusty contrib > /etc/apt/sources.list.d/htcondor.list" && \ sh -c "wget -qO - http://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add -" && \ @@ -103,6 +102,7 @@ RUN mkdir $GALAXY_ROOT && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_DIR && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install pip --upgrade && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install --extra-index-url https://wheels.galaxyproject.org/ uwsgi -v --pre && \ + chown galaxy:galaxy -R $GALAXY_VIRTUAL_ENV && \ rm -rf ~/.cache/ RUN su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" From a453508a8479ce41a436253c34be39fb9d6baffc Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 25 Feb 2018 12:48:13 +0100 Subject: [PATCH 22/60] remove nodejs and npm --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 9b52be794..1b3cc892c 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -72,7 +72,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-get update -qq && apt-get upgrade -y && \ apt-get install --no-install-recommends -y postgresql-9.3 sudo python-virtualenv \ nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 docker-ce=17.09.0~ce-0~ubuntu slurm-llnl slurm-llnl-torque \ - slurm-drmaa-dev proftpd proftpd-mod-pgsql libyaml-dev nodejs-legacy npm ansible munge libmunge-dev \ + slurm-drmaa-dev proftpd proftpd-mod-pgsql libyaml-dev ansible munge libmunge-dev \ nano vim curl python-crypto python-pip python-psutil condor python-ldap autofs \ gridengine-common gridengine-drmaa1.0 rabbitmq-server libswitch-perl unattended-upgrades supervisor && \ # install and remove supervisor, so get the service config file From 404246ae61e72de274064e1e989afb10f3e03711 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 25 Feb 2018 17:06:09 +0100 Subject: [PATCH 23/60] track 18.01 branch of ansible-extras --- .gitmodules | 1 + galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 29623c5ae..e6612923a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "galaxy/roles/galaxyproject.galaxyextras"] path = galaxy/roles/galaxyprojectdotorg.galaxyextras url = https://github.com/galaxyproject/ansible-galaxy-extras + branch = 18.01 [submodule "galaxy/roles/galaxy-postgresql"] path = galaxy/roles/galaxy-postgresql url = https://github.com/galaxyproject/ansible-postgresql diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index d6c93d964..c4dd69878 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit d6c93d964a2add6ed1d5d61c8718d305b86aea9d +Subproject commit c4dd69878644b72687c5ddd324322f95b7eb26a4 From b6f8a8516f2fd0d8dc1b3620cc16a081e87ccaa5 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 25 Feb 2018 17:48:15 +0100 Subject: [PATCH 24/60] track special 18.01 branches from upstream --- compose/buildlocal.sh | 9 +++++---- compose/galaxy-base/Dockerfile | 4 ++-- compose/galaxy-init/Dockerfile | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index e776d1f58..d447071ea 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -2,15 +2,16 @@ set -x -e ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ANSIBLE_RELEASE=5d9c8be9b496a6890cf04a858ed650f0211baf89 +ANSIBLE_RELEASE=18.01 -GALAXY_RELEASE=dev +GALAXY_RELEASE=release_18.01 GALAXY_REPO=galaxyproject/galaxy -DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache" +DOCKER_ADDITIONAL_BUILD_ARGS="" +#"--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -TAG=":latest" +TAG=":dev" docker pull postgres diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 843c87492..6c128cd89 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -7,7 +7,7 @@ FROM ubuntu:14.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ARG ANSIBLE_RELEASE=5d9c8be9b496a6890cf04a858ed650f0211baf89 +ARG ANSIBLE_RELEASE=18.01 ENV DEBIAN_FRONTEND=noninteractive \ GALAXY_USER=galaxy \ @@ -72,7 +72,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-get update -qq && apt-get upgrade -y && \ apt-get install --no-install-recommends -y sudo python-virtualenv \ supervisor linux-image-extra-virtual munge \ - nodejs-legacy npm ansible nano python-pip \ + ansible nano python-pip \ unattended-upgrades \ gridengine-drmaa1.0 && \ wget -q --no-check-certificate https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \ diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 7d829c48a..60bc7fe83 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -6,7 +6,7 @@ FROM quay.io/bgruening/galaxy-base:dev MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com -ARG GALAXY_RELEASE=dev +ARG GALAXY_RELEASE=release_18.01 ARG GALAXY_REPO=galaxyproject/galaxy # Create these folders and link to target directory for installation From 0ea3a3e267882d32648b0dd8208c1484afb88590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 25 Feb 2018 18:59:09 +0100 Subject: [PATCH 25/60] remove install-repository This wrapper script is since a long time depreacted and now its not broken since 18.01, time to remove it. --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a78097e2a..ca21f1ff9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -267,8 +267,6 @@ script: # Run a ton of BioBlend test against our servers. - cd $TRAVIS_BUILD_DIR/test/bioblend/ && . ./test.sh && cd $WORKING_DIR/ - # Test the 'old' tool installation script - - docker_exec_run bash -c "install-repository $INSTALL_REPO_ARG '--url https://toolshed.g2.bx.psu.edu -o iuc --name samtools_sort --panel-section-name SAMtools'" - | if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] then From aafd5fb63da935fc9ff416ca0e614fc4671f684f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 25 Feb 2018 19:59:11 +0100 Subject: [PATCH 26/60] deactivate an other install test --- .travis.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ca21f1ff9..b263b1a4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -267,13 +267,22 @@ script: # Run a ton of BioBlend test against our servers. - cd $TRAVIS_BUILD_DIR/test/bioblend/ && . ./test.sh && cd $WORKING_DIR/ - - | - if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] - then - # Test without install-repository wrapper - sleep 10 - docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools' - fi + # not working anymore in 18.01 + # executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid + # [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml + # /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log' + # getopt_long() error + # cat: galaxy_install.pid: No such file or directory + # tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory + #- | + # if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] + # then + # # Test without install-repository wrapper + # sleep 10 + # docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools' + # fi + + # Test the 'new' tool installation script # - docker_exec bash -c "install-tools $GALAXY_HOME/ephemeris/sample_tool_list.yaml" - | From f13aa40927ab6d5fe234221dd8c5d3625e075e77 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 14:37:24 +0100 Subject: [PATCH 27/60] use new uwsgi based startup method --- galaxy/install_tools_wrapper.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index 83c2e4220..8b587da90 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -24,7 +24,7 @@ else done echo "starting Galaxy" - sudo -E -u galaxy ./run.sh --daemon --log-file=$install_log --pid-file=galaxy_install.pid + sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid galaxy_install_pid=`cat galaxy_install.pid` @@ -62,7 +62,7 @@ fi if ! pgrep "supervisord" > /dev/null then # stop everything - sudo -E -u galaxy ./run.sh --stop-daemon --log-file=$install_log --pid-file=galaxy_install.pid + sudo -E -u galaxy ./run.sh --stop --pidfile galaxy_install.pid rm $install_log service postgresql stop fi From c1dad2c523078d519c00712705edc516ebd907ae Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 15:21:23 +0100 Subject: [PATCH 28/60] use galaxy-wait to wait for startup --- galaxy/install_tools_wrapper.sh | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index 8b587da90..c8a34498e 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -27,24 +27,7 @@ else sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid galaxy_install_pid=`cat galaxy_install.pid` - - while : ; do - tail -n 2 $install_log | grep -E -q "Removing PID file galaxy_install.pid|Daemon is already running" - if [ $? -eq 0 ] ; then - echo "Galaxy could not be started." - echo "More information about this failure may be found in the following log snippet from galaxy_install.log:" - echo "========================================" - tail -n 60 $install_log - echo "========================================" - echo $1 - exit 1 - fi - tail -n 2 $install_log | grep -q "Starting server in PID $galaxy_install_pid" - if [ $? -eq 0 ] ; then - echo "Galaxy is running." - break - fi - done + galaxy-wait -g http://127.0.0.1:8080 -v --timeout 120 fi shed-tools install -g "http://localhost:$PORT" -a admin -t "$1" From b6043c692712a38ab3ed2e1dd1f477914847c52b Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 15:29:22 +0100 Subject: [PATCH 29/60] fetch prebuild eggs and miniconda --- compose/galaxy-init/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 4d29b2c45..dad72aa42 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -71,12 +71,11 @@ WORKDIR $GALAXY_ROOT # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments RUN ./scripts/common_startup.sh && \ + . $GALAXY_VIRTUAL_ENV/bin/activate && \ + python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ + # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ - npm install && \ - wget -q https://repo.continuum.io/miniconda/Miniconda3-4.0.5-Linux-x86_64.sh && \ - bash Miniconda3-4.0.5-Linux-x86_64.sh -b -p $GALAXY_CONDA_PREFIX && \ - rm Miniconda3-4.0.5-Linux-x86_64.sh && \ - $GALAXY_CONDA_PREFIX/bin/conda install -y conda==3.19.3 + npm install # the following can be removed with 18.01 RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel bioconda --channel conda-forge --channel defaults --channel r --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5 From b409bdf121011e6ffac37d20bd30d22f57f68eea Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 15:58:59 +0100 Subject: [PATCH 30/60] install latest ephemeris release --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 73e168c49..bf7f6919b 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -81,7 +81,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ && \ mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \ pip install --upgrade pip && \ - pip install "ephemeris<0.8" supervisor --upgrade && \ + pip install ephemeris supervisor --upgrade && \ ln -s /usr/local/bin/supervisord /usr/bin/supervisord RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ From 26e724420bc2939de81f52fa2e495d7db8a9e6e3 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 15:59:29 +0100 Subject: [PATCH 31/60] install latest ephemeris release --- compose/galaxy-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 77ea3eb02..6c128cd89 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -79,7 +79,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \ rm condor_8.6.3_linux_all.deb && \ pip install --upgrade pip && \ - pip install "ephemeris<0.8" && \ + pip install ephemeris && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ From 7b158d17cf3c269e26a08e5bc965ab0467d4e9e3 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 19:22:53 +0100 Subject: [PATCH 32/60] force skip client side build --- compose/galaxy-init/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index dad72aa42..c181f0745 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -22,7 +22,7 @@ RUN mkdir -p /export /galaxy-export && \ ln -s -f /galaxy-export/venv /export/venv && \ ln -s -f /galaxy-export/galaxy-central /export/galaxy-central && \ chown -R $GALAXY_USER:$GALAXY_USER /galaxy-export /export && \ - wget -q -O - https://api.github.com/repos/$GALAXY_REPO/tarball/$GALAXY_RELEASE | tar xz --strip-components=1 -C $GALAXY_ROOT && \ + wget -q -O - https://api.github.com/repos/$GALAXY_REPO/tarball/release_18.01 | tar xz --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ . $GALAXY_VIRTUAL_ENV/bin/activate && pip install pip --upgrade && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV/* && \ @@ -70,7 +70,7 @@ WORKDIR $GALAXY_ROOT # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments -RUN ./scripts/common_startup.sh && \ +RUN ./scripts/common_startup.sh --skip-client-build && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments From 298c2f5d27cd57c37b4e00af284c3ad8e3d4eb13 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 21:20:33 +0100 Subject: [PATCH 33/60] yml changes --- compose/galaxy-base/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 6c128cd89..9d843a4cc 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -17,13 +17,14 @@ GALAXY_HOME=/home/galaxy \ GALAXY_LOGS_DIR=/home/galaxy/logs \ GALAXY_ROOT=/export/galaxy-central \ GALAXY_VIRTUAL_ENV=/export/venv \ +GALAXY_CONFIG_DIR=/etc/galaxy \ EXPORT_DIR=/export \ # Setting a standard encoding. This can get important for things like the unix sort tool. LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 ENV \ -GALAXY_CONFIG_FILE=/export/config/galaxy.ini \ +GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \ GALAXY_CONFIG_STATIC_ENABLED=False \ GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://localhost/galaxy?client_encoding=utf8 \ GALAXY_CONFIG_TEMPLATE_CACHE_PATH=/export/galaxy-central/database/compiled_templates \ From 454e8b94453a271a096070e6008dc28ea2033613 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 21:45:19 +0100 Subject: [PATCH 34/60] copy example yml file into the right spot --- compose/docker-compose.yml | 8 +++----- compose/galaxy-init/Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 12fe3e2a6..a95bbea0e 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -34,7 +34,7 @@ services: replicas: 1 placement: constraints: [node.role == manager] - labels: + labels: kompose.service.type: nodeport galaxy-postgres: @@ -248,11 +248,11 @@ services: - GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True - GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:chaopagoosaequuashie@galaxy-postgres/galaxy?client_encoding=utf8 - GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml - # Configurate admin and master api key. Can be overridden in galaxy.ini + # Configurate admin and master api key. Can be overridden in galaxy.yml - GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org - GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 - GALAXY_CONFIG_DATABASE_AUTO_MIGRATE=${GALAXY_CONFIG_DATABASE_AUTO_MIGRATE:-false} - - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy + - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy # Condor - GALAXY_DESTINATIONS_DEFAULT=${GALAXY_DESTINATIONS_DEFAULT:-slurm_cluster} - GALAXY_RUNNERS_ENABLE_CONDOR=${GALAXY_RUNNERS_ENABLE_CONDOR:-False} @@ -304,5 +304,3 @@ services: networks: galaxy: - - diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index c181f0745..8a2c9db34 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -24,9 +24,11 @@ RUN mkdir -p /export /galaxy-export && \ chown -R $GALAXY_USER:$GALAXY_USER /galaxy-export /export && \ wget -q -O - https://api.github.com/repos/$GALAXY_REPO/tarball/release_18.01 | tar xz --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ + cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE && \ . $GALAXY_VIRTUAL_ENV/bin/activate && pip install pip --upgrade && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV/* && \ - chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_ROOT/* + chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_ROOT/* && \ + chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_FILE ADD config $GALAXY_ROOT/config ADD welcome /galaxy-export/welcome From 422ce8e3ac557e6f2d0263e77cea084c28fef445 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 21:58:07 +0100 Subject: [PATCH 35/60] remove old config --- compose/galaxy-init/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 8a2c9db34..f69323a90 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -91,8 +91,7 @@ RUN mv $GALAXY_ROOT/config /galaxy-export/config && ln -s -f /export/config $GAL mv $GALAXY_ROOT/tool-data /galaxy-export/tool-data && ln -s -f /export/tool-data $GALAXY_ROOT/tool-data && \ mv $GALAXY_ROOT/display_applications /galaxy-export/display_applications && ln -s -f /export/display_applications $GALAXY_ROOT/display_applications && \ mv $GALAXY_ROOT/database /galaxy-export/database && ln -s -f /export/database $GALAXY_ROOT/database && \ - rm -rf $GALARY_ROOT/.venv && ln -s -f /export/venv $GALAXY_ROOT/.venv && ln -s -f /export/venv $GALAXY_ROOT/venv && \ - cp /galaxy-export/config/galaxy.ini.sample /galaxy-export/config/galaxy.ini + rm -rf $GALARY_ROOT/.venv && ln -s -f /export/venv $GALAXY_ROOT/.venv && ln -s -f /export/venv $GALAXY_ROOT/venv WORKDIR /galaxy-export From 7eec7f6692f05e75d8a61d2b8c473a06cd489352 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 22:24:57 +0100 Subject: [PATCH 36/60] run dev images in compose --- compose/buildlocal.sh | 2 +- compose/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index d447071ea..724591f87 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -11,7 +11,7 @@ DOCKER_ADDITIONAL_BUILD_ARGS="" #"--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -TAG=":dev" +export TAG=":dev" docker pull postgres diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index a95bbea0e..614b9a1a3 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -157,7 +157,7 @@ services: galaxy-htcondor-executor-big: - image: quay.io/bgruening/galaxy-htcondor-executor + image: quay.io/bgruening/galaxy-htcondor-executor:${TAG:-latest} # privileged is needed to run Docker jobs, moreover HTCondor wants # a RW cgroups mount. A RO cgroups mount throws an error in ProcD for running jobs. privileged: True From 34d8081900419caff1fafd00b264b3e590677f71 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 22:44:30 +0100 Subject: [PATCH 37/60] add dev tag --- compose/.env_htcondor_docker | 2 ++ compose/.env_k8_native | 2 ++ compose/.env_slurm | 2 +- compose/.env_slurm_singularity | 2 ++ compose/buildlocal.sh | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/compose/.env_htcondor_docker b/compose/.env_htcondor_docker index 3f3685548..ae7e59c4c 100644 --- a/compose/.env_htcondor_docker +++ b/compose/.env_htcondor_docker @@ -6,6 +6,8 @@ # Galaxy deployments. # ln -sf .env_htcondor_docker .env +TAG=dev + GALAXY_DESTINATIONS_DEFAULT=condor_cluster GALAXY_RUNNERS_ENABLE_CONDOR=True diff --git a/compose/.env_k8_native b/compose/.env_k8_native index f59f86bef..269522585 100644 --- a/compose/.env_k8_native +++ b/compose/.env_k8_native @@ -3,3 +3,5 @@ GALAXY_DESTINATIONS_DEFAULT=docker_dispatch GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8_default GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml + +TAG=dev diff --git a/compose/.env_slurm b/compose/.env_slurm index c42d24ff7..033c087f3 100644 --- a/compose/.env_slurm +++ b/compose/.env_slurm @@ -6,6 +6,7 @@ # Galaxy deployments. # ln -sf .env_slurm .env +TAG=dev # default destination is our SLURM cluster GALAXY_DESTINATIONS_DEFAULT=slurm_cluster @@ -17,4 +18,3 @@ GALAXY_DOCKER_ENABLED=false GALAXY_CONFIG_CLEANUP_JOB=onsuccess GALAXY_PORT=8080 - diff --git a/compose/.env_slurm_singularity b/compose/.env_slurm_singularity index 11af85a8a..e169c5b76 100644 --- a/compose/.env_slurm_singularity +++ b/compose/.env_slurm_singularity @@ -6,6 +6,8 @@ # Galaxy deployments. # ln -sf .env_slurm .env +TAG=dev + # default destination is our SLURM cluster with Singularity enabled GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity GALAXY_RUNNERS_ENABLE_SLURM=True diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 724591f87..d447071ea 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -11,7 +11,7 @@ DOCKER_ADDITIONAL_BUILD_ARGS="" #"--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -export TAG=":dev" +TAG=":dev" docker pull postgres From 5143c28c3006940d5e4e4ce2a52f9f364b254272 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Mar 2018 23:09:29 +0100 Subject: [PATCH 38/60] two tags missing --- compose/buildlocal.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index d447071ea..95f801fc3 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -24,10 +24,10 @@ docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/bgruening/galaxy-web$TAG . docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/galaxy/proftpd$TAG ./galaxy-proftpd # Build the postgres container -docker build -t quay.io/galaxy/postgres ./galaxy-postgres +docker build -t quay.io/galaxy/postgres$TAG ./galaxy-postgres # The SLURM cluster -docker build -t quay.io/galaxy/slurm ./galaxy-slurm +docker build -t quay.io/galaxy/slurm$TAG ./galaxy-slurm # we build a common HTCondor and derive from that laster docker build -t quay.io/bgruening/galaxy-htcondor-base$TAG ./galaxy-htcondor-base From 740040894a165d1a6c08180c3e3a0965460d49bc Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 00:24:45 +0100 Subject: [PATCH 39/60] fix galaxy.yml --- compose/galaxy-base/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 9d843a4cc..8c48ff024 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -86,7 +86,9 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ mv /tmp/download/docker/docker /usr/bin/ && \ rm -rf /tmp/download && \ - rm -rf ~/.cache/ + rm -rf ~/.cache/ && \ + cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE && \ + chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_FILE RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \ From 6cb236ff33b71df92063f744632596c68a952745 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 00:39:51 +0100 Subject: [PATCH 40/60] fix galaxy.yml --- compose/galaxy-base/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 8c48ff024..9d843a4cc 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -86,9 +86,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ mv /tmp/download/docker/docker /usr/bin/ && \ rm -rf /tmp/download && \ - rm -rf ~/.cache/ && \ - cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE && \ - chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_FILE + rm -rf ~/.cache/ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \ From 3a38095db4dca686dddeb1e4ed1ebebc539ed4dc Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 00:47:59 +0100 Subject: [PATCH 41/60] fix galaxy.yml --- compose/galaxy-web/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 2ba9aa9c7..a0aeea5af 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -37,7 +37,9 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \ pip install --upgrade pip && \ pip install "ephemeris<0.8" supervisor --upgrade && \ - ln -s /usr/local/bin/supervisord /usr/bin/supervisord + ln -s /usr/local/bin/supervisord /usr/bin/supervisord && \ + cp /galaxy-config/config/galaxy.yml.sample $GALAXY_CONFIG_FILE && \ + chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_FILE RUN rm -f /usr/bin/startup && \ ansible-playbook /ansible/provision.yml \ From 6934a05f0944101af4cd70e6e6cf47710f27b2e5 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 01:49:12 +0100 Subject: [PATCH 42/60] database update --- compose/docker-compose.yml | 1 + compose/dumpsql.sh | 6 +- compose/galaxy-postgres/init-galaxy-db.sql.in | 119 ++++++++++++++++-- 3 files changed, 117 insertions(+), 9 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 614b9a1a3..c68bd272d 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -290,6 +290,7 @@ services: volumes: # This is the directory where all your files from Galaxy will be stored # on your host system + - ${EXPORT_DIR:-/export}/galaxy-central/config/galaxy.yml.sample:/etc/galaxy/galaxy.yml - ${EXPORT_DIR:-/export}/:/export/ - /var/run/docker.sock:/var/run/docker.sock depends_on: diff --git a/compose/dumpsql.sh b/compose/dumpsql.sh index 8a2a6a4c3..ae5a6d413 100755 --- a/compose/dumpsql.sh +++ b/compose/dumpsql.sh @@ -1,5 +1,7 @@ #!/bin/bash +TAG=dev + # Sets the image of postgres to use POSTGRES=postgres:9.6.5 @@ -22,9 +24,9 @@ pg_end=`date +%s` init_start=`date +%s` docker run -i --rm --name "dumpsql_galaxy_installdb" \ - -e "GALAXY_CONFIG_FILE=/galaxy-export/config/galaxy.ini.sample" \ + -e "GALAXY_CONFIG_FILE=/etc/galaxy/galaxy.yml" \ -e "GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB?client_encoding=utf8" \ - --link "dumpsql_postgres:db" quay.io/bgruening/galaxy-init install_db.sh + --link "dumpsql_postgres:db" quay.io/bgruening/galaxy-init:$TAG install_db.sh init_end=`date +%s` dump_start=`date +%s` diff --git a/compose/galaxy-postgres/init-galaxy-db.sql.in b/compose/galaxy-postgres/init-galaxy-db.sql.in index 4af8ee621..77c16f4ce 100644 --- a/compose/galaxy-postgres/init-galaxy-db.sql.in +++ b/compose/galaxy-postgres/init-galaxy-db.sql.in @@ -1327,7 +1327,8 @@ CREATE TABLE history_dataset_association ( purged boolean, tool_version text, extended_metadata_id integer, - hidden_beneath_collection_instance_id integer + hidden_beneath_collection_instance_id integer, + version integer ); @@ -1395,6 +1396,41 @@ CREATE SEQUENCE history_dataset_association_display_at_authorization_id_seq ALTER SEQUENCE history_dataset_association_display_at_authorization_id_seq OWNED BY history_dataset_association_display_at_authorization.id; +-- +-- Name: history_dataset_association_history; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE history_dataset_association_history ( + id integer NOT NULL, + history_dataset_association_id integer, + update_time timestamp without time zone, + version integer, + name character varying(255), + extension character varying(64), + metadata bytea, + extended_metadata_id integer +); + + +-- +-- Name: history_dataset_association_history_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE history_dataset_association_history_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: history_dataset_association_history_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE history_dataset_association_history_id_seq OWNED BY history_dataset_association_history.id; + + -- -- Name: history_dataset_association_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -1917,7 +1953,8 @@ CREATE TABLE job ( exit_code integer, destination_id character varying(255), destination_params bytea, - dependencies bytea + dependencies bytea, + copied_from_job_id integer ); @@ -2211,7 +2248,8 @@ CREATE TABLE job_to_input_dataset ( id integer NOT NULL, job_id integer, dataset_id integer, - name character varying(255) + name character varying(255), + dataset_version integer ); @@ -5710,6 +5748,13 @@ ALTER TABLE ONLY history_dataset_association_annotation_association ALTER COLUMN ALTER TABLE ONLY history_dataset_association_display_at_authorization ALTER COLUMN id SET DEFAULT nextval('history_dataset_association_display_at_authorization_id_seq'::regclass); +-- +-- Name: history_dataset_association_history id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_dataset_association_history ALTER COLUMN id SET DEFAULT nextval('history_dataset_association_history_id_seq'::regclass); + + -- -- Name: history_dataset_association_rating_association id; Type: DEFAULT; Schema: public; Owner: - -- @@ -7118,7 +7163,7 @@ SELECT pg_catalog.setval('history_annotation_association_id_seq', 1, false); -- Data for Name: history_dataset_association; Type: TABLE DATA; Schema: public; Owner: - -- -COPY history_dataset_association (id, history_id, dataset_id, create_time, update_time, copied_from_history_dataset_association_id, hid, name, info, blurb, peek, extension, metadata, parent_id, designation, deleted, visible, copied_from_library_dataset_dataset_association_id, state, purged, tool_version, extended_metadata_id, hidden_beneath_collection_instance_id) FROM stdin; +COPY history_dataset_association (id, history_id, dataset_id, create_time, update_time, copied_from_history_dataset_association_id, hid, name, info, blurb, peek, extension, metadata, parent_id, designation, deleted, visible, copied_from_library_dataset_dataset_association_id, state, purged, tool_version, extended_metadata_id, hidden_beneath_collection_instance_id, version) FROM stdin; \. @@ -7152,6 +7197,21 @@ COPY history_dataset_association_display_at_authorization (id, create_time, upda SELECT pg_catalog.setval('history_dataset_association_display_at_authorization_id_seq', 1, false); +-- +-- Data for Name: history_dataset_association_history; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY history_dataset_association_history (id, history_dataset_association_id, update_time, version, name, extension, metadata, extended_metadata_id) FROM stdin; +\. + + +-- +-- Name: history_dataset_association_history_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('history_dataset_association_history_id_seq', 1, false); + + -- -- Name: history_dataset_association_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - -- @@ -7380,7 +7440,7 @@ SELECT pg_catalog.setval('implicitly_created_dataset_collection_inputs_id_seq', -- Data for Name: job; Type: TABLE DATA; Schema: public; Owner: - -- -COPY job (id, create_time, update_time, history_id, tool_id, tool_version, state, info, command_line, param_filename, runner_name, stdout, stderr, traceback, session_id, job_runner_name, job_runner_external_id, library_folder_id, user_id, imported, object_store_id, params, handler, exit_code, destination_id, destination_params, dependencies) FROM stdin; +COPY job (id, create_time, update_time, history_id, tool_id, tool_version, state, info, command_line, param_filename, runner_name, stdout, stderr, traceback, session_id, job_runner_name, job_runner_external_id, library_folder_id, user_id, imported, object_store_id, params, handler, exit_code, destination_id, destination_params, dependencies, copied_from_job_id) FROM stdin; \. @@ -7515,7 +7575,7 @@ SELECT pg_catalog.setval('job_to_implicit_output_dataset_collection_id_seq', 1, -- Data for Name: job_to_input_dataset; Type: TABLE DATA; Schema: public; Owner: - -- -COPY job_to_input_dataset (id, job_id, dataset_id, name) FROM stdin; +COPY job_to_input_dataset (id, job_id, dataset_id, name, dataset_version) FROM stdin; \. @@ -7870,7 +7930,7 @@ GalaxyTools lib/tool_shed/galaxy_install/migrate 1 -- COPY migrate_version (repository_id, repository_path, version) FROM stdin; -Galaxy lib/galaxy/model/migrate 136 +Galaxy lib/galaxy/model/migrate 140 \. @@ -9274,6 +9334,14 @@ ALTER TABLE ONLY history_dataset_association_display_at_authorization ADD CONSTRAINT history_dataset_association_display_at_authorization_pkey PRIMARY KEY (id); +-- +-- Name: history_dataset_association_history history_dataset_association_history_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_dataset_association_history + ADD CONSTRAINT history_dataset_association_history_pkey PRIMARY KEY (id); + + -- -- Name: history_dataset_association history_dataset_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -10973,6 +11041,20 @@ CREATE INDEX ix_history_dataset_association_display_at_authorization_a293 ON his CREATE INDEX ix_history_dataset_association_display_at_authorization_user_id ON history_dataset_association_display_at_authorization USING btree (user_id); +-- +-- Name: ix_history_dataset_association_history_extended_metadata_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_history_dataset_association_history_extended_metadata_id ON history_dataset_association_history USING btree (extended_metadata_id); + + +-- +-- Name: ix_history_dataset_association_history_history_dataset__5f1c; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_history_dataset_association_history_history_dataset__5f1c ON history_dataset_association_history USING btree (history_dataset_association_id); + + -- -- Name: ix_history_dataset_association_history_id; Type: INDEX; Schema: public; Owner: - -- @@ -10980,6 +11062,13 @@ CREATE INDEX ix_history_dataset_association_display_at_authorization_user_id ON CREATE INDEX ix_history_dataset_association_history_id ON history_dataset_association USING btree (history_id); +-- +-- Name: ix_history_dataset_association_history_version; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_history_dataset_association_history_version ON history_dataset_association_history USING btree (version); + + -- -- Name: ix_history_dataset_association_purged; Type: INDEX; Schema: public; Owner: - -- @@ -14078,6 +14167,14 @@ ALTER TABLE ONLY history_dataset_association ADD CONSTRAINT history_dataset_association_extended_metadata_id_fkey FOREIGN KEY (extended_metadata_id) REFERENCES extended_metadata(id); +-- +-- Name: history_dataset_association_history history_dataset_association_h_history_dataset_association__fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_dataset_association_history + ADD CONSTRAINT history_dataset_association_h_history_dataset_association__fkey FOREIGN KEY (history_dataset_association_id) REFERENCES history_dataset_association(id); + + -- -- Name: history_dataset_association history_dataset_association_hidden_beneath_collection_inst_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -14086,6 +14183,14 @@ ALTER TABLE ONLY history_dataset_association ADD CONSTRAINT history_dataset_association_hidden_beneath_collection_inst_fkey FOREIGN KEY (hidden_beneath_collection_instance_id) REFERENCES history_dataset_collection_association(id); +-- +-- Name: history_dataset_association_history history_dataset_association_history_extended_metadata_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_dataset_association_history + ADD CONSTRAINT history_dataset_association_history_extended_metadata_id_fkey FOREIGN KEY (extended_metadata_id) REFERENCES extended_metadata(id); + + -- -- Name: history_dataset_association history_dataset_association_history_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- From 0ab0fc1e39b3d9ec032b0538229504d35566b365 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 01:57:10 +0100 Subject: [PATCH 43/60] fix galaxy.yml --- compose/galaxy-web/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index a0aeea5af..2ba9aa9c7 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -37,9 +37,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \ pip install --upgrade pip && \ pip install "ephemeris<0.8" supervisor --upgrade && \ - ln -s /usr/local/bin/supervisord /usr/bin/supervisord && \ - cp /galaxy-config/config/galaxy.yml.sample $GALAXY_CONFIG_FILE && \ - chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_FILE + ln -s /usr/local/bin/supervisord /usr/bin/supervisord RUN rm -f /usr/bin/startup && \ ansible-playbook /ansible/provision.yml \ From 6e3ad01a0813f66051d8209fbef42e4ba8a9280e Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 09:39:48 +0100 Subject: [PATCH 44/60] small fixes to get FTP working --- compose/buildlocal.sh | 4 ++-- compose/docker-compose.yml | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 95f801fc3..b858893d3 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -24,10 +24,10 @@ docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/bgruening/galaxy-web$TAG . docker build $DOCKER_ADDITIONAL_BUILD_ARGS --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t quay.io/galaxy/proftpd$TAG ./galaxy-proftpd # Build the postgres container -docker build -t quay.io/galaxy/postgres$TAG ./galaxy-postgres +docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/galaxy/postgres$TAG ./galaxy-postgres # The SLURM cluster -docker build -t quay.io/galaxy/slurm$TAG ./galaxy-slurm +docker build $DOCKER_ADDITIONAL_BUILD_ARGS -t quay.io/galaxy/slurm$TAG ./galaxy-slurm # we build a common HTCondor and derive from that laster docker build -t quay.io/bgruening/galaxy-htcondor-base$TAG ./galaxy-htcondor-base diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index c68bd272d..fc0f9f3f9 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -75,6 +75,8 @@ services: restart: unless-stopped networks: - galaxy + depends_on: + - galaxy-postgres deploy: placement: constraints: [node.role == manager] @@ -237,7 +239,8 @@ services: environment: # In case the galaxy-init container is not used, you need to include "sleeplock" to NONUSE - NONUSE=proftpd,postgres,slurmd,slurmctld - - GALAXY_DEFAULT_ADMIN_USER=admin@galaxy.org + - GALAXY_DEFAULT_ADMIN_USER=admin + - GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org - GALAXY_DEFAULT_ADMIN_PASSWORD=admin - GALAXY_DEFAULT_ADMIN_KEY=admin - GALAXY_HANDLER_NUMPROCS=2 From 3144d7fa7a0c5004ebd06ced19bcdf5265300781 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 10:14:32 +0100 Subject: [PATCH 45/60] install latest ephemeris --- compose/galaxy-web/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 2ba9aa9c7..a2d3350aa 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -36,7 +36,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ gpasswd -a $GALAXY_USER docker && \ mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \ pip install --upgrade pip && \ - pip install "ephemeris<0.8" supervisor --upgrade && \ + pip install ephemeris supervisor --upgrade && \ ln -s /usr/local/bin/supervisord /usr/bin/supervisord RUN rm -f /usr/bin/startup && \ From 7fb9bb0f0bf777536bfef313838792de0cd9872e Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 12:12:22 +0100 Subject: [PATCH 46/60] tag 18.01 release --- compose/.env_htcondor_docker | 2 +- compose/.env_k8_native | 2 +- compose/.env_slurm_singularity | 2 +- compose/buildlocal.sh | 2 +- compose/dumpsql.sh | 2 +- compose/galaxy-htcondor-executor/Dockerfile | 2 +- compose/galaxy-init/Dockerfile | 2 +- compose/galaxy-web/Dockerfile | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compose/.env_htcondor_docker b/compose/.env_htcondor_docker index ae7e59c4c..12182a726 100644 --- a/compose/.env_htcondor_docker +++ b/compose/.env_htcondor_docker @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_htcondor_docker .env -TAG=dev +TAG=18.01 GALAXY_DESTINATIONS_DEFAULT=condor_cluster diff --git a/compose/.env_k8_native b/compose/.env_k8_native index 269522585..a83eade96 100644 --- a/compose/.env_k8_native +++ b/compose/.env_k8_native @@ -4,4 +4,4 @@ GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8_default GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml -TAG=dev +TAG=18.01 diff --git a/compose/.env_slurm_singularity b/compose/.env_slurm_singularity index e169c5b76..0dcd70a00 100644 --- a/compose/.env_slurm_singularity +++ b/compose/.env_slurm_singularity @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_slurm .env -TAG=dev +TAG=18.01 # default destination is our SLURM cluster with Singularity enabled GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index b858893d3..3e86c2ee6 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -11,7 +11,7 @@ DOCKER_ADDITIONAL_BUILD_ARGS="" #"--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -TAG=":dev" +TAG=":18.01" docker pull postgres diff --git a/compose/dumpsql.sh b/compose/dumpsql.sh index ae5a6d413..fa8b16789 100755 --- a/compose/dumpsql.sh +++ b/compose/dumpsql.sh @@ -1,6 +1,6 @@ #!/bin/bash -TAG=dev +TAG=18.01 # Sets the image of postgres to use POSTGRES=postgres:9.6.5 diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index 3ae0255d8..b2992e778 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base:dev +FROM quay.io/bgruening/galaxy-htcondor-base:18.01 ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index f69323a90..ae570e91e 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base:dev +FROM quay.io/bgruening/galaxy-base:18.01 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index a2d3350aa..640dbbcfb 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base:dev +FROM quay.io/bgruening/galaxy-base:18.01 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com From dcf962d7de944b6eb80bb1ff4538c983a095ff5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 5 Mar 2018 13:12:47 +0100 Subject: [PATCH 47/60] use http://localhost:$PORT for galaxy-wait --- galaxy/install_tools_wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index c8a34498e..8b35863d2 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -27,7 +27,7 @@ else sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid galaxy_install_pid=`cat galaxy_install.pid` - galaxy-wait -g http://127.0.0.1:8080 -v --timeout 120 + galaxy-wait -g http://localhost:$PORT -v --timeout 120 fi shed-tools install -g "http://localhost:$PORT" -a admin -t "$1" From aa0256ff995d4a97c9e0076a1d2683690cf056aa Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 5 Mar 2018 13:23:13 +0100 Subject: [PATCH 48/60] fix slurm test --- compose/.env_slurm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/.env_slurm b/compose/.env_slurm index 033c087f3..7cd741493 100644 --- a/compose/.env_slurm +++ b/compose/.env_slurm @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_slurm .env -TAG=dev +TAG=18.01 # default destination is our SLURM cluster GALAXY_DESTINATIONS_DEFAULT=slurm_cluster From e0dee3cbab56aff1486747c09c13c1a669ed536f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Wed, 7 Mar 2018 17:59:10 +0100 Subject: [PATCH 49/60] GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ --- compose/galaxy-base/Dockerfile | 1 + galaxy/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 9d843a4cc..f0bd236d3 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -30,6 +30,7 @@ GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://localhost/galaxy?client_encoding= GALAXY_CONFIG_TEMPLATE_CACHE_PATH=/export/galaxy-central/database/compiled_templates \ GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=/export/galaxy-central/database/citations/data \ GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=/export/galaxy-central/database/pbs \ +GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ GALAXY_CONFIG_FTP_UPLOAD_DIR=/export/ftp \ GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \ GALAXY_CONFIG_USE_PBKDF2=False \ diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index bf7f6919b..c55901dc2 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -20,6 +20,7 @@ ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \ GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \ +GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \ GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \ GALAXY_VIRTUAL_ENV=/galaxy_venv \ From 52d688cf5f1deab3a99168a6394c75c5b4b0f42a Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 20 Mar 2018 10:07:25 +0100 Subject: [PATCH 50/60] update role --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index c4dd69878..a8f4e95eb 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit c4dd69878644b72687c5ddd324322f95b7eb26a4 +Subproject commit a8f4e95eb1bbd9598b19357aadd5fbb37214601b From 0e25932319098664ad3be256f54bca83ea0d14e2 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 22 Mar 2018 13:40:18 +0100 Subject: [PATCH 51/60] add http-timeout --- galaxy/install_tools_wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index 8b35863d2..ff170f5f1 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -24,7 +24,7 @@ else done echo "starting Galaxy" - sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid + sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid --http-timeout 3000 galaxy_install_pid=`cat galaxy_install.pid` galaxy-wait -g http://localhost:$PORT -v --timeout 120 From a6dc89eb9b05cd24489963aaf38a745d57e4c2e2 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 22 Mar 2018 13:40:40 +0100 Subject: [PATCH 52/60] update role --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index a8f4e95eb..ec5efbe4c 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit a8f4e95eb1bbd9598b19357aadd5fbb37214601b +Subproject commit ec5efbe4cf3122c92d4b8e29302c78567f49d714 From 47baa736ccc0ec4faab23b42c0ce5e1bb6dacb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 24 Mar 2018 23:38:58 +0100 Subject: [PATCH 53/60] deactivat SLURM for the moment --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b263b1a4e..d99b01359 100644 --- a/.travis.yml +++ b/.travis.yml @@ -233,7 +233,7 @@ script: if [ ! "${COMPOSE_SLURM}" ] && [ ! "${KUBE}" ] && [ ! "${COMPOSE_CONDOR_DOCKER}" ] && [ ! "${COMPOSE_SLURM_SINGULARITY}" ] then # For compose slurm is already included and thus tested - cd $TRAVIS_BUILD_DIR/test/slurm/ && bash test.sh && cd $WORKING_DIR + sleep 1 ##cd $TRAVIS_BUILD_DIR/test/slurm/ && bash test.sh && cd $WORKING_DIR fi # Test submitting jobs to an external gridengine cluster - | From c5edcf57bb4f58d5698dcbb2e8d937d65a88feae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 25 Mar 2018 15:04:36 +0200 Subject: [PATCH 54/60] enable SLURM again --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d99b01359..b263b1a4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -233,7 +233,7 @@ script: if [ ! "${COMPOSE_SLURM}" ] && [ ! "${KUBE}" ] && [ ! "${COMPOSE_CONDOR_DOCKER}" ] && [ ! "${COMPOSE_SLURM_SINGULARITY}" ] then # For compose slurm is already included and thus tested - sleep 1 ##cd $TRAVIS_BUILD_DIR/test/slurm/ && bash test.sh && cd $WORKING_DIR + cd $TRAVIS_BUILD_DIR/test/slurm/ && bash test.sh && cd $WORKING_DIR fi # Test submitting jobs to an external gridengine cluster - | From ecb662a10470700f52d8af5c50fef9ba7e46f407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 25 Mar 2018 22:36:26 +0200 Subject: [PATCH 55/60] increase sleep time --- test/slurm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/slurm/test.sh b/test/slurm/test.sh index 5af216046..12e584f74 100644 --- a/test/slurm/test.sh +++ b/test/slurm/test.sh @@ -18,7 +18,7 @@ docker run -d -e "NONUSE=slurmd,slurmctld" \ --link slurm --name galaxy-slurm-test -h galaxy \ -p 80:80 -v "$EXPORT":/export quay.io/bgruening/galaxy # We wait for the creation of the /galaxy-central/config/ if it does not exist yet -sleep 60s +sleep 120s # We restart galaxy docker stop galaxy-slurm-test docker rm galaxy-slurm-test From 095cbacd48d8e59d6c5d821ec7a0e89fe90e9e88 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 25 Mar 2018 23:03:52 +0200 Subject: [PATCH 56/60] small changes to SLURM test --- test/slurm/startup.sh | 1 + test/slurm/test.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/slurm/startup.sh b/test/slurm/startup.sh index 6fa5e1d62..605d52ba0 100644 --- a/test/slurm/startup.sh +++ b/test/slurm/startup.sh @@ -18,6 +18,7 @@ if [ ! -f "$SLURM_CONF_PATH" ] then python /usr/local/bin/configure_slurm.py cp /etc/slurm-llnl/slurm.conf "$SLURM_CONF_PATH" + rm /etc/slurm-llnl/slurm.conf fi if [ ! -f "$GALAXY_DIR"/.venv ] then diff --git a/test/slurm/test.sh b/test/slurm/test.sh index 5af216046..23e265b73 100644 --- a/test/slurm/test.sh +++ b/test/slurm/test.sh @@ -18,7 +18,7 @@ docker run -d -e "NONUSE=slurmd,slurmctld" \ --link slurm --name galaxy-slurm-test -h galaxy \ -p 80:80 -v "$EXPORT":/export quay.io/bgruening/galaxy # We wait for the creation of the /galaxy-central/config/ if it does not exist yet -sleep 60s +sleep 180s # We restart galaxy docker stop galaxy-slurm-test docker rm galaxy-slurm-test @@ -32,7 +32,7 @@ docker run -d -e "NONUSE=slurmd,slurmctld" \ -p 80:80 -v "$EXPORT":/export quay.io/bgruening/galaxy # Let's submit a job from the galaxy container and check it runs in the slurm container sleep 60s -docker exec galaxy-slurm-test su - galaxy -c 'srun hostname' | grep slurm && \ -docker stop galaxy-slurm-test slurm && \ +docker exec galaxy-slurm-test su - galaxy -c 'srun hostname' | grep slurm +docker stop galaxy-slurm-test slurm docker rm galaxy-slurm-test slurm # TODO: Run a galaxy tool and check it runs on the cluster From 094cb36388d715c747d10b36f9ced10790b38c89 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 25 Mar 2018 23:54:27 +0200 Subject: [PATCH 57/60] role update --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index ec5efbe4c..f0a9de96b 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit ec5efbe4cf3122c92d4b8e29302c78567f49d714 +Subproject commit f0a9de96b20853b9abdf8fab1b0d81c3f8ad0849 From fe945a2224d14fd9c57c2171c3fc8479e6cedc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 25 Mar 2018 23:58:42 +0200 Subject: [PATCH 58/60] Update Dockerfile --- compose/galaxy-htcondor/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index af80dbaa3..401373a3f 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base:dev +FROM quay.io/bgruening/galaxy-htcondor-base:18.01 ADD condor_config.local /etc/condor/condor_config.local ADD supervisord.conf /etc/supervisord.conf From 34af07cd5d2c6b84ab17c656268c683499307da0 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 26 Mar 2018 08:53:40 +0200 Subject: [PATCH 59/60] role update --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index f0a9de96b..e896d2e51 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit f0a9de96b20853b9abdf8fab1b0d81c3f8ad0849 +Subproject commit e896d2e518624bbe6bceed1806e6b78a3d7f0644 From 3aea64fcee22c9cd5cfb269fbd45cfd9fc30c9d3 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 9 Apr 2018 11:27:00 +0200 Subject: [PATCH 60/60] update role --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index e896d2e51..01bb63f22 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit e896d2e518624bbe6bceed1806e6b78a3d7f0644 +Subproject commit 01bb63f22669eb5118e7bd89649fb5244f368972