From 254a0d5ada7fc575bbe4913366458ecf45be25de Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 25 Apr 2017 15:49:53 -0400 Subject: [PATCH 1/8] Add instructions and script for converting compose to k8. --- compose/k8/convert.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 compose/k8/convert.py diff --git a/compose/k8/convert.py b/compose/k8/convert.py new file mode 100644 index 000000000..54e9659ed --- /dev/null +++ b/compose/k8/convert.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Install kompose (e.g. brew install kompose). +# Install minikube (e.g. brew install minikube). +# $ minikube start # Build a k8 cluster in a VM. +# $ eval $(minikube docker-env) # Target Docker commands at the VM's Docker host. +# $ cd ..; bash buildlocal.sh; cd k8 # Build Docker containers required for k8 setup. +# $ python convert.py # Execute this wrapper around kompose to build native k8 artifacts. +# $ kompose -f docker-compose-for-kompose.yml up + +import os +import subprocess +import yaml + +DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +COMPOSE_TARGET = os.path.abspath(os.path.join(DIRECTORY, "..", "docker-compose.yml")) +KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml") + + +def main(): + with open(COMPOSE_TARGET, "r") as f: + raw_compose_def = yaml.load(f) + + _hack_for_kompose(raw_compose_def) + with open(KOMPOSE_TARGET, "w") as f: + yaml.dump(raw_compose_def, f) + + subprocess.check_call(["kompose", "-f", KOMPOSE_TARGET, "convert"]) + + +def _hack_for_kompose(raw_compose_def): + ftp_ports = raw_compose_def["services"]["proftpd"]["ports"] + del ftp_ports[2] + for i in range(10): + # Replace "30000-30010:30000-30010" with individual entries. + ftp_ports.append("%d:%d" % (30000 + i, 30000 + i)) + + +if __name__ == "__main__": + main() From 93788a3b07ee99192f2007ba01ea6aa70d541e32 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 13:55:27 -0400 Subject: [PATCH 2/8] More progress toward k8 compat. for compose setup. - Use volumes_from instead of volumes so kompose can match up paths properly. - Mount pgadmin4 without volumes when using kompose for now. - Annotate the services that should be exposed externally with kompose labels in the source compose files. --- compose/docker-compose.yml | 20 ++++++++++++-------- compose/k8/convert.py | 7 ++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 9568172ef..4cce0acc1 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -15,8 +15,8 @@ services: - proftpd_passive_port_high=30010 container_name: galaxy-proftpd hostname: galaxy-proftpd - volumes: - - ./galaxy-storage/ftp:/export/ftp + volumes_from: + - galaxy-web expose: - 21 - 22 @@ -25,6 +25,8 @@ services: - "8022:22" - "30000-30010:30000-30010" restart: always + labels: + kompose.service.type: nodeport # Using the official postgres image. This needs to be populated by calling # docker-compose run galaxy install_db.sh @@ -54,6 +56,8 @@ services: volumes: - ./pgadmin:/pgadmin restart: unless-stopped + labels: + kompose.service.type: nodeport # slurm container @@ -63,8 +67,8 @@ services: environment: {} container_name: galaxy-slurm hostname: galaxy-slurm - volumes: - - ./galaxy-storage/:/export + volumes_from: + - galaxy-web restart: always # This container initializes the galaxy export. @@ -73,10 +77,8 @@ services: image: quay.io/bgruening/galaxy-init:compose container_name: galaxy-init hostname: galaxy-init - volumes: - # This is the directory where all your files from Galaxy will be stored - # on your host system - - ./galaxy-storage/:/export/ + volumes_from: + - galaxy-web # This container provides the galaxy uwsgi webhandlers, job handlers, nginx galaxy-web: @@ -107,6 +109,8 @@ services: # on your host system - ./galaxy-storage/:/export/ privileged: True + labels: + kompose.service.type: nodeport # Use the monolith galaxy container instead # galaxy: diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 54e9659ed..907bf879c 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -17,10 +17,11 @@ KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml") + def main(): with open(COMPOSE_TARGET, "r") as f: raw_compose_def = yaml.load(f) - + _hack_for_kompose(raw_compose_def) with open(KOMPOSE_TARGET, "w") as f: yaml.dump(raw_compose_def, f) @@ -35,6 +36,10 @@ def _hack_for_kompose(raw_compose_def): # Replace "30000-30010:30000-30010" with individual entries. ftp_ports.append("%d:%d" % (30000 + i, 30000 + i)) + # pgadmin can run without volumes and gets permission errors if not started this way in + # minikube. + del raw_compose_def["services"]["pgadmin4"]["volumes"] + if __name__ == "__main__": main() From b805504b3d181175d83c83d333a589ff31d33f78 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 14:37:07 -0400 Subject: [PATCH 3/8] Kompose does not respect "hostname" - so rewrite this to be service name when specified. --- compose/k8/convert.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 907bf879c..099a8a6af 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -17,7 +17,6 @@ KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml") - def main(): with open(COMPOSE_TARGET, "r") as f: raw_compose_def = yaml.load(f) @@ -40,6 +39,17 @@ def _hack_for_kompose(raw_compose_def): # minikube. del raw_compose_def["services"]["pgadmin4"]["volumes"] + services = raw_compose_def["services"] + for service_name in list(services.keys()): + service_def = services[service_name] + if "hostname" in service_def: + hostname = service_def["hostname"] + # These need to be same for Kompose it seems + if hostname != service_name: + raw_compose_def[hostname] = service_def + del services[service_name] + del service_def["hostname"] + if __name__ == "__main__": main() From a7400b8e414719f24f68e92aa9135ebd50230af1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 15:24:33 -0400 Subject: [PATCH 4/8] Switch docker-compose.yml to match up service and hostnames. This is helpful for Kompose. --- compose/docker-compose.yml | 9 ++++----- compose/k8/convert.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 4cce0acc1..eb378415b 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: # proftpd container - proftpd: + galaxy-proftpd: # build: galaxy-proftpd image: quay.io/galaxy/proftpd:compose environment: @@ -31,7 +31,7 @@ services: # Using the official postgres image. This needs to be populated by calling # docker-compose run galaxy install_db.sh # on first run - postgres: + galaxy-postgres: # image: postgres # This comes with an initialization to quickly populate the database on first start # build: galaxy-postgres @@ -52,7 +52,7 @@ services: ports: - "5050:5050" links: - - postgres + - galaxy-postgres volumes: - ./pgadmin:/pgadmin restart: unless-stopped @@ -61,7 +61,7 @@ services: # slurm container - slurm: + galaxy-slurm: # build: galaxy-slurm image: quay.io/galaxy/slurm:compose environment: {} @@ -101,7 +101,6 @@ services: - GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org - GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 container_name: galaxy-web - hostname: galaxy ports: - "8080:80" # nginx volumes: diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 099a8a6af..17bf17113 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -46,9 +46,15 @@ def _hack_for_kompose(raw_compose_def): hostname = service_def["hostname"] # These need to be same for Kompose it seems if hostname != service_name: - raw_compose_def[hostname] = service_def + services[hostname] = service_def del services[service_name] - del service_def["hostname"] + for service in services.values(): + links = service.get("links", []) + if service_name in links: + links.remove(service_name) + links.append(hostname) + + del service_def["hostname"] if __name__ == "__main__": From fde887c7a05bd763b1568db42875c3091b34cdc9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 16:57:02 -0400 Subject: [PATCH 5/8] Baby steps toward a working Kubernetes setup. --- compose/galaxy-slurm/Dockerfile | 1 + compose/k8/convert.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 2fede778d..bb2684805 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -38,6 +38,7 @@ ENV GALAXY_DIR=/export/galaxy-central \ SLURM_GID=1450 \ SLURM_PARTITION_NAME=work \ SLURM_CLUSTER_NAME=Cluster \ + SLURM_CONTROL_MACHINE=galaxy-slurm \ SLURMD_AUTOSTART=True \ SLURMCTLD_AUTOSTART=True \ SLURM_CONF_PATH=/export/slurm.conf \ diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 17bf17113..81495cfb8 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -29,7 +29,7 @@ def main(): def _hack_for_kompose(raw_compose_def): - ftp_ports = raw_compose_def["services"]["proftpd"]["ports"] + ftp_ports = raw_compose_def["services"]["galaxy-proftpd"]["ports"] del ftp_ports[2] for i in range(10): # Replace "30000-30010:30000-30010" with individual entries. From 191c59f591dc25492016fb85d519c4c58366eff6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 29 Apr 2017 17:26:21 -0400 Subject: [PATCH 6/8] Adapt configure_slurm.py to allow Kubernetes style naming. Relevant pieces of SLURM docs: ``` Node names can have up to three name specifications: NodeName is the name used by all Slurm tools when referring to the node, NodeAddr is the name or IP address Slurm uses to communicate with the node, and NodeHostname is the name returned by the command /bin/hostname -s. Only NodeName is required (the others default to the same name), although supporting all three parameters provides complete control over naming and addressing the nodes. See the slurm.conf man page for details on all configuration parameters. ``` ``` ControlAddr Name that ControlMachine should be referred to in establishing a communications path. This name will be used as an argument to the gethostbyname() function for identification. For example, "elx0000" might be used to designate the Ethernet address for node "lx0000". By default the ControlAddr will be identical in value to ControlMachine. ``` --- compose/galaxy-slurm/Dockerfile | 3 ++- compose/galaxy-slurm/configure_slurm.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index bb2684805..037bcfed5 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -38,7 +38,8 @@ ENV GALAXY_DIR=/export/galaxy-central \ SLURM_GID=1450 \ SLURM_PARTITION_NAME=work \ SLURM_CLUSTER_NAME=Cluster \ - SLURM_CONTROL_MACHINE=galaxy-slurm \ + SLURM_CONTROL_ADDR=galaxy-slurm \ + SLURM_NODE_NAME=galaxy-slurm \ SLURMD_AUTOSTART=True \ SLURMCTLD_AUTOSTART=True \ SLURM_CONF_PATH=/export/slurm.conf \ diff --git a/compose/galaxy-slurm/configure_slurm.py b/compose/galaxy-slurm/configure_slurm.py index 81859bed2..307c6dfd8 100644 --- a/compose/galaxy-slurm/configure_slurm.py +++ b/compose/galaxy-slurm/configure_slurm.py @@ -76,7 +76,7 @@ #SlurmctldLogFile= SlurmdDebug=3 #SlurmdLogFile= -NodeName=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN +NodeName=$node_name NodeAddr=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN PartitionName=$partition_name Nodes=$nodes Default=YES MaxTime=INFINITE State=UP Shared=YES ''' @@ -91,9 +91,11 @@ def main(): hostname = gethostname() + node_name = environ.get('SLURM_NODE_NAME', hostname) template_params = { "hostname": hostname, - "nodes": ",".join(environ.get('SLURM_NODES', hostname).split(',')), + "node_name": node_name, + "nodes": ",".join(environ.get('SLURM_NODES', node_name).split(',')), "cluster_name": environ.get('SLURM_CLUSTER_NAME', 'Cluster'), "control_machine": environ.get('SLURM_CONTROL_MACHINE', hostname), "user": environ.get('SLURM_USER_NAME', '{{ galaxy_user_name }}'), @@ -102,6 +104,9 @@ def main(): "memory": environ.get("SLURM_MEMORY", int(mem / (1024 * 1024))) } config_contents = Template(SLURM_CONFIG_TEMPLATE).substitute(template_params) + control_addr = environ.get('SLURM_CONTROL_ADDR', None) + if control_addr: + config_contents = config_contents.replace("#ControlAddr=", "ControlAddr=%s" % control_addr) open("/etc/slurm-llnl/slurm.conf", "w").write(config_contents) if __name__ == "__main__": From 878059df8016200b5287ba722dd0dbe1dc11ef4d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 29 Apr 2017 17:52:33 -0400 Subject: [PATCH 7/8] Last commit worked for slurmctld but not slurmd, fix that. Add TODO to tweak NodeAddr in the Kubernetes case for Kubernetes case. --- compose/galaxy-slurm/configure_slurm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/configure_slurm.py b/compose/galaxy-slurm/configure_slurm.py index 307c6dfd8..9e389496d 100644 --- a/compose/galaxy-slurm/configure_slurm.py +++ b/compose/galaxy-slurm/configure_slurm.py @@ -76,7 +76,7 @@ #SlurmctldLogFile= SlurmdDebug=3 #SlurmdLogFile= -NodeName=$node_name NodeAddr=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN +NodeName=$node_name NodeAddr=$hostname NodeHostname=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN PartitionName=$partition_name Nodes=$nodes Default=YES MaxTime=INFINITE State=UP Shared=YES ''' @@ -107,6 +107,7 @@ def main(): control_addr = environ.get('SLURM_CONTROL_ADDR', None) if control_addr: config_contents = config_contents.replace("#ControlAddr=", "ControlAddr=%s" % control_addr) + # TODO: NodeAddr should probably galaxy-slurm in the Kubernetes case. open("/etc/slurm-llnl/slurm.conf", "w").write(config_contents) if __name__ == "__main__": From 036f0b08e384830177240be49306d641e6ce5382 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 5 May 2017 13:57:06 -0400 Subject: [PATCH 8/8] Enable a Kubernetes job execution. --- compose/docker-compose.yml | 9 +++++++++ compose/galaxy-init/Dockerfile | 7 ++++++- compose/k8/Makefile | 10 ++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 compose/k8/Makefile diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index eb378415b..093aee1ee 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -100,6 +100,15 @@ services: # Configurate admin and master api key. Can be overridden in galaxy.ini - GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org - GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 + - GALAXY_CONFIG_DATABASE_AUTO_MIGRATE=True + - GALAXY_DESTINATIONS_DEFAULT=docker_dispatch + - GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8_default + - GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container + - GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml + # Hack for pykube - https://github.com/kelproject/pykube/issues/29 + - PYKUBE_KUBERNETES_SERVICE_HOST=kubernetes + # Just for testing... + - GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE=/export/galaxy-central/test/functional/tools/samples_tool_conf.xml container_name: galaxy-web ports: - "8080:80" # nginx diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 8b77a0247..c1deff485 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -40,11 +40,16 @@ RUN ansible-playbook /ansible/provision.yml \ --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \ --extra-vars galaxy_extras_config_condor=True \ --extra-vars galaxy_extras_config_condor_docker=True \ + --extra-vars galaxy_extras_config_k8_jobs=True \ --extra-vars galaxy_extras_config_rabbitmq=False \ --extra-vars nginx_upload_store_path=/export/nginx_upload_store \ --extra-vars nginx_welcome_location=$NGINX_WELCOME_LOCATION \ --extra-vars nginx_welcome_path=$NGINX_WELCOME_PATH \ - --tags=ie,pbs,slurm,uwsgi,metrics -c local && \ + --extra-vars galaxy_extras_config_container_resolution=True \ + --extra-vars container_resolution_explicit=True \ + --extra-vars container_resolution_cached_mulled=False \ + --extra-vars container_resolution_build_mulled=False \ + --tags=ie,pbs,slurm,uwsgi,metrics,k8 -c local && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/compose/k8/Makefile b/compose/k8/Makefile new file mode 100644 index 000000000..797636580 --- /dev/null +++ b/compose/k8/Makefile @@ -0,0 +1,10 @@ + + +convert: + python convert.py + +down: + kompose -f docker-compose-for-kompose.yml down + +up: + kompose -f docker-compose-for-kompose.yml up \ No newline at end of file