From 254a0d5ada7fc575bbe4913366458ecf45be25de Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 25 Apr 2017 15:49:53 -0400 Subject: [PATCH 1/6] Add instructions and script for converting compose to k8. --- compose/k8/convert.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 compose/k8/convert.py diff --git a/compose/k8/convert.py b/compose/k8/convert.py new file mode 100644 index 000000000..54e9659ed --- /dev/null +++ b/compose/k8/convert.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Install kompose (e.g. brew install kompose). +# Install minikube (e.g. brew install minikube). +# $ minikube start # Build a k8 cluster in a VM. +# $ eval $(minikube docker-env) # Target Docker commands at the VM's Docker host. +# $ cd ..; bash buildlocal.sh; cd k8 # Build Docker containers required for k8 setup. +# $ python convert.py # Execute this wrapper around kompose to build native k8 artifacts. +# $ kompose -f docker-compose-for-kompose.yml up + +import os +import subprocess +import yaml + +DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +COMPOSE_TARGET = os.path.abspath(os.path.join(DIRECTORY, "..", "docker-compose.yml")) +KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml") + + +def main(): + with open(COMPOSE_TARGET, "r") as f: + raw_compose_def = yaml.load(f) + + _hack_for_kompose(raw_compose_def) + with open(KOMPOSE_TARGET, "w") as f: + yaml.dump(raw_compose_def, f) + + subprocess.check_call(["kompose", "-f", KOMPOSE_TARGET, "convert"]) + + +def _hack_for_kompose(raw_compose_def): + ftp_ports = raw_compose_def["services"]["proftpd"]["ports"] + del ftp_ports[2] + for i in range(10): + # Replace "30000-30010:30000-30010" with individual entries. + ftp_ports.append("%d:%d" % (30000 + i, 30000 + i)) + + +if __name__ == "__main__": + main() From 93788a3b07ee99192f2007ba01ea6aa70d541e32 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 13:55:27 -0400 Subject: [PATCH 2/6] More progress toward k8 compat. for compose setup. - Use volumes_from instead of volumes so kompose can match up paths properly. - Mount pgadmin4 without volumes when using kompose for now. - Annotate the services that should be exposed externally with kompose labels in the source compose files. --- compose/docker-compose.yml | 20 ++++++++++++-------- compose/k8/convert.py | 7 ++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 9568172ef..4cce0acc1 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -15,8 +15,8 @@ services: - proftpd_passive_port_high=30010 container_name: galaxy-proftpd hostname: galaxy-proftpd - volumes: - - ./galaxy-storage/ftp:/export/ftp + volumes_from: + - galaxy-web expose: - 21 - 22 @@ -25,6 +25,8 @@ services: - "8022:22" - "30000-30010:30000-30010" restart: always + labels: + kompose.service.type: nodeport # Using the official postgres image. This needs to be populated by calling # docker-compose run galaxy install_db.sh @@ -54,6 +56,8 @@ services: volumes: - ./pgadmin:/pgadmin restart: unless-stopped + labels: + kompose.service.type: nodeport # slurm container @@ -63,8 +67,8 @@ services: environment: {} container_name: galaxy-slurm hostname: galaxy-slurm - volumes: - - ./galaxy-storage/:/export + volumes_from: + - galaxy-web restart: always # This container initializes the galaxy export. @@ -73,10 +77,8 @@ services: image: quay.io/bgruening/galaxy-init:compose container_name: galaxy-init hostname: galaxy-init - volumes: - # This is the directory where all your files from Galaxy will be stored - # on your host system - - ./galaxy-storage/:/export/ + volumes_from: + - galaxy-web # This container provides the galaxy uwsgi webhandlers, job handlers, nginx galaxy-web: @@ -107,6 +109,8 @@ services: # on your host system - ./galaxy-storage/:/export/ privileged: True + labels: + kompose.service.type: nodeport # Use the monolith galaxy container instead # galaxy: diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 54e9659ed..907bf879c 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -17,10 +17,11 @@ KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml") + def main(): with open(COMPOSE_TARGET, "r") as f: raw_compose_def = yaml.load(f) - + _hack_for_kompose(raw_compose_def) with open(KOMPOSE_TARGET, "w") as f: yaml.dump(raw_compose_def, f) @@ -35,6 +36,10 @@ def _hack_for_kompose(raw_compose_def): # Replace "30000-30010:30000-30010" with individual entries. ftp_ports.append("%d:%d" % (30000 + i, 30000 + i)) + # pgadmin can run without volumes and gets permission errors if not started this way in + # minikube. + del raw_compose_def["services"]["pgadmin4"]["volumes"] + if __name__ == "__main__": main() From b805504b3d181175d83c83d333a589ff31d33f78 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 14:37:07 -0400 Subject: [PATCH 3/6] Kompose does not respect "hostname" - so rewrite this to be service name when specified. --- compose/k8/convert.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 907bf879c..099a8a6af 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -17,7 +17,6 @@ KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml") - def main(): with open(COMPOSE_TARGET, "r") as f: raw_compose_def = yaml.load(f) @@ -40,6 +39,17 @@ def _hack_for_kompose(raw_compose_def): # minikube. del raw_compose_def["services"]["pgadmin4"]["volumes"] + services = raw_compose_def["services"] + for service_name in list(services.keys()): + service_def = services[service_name] + if "hostname" in service_def: + hostname = service_def["hostname"] + # These need to be same for Kompose it seems + if hostname != service_name: + raw_compose_def[hostname] = service_def + del services[service_name] + del service_def["hostname"] + if __name__ == "__main__": main() From a7400b8e414719f24f68e92aa9135ebd50230af1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 15:24:33 -0400 Subject: [PATCH 4/6] Switch docker-compose.yml to match up service and hostnames. This is helpful for Kompose. --- compose/docker-compose.yml | 9 ++++----- compose/k8/convert.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 4cce0acc1..eb378415b 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: # proftpd container - proftpd: + galaxy-proftpd: # build: galaxy-proftpd image: quay.io/galaxy/proftpd:compose environment: @@ -31,7 +31,7 @@ services: # Using the official postgres image. This needs to be populated by calling # docker-compose run galaxy install_db.sh # on first run - postgres: + galaxy-postgres: # image: postgres # This comes with an initialization to quickly populate the database on first start # build: galaxy-postgres @@ -52,7 +52,7 @@ services: ports: - "5050:5050" links: - - postgres + - galaxy-postgres volumes: - ./pgadmin:/pgadmin restart: unless-stopped @@ -61,7 +61,7 @@ services: # slurm container - slurm: + galaxy-slurm: # build: galaxy-slurm image: quay.io/galaxy/slurm:compose environment: {} @@ -101,7 +101,6 @@ services: - GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org - GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 container_name: galaxy-web - hostname: galaxy ports: - "8080:80" # nginx volumes: diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 099a8a6af..17bf17113 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -46,9 +46,15 @@ def _hack_for_kompose(raw_compose_def): hostname = service_def["hostname"] # These need to be same for Kompose it seems if hostname != service_name: - raw_compose_def[hostname] = service_def + services[hostname] = service_def del services[service_name] - del service_def["hostname"] + for service in services.values(): + links = service.get("links", []) + if service_name in links: + links.remove(service_name) + links.append(hostname) + + del service_def["hostname"] if __name__ == "__main__": From fde887c7a05bd763b1568db42875c3091b34cdc9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Apr 2017 16:57:02 -0400 Subject: [PATCH 5/6] Baby steps toward a working Kubernetes setup. --- compose/galaxy-slurm/Dockerfile | 1 + compose/k8/convert.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 2fede778d..bb2684805 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -38,6 +38,7 @@ ENV GALAXY_DIR=/export/galaxy-central \ SLURM_GID=1450 \ SLURM_PARTITION_NAME=work \ SLURM_CLUSTER_NAME=Cluster \ + SLURM_CONTROL_MACHINE=galaxy-slurm \ SLURMD_AUTOSTART=True \ SLURMCTLD_AUTOSTART=True \ SLURM_CONF_PATH=/export/slurm.conf \ diff --git a/compose/k8/convert.py b/compose/k8/convert.py index 17bf17113..81495cfb8 100644 --- a/compose/k8/convert.py +++ b/compose/k8/convert.py @@ -29,7 +29,7 @@ def main(): def _hack_for_kompose(raw_compose_def): - ftp_ports = raw_compose_def["services"]["proftpd"]["ports"] + ftp_ports = raw_compose_def["services"]["galaxy-proftpd"]["ports"] del ftp_ports[2] for i in range(10): # Replace "30000-30010:30000-30010" with individual entries. From 191c59f591dc25492016fb85d519c4c58366eff6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 29 Apr 2017 17:26:21 -0400 Subject: [PATCH 6/6] Adapt configure_slurm.py to allow Kubernetes style naming. Relevant pieces of SLURM docs: ``` Node names can have up to three name specifications: NodeName is the name used by all Slurm tools when referring to the node, NodeAddr is the name or IP address Slurm uses to communicate with the node, and NodeHostname is the name returned by the command /bin/hostname -s. Only NodeName is required (the others default to the same name), although supporting all three parameters provides complete control over naming and addressing the nodes. See the slurm.conf man page for details on all configuration parameters. ``` ``` ControlAddr Name that ControlMachine should be referred to in establishing a communications path. This name will be used as an argument to the gethostbyname() function for identification. For example, "elx0000" might be used to designate the Ethernet address for node "lx0000". By default the ControlAddr will be identical in value to ControlMachine. ``` --- compose/galaxy-slurm/Dockerfile | 3 ++- compose/galaxy-slurm/configure_slurm.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index bb2684805..037bcfed5 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -38,7 +38,8 @@ ENV GALAXY_DIR=/export/galaxy-central \ SLURM_GID=1450 \ SLURM_PARTITION_NAME=work \ SLURM_CLUSTER_NAME=Cluster \ - SLURM_CONTROL_MACHINE=galaxy-slurm \ + SLURM_CONTROL_ADDR=galaxy-slurm \ + SLURM_NODE_NAME=galaxy-slurm \ SLURMD_AUTOSTART=True \ SLURMCTLD_AUTOSTART=True \ SLURM_CONF_PATH=/export/slurm.conf \ diff --git a/compose/galaxy-slurm/configure_slurm.py b/compose/galaxy-slurm/configure_slurm.py index 81859bed2..307c6dfd8 100644 --- a/compose/galaxy-slurm/configure_slurm.py +++ b/compose/galaxy-slurm/configure_slurm.py @@ -76,7 +76,7 @@ #SlurmctldLogFile= SlurmdDebug=3 #SlurmdLogFile= -NodeName=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN +NodeName=$node_name NodeAddr=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN PartitionName=$partition_name Nodes=$nodes Default=YES MaxTime=INFINITE State=UP Shared=YES ''' @@ -91,9 +91,11 @@ def main(): hostname = gethostname() + node_name = environ.get('SLURM_NODE_NAME', hostname) template_params = { "hostname": hostname, - "nodes": ",".join(environ.get('SLURM_NODES', hostname).split(',')), + "node_name": node_name, + "nodes": ",".join(environ.get('SLURM_NODES', node_name).split(',')), "cluster_name": environ.get('SLURM_CLUSTER_NAME', 'Cluster'), "control_machine": environ.get('SLURM_CONTROL_MACHINE', hostname), "user": environ.get('SLURM_USER_NAME', '{{ galaxy_user_name }}'), @@ -102,6 +104,9 @@ def main(): "memory": environ.get("SLURM_MEMORY", int(mem / (1024 * 1024))) } config_contents = Template(SLURM_CONFIG_TEMPLATE).substitute(template_params) + control_addr = environ.get('SLURM_CONTROL_ADDR', None) + if control_addr: + config_contents = config_contents.replace("#ControlAddr=", "ControlAddr=%s" % control_addr) open("/etc/slurm-llnl/slurm.conf", "w").write(config_contents) if __name__ == "__main__":