[WIP] Enable K8 Job Execution

compose/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   # proftpd container
-  proftpd:
+  galaxy-proftpd:
 #    build: galaxy-proftpd
     image: quay.io/galaxy/proftpd:compose
     environment:
@@ -15,8 +15,8 @@ services:
         - proftpd_passive_port_high=30010
     container_name: galaxy-proftpd
     hostname: galaxy-proftpd
-    volumes:
-        - ./galaxy-storage/ftp:/export/ftp
+    volumes_from:
+        - galaxy-web
     expose:
         - 21
         - 22
@@ -25,11 +25,13 @@ services:
         - "8022:22"
         - "30000-30010:30000-30010"
     restart: always
+    labels: 
+        kompose.service.type: nodeport
 
   # Using the official postgres image. This needs to be populated by calling
   # docker-compose run galaxy install_db.sh
   # on first run
-  postgres:
+  galaxy-postgres:
 #    image: postgres
 # This comes with an initialization to quickly populate the database on first start
 #    build: galaxy-postgres
@@ -50,21 +52,23 @@ services:
     ports:
         - "5050:5050"
     links:
-        - postgres
+        - galaxy-postgres
     volumes:
         - ./pgadmin:/pgadmin
     restart: unless-stopped
+    labels: 
+        kompose.service.type: nodeport
 
 
   # slurm container
-  slurm:
+  galaxy-slurm:
 #    build: galaxy-slurm
     image: quay.io/galaxy/slurm:compose
     environment: {}
     container_name: galaxy-slurm
     hostname: galaxy-slurm
-    volumes:
-        - ./galaxy-storage/:/export
+    volumes_from:
+        - galaxy-web
     restart: always
 
   # This container initializes the galaxy export.
@@ -73,10 +77,8 @@ services:
     image: quay.io/bgruening/galaxy-init:compose
     container_name: galaxy-init
     hostname: galaxy-init
-    volumes:
-        # This is the directory where all your files from Galaxy will be stored
-        # on your host system
-        - ./galaxy-storage/:/export/
+    volumes_from:
+        - galaxy-web
 
   # This container provides the galaxy uwsgi webhandlers, job handlers, nginx
   galaxy-web:
@@ -98,15 +100,25 @@ services:
         # Configurate admin and master api key. Can be overridden in galaxy.ini
         - [email protected]
         - GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3
+        - GALAXY_CONFIG_DATABASE_AUTO_MIGRATE=True
+        - GALAXY_DESTINATIONS_DEFAULT=docker_dispatch
+        - GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8_default
+        - GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container
+        - GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml
+        # Hack for pykube - https://github.com/kelproject/pykube/issues/29
+        - PYKUBE_KUBERNETES_SERVICE_HOST=kubernetes
+        # Just for testing...
+        - GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE=/export/galaxy-central/test/functional/tools/samples_tool_conf.xml
     container_name: galaxy-web
-    hostname: galaxy
     ports:
         - "8080:80" # nginx
     volumes:
         # This is the directory where all your files from Galaxy will be stored
         # on your host system
         - ./galaxy-storage/:/export/
     privileged: True
+    labels: 
+        kompose.service.type: nodeport
 
   # Use the monolith galaxy container instead
 #  galaxy:

compose/galaxy-init/Dockerfile

@@ -40,11 +40,16 @@ RUN ansible-playbook /ansible/provision.yml \
     --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \
     --extra-vars galaxy_extras_config_condor=True \
     --extra-vars galaxy_extras_config_condor_docker=True \
+    --extra-vars galaxy_extras_config_k8_jobs=True \
     --extra-vars galaxy_extras_config_rabbitmq=False \
     --extra-vars nginx_upload_store_path=/export/nginx_upload_store \
     --extra-vars nginx_welcome_location=$NGINX_WELCOME_LOCATION \
     --extra-vars nginx_welcome_path=$NGINX_WELCOME_PATH \
-    --tags=ie,pbs,slurm,uwsgi,metrics -c local && \
+    --extra-vars galaxy_extras_config_container_resolution=True \
+    --extra-vars container_resolution_explicit=True \
+    --extra-vars container_resolution_cached_mulled=False \
+    --extra-vars container_resolution_build_mulled=False \
+    --tags=ie,pbs,slurm,uwsgi,metrics,k8 -c local && \
     apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 

compose/galaxy-slurm/Dockerfile

@@ -38,6 +38,8 @@ ENV GALAXY_DIR=/export/galaxy-central \
     SLURM_GID=1450 \
     SLURM_PARTITION_NAME=work \
     SLURM_CLUSTER_NAME=Cluster \
+    SLURM_CONTROL_ADDR=galaxy-slurm \
+    SLURM_NODE_NAME=galaxy-slurm \
     SLURMD_AUTOSTART=True \
     SLURMCTLD_AUTOSTART=True \
     SLURM_CONF_PATH=/export/slurm.conf \

compose/galaxy-slurm/configure_slurm.py

@@ -76,7 +76,7 @@
 #SlurmctldLogFile=
 SlurmdDebug=3
 #SlurmdLogFile=
-NodeName=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN
+NodeName=$node_name NodeAddr=$hostname NodeHostname=$hostname CPUs=$cpus RealMemory=$memory State=UNKNOWN
 PartitionName=$partition_name Nodes=$nodes Default=YES MaxTime=INFINITE State=UP Shared=YES
 '''
 
@@ -91,9 +91,11 @@
 
 def main():
     hostname = gethostname()
+    node_name = environ.get('SLURM_NODE_NAME', hostname)
     template_params = {
         "hostname": hostname,
-        "nodes": ",".join(environ.get('SLURM_NODES', hostname).split(',')),
+        "node_name": node_name,
+        "nodes": ",".join(environ.get('SLURM_NODES', node_name).split(',')),
         "cluster_name": environ.get('SLURM_CLUSTER_NAME', 'Cluster'),
         "control_machine": environ.get('SLURM_CONTROL_MACHINE', hostname),
         "user": environ.get('SLURM_USER_NAME', '{{ galaxy_user_name }}'),
@@ -102,6 +104,10 @@ def main():
         "memory": environ.get("SLURM_MEMORY", int(mem / (1024 * 1024)))
     }
     config_contents = Template(SLURM_CONFIG_TEMPLATE).substitute(template_params)
+    control_addr = environ.get('SLURM_CONTROL_ADDR', None)
+    if control_addr:
+        config_contents = config_contents.replace("#ControlAddr=", "ControlAddr=%s" % control_addr)
+    # TODO: NodeAddr should probably galaxy-slurm in the Kubernetes case.
     open("/etc/slurm-llnl/slurm.conf", "w").write(config_contents)
 
 if __name__ == "__main__":

compose/k8/Makefile

@@ -0,0 +1,10 @@
+
+
+convert:
+	python convert.py
+
+down:
+	kompose -f docker-compose-for-kompose.yml down
+
+up:
+	kompose -f docker-compose-for-kompose.yml up

compose/k8/convert.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+# Install kompose (e.g. brew install kompose).
+# Install minikube (e.g. brew install minikube).
+# $ minikube start  # Build a k8 cluster in a VM.
+# $ eval $(minikube docker-env) # Target Docker commands at the VM's Docker host.
+# $ cd ..; bash buildlocal.sh; cd k8 # Build Docker containers required for k8 setup.
+# $ python convert.py  # Execute this wrapper around kompose to build native k8 artifacts.
+# $ kompose -f docker-compose-for-kompose.yml up
+
+import os
+import subprocess
+import yaml
+
+DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+COMPOSE_TARGET = os.path.abspath(os.path.join(DIRECTORY, "..", "docker-compose.yml"))
+KOMPOSE_TARGET = os.path.join(DIRECTORY, "docker-compose-for-kompose.yml")
+
+
+def main():
+    with open(COMPOSE_TARGET, "r") as f:
+        raw_compose_def = yaml.load(f)
+
+    _hack_for_kompose(raw_compose_def)
+    with open(KOMPOSE_TARGET, "w") as f:
+        yaml.dump(raw_compose_def, f)
+
+    subprocess.check_call(["kompose", "-f", KOMPOSE_TARGET, "convert"])
+
+
+def _hack_for_kompose(raw_compose_def):
+    ftp_ports = raw_compose_def["services"]["galaxy-proftpd"]["ports"]
+    del ftp_ports[2]
+    for i in range(10):
+        # Replace "30000-30010:30000-30010" with individual entries.
+        ftp_ports.append("%d:%d" % (30000 + i, 30000 + i))
+
+    # pgadmin can run without volumes and gets permission errors if not started this way in
+    # minikube.
+    del raw_compose_def["services"]["pgadmin4"]["volumes"]
+
+    services = raw_compose_def["services"]
+    for service_name in list(services.keys()):
+        service_def = services[service_name]
+        if "hostname" in service_def:
+            hostname = service_def["hostname"]
+            # These need to be same for Kompose it seems
+            if hostname != service_name:
+                services[hostname] = service_def
+                del services[service_name]
+            for service in services.values():
+                links = service.get("links", [])
+                if service_name in links:
+                    links.remove(service_name)
+                    links.append(hostname)
+
+            del service_def["hostname"]
+
+
+if __name__ == "__main__":
+    main()