Add helm and Jenkins

.gitignore

@@ -30,6 +30,7 @@ htmlcov/
 .tox/
 .coverage
 .cache
+.local/
 nosetests.xml
 coverage.xml
 swagger_client

Jenkinsfile

@@ -1,6 +1,6 @@
 #!groovy
 
-@Library('cdis-jenkins-lib@master') _
-
-testPipeline { 
-}
+library identifier: "jenkins-lib@develop"
+dockerPipeline{
+    // testBranches = '(develop|master|release.*)'
+}

docker-build-jenkins.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+
+rm -rf build/src
+mkdir -p build/src
+
+pip install \
+    --user \
+    --src build/src \
+    --no-deps \
+    $(grep '^-e.*egg=' requirements.txt)
+
+cp requirements.txt build/requirements.txt
+sed -i.tmp \
+  -e 's;^-e.*egg=;build/src/;' \
+build/requirements.txt
+
+
+# avoid installing git
+COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >indexd/index/version_data.py
+VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>indexd/index/version_data.py
+
+# Build and push docker image.
+docker build -t ${REGISTRY}/${REPO}:${GIT_TAG} . --build-arg http_proxy=https://cloud-proxy:3128 --build-arg https_proxy=https://cloud-proxy:3128
+docker push ${REGISTRY}/${REPO}:${GIT_TAG}

docker-compose-ci.yaml

@@ -0,0 +1,23 @@
+version: "3.3"
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment: 
+      POSTGRES_HOST_AUTH_METHOD: trust
+
+  app:
+    image: quay.io/ncigdc/jenkins-agent:multipython
+    environment:
+      SWAGGER_URL: "https://nexus.osdc.io/repository/maven-public/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar"
+      SSH_AUTH_SOCK: $SSH_AUTH_SOCK
+      HTTPS_PROXY: http://cloud-proxy:3128
+      HTTP_PROXY: http://cloud-proxy:3128
+      NO_PROXY: localhost
+    volumes:
+    - .:/home/jenkins
+    - $SSH_AUTH_SOCK:$SSH_AUTH_SOCK
+    command: bash -c "./wait-for-it.sh localhost:5432 -t 120 && tox --recreate"
+    network_mode: "service:postgres"
+    depends_on: # other containers it needs in order to run
+      - postgres

helm/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

helm/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: indexd
+description: A Helm chart to package your application container for Kubernetes
+# This will be updated with the release tag in the CI/CD pipeline before publishing. This has to be a valid semver for
+# the linter to accept.
+version: 0.0.1-replace
+home: https://github.com/gruntwork-io/helm-kubernetes-services
+maintainers:
+  - name: Center for Translational Data Science
+    email: [email protected]
+    url: https://ctds.uchicago.edu

helm/files/create_indexd_user

@@ -0,0 +1,6 @@
+#!/bin/bash
+# these are set in supervisor's environment by upstart (presumably), but we don't want them so we unset them here
+unset http_proxy
+unset https_proxy
+
+exec /usr/bin/python /indexd/bin/index_admin.py create --username $INDEXD_USER --password $INDEXD_PASSWORD

helm/files/index_table_migration

@@ -0,0 +1,9 @@
+#!/bin/bash
+# these are set in supervisor's environment by upstart (presumably),
+# but we don't want them so we unset them here
+unset http_proxy
+unset https_proxy
+
+PYTHON=/usr/bin/python
+SCRIPT=/indexd/bin/migrate_index.py
+exec $PYTHON $SCRIPT

helm/files/indexd.conf

@@ -0,0 +1,24 @@
+<VirtualHost *:80>
+    ServerName indexd.{{.Release.Namespace}}.svc.cluster.local
+
+    Include /etc/apache2/sites-available/logs.conf
+    LogLevel warn
+    ErrorLog /dev/stderr
+    CustomLog /dev/stdout json
+
+    WSGIProcessGroup indexd
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIScriptReloading On
+    WSGIPassAuthorization On
+    WSGIDaemonProcess indexd processes=4 threads=16 home=/var/www/indexd/ python-path=/usr/local/lib/python2.7/dist-packages/:/var/www/indexd
+    WSGIScriptAlias / /var/www/indexd/wsgi.py
+
+    DocumentRoot /var/www/indexd/
+    <Directory /var/www/indexd/>
+        Order Deny,Allow
+        Allow from all
+        WSGIApplicationGroup %{GLOBAL}
+        Options +ExecCGI
+        Header set Access-Control-Allow-Origin "*"
+    </Directory>
+</VirtualHost>

helm/files/logs.conf

@@ -0,0 +1,2 @@
+LogFormat "{ \"time\":\"%{%Y-%m-%dT%T}t.%{usec_frac}t%{%z}t\", \"remoteIP\":\"%a\", \"host\":\"%V\", \"requestPath\":\"%U\", \"query\":\"%q\", \"method\":\"%m\", \"status\":\"%>s\", \"userAgent\":\"%{User-agent}i\", \"referer\":\"%{Referer}i\", \"timetaken\":\"%D\"  }" json
+ErrorLogFormat "{ \"time\":\"%{cu}tZ\", \"severity\" : \"%l\", \"message\" : \"%M\"}"

helm/templates/NOTES.txt

@@ -0,0 +1,43 @@
+
+Check the status of your Deployment by running this comamnd:
+
+kubectl get deployments --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "k8s-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
+
+
+List the related Pods with the following command:
+
+kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "k8s-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
+
+
+Use the following command to view information about the Service:
+
+kubectl get services --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "k8s-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
+
+
+{{ if .Values.containerPorts -}}
+{{- $serviceType := .Values.service.type | default "ClusterIP" -}}
+Get the application URL by running these commands:
+
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }}
+{{- end }}
+{{- else if contains "NodePort" $serviceType }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "k8s-service.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" $serviceType }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ include "k8s-service.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "k8s-service.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" $serviceType }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "k8s-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  {{- range $portName, $portSpec := .Values.containerPorts }}
+  {{- if not $portSpec.disabled }}
+  echo "Visit http://127.0.0.1:80{{ $portSpec.port }} to use your application container serving port {{ $portName }}"
+  kubectl port-forward $POD_NAME 80{{ $portSpec.port }}:{{ $portSpec.port }}
+  {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}

helm/templates/_helpers.tpl

@@ -0,0 +1,71 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "k8s-service.name" -}}
+  {{- .Values.applicationName | required "applicationName is required" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "k8s-service.fullname" -}}
+  {{- $name := required "applicationName is required" .Values.applicationName -}}
+  {{- if contains $name .Release.Name -}}
+    {{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+  {{- else -}}
+    {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "k8s-service.chart" -}}
+  {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Convert octal to decimal (e.g 644 => 420). For file permission modes, many people are more familiar with octal notation.
+However, due to yaml/json limitations, all the Kubernetes resources require file modes to be reported in decimal.
+*/}}
+{{- define "k8s-service.fileModeOctalToDecimal" -}}
+  {{- $digits := splitList "" (toString .) -}}
+
+  {{/* Make sure there are exactly 3 digits */}}
+  {{- if ne (len $digits) 3 -}}
+    {{- fail (printf "File mode octal expects exactly 3 digits: %s" .) -}}
+  {{- end -}}
+
+  {{/* Go Templates do not support variable updating, so we simulate it using dictionaries */}}
+  {{- $accumulator := dict "res" 0 -}}
+  {{- range $idx, $digit := $digits -}}
+    {{- $digitI := atoi $digit -}}
+
+    {{/* atoi from sprig swallows conversion errors, so we double check to make sure it is a valid conversion */}}
+    {{- if and (eq $digitI 0) (ne $digit "0") -}}
+      {{- fail (printf "Digit %d of %s is not a number: %s" $idx . $digit) -}}
+    {{- end -}}
+
+    {{/*  Make sure each digit is less than 8 */}}
+    {{- if ge $digitI 8 -}}
+      {{- fail (printf "%s is not a valid octal digit" $digit) -}}
+    {{- end -}}
+
+    {{/* Since we don't have math.Pow, we hard code */}}
+    {{- if eq $idx 0 -}}
+      {{/* 8^2 */}}
+      {{- $_ := set $accumulator "res" (add (index $accumulator "res") (mul $digitI 64)) -}}
+    {{- else if eq $idx 1 -}}
+      {{/* 8^1 */}}
+      {{- $_ := set $accumulator "res" (add (index $accumulator "res") (mul $digitI 8)) -}}
+    {{- else -}}
+      {{/* 8^0 */}}
+      {{- $_ := set $accumulator "res" (add (index $accumulator "res") (mul $digitI 1)) -}}
+    {{- end -}}
+  {{- end -}}
+  {{- "res" | index $accumulator | toString | printf -}}
+{{- end -}}

helm/templates/configmap.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: indexd-conf
+    labels:
+        app.kubernetes.io/name: {{ include "k8s-service.name" . }}
+        helm.sh/chart: {{ include "k8s-service.chart" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+{{ (tpl (.Files.Glob "files/indexd.conf").AsConfig .) | indent 4 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: indexd-initialization
+    labels:
+        app.kubernetes.io/name: {{ include "k8s-service.name" . }}
+        helm.sh/chart: {{ include "k8s-service.chart" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+{{ (.Files.Glob "files/create_indexd_user").AsConfig | indent 4 }}
+{{ (.Files.Glob "files/index_table_migration").AsConfig | indent 4 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: indexd-local-settings
+    labels:
+        app.kubernetes.io/name: {{ include "k8s-service.name" . }}
+        helm.sh/chart: {{ include "k8s-service.chart" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+{{ (.Files.Glob "files/local_settings.py").AsConfig | indent 4 }}