add partial charts

distributed/charts/autotm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

distributed/charts/autotm/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: autotm
+description: A Helm chart for AutoTM
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.0.0"

distributed/charts/autotm/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "st-workspace.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "st-workspace.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "st-workspace.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "st-workspace.labels" -}}
+helm.sh/chart: {{ include "st-workspace.chart" . }}
+{{ include "st-workspace.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "st-workspace.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "st-workspace.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "st-workspace.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "st-workspace.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

distributed/charts/autotm/templates/configmaps.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rabbitmq-config
+data:
+  consumer-settings.conf: |
+    ## Consumer timeout
+    ## If a message delivered to a consumer has not been acknowledge before this timer
+    ## triggers the channel will be force closed by the broker. This ensure that
+    ## faultly consumers that never ack will not hold on to messages indefinitely.
+    ##
+    consumer_timeout = 1800000
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fitness-worker-config
+data:
+  datasets-config.yaml: |
+    {{ datasets_config_content | indent( width=4, first=False) }}

distributed/charts/autotm/templates/deployments.yaml

@@ -0,0 +1,151 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rabbitmq
+  template:
+    metadata:
+      annotations:
+        "sidecar.istio.io/inject": "false"
+      labels:
+        app: rabbitmq
+    spec:
+      volumes:
+        - name: config-volume
+          configMap:
+            name: rabbitmq-config
+      containers:
+      - name: rabbitmq
+        image: node2.bdcl:5000/rabbitmq:3.8-management-alpine
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 5672
+        volumeMounts:
+          - name: config-volume
+            mountPath: /etc/rabbitmq/conf.d/consumer-settings.conf
+            subPath: consumer-settings.conf
+---
+apiVersion: apps/v1 #  for k8s versions before 1.9.0 use apps/v1beta2  and before 1.8.0 use extensions/v1beta1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      annotations:
+        "sidecar.istio.io/inject": "false"
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: node2.bdcl:5000/redis:6.2
+        imagePullPolicy: IfNotPresent
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        ports:
+        - containerPort: 6379
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-flower
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: celery-flower
+  template:
+    metadata:
+      annotations:
+        "sidecar.istio.io/inject": "false"
+      labels:
+        app: celery-flower
+    spec:
+      containers:
+      - name: flower
+        image: {{ flower_image }}
+        imagePullPolicy: {{ pull_policy }}
+        ports:
+        - containerPort: 5555
+        env:
+          - name: CELERY_BROKER_URL
+            value: "amqp://guest:guest@rabbitmq-service:5672"
+          - name: CELERY_RESULT_BACKEND
+            value: "redis://redis:6379/1" # "rpc://"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fitness-worker
+  labels:
+    app: fitness-worker
+spec:
+  replicas: {{ worker_count }}
+  selector:
+    matchLabels:
+      app: fitness-worker
+  template:
+    metadata:
+      annotations:
+        "sidecar.istio.io/inject": "false"
+      labels:
+        app: fitness-worker
+    spec:
+      volumes:
+        - name: dataset
+          hostPath:
+            path: {{ host_data_dir }}
+            type: Directory
+        - name: config-volume
+          configMap:
+            name: fitness-worker-config
+        - name: mlflow-vol
+          persistentVolumeClaim:
+            claimName: mlflow-artifact-store-pvc
+      containers:
+      - name: worker
+        image: {{ image }}
+        imagePullPolicy: {{ pull_policy }}
+        volumeMounts:
+          - name: dataset
+            mountPath: {{ host_data_dir_mount_path }}
+          - name: config-volume
+            mountPath: /etc/fitness/datasets-config.yaml
+            subPath: datasets-config.yaml
+          - mountPath: "/var/lib/mlruns"
+            name: mlflow-vol
+        env:
+          - name: CELERY_BROKER_URL
+            value: "amqp://guest:guest@rabbitmq-service:5672"
+          - name: CELERY_RESULT_BACKEND
+            value: "redis://redis:6379/1" # "rpc://"
+          - name: NUM_PROCESSORS
+            value: "{{ worker_cpu }}"
+          - name: DATASETS_CONFIG
+            value: /etc/fitness/datasets-config.yaml
+          - name: MLFLOW_TRACKING_URI
+            value: mysql+pymysql://mlflow:mlflow@mlflow-db:3306/mlflow
+          # see: https://github.com/mongodb/mongo-python-driver/blob/c8d920a46bfb7b054326b3e983943bfc794cb676/pymongo/mongo_client.py
+          - name: MONGO_URI
+            value: mongodb://mongoadmin:secret@mongo-tm-experiments-db:27017
+          - name: MONGO_COLLECTION
+            value: "{{ mongo_collection or 'tm_stats' }}"
+        resources:
+          requests:
+            memory: "{{ worker_mem }}"
+            cpu: "{{ worker_cpu }}"
+          limits:
+            memory: "{{ worker_mem }}"
+            cpu: "{{ worker_cpu }}"

distributed/charts/autotm/templates/pvc.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  namespace: {{ .Values.user }}
+  name: pv-{{- .Values.user }}
+spec:
+  capacity:
+    storage: 10Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: slow
+  mountOptions:
+    - relatime
+  nfs:
+    path: /nfs_storage/home/{{- .Values.user }}
+    readOnly: no
+    server: 10.32.7.103
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: {{ .Values.user }}
+  name: pvc-{{- .Values.user }}
+spec:
+  volumeName: pv-{{- .Values.user }}
+  storageClassName: slow
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  namespace: {{ .Values.user }}
+  name: pv-shared-{{- .Values.user }}
+spec:
+  capacity:
+    storage: 100Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadOnlyMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: slow
+  mountOptions:
+    - relatime
+  nfs:
+    path: /nfs_storage/shared
+    readOnly: no
+    server: 10.32.7.103
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: {{ .Values.user }}
+  name: pvc-shared-{{- .Values.user }}
+spec:
+  volumeName: pv-shared-{{- .Values.user }}
+  storageClassName: slow
+  accessModes:
+    - ReadOnlyMany
+  resources:
+    requests:
+      storage: 100Gi

distributed/charts/autotm/templates/services.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rabbitmq-service
+spec:
+  ports:
+  - port: 5672
+  selector:
+    app: rabbitmq
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  ports:
+  - port: 6379
+  selector:
+    app: redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: celery-flower-service
+spec:
+  type: NodePort
+  ports:
+  - port: 5555
+  selector:
+    app: celery-flower

distributed/charts/autotm/values.yaml

@@ -0,0 +1,8 @@
+ns_cpu_limit: '16'
+ns_mem_limit: '64Gi'
+
+jupyter_cpu_limits: '2'
+jupyter_mem_limits: '8Gi'
+
+# On install one also need to set
+# user: 'tutor'