Merge pull request #200 from uc-cdis/feat/GPE-1454

adding in the es-garbage-collect cronjob to helm

helm/etl/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.2
+version: 0.1.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/etl/README.md

@@ -1,6 +1,6 @@
 # etl
 
-![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.3](https://img.shields.io/badge/Version-0.1.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 etl
 
@@ -9,6 +9,11 @@ A Helm chart for gen3 etl
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | esEndpoint | string | `"gen3-elasticsearch-master"` |  |
+| esGarbageCollect | map | `{"custom_image":null,"enabled":false,"schedule":"0 0 * * *","slack_webhook":"None"}` | Configuration options for es garbage cronjob. |
+| esGarbageCollect.custom_image | string | `nil` | To set a custom image for the es garbage collect cronjob. Default is the Gen3 Awshelper image. |
+| esGarbageCollect.enabled | bool | `false` | Whether to create es garbage collect cronjob. |
+| esGarbageCollect.schedule | string | `"0 0 * * *"` | The cron schedule expression to use in the es garbage collect cronjob. Runs once a day by default. |
+| esGarbageCollect.slack_webhook | string | `"None"` | Slack webhook endpoint to use for cronjob. |
 | etlMapping.mappings[0].aggregated_props[0].fn | string | `"count"` |  |
 | etlMapping.mappings[0].aggregated_props[0].name | string | `"_samples_count"` |  |
 | etlMapping.mappings[0].aggregated_props[0].path | string | `"samples"` |  |

helm/etl/templates/es-garbage-collect-cronjob.yaml

@@ -0,0 +1,85 @@
+{{- if .Values.esGarbageCollect.enabled -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitops-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitops-sa-role
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces","services"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitops-sa-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gitops-sa-role
+subjects:
+  - kind: ServiceAccount
+    name: gitops-sa
+    namespace: default
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: es-garbage
+spec:
+  schedule: {{ .Values.esGarbageCollect.schedule | quote }}
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 1
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 4
+      template:
+        metadata:
+          labels:
+            app: gen3job
+        spec:
+          restartPolicy: Never
+          serviceAccountName: gitops-sa
+          securityContext:
+            fsGroup: 1000
+          containers:
+          - name: awshelper
+            image: {{ .Values.esGarbageCollect.custom_image | default "quay.io/cdis/awshelper:master" }}
+            imagePullPolicy: Always
+            env:
+            - name: AWS_STS_REGIONAL_ENDPOINTS
+              value: regional
+            - name: ESHOST
+              value: {{ printf "%s:9200" .Values.esEndpoint | quote }}
+            - name: slackWebHook
+              value: {{ .Values.esGarbageCollect.slack_webhook | quote }}
+            command: ["/bin/bash" ]
+            args:
+              - "-c"
+              - |
+                export GEN3_HOME="$HOME/cloud-automation"
+                source "$GEN3_HOME/gen3/gen3setup.sh"
+                echo $ESHOST
+                if gen3 klock lock es-garbage-job gitops 900; then
+                    repoList="$(gen3 es garbage)"
+                    for indexName in $repoList; do
+                    echo "deleting index $indexName"
+                    gen3 es delete "$indexName"
+                    done
+                    if [[ -n "$repoList" && -n "$slackWebHook" && "$slackWebHook" != "None" ]]; then
+                    curl -X POST --data-urlencode "payload={\"text\": \"es-garbage-collect in $(gen3 api hostname): \n\`\`\`\n${repoList}\n\`\`\`\"}" "${slackWebHook}"
+                    fi
+                    gen3 klock unlock es-garbage-job gitops
+                else
+                    echo "Failed to acquire es-garbage-job lock: exiting without attempting to sync"
+                fi
+                echo "Exit code: $?"
+{{- end }}

helm/etl/values.yaml

@@ -143,3 +143,14 @@ etlMapping:
       target_nodes:
         - name: slide_image
           path: slides.samples.cases
+
+# -- (map) Configuration options for es garbage cronjob.
+esGarbageCollect:
+  # -- (bool) Whether to create es garbage collect cronjob.
+  enabled: false
+  # -- (string) The cron schedule expression to use in the es garbage collect cronjob. Runs once a day by default.
+  schedule: "0 0 * * *"
+  # -- (string) To set a custom image for the es garbage collect cronjob. Default is the Gen3 Awshelper image.
+  custom_image:
+  # -- (string) Slack webhook endpoint to use for cronjob.
+  slack_webhook: None