diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml index 3a862865..497180b1 100644 --- a/helm/etl/Chart.yaml +++ b/helm/etl/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.1.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/etl/README.md b/helm/etl/README.md index 9d9640e3..4347fd4e 100644 --- a/helm/etl/README.md +++ b/helm/etl/README.md @@ -1,6 +1,6 @@ # etl -![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.3](https://img.shields.io/badge/Version-0.1.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 etl @@ -9,6 +9,11 @@ A Helm chart for gen3 etl | Key | Type | Default | Description | |-----|------|---------|-------------| | esEndpoint | string | `"gen3-elasticsearch-master"` | | +| esGarbageCollect | map | `{"custom_image":null,"enabled":false,"schedule":"0 0 * * *","slack_webhook":"None"}` | Configuration options for es garbage cronjob. | +| esGarbageCollect.custom_image | string | `nil` | To set a custom image for the es garbage collect cronjob. Default is the Gen3 Awshelper image. | +| esGarbageCollect.enabled | bool | `false` | Whether to create es garbage collect cronjob. | +| esGarbageCollect.schedule | string | `"0 0 * * *"` | The cron schedule expression to use in the es garbage collect cronjob. Runs once a day by default. | +| esGarbageCollect.slack_webhook | string | `"None"` | Slack webhook endpoint to use for cronjob. | | etlMapping.mappings[0].aggregated_props[0].fn | string | `"count"` | | | etlMapping.mappings[0].aggregated_props[0].name | string | `"_samples_count"` | | | etlMapping.mappings[0].aggregated_props[0].path | string | `"samples"` | | diff --git a/helm/etl/templates/es-garbage-collect-cronjob.yaml b/helm/etl/templates/es-garbage-collect-cronjob.yaml new file mode 100644 index 00000000..786172a7 --- /dev/null +++ b/helm/etl/templates/es-garbage-collect-cronjob.yaml @@ -0,0 +1,85 @@ +{{- if .Values.esGarbageCollect.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitops-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitops-sa-role +rules: + - apiGroups: [""] + resources: ["namespaces","services"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitops-sa-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitops-sa-role +subjects: + - kind: ServiceAccount + name: gitops-sa + namespace: default +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: es-garbage +spec: + schedule: {{ .Values.esGarbageCollect.schedule | quote }} + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 4 + template: + metadata: + labels: + app: gen3job + spec: + restartPolicy: Never + serviceAccountName: gitops-sa + securityContext: + fsGroup: 1000 + containers: + - name: awshelper + image: {{ .Values.esGarbageCollect.custom_image | default "quay.io/cdis/awshelper:master" }} + imagePullPolicy: Always + env: + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional + - name: ESHOST + value: {{ printf "%s:9200" .Values.esEndpoint | quote }} + - name: slackWebHook + value: {{ .Values.esGarbageCollect.slack_webhook | quote }} + command: ["/bin/bash" ] + args: + - "-c" + - | + export GEN3_HOME="$HOME/cloud-automation" + source "$GEN3_HOME/gen3/gen3setup.sh" + echo $ESHOST + if gen3 klock lock es-garbage-job gitops 900; then + repoList="$(gen3 es garbage)" + for indexName in $repoList; do + echo "deleting index $indexName" + gen3 es delete "$indexName" + done + if [[ -n "$repoList" && -n "$slackWebHook" && "$slackWebHook" != "None" ]]; then + curl -X POST --data-urlencode "payload={\"text\": \"es-garbage-collect in $(gen3 api hostname): \n\`\`\`\n${repoList}\n\`\`\`\"}" "${slackWebHook}" + fi + gen3 klock unlock es-garbage-job gitops + else + echo "Failed to acquire es-garbage-job lock: exiting without attempting to sync" + fi + echo "Exit code: $?" +{{- end }} \ No newline at end of file diff --git a/helm/etl/values.yaml b/helm/etl/values.yaml index 1db9765e..718310b5 100644 --- a/helm/etl/values.yaml +++ b/helm/etl/values.yaml @@ -143,3 +143,14 @@ etlMapping: target_nodes: - name: slide_image path: slides.samples.cases + +# -- (map) Configuration options for es garbage cronjob. +esGarbageCollect: + # -- (bool) Whether to create es garbage collect cronjob. + enabled: false + # -- (string) The cron schedule expression to use in the es garbage collect cronjob. Runs once a day by default. + schedule: "0 0 * * *" + # -- (string) To set a custom image for the es garbage collect cronjob. Default is the Gen3 Awshelper image. + custom_image: + # -- (string) Slack webhook endpoint to use for cronjob. + slack_webhook: None