CHG complete volume backup

api/anubis/jobs/volume_backup.py

@@ -6,11 +6,11 @@
 from datetime import datetime
 import argparse
 from pathlib import Path
-from tqdm import tqdm
+# from tqdm import tqdm
 
-volumes = json.load(open("./volumes.json"))
+volumes = json.load(open("/tmp/volumes.json"))
 netids = [parse("ide-volume-{}", v)[0] for v in volumes]
-jobs_dir = Path("jobs/")
+jobs_dir = Path("/tmp/jobs/")
 now = datetime.now().strftime("%Y%m%d-%H%M%S")
 
 
@@ -176,7 +176,9 @@ def backup_restore(args, label: str):
     print(f"Using latest:")
     print(f"  {latest_dir=}")
     print(f"Starting jobs")
-    for index, job_file in tqdm(enumerate(job_files), total=len(job_files)):
+    total = len(job_files)
+    for index, job_file in enumerate(job_files):
+        print(f'{index} / {total}')
         cmd = f"kubectl apply -f {str(job_file)} --wait=false 1> /dev/null"
         os.system(cmd)
         if (index + 1) % args.jobs == 0:
@@ -203,6 +205,4 @@ def main() -> int | None:
 
 
 if __name__ == "__main__":
-    # TODO: add logging
-    # Do we need refactoring here? -> In terms of renaming main to something else
     main()

k8s/chart/templates/backup-volumes-cron.yml

@@ -0,0 +1,88 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "chart.fullname" . }}-volume-backup
+  labels:
+      {{- include "chart.labels" . | nindent 4 }}
+    component: volume-backup
+{{- if .Values.imagePullSecret }}
+imagePullSecrets:
+  - name: {{ .Values.imagePullSecret }}
+{{- end }}
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "chart.fullname" . }}-volume-backup
+  labels:
+      {{- include "chart.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [ "v1" ]
+    resources: [ "PersistentVolumeClaim" ]
+    verbs: [ "get", "list", "create" ]
+---
+
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "chart.fullname" . }}-volume-backup
+  labels:
+      {{- include "chart.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "chart.fullname" . }}-volume-backup
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "chart.fullname" . }}-volume-backup
+    namespace: anubis
+---
+
+{{- if .Values.backup.enable }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "chart.fullname" . }}-volume-backup
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+    component: volume-backup
+spec:
+  schedule: {{ .Values.backup.volume_schedule | quote }}
+  concurrencyPolicy: Replace
+  suspend: {{ .Values.backup.suspend }}
+  jobTemplate:
+    metadata:
+      labels:
+        {{- include "chart.selectorLabels" . | nindent 8 }}
+        component: volume-backup
+    spec:
+      template:
+        spec:
+          {{- if and .Values.nodeSelector (not .Values.debug) }}
+          nodeSelector:
+            {{ .Values.nodeSelector | toYaml }}
+          {{- end }}
+          restartPolicy: Never
+          securityContext:
+            runAsUser: 1000
+          containers:
+          - name: backup
+            image: "{{ .Values.api.image }}:{{ .Values.tag }}"
+            imagePullPolicy: {{ .Values.imagePullPolicy }}
+            command:
+            - |
+              kubectl get pvc -n anubis | awk '/ide-volume/ {print "\""$1"\""}' | jq -s | tee /tmp/volumes.json
+            - ./anubis/jobs/volume_backup.py gen
+            - ./anubis/jobs/volume_backup.py backup --jobs 30 --wait 60 --yes
+            env:
+            {{- include "api.env" . | nindent 12 }}
+            volumeMounts:
+            - name: {{ .Values.backup.secretName }}
+              mountPath: "/home/anubis/.ssh"
+          volumes:
+          - name: {{ include "chart.fullname" . }}-backup-creds
+            secret:
+              secretName: {{ .Values.backup.secretName }}
+              defaultMode: 0644
+{{- end }}

k8s/chart/values.yaml

@@ -151,6 +151,7 @@ backup:
   enable: true
   suspend: false
   schedule: "0 */6 * * *" # Midnight
+  volume_schedule: "0 */6 * * *" # Midnight
   secretName: anubis-backup-creds
 
 email: