Switch vo-cutouts to use Wobbly for jobs

Instead of directly connecting to a database, change the vo-cutouts application to use Wobbly as the backend for job storage. Remove all the schema management and database connection logic from vo-cutouts, which will no longer require it. Bump the application version to the new version that supports Wobbly.
lsst-sqre · Dec 12, 2024 · 323cb27 · 323cb27
1 parent 608f538
commit 323cb27
Show file tree

Hide file tree

Showing 13 changed files with 8 additions and 278 deletions.
diff --git a/applications/vo-cutouts/Chart.yaml b/applications/vo-cutouts/Chart.yaml
@@ -4,7 +4,7 @@ version: 1.0.0
 description: "Image cutout service complying with IVOA SODA"
 sources:
   - "https://github.com/lsst-sqre/vo-cutouts"
-appVersion: 3.2.0
+appVersion: 4.0.0
 
 dependencies:
   - name: redis

diff --git a/applications/vo-cutouts/README.md b/applications/vo-cutouts/README.md
@@ -10,24 +10,15 @@ Image cutout service complying with IVOA SODA
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| cloudsql.enabled | bool | `false` | Enable the Cloud SQL Auth Proxy sidecar, used with Cloud SQL databases on Google Cloud |
-| cloudsql.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for Cloud SQL Auth Proxy images |
-| cloudsql.image.repository | string | `"gcr.io/cloudsql-docker/gce-proxy"` | Cloud SQL Auth Proxy image to use |
-| cloudsql.image.schemaUpdateTagSuffix | string | `"-alpine"` | Tag suffix to use for the proxy for schema updates |
-| cloudsql.image.tag | string | `"1.37.2"` | Cloud SQL Auth Proxy tag to use |
-| cloudsql.instanceConnectionName | string | None, must be set if Cloud SQL is used | Instance connection name for a Cloud SQL PostgreSQL instance |
-| cloudsql.resources | object | See `values.yaml` | Resource limits and requests for the Cloud SQL Proxy container |
-| config.databaseUrl | string | None, must be set if `cloudsql.enabled` is false | URL for the PostgreSQL database if Cloud SQL is not in use |
 | config.gracePeriod | int | `60` | Grace period in seconds to wait for cutout worker jobs to finish |
 | config.lifetime | string | `"30d"` | Lifetime of job results in Safir `parse_timedelta` format |
 | config.loglevel | string | `"INFO"` | Choose from the text form of Python logging levels |
 | config.pathPrefix | string | `"/api/cutout"` | URL path prefix for the cutout API |
-| config.serviceAccount | string | None, must be set | Google service account with an IAM binding to the `vo-cutouts` Kubernetes service accounts and has the `cloudsql.client` role, access to write to the GCS bucket, and ability to sign URLs as itself |
+| config.serviceAccount | string | None, must be set | Google service account with an IAM binding to the `vo-cutouts` Kubernetes service accounts and has access to write to the GCS bucket and ability to sign URLs as itself |
 | config.slackAlerts | bool | `true` | Whether to send Slack alerts for unexpected failures |
 | config.storageBucketUrl | string | None, must be set | URL for the GCS bucket for results (must start with `gs`) |
 | config.syncTimeout | string | `"1m"` | Timeout for results from a sync cutout in Safir `parse_timedelta` format |
 | config.timeout | int | 600 (10 minutes) | Timeout for a single cutout job in seconds |
-| config.updateSchema | bool | `false` | Whether to automatically update the vo-cutouts database schema |
 | cutoutWorker.affinity | object | `{}` | Affinity rules for the cutout worker pod |
 | cutoutWorker.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for cutout workers |
 | cutoutWorker.image.repository | string | `"ghcr.io/lsst-sqre/vo-cutouts-worker"` | Stack image to use for cutouts |

diff --git a/applications/vo-cutouts/secrets.yaml b/applications/vo-cutouts/secrets.yaml
@@ -1,7 +1,3 @@
-database-password:
-  description: >-
-    Password used to authenticate to the PostgreSQL database used to store job
-    information. This password may be changed at any time.
 redis-password:
   description: >-
     Password used to authenticate vo-cutouts to its internal Redis server,

diff --git a/applications/vo-cutouts/templates/configmap.yaml b/applications/vo-cutouts/templates/configmap.yaml
@@ -2,25 +2,17 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: vo-cutouts
-  {{- if .Values.config.updateSchema }}
-  annotations:
-    helm.sh/hook: "pre-install,pre-upgrade"
-    helm.sh/hook-delete-policy: "before-hook-creation"
-    helm.sh/hook-weight: "0"
-  {{- end }}
   labels:
     {{- include "vo-cutouts.labels" . | nindent 4 }}
 data:
   CUTOUT_ARQ_QUEUE_URL: "redis://vo-cutouts-redis.{{ .Release.Namespace }}"
-  {{- if .Values.cloudsql.enabled }}
-  CUTOUT_DATABASE_URL: "postgresql://vo-cutouts@localhost/vo-cutouts"
-  {{- end }}
   CUTOUT_GRACE_PERIOD: {{ .Values.config.gracePeriod | quote }}
   CUTOUT_LIFETIME: {{ .Values.config.lifetime | quote }}
+  CUTOUT_LOG_LEVEL: {{ .Values.config.loglevel | quote }}
+  CUTOUT_PATH_PREFIX: {{ .Values.config.pathPrefix | quote }}
+  CUTOUT_PROFILE: "production"
   CUTOUT_SERVICE_ACCOUNT: {{ required "config.serviceAccount must be set" .Values.config.serviceAccount | quote }}
   CUTOUT_STORAGE_URL: {{ required "config.storageBucketUrl must be set" .Values.config.storageBucketUrl | quote }}
   CUTOUT_SYNC_TIMEOUT: {{ .Values.config.syncTimeout | quote }}
   CUTOUT_TIMEOUT: {{ .Values.config.timeout | quote }}
-  CUTOUT_PATH_PREFIX: {{ .Values.config.pathPrefix | quote }}
-  CUTOUT_PROFILE: "production"
-  CUTOUT_LOG_LEVEL: {{ .Values.config.loglevel | quote }}
+  CUTOUT_WOBBLY_URL: "{{ .Values.global.baseUrl }}/wobbly"
diff --git a/applications/vo-cutouts/templates/db-worker-deployment.yaml b/applications/vo-cutouts/templates/db-worker-deployment.yaml
@@ -26,32 +26,8 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.cloudsql.enabled }}
-      serviceAccountName: "vo-cutouts"
-      {{- else }}
       automountServiceAccountToken: false
-      {{- end }}
       containers:
-        {{- if .Values.cloudsql.enabled }}
-        - name: "cloud-sql-proxy"
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - "all"
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            runAsUser: 65532
-            runAsGroup: 65532
-          image: "{{ .Values.cloudsql.image.repository }}:{{ .Values.cloudsql.image.tag }}"
-          imagePullPolicy: {{ .Values.cloudsql.image.pullPolicy | quote }}
-          command:
-            - "/cloud_sql_proxy"
-            - "-ip_address_types=PRIVATE"
-            - "-log_debug_stdout=true"
-            - "-structured_logs=true"
-            - "-instances={{ required "cloudsql.instanceConnectionName must be specified" .Values.cloudsql.instanceConnectionName }}=tcp:5432"
-        {{- end }}
         - name: "db-worker"
           securityContext:
             allowPrivilegeEscalation: false
@@ -70,11 +46,6 @@ spec:
                 secretKeyRef:
                   name: "vo-cutouts"
                   key: "redis-password"
-            - name: "CUTOUT_DATABASE_PASSWORD"
-              valueFrom:
-                secretKeyRef:
-                  name: "vo-cutouts"
-                  key: "database-password"
             {{- if .Values.config.slackAlerts }}
             - name: "CUTOUT_SLACK_WEBHOOK"
               valueFrom:

diff --git a/applications/vo-cutouts/templates/deployment.yaml b/applications/vo-cutouts/templates/deployment.yaml
@@ -26,40 +26,12 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.cloudsql.enabled }}
       serviceAccountName: "vo-cutouts"
-      {{- else }}
-      automountServiceAccountToken: false
-      {{- end }}
       securityContext:
         runAsNonRoot: true
         runAsUser: 1000
         runAsGroup: 1000
       containers:
-        {{- if .Values.cloudsql.enabled }}
-        - name: "cloud-sql-proxy"
-          command:
-            - "/cloud_sql_proxy"
-            - "-ip_address_types=PRIVATE"
-            - "-log_debug_stdout=true"
-            - "-structured_logs=true"
-            - "-instances={{ required "cloudsql.instanceConnectionName must be specified" .Values.cloudsql.instanceConnectionName }}=tcp:5432"
-          image: "{{ .Values.cloudsql.image.repository }}:{{ .Values.cloudsql.image.tag }}"
-          imagePullPolicy: {{ .Values.cloudsql.image.pullPolicy | quote }}
-          {{- with .Values.cloudsql.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - "all"
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            runAsUser: 65532
-            runAsGroup: 65532
-        {{- end }}
         - name: "vo-cutouts"
           securityContext:
             allowPrivilegeEscalation: false
@@ -79,11 +51,6 @@ spec:
                 secretKeyRef:
                   name: "vo-cutouts"
                   key: "redis-password"
-            - name: "CUTOUT_DATABASE_PASSWORD"
-              valueFrom:
-                secretKeyRef:
-                  name: "vo-cutouts"
-                  key: "database-password"
             {{- if .Values.config.slackAlerts }}
             - name: "CUTOUT_SLACK_WEBHOOK"
               valueFrom:

diff --git a/applications/vo-cutouts/templates/job-schea-update.yaml b/applications/vo-cutouts/templates/job-schea-update.yaml
diff --git a/applications/vo-cutouts/templates/serviceaccount.yaml b/applications/vo-cutouts/templates/serviceaccount.yaml
@@ -1,4 +1,3 @@
-{{- if (or .Values.config.serviceAccount .Values.cloudsql.enabled) }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -12,4 +11,3 @@ metadata:
     helm.sh/hook-weight: "0"
     {{- end }}
     iam.gke.io/gcp-service-account: {{ required "config.serviceAccount must be set to a valid Google service account" .Values.config.serviceAccount | quote }}
-{{- end }}
diff --git a/applications/vo-cutouts/templates/worker-deployment.yaml b/applications/vo-cutouts/templates/worker-deployment.yaml
@@ -26,11 +26,7 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.config.serviceAccount }}
       serviceAccountName: "vo-cutouts"
-      {{- else }}
-      automountServiceAccountToken: false
-      {{- end }}
       containers:
         - name: "worker"
           securityContext:

diff --git a/applications/vo-cutouts/values-idfdev.yaml b/applications/vo-cutouts/values-idfdev.yaml
@@ -1,7 +1,3 @@
 config:
   serviceAccount: "[email protected]"
   storageBucketUrl: "gs://rubin-cutouts-dev-us-central1-output/"
-
-cloudsql:
-  enabled: true
-  instanceConnectionName: "science-platform-dev-7696:us-central1:science-platform-dev-e9e11de2"
diff --git a/applications/vo-cutouts/values-idfint.yaml b/applications/vo-cutouts/values-idfint.yaml
@@ -1,7 +1,3 @@
 config:
   serviceAccount: "[email protected]"
   storageBucketUrl: "gs://rubin-cutouts-int-us-central1-output/"
-
-cloudsql:
-  enabled: true
-  instanceConnectionName: "science-platform-int-dc5d:us-central1:science-platform-int-8f439af2"
diff --git a/applications/vo-cutouts/values-idfprod.yaml b/applications/vo-cutouts/values-idfprod.yaml
@@ -1,7 +1,3 @@
 config:
   serviceAccount: "[email protected]"
   storageBucketUrl: "gs://rubin-cutouts-stable-us-central1-output/"
-
-cloudsql:
-  enabled: true
-  instanceConnectionName: "science-platform-stable-6994:us-central1:science-platform-stable-0c29612b"