Merge pull request #3794 from lsst-sqre/tickets/DM-45906

DM-45906: Enable SIA over Butler application in phalanx

applications/sia/Chart.yaml

@@ -0,0 +1,13 @@
+apiVersion: v2
+appVersion: 0.1.2
+description: Simple Image Access (SIA) IVOA Service using Butler
+name: sia
+sources:
+- https://github.com/lsst-sqre/sia
+type: application
+version: 1.0.0
+annotations:
+  phalanx.lsst.io/docs: |
+    - id: "SQR-095"
+      title: "SIAv2 over Butler FastAPI service"
+      url: "https://sqr-095.lsst.io"

applications/sia/README.md

@@ -0,0 +1,35 @@
+# sia
+
+Simple Image Access (SIA) IVOA Service using Butler
+
+## Source Code
+
+* <https://github.com/lsst-sqre/sia>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Affinity rules for the sia deployment pod |
+| config.butlerDataCollections | list | `[]` | List of data (Butler) Collections Expected attributes: `config`, `label`, `name`, `butler_type`, `repository`, `datalink_url` & `default_instrument` |
+| config.directButlerEnabled | bool | `false` | Whether direct butler access is enabled |
+| config.logLevel | string | `"INFO"` | Logging level |
+| config.logProfile | string | `"production"` | Logging profile (`production` for JSON, `development` for human-friendly) |
+| config.pathPrefix | string | `"/api/sia"` | URL path prefix |
+| config.pgUser | string | `"rubin"` | User to use from the PGPASSFILE if sia is using a direct Butler connection |
+| config.slackAlerts | bool | `false` | Whether to send alerts and status to Slack. |
+| fullnameOverride | string | `""` | Override the full name for resources (includes the release name) |
+| global.baseUrl | string | Set by Argo CD | Base URL for the environment |
+| global.host | string | Set by Argo CD | Host name for ingress |
+| global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |
+| image.pullPolicy | string | `"IfNotPresent"` | Pull policy for the sia image |
+| image.repository | string | `"ghcr.io/lsst-sqre/sia"` | Image to use in the sia deployment |
+| image.tag | string | The appVersion of the chart | Tag of image to use |
+| ingress.annotations | object | `{}` | Additional annotations for the ingress rule |
+| ingress.path | string | `"/api/sia"` | Path prefix where app is hosted |
+| nameOverride | string | `""` | Override the base name for resources |
+| nodeSelector | object | `{}` | Node selection rules for the sia deployment pod |
+| podAnnotations | object | `{}` | Annotations for the sia deployment pod |
+| replicaCount | int | `1` | Number of web deployment pods to start |
+| resources | object | See `values.yaml` | Resource limits and requests for the sia deployment pod |
+| tolerations | list | `[]` | Tolerations for the sia deployment pod |

applications/sia/secrets.yaml

@@ -0,0 +1,23 @@
+"aws-credentials.ini":
+  if: config.directButlerEnabled
+  copy:
+    application: nublado
+    key: "aws-credentials.ini"
+"butler-gcs-idf-creds.json":
+  if: config.directButlerEnabled
+  copy:
+    application: nublado
+    key: "butler-gcs-idf-creds.json"
+"postgres-credentials.txt":
+  if: config.directButlerEnabled
+  copy:
+    application: nublado
+    key: "postgres-credentials.txt"
+slack-webhook:
+  description: >-
+    Slack web hook used to report internal errors to Slack. This secret may be
+    changed at any time.
+  if: config.slackAlerts
+  copy:
+    application: mobu
+    key: app-alert-webhook

applications/sia/templates/_helpers.tpl

@@ -0,0 +1,52 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "sia.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "sia.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "sia.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "sia.labels" -}}
+helm.sh/chart: {{ include "sia.chart" . }}
+{{ include "sia.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "sia.selectorLabels" -}}
+app.kubernetes.io/name: "sia"
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+

applications/sia/templates/configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "sia"
+  labels:
+    {{- include "sia.labels" . | nindent 4 }}
+data:
+  SIA_LOG_LEVEL: {{ .Values.config.logLevel | quote }}
+  SIA_PATH_PREFIX: {{ .Values.config.pathPrefix | quote }}
+  SIA_PROFILE: {{ .Values.config.logProfile | quote }}

applications/sia/templates/deployment.yaml

@@ -0,0 +1,117 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "sia"
+  labels:
+    {{- include "sia.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "sia.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "sia.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      automountServiceAccountToken: false
+      {{- if .Values.config.directButlerEnabled }}
+      initContainers:
+        - name: fix-secret-permissions
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          command:
+            - "/bin/sh"
+            - "-c"
+            - |
+              cp -RL /tmp/secrets-raw/* /etc/butler/secrets/
+              chmod 0400 /etc/butler/secrets/*
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "all"
+          volumeMounts:
+            - name: "secrets"
+              mountPath: "/etc/butler/secrets"
+            - name: "secrets-raw"
+              mountPath: "/tmp/secrets-raw"
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          envFrom:
+            - configMapRef:
+                name: "sia"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: "http"
+              containerPort: 8080
+              protocol: "TCP"
+          readinessProbe:
+            httpGet:
+              path: {{ .Values.config.pathPrefix }}
+              port: "http"
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          env:
+            - name: "SIA_BUTLER_DATA_COLLECTIONS"
+              value: {{ .Values.config.butlerDataCollections | toJson | quote }}
+            {{- if .Values.config.slackAlerts }}
+            - name: "SIA_SLACK_WEBHOOK"
+              valueFrom:
+                secretKeyRef:
+                  name: "sia"
+                  key: "slack-webhook"
+            {{- end }}
+            {{- if .Values.config.directButlerEnabled }}
+            - name: "AWS_SHARED_CREDENTIALS_FILE"
+              value: "/tmp/secrets/aws-credentials.ini"
+            - name: "PGUSER"
+              value: {{ .Values.config.pgUser }}
+            - name: "PGPASSFILE"
+              value: "/etc/butler/secrets/postgres-credentials.txt"
+            - name: "GOOGLE_APPLICATION_CREDENTIALS"
+              value: "/tmp/secrets/butler-gcs-idf-creds.json"
+            {{- end }}
+          {{- if .Values.config.directButlerEnabled }}
+          volumeMounts:
+            - name: "secrets"
+              mountPath: "/etc/butler/secrets"
+              readOnly: true
+          {{- end }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "all"
+            readOnlyRootFilesystem: false
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.config.directButlerEnabled }}
+      volumes:
+        - name: "secrets-raw"
+          secret:
+            secretName: "sia"
+        - name: "secrets"
+          emptyDir: {}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000

applications/sia/templates/ingress-anonymous.yaml

@@ -0,0 +1,44 @@
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
+metadata:
+  name: {{ template "sia.fullname" . }}-anonymous
+  labels:
+    {{- include "sia.labels" . | nindent 4 }}
+config:
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  scopes:
+    anonymous: true
+template:
+  metadata:
+    name: {{ template "sia.fullname" . }}-anonymous
+    {{- with .Values.ingress.annotations }}
+    annotations:
+      nginx.ingress.kubernetes.io/use-regex: "true"
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  spec:
+    rules:
+      - host: {{ .Values.global.host | quote }}
+        http:
+          paths:
+            - path: "{{ .Values.ingress.path }}/openapi.json"
+              pathType: "Exact"
+              backend:
+                service:
+                  name: {{ template "sia.fullname" . }}
+                  port:
+                    number: 8080
+            - path: "{{ .Values.ingress.path }}/.+/capabilities"
+              pathType: "ImplementationSpecific"
+              backend:
+                service:
+                  name: {{ template "sia.fullname" . }}
+                  port:
+                    number: 8080
+            - path: "{{ .Values.ingress.path }}/.+/availability"
+              pathType: "ImplementationSpecific"
+              backend:
+                service:
+                  name: {{ template "sia.fullname" . }}
+                  port:
+                    number: 8080

applications/sia/templates/ingress.yaml

@@ -0,0 +1,35 @@
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
+metadata:
+  name: {{ template "sia.fullname" . }}
+  labels:
+    {{- include "sia.labels" . | nindent 4 }}
+config:
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  scopes:
+    all:
+      - "read:image"
+  delegate:
+    internal:
+      service: "sia"
+      scopes:
+        - "read:image"
+template:
+  metadata:
+    name: {{ template "sia.fullname" . }}
+    {{- with .Values.ingress.annotations }}
+    annotations:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  spec:
+    rules:
+      - host: {{ required "global.host must be set" .Values.global.host | quote }}
+        http:
+          paths:
+            - path: {{ .Values.config.pathPrefix | quote }}
+              pathType: "Prefix"
+              backend:
+                service:
+                  name: "sia"
+                  port:
+                    number: 8080

applications/sia/templates/networkpolicy.yaml

@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: "sia"
+spec:
+  podSelector:
+    matchLabels:
+      {{- include "sia.selectorLabels" . | nindent 6 }}
+  policyTypes:
+    - "Ingress"
+  ingress:
+    # Allow inbound access from pods (in any namespace) labeled
+    # gafaelfawr.lsst.io/ingress: true.
+    - from:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              gafaelfawr.lsst.io/ingress: "true"
+      ports:
+        - protocol: "TCP"
+          port: 8080

applications/sia/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: "sia"
+  labels:
+    {{- include "sia.labels" . | nindent 4 }}
+spec:
+  type: "ClusterIP"
+  ports:
+    - port: 8080
+      targetPort: "http"
+      protocol: "TCP"
+      name: "http"
+  selector:
+    {{- include "sia.selectorLabels" . | nindent 4 }}

applications/sia/templates/vault-secrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: ricoberger.de/v1alpha1
+kind: VaultSecret
+metadata:
+  name: "sia"
+  labels:
+    {{- include "sia.labels" . | nindent 4 }}
+spec:
+  path: "{{ .Values.global.vaultSecretsPath }}/sia"
+  type: Opaque

applications/sia/values-idfdev.yaml

@@ -0,0 +1,11 @@
+config:
+
+  # Data (Butler) Collections
+  butlerDataCollections:
+    - config: "https://raw.githubusercontent.com/lsst-dm/dax_obscore/refs/heads/main/configs/dp02.yaml"
+      label: "LSST.DP02"
+      name: "dp02"
+      butler_type: "REMOTE"
+      repository: "https://data-dev.lsst.cloud/api/butler/repo/dp02/butler.yaml"
+      datalink_url: "https://data-dev.lsst.cloud/api/datalink/links?ID=butler%3A//dp02/{id}"
+      default_instrument: "LSSTCam-imSim"