Merge pull request #3887 from lsst-sqre/tickets/DM-47606

DM-47606: Add s3proxy application.

applications/s3proxy/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

applications/s3proxy/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+appVersion: 1.0.0
+description: Simple application to gateway S3 URLs to HTTPS
+name: s3proxy
+sources:
+- https://github.com/lsst-dm/s3proxy
+type: application
+version: 1.0.0

applications/s3proxy/README.md

@@ -0,0 +1,30 @@
+# s3proxy
+
+Simple application to gateway S3 URLs to HTTPS
+
+## Source Code
+
+* <https://github.com/lsst-dm/s3proxy>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Affinity rules for the s3proxy deployment pod |
+| config.logLevel | string | `"INFO"` | Logging level |
+| config.logProfile | string | `"production"` | Logging profile (`production` for JSON, `development` for human-friendly) |
+| config.pathPrefix | string | `"/s3proxy"` | URL path prefix |
+| config.profiles | list | `[]` | Profiles using different endpoint URLs and credentials |
+| config.s3EndpointUrl | string | `""` | Default S3 endpoint URL |
+| global.baseUrl | string | Set by Argo CD | Base URL for the environment |
+| global.host | string | Set by Argo CD | Host name for ingress |
+| global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |
+| image.pullPolicy | string | `"IfNotPresent"` | Pull policy for the s3proxy image |
+| image.repository | string | `"ghcr.io/lsst-sqre/s3proxy"` | Image to use in the s3proxy deployment |
+| image.tag | string | The appVersion of the chart | Tag of image to use |
+| ingress.annotations | object | `{}` | Additional annotations for the ingress rule |
+| nodeSelector | object | `{}` | Node selection rules for the s3proxy deployment pod |
+| podAnnotations | object | `{}` | Annotations for the s3proxy deployment pod |
+| replicaCount | int | `1` | Number of web deployment pods to start |
+| resources | object | See `values.yaml` | Resource limits and requests for the s3proxy deployment pod |
+| tolerations | list | `[]` | Tolerations for the s3proxy deployment pod |

applications/s3proxy/secrets.yaml

@@ -0,0 +1,8 @@
+"aws-credentials.ini":
+  description: |
+    S3 credentials in AWS INI format.
+    Each section corresponds to a profile.
+    Each section contains an aws_access_key_id and an aws_secret_access_key.
+  copy:
+    application: nublado
+    key: "aws-credentials.ini"

applications/s3proxy/templates/_helpers.tpl

@@ -0,0 +1,26 @@
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "s3proxy.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "s3proxy.labels" -}}
+helm.sh/chart: {{ include "s3proxy.chart" . }}
+{{ include "s3proxy.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "s3proxy.selectorLabels" -}}
+app.kubernetes.io/name: "s3proxy"
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

applications/s3proxy/templates/configmap.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "s3proxy"
+  labels:
+    {{- include "s3proxy.labels" . | nindent 4 }}
+data:
+  S3PROXY_LOG_LEVEL: {{ .Values.config.logLevel | quote }}
+  S3PROXY_PATH_PREFIX: {{ .Values.config.pathPrefix | quote }}
+  S3PROXY_PROFILE: {{ .Values.config.logProfile | quote }}
+  S3_ENDPOINT_URL: {{ .Values.config.s3EndpointUrl | quote }}
+  {{- range .Values.config.profiles }}
+  LSST_RESOURCES_S3_PROFILE_{{ .name | quote }}: {{ .url | quote }}
+  {{- end }}

applications/s3proxy/templates/deployment.yaml

@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "s3proxy"
+  labels:
+    {{- include "s3proxy.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "s3proxy.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "s3proxy.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      automountServiceAccountToken: false
+      containers:
+        - name: {{ .Chart.Name }}
+          env:
+            - name: AWS_SHARED_CREDENTIALS_FILE
+              value: /pod-secrets/aws-credentials.ini
+          envFrom:
+            - configMapRef:
+                name: "s3proxy"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: "http"
+              containerPort: 8080
+              protocol: "TCP"
+          readinessProbe:
+            httpGet:
+              path: "/"
+              port: "http"
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "all"
+            readOnlyRootFilesystem: true
+          volumes:
+            - name: pod-secrets
+              mountPath: /pod-secrets
+      initContainers:
+        - name: secret-setup
+          command:
+            - /bin/ash
+            - "-c"
+            - |
+              cp -R /secrets /pod-secrets
+              chmod -R go-rwx /pod-secrets
+          image: alpine
+          volumeMounts:
+            - name: secret-volume
+              mountPath: /secrets
+              readOnly: true
+            - name: pod-secrets
+              mountPath: /pod-secrets
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+      volumes:
+        - name: pod-secrets
+          emptyDir:
+            sizeLimit: 1Mi
+        - name: secret-volume
+          secret:
+            secretName: "aws-credentials.ini"

applications/s3proxy/templates/ingress.yaml

@@ -0,0 +1,34 @@
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
+metadata:
+  name: "s3proxy"
+  labels:
+    {{- include "s3proxy.labels" . | nindent 4 }}
+config:
+  authCacheDuration: 5m
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  loginRedirect: true
+  onlyServices:
+    - portal
+  scopes:
+    all:
+      - "read:image"
+template:
+  metadata:
+    name: "s3proxy"
+    {{- with .Values.ingress.annotations }}
+    annotations:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  spec:
+    rules:
+      - host: {{ required "global.host must be set" .Values.global.host | quote }}
+        http:
+          paths:
+            - path: {{ .Values.config.pathPrefix | quote }}
+              pathType: "Prefix"
+              backend:
+                service:
+                  name: "s3proxy"
+                  port:
+                    number: 8080

applications/s3proxy/templates/networkpolicy.yaml

@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: "s3proxy"
+spec:
+  podSelector:
+    matchLabels:
+      {{- include "s3proxy.selectorLabels" . | nindent 6 }}
+  policyTypes:
+    - "Ingress"
+  ingress:
+    # Allow inbound access from pods (in any namespace) labeled
+    # gafaelfawr.lsst.io/ingress: true.
+    - from:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              gafaelfawr.lsst.io/ingress: "true"
+      ports:
+        - protocol: "TCP"
+          port: 8080

applications/s3proxy/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: "s3proxy"
+  labels:
+    {{- include "s3proxy.labels" . | nindent 4 }}
+spec:
+  type: "ClusterIP"
+  ports:
+    - port: 8080
+      targetPort: "http"
+      protocol: "TCP"
+      name: "http"
+  selector:
+    {{- include "s3proxy.selectorLabels" . | nindent 4 }}

applications/s3proxy/templates/vault-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: ricoberger.de/v1alpha1
+kind: VaultSecret
+metadata:
+  name: "s3proxy"
+  labels:
+    {{- include "s3proxy.labels" . | nindent 4 }}
+spec:
+  path: "{{ .Values.global.vaultSecretsPath }}/s3proxy"
+  type: Opaque

applications/s3proxy/values-usdfdev.yaml

@@ -0,0 +1,13 @@
+config:
+  profiles:
+    - name: embargo
+      url: "https://sdfembs3.sdf.slac.stanford.edu/"
+  s3EndpointUrl: "https://s3dfrgw.slac.stanford.edu/"
+
+resources:
+  limits:
+    compute: 500m
+    memory: 1Gi
+  requests:
+    compute: 200m
+    memory: 100Mi

applications/s3proxy/values.yaml

@@ -0,0 +1,69 @@
+# Default values for s3proxy.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# -- Number of web deployment pods to start
+replicaCount: 1
+
+image:
+  # -- Image to use in the s3proxy deployment
+  repository: "ghcr.io/lsst-sqre/s3proxy"
+
+  # -- Pull policy for the s3proxy image
+  pullPolicy: "IfNotPresent"
+
+  # -- Tag of image to use
+  # @default -- The appVersion of the chart
+  tag: 0.1.0
+
+config:
+  # -- Logging level
+  logLevel: "INFO"
+
+  # -- Logging profile (`production` for JSON, `development` for
+  # human-friendly)
+  logProfile: "production"
+
+  # -- URL path prefix
+  pathPrefix: "/s3proxy"
+
+  # -- Profiles using different endpoint URLs and credentials
+  profiles: []
+
+  # -- Default S3 endpoint URL
+  s3EndpointUrl: ""
+
+ingress:
+  # -- Additional annotations for the ingress rule
+  annotations: {}
+
+# -- Affinity rules for the s3proxy deployment pod
+affinity: {}
+
+# -- Node selection rules for the s3proxy deployment pod
+nodeSelector: {}
+
+# -- Annotations for the s3proxy deployment pod
+podAnnotations: {}
+
+# -- Resource limits and requests for the s3proxy deployment pod
+# @default -- See `values.yaml`
+resources: {}
+
+# -- Tolerations for the s3proxy deployment pod
+tolerations: []
+
+# The following will be set by parameters injected by Argo CD and should not
+# be set in the individual environment values files.
+global:
+  # -- Base URL for the environment
+  # @default -- Set by Argo CD
+  baseUrl: null
+
+  # -- Host name for ingress
+  # @default -- Set by Argo CD
+  host: null
+
+  # -- Base path for Vault secrets
+  # @default -- Set by Argo CD
+  vaultSecretsPath: null

docs/applications/rubin.rst

@@ -23,4 +23,5 @@ Argo CD project: ``rubin``
    rapid-analysis/index
    rubintv/index
    rubintv-dev/index
+   s3proxy/index
    schedview-snapshot/index

docs/applications/s3proxy/index.rst

@@ -0,0 +1,19 @@
+.. px-app:: s3proxy
+
+########################################################
+s3proxy — Simple application to gateway S3 URLs to HTTPS
+########################################################
+
+This application provides authenticated internal links to S3 resources.
+It is intended for deployment only at the USDF, but it could be used elsewhere.
+
+.. jinja:: s3proxy
+   :file: applications/_summary.rst.jinja
+
+Guides
+======
+
+.. toctree::
+   :maxdepth: 1
+
+   values