diff --git a/applications/s3proxy/.helmignore b/applications/s3proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/applications/s3proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/applications/s3proxy/Chart.yaml b/applications/s3proxy/Chart.yaml new file mode 100644 index 0000000000..a165868d44 --- /dev/null +++ b/applications/s3proxy/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +appVersion: 1.0.0 +description: Simple application to gateway S3 URLs to HTTPS +name: s3proxy +sources: +- https://github.com/lsst-dm/s3proxy +type: application +version: 1.0.0 diff --git a/applications/s3proxy/README.md b/applications/s3proxy/README.md new file mode 100644 index 0000000000..d8c3df29e6 --- /dev/null +++ b/applications/s3proxy/README.md @@ -0,0 +1,30 @@ +# s3proxy + +Simple application to gateway S3 URLs to HTTPS + +## Source Code + +* + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | Affinity rules for the s3proxy deployment pod | +| config.logLevel | string | `"INFO"` | Logging level | +| config.logProfile | string | `"production"` | Logging profile (`production` for JSON, `development` for human-friendly) | +| config.pathPrefix | string | `"/s3proxy"` | URL path prefix | +| config.profiles | list | `[]` | Profiles using different endpoint URLs and credentials | +| config.s3EndpointUrl | string | `""` | Default S3 endpoint URL | +| global.baseUrl | string | Set by Argo CD | Base URL for the environment | +| global.host | string | Set by Argo CD | Host name for ingress | +| global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets | +| image.pullPolicy | string | `"IfNotPresent"` | Pull policy for the s3proxy image | +| image.repository | string | `"ghcr.io/lsst-sqre/s3proxy"` | Image to use in the s3proxy deployment | +| image.tag | string | The appVersion of the chart | Tag of image to use | +| ingress.annotations | object | `{}` | Additional annotations for the ingress rule | +| nodeSelector | object | `{}` | Node selection rules for the s3proxy deployment pod | +| podAnnotations | object | `{}` | Annotations for the s3proxy deployment pod | +| replicaCount | int | `1` | Number of web deployment pods to start | +| resources | object | See `values.yaml` | Resource limits and requests for the s3proxy deployment pod | +| tolerations | list | `[]` | Tolerations for the s3proxy deployment pod | diff --git a/applications/s3proxy/secrets.yaml b/applications/s3proxy/secrets.yaml new file mode 100644 index 0000000000..fd1f46e6ee --- /dev/null +++ b/applications/s3proxy/secrets.yaml @@ -0,0 +1,8 @@ +"aws-credentials.ini": + description: | + S3 credentials in AWS INI format. + Each section corresponds to a profile. + Each section contains an aws_access_key_id and an aws_secret_access_key. + copy: + application: nublado + key: "aws-credentials.ini" diff --git a/applications/s3proxy/templates/_helpers.tpl b/applications/s3proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..67e025f3fc --- /dev/null +++ b/applications/s3proxy/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "s3proxy.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "s3proxy.labels" -}} +helm.sh/chart: {{ include "s3proxy.chart" . }} +{{ include "s3proxy.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "s3proxy.selectorLabels" -}} +app.kubernetes.io/name: "s3proxy" +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/applications/s3proxy/templates/configmap.yaml b/applications/s3proxy/templates/configmap.yaml new file mode 100644 index 0000000000..261cbd8bf3 --- /dev/null +++ b/applications/s3proxy/templates/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "s3proxy" + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +data: + S3PROXY_LOG_LEVEL: {{ .Values.config.logLevel | quote }} + S3PROXY_PATH_PREFIX: {{ .Values.config.pathPrefix | quote }} + S3PROXY_PROFILE: {{ .Values.config.logProfile | quote }} + S3_ENDPOINT_URL: {{ .Values.config.s3EndpointUrl | quote }} + {{- range .Values.config.profiles }} + LSST_RESOURCES_S3_PROFILE_{{ .name | quote }}: {{ .url | quote }} + {{- end }} diff --git a/applications/s3proxy/templates/deployment.yaml b/applications/s3proxy/templates/deployment.yaml new file mode 100644 index 0000000000..0a93824c54 --- /dev/null +++ b/applications/s3proxy/templates/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "s3proxy" + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "s3proxy.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "s3proxy.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: false + containers: + - name: {{ .Chart.Name }} + env: + - name: AWS_SHARED_CREDENTIALS_FILE + value: /pod-secrets/aws-credentials.ini + envFrom: + - configMapRef: + name: "s3proxy" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: "http" + containerPort: 8080 + protocol: "TCP" + readinessProbe: + httpGet: + path: "/" + port: "http" + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "all" + readOnlyRootFilesystem: true + volumes: + - name: pod-secrets + mountPath: /pod-secrets + initContainers: + - name: secret-setup + command: + - /bin/ash + - "-c" + - | + cp -R /secrets /pod-secrets + chmod -R go-rwx /pod-secrets + image: alpine + volumeMounts: + - name: secret-volume + mountPath: /secrets + readOnly: true + - name: pod-secrets + mountPath: /pod-secrets + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + - name: pod-secrets + emptyDir: + sizeLimit: 1Mi + - name: secret-volume + secret: + secretName: "aws-credentials.ini" diff --git a/applications/s3proxy/templates/ingress.yaml b/applications/s3proxy/templates/ingress.yaml new file mode 100644 index 0000000000..61ce54a2ff --- /dev/null +++ b/applications/s3proxy/templates/ingress.yaml @@ -0,0 +1,34 @@ +apiVersion: gafaelfawr.lsst.io/v1alpha1 +kind: GafaelfawrIngress +metadata: + name: "s3proxy" + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +config: + authCacheDuration: 5m + baseUrl: {{ .Values.global.baseUrl | quote }} + loginRedirect: true + onlyServices: + - portal + scopes: + all: + - "read:image" +template: + metadata: + name: "s3proxy" + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 6 }} + {{- end }} + spec: + rules: + - host: {{ required "global.host must be set" .Values.global.host | quote }} + http: + paths: + - path: {{ .Values.config.pathPrefix | quote }} + pathType: "Prefix" + backend: + service: + name: "s3proxy" + port: + number: 8080 diff --git a/applications/s3proxy/templates/networkpolicy.yaml b/applications/s3proxy/templates/networkpolicy.yaml new file mode 100644 index 0000000000..70d2c1e80d --- /dev/null +++ b/applications/s3proxy/templates/networkpolicy.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: "s3proxy" +spec: + podSelector: + matchLabels: + {{- include "s3proxy.selectorLabels" . | nindent 6 }} + policyTypes: + - "Ingress" + ingress: + # Allow inbound access from pods (in any namespace) labeled + # gafaelfawr.lsst.io/ingress: true. + - from: + - namespaceSelector: {} + podSelector: + matchLabels: + gafaelfawr.lsst.io/ingress: "true" + ports: + - protocol: "TCP" + port: 8080 diff --git a/applications/s3proxy/templates/service.yaml b/applications/s3proxy/templates/service.yaml new file mode 100644 index 0000000000..ffe65b1110 --- /dev/null +++ b/applications/s3proxy/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: "s3proxy" + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +spec: + type: "ClusterIP" + ports: + - port: 8080 + targetPort: "http" + protocol: "TCP" + name: "http" + selector: + {{- include "s3proxy.selectorLabels" . | nindent 4 }} diff --git a/applications/s3proxy/templates/vault-secret.yaml b/applications/s3proxy/templates/vault-secret.yaml new file mode 100644 index 0000000000..f1db384244 --- /dev/null +++ b/applications/s3proxy/templates/vault-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: ricoberger.de/v1alpha1 +kind: VaultSecret +metadata: + name: "s3proxy" + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +spec: + path: "{{ .Values.global.vaultSecretsPath }}/s3proxy" + type: Opaque diff --git a/applications/s3proxy/values-usdfdev.yaml b/applications/s3proxy/values-usdfdev.yaml new file mode 100644 index 0000000000..13cfe53903 --- /dev/null +++ b/applications/s3proxy/values-usdfdev.yaml @@ -0,0 +1,13 @@ +config: + profiles: + - name: embargo + url: "https://sdfembs3.sdf.slac.stanford.edu/" + s3EndpointUrl: "https://s3dfrgw.slac.stanford.edu/" + +resources: + limits: + compute: 500m + memory: 1Gi + requests: + compute: 200m + memory: 100Mi diff --git a/applications/s3proxy/values.yaml b/applications/s3proxy/values.yaml new file mode 100644 index 0000000000..ed8ba92582 --- /dev/null +++ b/applications/s3proxy/values.yaml @@ -0,0 +1,69 @@ +# Default values for s3proxy. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Number of web deployment pods to start +replicaCount: 1 + +image: + # -- Image to use in the s3proxy deployment + repository: "ghcr.io/lsst-sqre/s3proxy" + + # -- Pull policy for the s3proxy image + pullPolicy: "IfNotPresent" + + # -- Tag of image to use + # @default -- The appVersion of the chart + tag: 0.1.0 + +config: + # -- Logging level + logLevel: "INFO" + + # -- Logging profile (`production` for JSON, `development` for + # human-friendly) + logProfile: "production" + + # -- URL path prefix + pathPrefix: "/s3proxy" + + # -- Profiles using different endpoint URLs and credentials + profiles: [] + + # -- Default S3 endpoint URL + s3EndpointUrl: "" + +ingress: + # -- Additional annotations for the ingress rule + annotations: {} + +# -- Affinity rules for the s3proxy deployment pod +affinity: {} + +# -- Node selection rules for the s3proxy deployment pod +nodeSelector: {} + +# -- Annotations for the s3proxy deployment pod +podAnnotations: {} + +# -- Resource limits and requests for the s3proxy deployment pod +# @default -- See `values.yaml` +resources: {} + +# -- Tolerations for the s3proxy deployment pod +tolerations: [] + +# The following will be set by parameters injected by Argo CD and should not +# be set in the individual environment values files. +global: + # -- Base URL for the environment + # @default -- Set by Argo CD + baseUrl: null + + # -- Host name for ingress + # @default -- Set by Argo CD + host: null + + # -- Base path for Vault secrets + # @default -- Set by Argo CD + vaultSecretsPath: null diff --git a/docs/applications/rubin.rst b/docs/applications/rubin.rst index 483f483438..ebd6ab8be3 100644 --- a/docs/applications/rubin.rst +++ b/docs/applications/rubin.rst @@ -23,4 +23,5 @@ Argo CD project: ``rubin`` rapid-analysis/index rubintv/index rubintv-dev/index + s3proxy/index schedview-snapshot/index diff --git a/docs/applications/s3proxy/index.rst b/docs/applications/s3proxy/index.rst new file mode 100644 index 0000000000..7ed9bd8ccc --- /dev/null +++ b/docs/applications/s3proxy/index.rst @@ -0,0 +1,19 @@ +.. px-app:: s3proxy + +######################################################## +s3proxy — Simple application to gateway S3 URLs to HTTPS +######################################################## + +This application provides authenticated internal links to S3 resources. +It is intended for deployment only at the USDF, but it could be used elsewhere. + +.. jinja:: s3proxy + :file: applications/_summary.rst.jinja + +Guides +====== + +.. toctree:: + :maxdepth: 1 + + values diff --git a/docs/applications/s3proxy/values.md b/docs/applications/s3proxy/values.md new file mode 100644 index 0000000000..1e546df95b --- /dev/null +++ b/docs/applications/s3proxy/values.md @@ -0,0 +1,12 @@ +```{px-app-values} s3proxy +``` + +# s3proxy Helm values reference + +Helm values reference table for the {px-app}`s3proxy` application. + +```{include} ../../../applications/s3proxy/README.md +--- +start-after: "## Values" +--- +``` \ No newline at end of file diff --git a/environments/README.md b/environments/README.md index e92de4f65a..3fce8c9d31 100644 --- a/environments/README.md +++ b/environments/README.md @@ -53,6 +53,7 @@ | applications.prompt-proto-service-lsstcomcamsim | bool | `false` | Enable the prompt-proto-service-lsstcomcamsim application | | applications.rubintv | bool | `false` | Enable the rubintv application | | applications.rubintv-dev | bool | `false` | Enable the rubintv-dev application | +| applications.s3proxy | bool | `false` | Enable the s3proxy application | | applications.sasquatch | bool | `false` | Enable the sasquatch application | | applications.sasquatch-backpack | bool | `false` | Enable the sasquatch-backpack application | | applications.schedview-snapshot | bool | `false` | Enable the schedview-snapshot application | diff --git a/environments/templates/applications/rubin/s3proxy.yaml b/environments/templates/applications/rubin/s3proxy.yaml new file mode 100644 index 0000000000..1f33f79960 --- /dev/null +++ b/environments/templates/applications/rubin/s3proxy.yaml @@ -0,0 +1,34 @@ +{{- if (index .Values "applications" "s3proxy") -}} +apiVersion: v1 +kind: Namespace +metadata: + name: "s3proxy" +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: "s3proxy" + namespace: "argocd" + finalizers: + - "resources-finalizer.argocd.argoproj.io" +spec: + destination: + namespace: "s3proxy" + server: "https://kubernetes.default.svc" + project: "rubin" + source: + path: "applications/s3proxy" + repoURL: {{ .Values.repoUrl | quote }} + targetRevision: {{ .Values.targetRevision | quote }} + helm: + parameters: + - name: "global.host" + value: {{ .Values.fqdn | quote }} + - name: "global.baseUrl" + value: "https://{{ .Values.fqdn }}" + - name: "global.vaultSecretsPath" + value: {{ .Values.vaultPathPrefix | quote }} + valueFiles: + - "values.yaml" + - "values-{{ .Values.name }}.yaml" +{{- end -}} \ No newline at end of file diff --git a/environments/values-usdfdev.yaml b/environments/values-usdfdev.yaml index d186cba8d2..ee2390b317 100644 --- a/environments/values-usdfdev.yaml +++ b/environments/values-usdfdev.yaml @@ -29,6 +29,7 @@ applications: postgres: true ppdb-replication: true rubintv: true + s3proxy: true sasquatch: true schedview-snapshot: true semaphore: true diff --git a/environments/values.yaml b/environments/values.yaml index b65965df03..25500c34d3 100644 --- a/environments/values.yaml +++ b/environments/values.yaml @@ -162,6 +162,9 @@ applications: # -- Enable the rubintv-dev application rubintv-dev: false + # -- Enable the s3proxy application + s3proxy: false + # -- Enable the sasquatch application sasquatch: false