From d45505b6447d190937b665ac34e533e8868db0a3 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Tue, 26 Nov 2024 14:09:03 -0800 Subject: [PATCH 1/2] Move Prompt Processing debugging flags into a debug block. This change keeps temporary "hack" settings together. --- .../values-usdfdev-prompt-processing.yaml | 5 +++-- .../values-usdfdev-prompt-processing.yaml | 5 +++-- .../values-usdfdev-prompt-processing.yaml | 5 +++-- charts/prompt-proto-service/README.md | 2 +- .../templates/prompt-proto-service.yaml | 2 +- charts/prompt-proto-service/values.yaml | 7 ++++--- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml index d7faeaf5c0..66dc8938bc 100644 --- a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml @@ -39,7 +39,8 @@ prompt-proto-service: endpointUrl: https://usdf-rsp-dev.slac.stanford.edu/sasquatch-rest-proxy auth_env: false - # A cache efficiency workaround breaks on RC2 tests; see DM-43205. - cacheCalibs: false + debug: + # A cache efficiency workaround breaks on RC2 tests; see DM-43205. + cacheCalibs: false fullnameOverride: "prompt-proto-service-hsc" diff --git a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml index fbc8b126c0..fd5ab7ce4f 100644 --- a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml @@ -40,7 +40,8 @@ prompt-proto-service: endpointUrl: https://usdf-rsp-dev.slac.stanford.edu/sasquatch-rest-proxy auth_env: false - # A cache efficiency workaround breaks when mixing observing dates; see DM-43205, DM-43913. - cacheCalibs: false + debug: + # A cache efficiency workaround breaks when mixing observing dates; see DM-43205, DM-43913. + cacheCalibs: false fullnameOverride: "prompt-proto-service-latiss" diff --git a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml index deacc0bbc8..d7b6b0847e 100644 --- a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml @@ -39,7 +39,8 @@ prompt-proto-service: endpointUrl: https://usdf-rsp-dev.slac.stanford.edu/sasquatch-rest-proxy auth_env: false - # A cache efficiency workaround breaks when mixing observing dates; see DM-43205, DM-43913. - cacheCalibs: false + debug: + # A cache efficiency workaround breaks when mixing observing dates; see DM-43205, DM-43913. + cacheCalibs: false fullnameOverride: "prompt-proto-service-lsstcomcam" diff --git a/charts/prompt-proto-service/README.md b/charts/prompt-proto-service/README.md index 41c08554c0..c50a573380 100644 --- a/charts/prompt-proto-service/README.md +++ b/charts/prompt-proto-service/README.md @@ -22,8 +22,8 @@ Event-driven processing of camera images | cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | -| cacheCalibs | bool | `true` | Whether or not calibs should be cached between runs of a pod. This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. | | containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | +| debug.cacheCalibs | bool | `true` | Whether or not calibs should be cached between runs of a pod. This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. | | fullnameOverride | string | `"prompt-proto-service"` | Override the full name for resources (includes the release name) | | image.pullPolicy | string | `IfNotPresent` in prod, `Always` in dev | Pull policy for the PP image | | image.repository | string | `"ghcr.io/lsst-dm/prompt-service"` | Image to use in the PP deployment | diff --git a/charts/prompt-proto-service/templates/prompt-proto-service.yaml b/charts/prompt-proto-service/templates/prompt-proto-service.yaml index 23eb47fd11..18eb30c683 100644 --- a/charts/prompt-proto-service/templates/prompt-proto-service.yaml +++ b/charts/prompt-proto-service/templates/prompt-proto-service.yaml @@ -122,7 +122,7 @@ spec: - name: FILTERS_WITH_CALIBS value: {{ .Values.cache.maxFilters | toString | quote }} - name: DEBUG_CACHE_CALIBS - value: {{ if .Values.cacheCalibs }}'1'{{ else }}'0'{{ end }} + value: {{ if .Values.debug.cacheCalibs }}'1'{{ else }}'0'{{ end }} volumeMounts: - mountPath: /tmp-butler name: ephemeral diff --git a/charts/prompt-proto-service/values.yaml b/charts/prompt-proto-service/values.yaml index 66fd0642e3..d32ef08132 100644 --- a/charts/prompt-proto-service/values.yaml +++ b/charts/prompt-proto-service/values.yaml @@ -168,9 +168,10 @@ fullnameOverride: "prompt-proto-service" # -- The number of Knative requests that can be handled simultaneously by one container containerConcurrency: 1 -# -- Whether or not calibs should be cached between runs of a pod. -# This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. -cacheCalibs: true +debug: + # -- Whether or not calibs should be cached between runs of a pod. + # This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. + cacheCalibs: true # -- Kubernetes YAML configs for extra container volume(s). # Any volumes required by other config options are automatically handled by the Helm chart. From 93ac0ead5a054150d45fdcc978736d4490e4ba26 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Tue, 26 Nov 2024 14:14:02 -0800 Subject: [PATCH 2/2] Add Prompt Processing config for turning off repo export. This flag is used for tests where we want to avoid conflicting updates to the central repo. --- charts/prompt-proto-service/README.md | 1 + .../prompt-proto-service/templates/prompt-proto-service.yaml | 2 ++ charts/prompt-proto-service/values.yaml | 3 +++ 3 files changed, 6 insertions(+) diff --git a/charts/prompt-proto-service/README.md b/charts/prompt-proto-service/README.md index c50a573380..f2d8e7f96d 100644 --- a/charts/prompt-proto-service/README.md +++ b/charts/prompt-proto-service/README.md @@ -24,6 +24,7 @@ Event-driven processing of camera images | cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | | containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | | debug.cacheCalibs | bool | `true` | Whether or not calibs should be cached between runs of a pod. This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. | +| debug.exportOutputs | bool | `true` | Whether or not pipeline outputs should be exported to the central repo. This flag does not turn off APDB writes or alert generation; those must be handled at the pipeline level or by setting up an alternative destination. | | fullnameOverride | string | `"prompt-proto-service"` | Override the full name for resources (includes the release name) | | image.pullPolicy | string | `IfNotPresent` in prod, `Always` in dev | Pull policy for the PP image | | image.repository | string | `"ghcr.io/lsst-dm/prompt-service"` | Image to use in the PP deployment | diff --git a/charts/prompt-proto-service/templates/prompt-proto-service.yaml b/charts/prompt-proto-service/templates/prompt-proto-service.yaml index 18eb30c683..ac77260360 100644 --- a/charts/prompt-proto-service/templates/prompt-proto-service.yaml +++ b/charts/prompt-proto-service/templates/prompt-proto-service.yaml @@ -123,6 +123,8 @@ spec: value: {{ .Values.cache.maxFilters | toString | quote }} - name: DEBUG_CACHE_CALIBS value: {{ if .Values.debug.cacheCalibs }}'1'{{ else }}'0'{{ end }} + - name: DEBUG_EXPORT_OUTPUTS + value: {{ if .Values.debug.exportOutputs }}'1'{{ else }}'0'{{ end }} volumeMounts: - mountPath: /tmp-butler name: ephemeral diff --git a/charts/prompt-proto-service/values.yaml b/charts/prompt-proto-service/values.yaml index d32ef08132..45eea955ed 100644 --- a/charts/prompt-proto-service/values.yaml +++ b/charts/prompt-proto-service/values.yaml @@ -172,6 +172,9 @@ debug: # -- Whether or not calibs should be cached between runs of a pod. # This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. cacheCalibs: true + # -- Whether or not pipeline outputs should be exported to the central repo. + # This flag does not turn off APDB writes or alert generation; those must be handled at the pipeline level or by setting up an alternative destination. + exportOutputs: true # -- Kubernetes YAML configs for extra container volume(s). # Any volumes required by other config options are automatically handled by the Helm chart.