From ec8d4af95747b4b31c460cb54a73b59ec38b9ebc Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Fri, 12 Jan 2024 11:03:20 -0800 Subject: [PATCH 1/2] Parameterize the postgres user for each prompt processing service Previously a shared "rubin" postgres user is set by default, and the butler config yaml of the dev central repo sets alternatives. Instead of setting the postgres user in the butler config yaml, let each prompt service set its own PGUSER. --- applications/prompt-proto-service-hsc/README.md | 1 + .../values-usdfdev-prompt-processing.yaml | 1 + applications/prompt-proto-service-hsc/values.yaml | 4 ++++ applications/prompt-proto-service-latiss/README.md | 1 + .../values-usdfdev-prompt-processing.yaml | 1 + .../values-usdfprod-prompt-processing.yaml | 1 + applications/prompt-proto-service-latiss/values.yaml | 4 ++++ applications/prompt-proto-service-lsstcam/README.md | 1 + .../values-usdfdev-prompt-processing.yaml | 1 + applications/prompt-proto-service-lsstcam/values.yaml | 4 ++++ applications/prompt-proto-service-lsstcomcam/README.md | 1 + .../values-usdfdev-prompt-processing.yaml | 1 + applications/prompt-proto-service-lsstcomcam/values.yaml | 4 ++++ .../prompt-proto-service/templates/prompt-proto-service.yaml | 2 +- 14 files changed, 26 insertions(+), 1 deletion(-) diff --git a/applications/prompt-proto-service-hsc/README.md b/applications/prompt-proto-service-hsc/README.md index 145a6d9cc3..0c5921e80d 100644 --- a/applications/prompt-proto-service-hsc/README.md +++ b/applications/prompt-proto-service-hsc/README.md @@ -22,6 +22,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.kafkaClusterAddress | string | None, must be set | Hostname and port of the Kafka provider | | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | +| prompt-proto-service.instrument.calibRepoPguser | string | None, must be set | Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. | | prompt-proto-service.instrument.name | string | `"HSC"` | The "short" name of the instrument | | prompt-proto-service.instrument.pipelines | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | | prompt-proto-service.instrument.skymap | string | `"hsc_rings_v1"` | Skymap to use with the instrument | diff --git a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml index 736e870385..a5bb1093c6 100644 --- a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml @@ -13,6 +13,7 @@ prompt-proto-service: instrument: pipelines: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml] calibRepo: s3://rubin:rubin-pp-users/central_repo/ + calibRepoPguser: hsc_prompt s3: imageBucket: rubin:rubin-pp diff --git a/applications/prompt-proto-service-hsc/values.yaml b/applications/prompt-proto-service-hsc/values.yaml index 8bc1a6f9e6..9208930e56 100644 --- a/applications/prompt-proto-service-hsc/values.yaml +++ b/applications/prompt-proto-service-hsc/values.yaml @@ -32,6 +32,10 @@ prompt-proto-service: # If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. # @default -- None, must be set calibRepo: "" + # -- Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. + # If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. + # @default -- None, must be set + calibRepoPguser: "" s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-latiss/README.md b/applications/prompt-proto-service-latiss/README.md index 82a0056ddc..5a8a15fbdf 100644 --- a/applications/prompt-proto-service-latiss/README.md +++ b/applications/prompt-proto-service-latiss/README.md @@ -22,6 +22,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.kafkaClusterAddress | string | None, must be set | Hostname and port of the Kafka provider | | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | +| prompt-proto-service.instrument.calibRepoPguser | string | None, must be set | Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. | | prompt-proto-service.instrument.name | string | `"LATISS"` | The "short" name of the instrument | | prompt-proto-service.instrument.pipelines | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | | prompt-proto-service.instrument.skymap | string | `"latiss_v1"` | Skymap to use with the instrument | diff --git a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml index 3d736c7bb2..367e905e3e 100644 --- a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml @@ -15,6 +15,7 @@ prompt-proto-service: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml, ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml] calibRepo: s3://rubin-pp-users/central_repo/ + calibRepoPguser: latiss_prompt s3: imageBucket: rubin-pp diff --git a/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml index 196d694efc..6ad883454d 100644 --- a/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml +++ b/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml @@ -26,6 +26,7 @@ prompt-proto-service: (survey="spec_pole_with_rotation")=[] (survey="")=[] calibRepo: s3://rubin-summit-users + calibRepoPguser: rubin s3: imageBucket: rubin-summit diff --git a/applications/prompt-proto-service-latiss/values.yaml b/applications/prompt-proto-service-latiss/values.yaml index a62ed6ce5b..34ae28ceee 100644 --- a/applications/prompt-proto-service-latiss/values.yaml +++ b/applications/prompt-proto-service-latiss/values.yaml @@ -33,6 +33,10 @@ prompt-proto-service: # If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. # @default -- None, must be set calibRepo: "" + # -- Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. + # If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. + # @default -- None, must be set + calibRepoPguser: "" s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-lsstcam/README.md b/applications/prompt-proto-service-lsstcam/README.md index ca3bd5b987..e07d659977 100644 --- a/applications/prompt-proto-service-lsstcam/README.md +++ b/applications/prompt-proto-service-lsstcam/README.md @@ -22,6 +22,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.kafkaClusterAddress | string | None, must be set | Hostname and port of the Kafka provider | | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | +| prompt-proto-service.instrument.calibRepoPguser | string | None, must be set | Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. | | prompt-proto-service.instrument.name | string | `""` | The "short" name of the instrument | | prompt-proto-service.instrument.pipelines | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | | prompt-proto-service.instrument.skymap | string | `""` | Skymap to use with the instrument | diff --git a/applications/prompt-proto-service-lsstcam/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-lsstcam/values-usdfdev-prompt-processing.yaml index 5b2fdab778..3dc7d35210 100644 --- a/applications/prompt-proto-service-lsstcam/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-lsstcam/values-usdfdev-prompt-processing.yaml @@ -12,6 +12,7 @@ prompt-proto-service: instrument: calibRepo: s3://rubin-summit-users/ + calibRepoPguser: rubin s3: imageBucket: rubin:rubin-pp diff --git a/applications/prompt-proto-service-lsstcam/values.yaml b/applications/prompt-proto-service-lsstcam/values.yaml index 9e41e9a8b4..4d70040810 100644 --- a/applications/prompt-proto-service-lsstcam/values.yaml +++ b/applications/prompt-proto-service-lsstcam/values.yaml @@ -32,6 +32,10 @@ prompt-proto-service: # If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. # @default -- None, must be set calibRepo: "" + # -- Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. + # If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. + # @default -- None, must be set + calibRepoPguser: "" s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-lsstcomcam/README.md b/applications/prompt-proto-service-lsstcomcam/README.md index 7b067af8a9..426556533e 100644 --- a/applications/prompt-proto-service-lsstcomcam/README.md +++ b/applications/prompt-proto-service-lsstcomcam/README.md @@ -21,6 +21,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.kafkaClusterAddress | string | None, must be set | Hostname and port of the Kafka provider | | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | +| prompt-proto-service.instrument.calibRepoPguser | string | None, must be set | Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. | | prompt-proto-service.instrument.name | string | `""` | The "short" name of the instrument | | prompt-proto-service.instrument.pipelines | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | | prompt-proto-service.instrument.skymap | string | `""` | Skymap to use with the instrument | diff --git a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml index ab8962cb4e..2b9f48237e 100644 --- a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml @@ -12,6 +12,7 @@ prompt-proto-service: instrument: calibRepo: s3://rubin-summit-users/ + calibRepoPguser: rubin s3: imageBucket: rubin:rubin-pp diff --git a/applications/prompt-proto-service-lsstcomcam/values.yaml b/applications/prompt-proto-service-lsstcomcam/values.yaml index 6fa3d7c6ba..7fa43f4768 100644 --- a/applications/prompt-proto-service-lsstcomcam/values.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values.yaml @@ -32,6 +32,10 @@ prompt-proto-service: # If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. # @default -- None, must be set calibRepo: "" + # -- Postgres username to access the shared butler repo for calibrations, templates, and pipeline outputs. + # If `registry.centralRepoFile` is set, a local redirect is used and its config may override this config. + # @default -- None, must be set + calibRepoPguser: "" s3: # -- Bucket containing the incoming raw images diff --git a/charts/prompt-proto-service/templates/prompt-proto-service.yaml b/charts/prompt-proto-service/templates/prompt-proto-service.yaml index f3f8379241..bd6e141981 100644 --- a/charts/prompt-proto-service/templates/prompt-proto-service.yaml +++ b/charts/prompt-proto-service/templates/prompt-proto-service.yaml @@ -54,7 +54,7 @@ spec: - name: IMAGE_TIMEOUT value: {{ .Values.imageNotifications.imageTimeout | quote }} - name: PGUSER - value: rubin + value: {{ .Values.instrument.calibRepoPguser }} - name: CALIB_REPO value: {{ .Values.instrument.calibRepo }} - name: LSST_DISABLE_BUCKET_VALIDATION From fe17d515c8dde6f66a8eae0c5e07de8c839668e0 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Fri, 12 Jan 2024 12:48:01 -0800 Subject: [PATCH 2/2] Add tenant prefix to repo in prompt processing dev LATISS service --- .../values-usdfdev-prompt-processing.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml index 367e905e3e..5d7718ff1c 100644 --- a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml @@ -14,12 +14,13 @@ prompt-proto-service: pipelines: >- (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml, ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml] - calibRepo: s3://rubin-pp-users/central_repo/ + calibRepo: s3://rubin:rubin-pp-users/central_repo/ calibRepoPguser: latiss_prompt s3: - imageBucket: rubin-pp + imageBucket: rubin:rubin-pp endpointUrl: https://s3dfrgw.slac.stanford.edu + disableBucketValidation: '1' imageNotifications: kafkaClusterAddress: prompt-processing-kafka-bootstrap.kafka:9092