From d7b0880629c3bbead336e4b4b05498885d0804f8 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Tue, 12 Nov 2024 14:25:07 -0800 Subject: [PATCH] Switch Prompt Processing pipelines config from Python-like to YAML. Delegating parsing to YAML makes it much easier to add more fields to the pipelines spec. The parsing is done by Prompt Processing instead of during Helm processing to keep the expanded configuration easier to read in Argo. --- .../prompt-proto-service-hsc-gpu/README.md | 4 +- .../values-usdfdev-prompt-processing.yaml | 8 +- .../prompt-proto-service-hsc-gpu/values.yaml | 6 +- .../prompt-proto-service-hsc/README.md | 4 +- .../values-usdfdev-prompt-processing.yaml | 8 +- .../prompt-proto-service-hsc/values.yaml | 6 +- .../prompt-proto-service-latiss/README.md | 4 +- .../values-usdfdev-prompt-processing.yaml | 13 +++- .../values-usdfprod-prompt-processing.yaml | 48 ++++++------ .../prompt-proto-service-latiss/values.yaml | 6 +- .../prompt-proto-service-lsstcam/README.md | 4 +- .../prompt-proto-service-lsstcam/values.yaml | 6 +- .../prompt-proto-service-lsstcomcam/README.md | 4 +- .../values-usdfdev-prompt-processing.yaml | 12 ++- .../values-usdfprod-prompt-processing.yaml | 73 ++++++++++--------- .../values.yaml | 6 +- .../README.md | 4 +- .../values-usdfdev-prompt-processing.yaml | 14 ++-- .../values.yaml | 6 +- charts/prompt-proto-service/README.md | 4 +- .../templates/prompt-proto-service.yaml | 6 +- charts/prompt-proto-service/values.yaml | 6 +- 22 files changed, 139 insertions(+), 113 deletions(-) diff --git a/applications/prompt-proto-service-hsc-gpu/README.md b/applications/prompt-proto-service-hsc-gpu/README.md index 7f119bce01..8d393c1e28 100644 --- a/applications/prompt-proto-service-hsc-gpu/README.md +++ b/applications/prompt-proto-service-hsc-gpu/README.md @@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | prompt-proto-service.instrument.name | string | `"HSC"` | The "short" name of the instrument | -| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | prompt-proto-service.instrument.preloadPadding | int | `42` | Number of arcseconds to pad the spatial region in preloading. | | prompt-proto-service.instrument.skymap | string | `"hsc_rings_v1"` | Skymap to use with the instrument | | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/applications/prompt-proto-service-hsc-gpu/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-hsc-gpu/values-usdfdev-prompt-processing.yaml index 33666a8bf6..0c3a27f4da 100644 --- a/applications/prompt-proto-service-hsc-gpu/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-hsc-gpu/values-usdfdev-prompt-processing.yaml @@ -12,8 +12,12 @@ prompt-proto-service: instrument: pipelines: - main: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml] - preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml] + main: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml'] + preprocessing: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml'] calibRepo: s3://rubin-pp-dev-users/central_repo_2 s3: diff --git a/applications/prompt-proto-service-hsc-gpu/values.yaml b/applications/prompt-proto-service-hsc-gpu/values.yaml index 4fbc4354e8..6cf07db2d8 100644 --- a/applications/prompt-proto-service-hsc-gpu/values.yaml +++ b/applications/prompt-proto-service-hsc-gpu/values.yaml @@ -38,11 +38,11 @@ prompt-proto-service: # -- The "short" name of the instrument name: HSC pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument diff --git a/applications/prompt-proto-service-hsc/README.md b/applications/prompt-proto-service-hsc/README.md index 33a4a6d8b7..3f6af2f7fb 100644 --- a/applications/prompt-proto-service-hsc/README.md +++ b/applications/prompt-proto-service-hsc/README.md @@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | prompt-proto-service.instrument.name | string | `"HSC"` | The "short" name of the instrument | -| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | prompt-proto-service.instrument.preloadPadding | int | `42` | Number of arcseconds to pad the spatial region in preloading. | | prompt-proto-service.instrument.skymap | string | `"hsc_rings_v1"` | Skymap to use with the instrument | | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml index 6ef44857a0..d7faeaf5c0 100644 --- a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml @@ -13,8 +13,12 @@ prompt-proto-service: instrument: pipelines: - main: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml] - preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml] + main: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml'] + preprocessing: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml'] calibRepo: s3://rubin-pp-dev-users/central_repo_2 s3: diff --git a/applications/prompt-proto-service-hsc/values.yaml b/applications/prompt-proto-service-hsc/values.yaml index f2e5eba4a8..9cfc798c4e 100644 --- a/applications/prompt-proto-service-hsc/values.yaml +++ b/applications/prompt-proto-service-hsc/values.yaml @@ -38,11 +38,11 @@ prompt-proto-service: # -- The "short" name of the instrument name: HSC pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument diff --git a/applications/prompt-proto-service-latiss/README.md b/applications/prompt-proto-service-latiss/README.md index b5028a2171..338d44300f 100644 --- a/applications/prompt-proto-service-latiss/README.md +++ b/applications/prompt-proto-service-latiss/README.md @@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | prompt-proto-service.instrument.name | string | `"LATISS"` | The "short" name of the instrument | -| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | prompt-proto-service.instrument.preloadPadding | int | `30` | Number of arcseconds to pad the spatial region in preloading. | | prompt-proto-service.instrument.skymap | string | `"latiss_v1"` | Skymap to use with the instrument | | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml index 4bd5b8032d..fbc8b126c0 100644 --- a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml @@ -11,10 +11,15 @@ prompt-proto-service: instrument: pipelines: - main: >- - (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml] - preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml] + main: |- + - survey: SURVEY + pipelines: + - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml + # Fallback for an upload.py image without templates + - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml + preprocessing: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml'] calibRepo: s3://rubin-pp-dev-users/central_repo_2 s3: diff --git a/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml index b8ff8138bc..4cf418992a 100644 --- a/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml +++ b/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml @@ -18,30 +18,30 @@ prompt-proto-service: instrument: pipelines: - # BLOCK-306 is photographic imaging - # BLOCK-T17 is daytime checkout - # BLOCK-271 is photon transfer curve calibrations - # BLOCK-295 is the daily calibration sequence as of May 27, 2024 - main: >- - (survey="BLOCK-306")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/SingleFrame.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml] - (survey="BLOCK-T17")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr-cal.yaml] - (survey="cwfs")=[] - (survey="cwfs-focus-sweep")=[] - (survey="spec-survey")=[] - (survey="BLOCK-271")=[] - (survey="BLOCK-295")=[] - (survey="")=[] - preprocessing: >- - (survey="BLOCK-306")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml] - (survey="BLOCK-T17")=[] - (survey="cwfs")=[] - (survey="cwfs-focus-sweep")=[] - (survey="spec-survey")=[] - (survey="BLOCK-271")=[] - (survey="BLOCK-295")=[] - (survey="")=[] + main: |- + - survey: BLOCK-306 # photographic imaging + pipelines: + - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/SingleFrame.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml + - survey: BLOCK-T17 # daytime checkout + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr-cal.yaml'] + - {survey: cwfs, pipelines: []} + - {survey: cwfs-focus-sweep, pipelines: []} + - {survey: spec-survey, pipelines: []} + - {survey: BLOCK-271, pipelines: []} # photon transfer curve calibrations + - {survey: BLOCK-295, pipelines: []} # the daily calibration sequence as of May 27, 2024 + - {survey: "", pipelines: []} + preprocessing: |- + - survey: BLOCK-306 + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml'] + - {survey: BLOCK-T17, pipelines: []} + - {survey: cwfs, pipelines: []} + - {survey: cwfs-focus-sweep, pipelines: []} + - {survey: spec-survey, pipelines: []} + - {survey: BLOCK-271, pipelines: []} + - {survey: BLOCK-295, pipelines: []} + - {survey: "", pipelines: []} calibRepo: s3://rubin-summit-users s3: diff --git a/applications/prompt-proto-service-latiss/values.yaml b/applications/prompt-proto-service-latiss/values.yaml index 9229ff3319..10e1fdcd2b 100644 --- a/applications/prompt-proto-service-latiss/values.yaml +++ b/applications/prompt-proto-service-latiss/values.yaml @@ -39,11 +39,11 @@ prompt-proto-service: # -- The "short" name of the instrument name: LATISS pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument diff --git a/applications/prompt-proto-service-lsstcam/README.md b/applications/prompt-proto-service-lsstcam/README.md index ca502f5749..9ad358da9f 100644 --- a/applications/prompt-proto-service-lsstcam/README.md +++ b/applications/prompt-proto-service-lsstcam/README.md @@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | prompt-proto-service.instrument.name | string | `""` | The "short" name of the instrument | -| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | prompt-proto-service.instrument.preloadPadding | int | `50` | Number of arcseconds to pad the spatial region in preloading. | | prompt-proto-service.instrument.skymap | string | `""` | Skymap to use with the instrument | | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/applications/prompt-proto-service-lsstcam/values.yaml b/applications/prompt-proto-service-lsstcam/values.yaml index 3eecc4bf32..d5c1b4d8e9 100644 --- a/applications/prompt-proto-service-lsstcam/values.yaml +++ b/applications/prompt-proto-service-lsstcam/values.yaml @@ -38,11 +38,11 @@ prompt-proto-service: # -- The "short" name of the instrument name: "" pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument diff --git a/applications/prompt-proto-service-lsstcomcam/README.md b/applications/prompt-proto-service-lsstcomcam/README.md index bb28592707..c08de7a78c 100644 --- a/applications/prompt-proto-service-lsstcomcam/README.md +++ b/applications/prompt-proto-service-lsstcomcam/README.md @@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | prompt-proto-service.instrument.name | string | `"LSSTComCam"` | The "short" name of the instrument | -| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | prompt-proto-service.instrument.preloadPadding | int | `50` | Number of arcseconds to pad the spatial region in preloading. | | prompt-proto-service.instrument.skymap | string | `"lsst_cells_v1"` | Skymap to use with the instrument | | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml index e8fbd39cd6..deacc0bbc8 100644 --- a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml @@ -11,10 +11,14 @@ prompt-proto-service: instrument: pipelines: - main: >- - (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/ApPipe.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr.yaml] - preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Preprocessing.yaml] + main: |- + - survey: SURVEY + pipelines: + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/ApPipe.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr.yaml + preprocessing: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Preprocessing.yaml'] calibRepo: s3://rubin-pp-dev-users/central_repo_2 s3: diff --git a/applications/prompt-proto-service-lsstcomcam/values-usdfprod-prompt-processing.yaml b/applications/prompt-proto-service-lsstcomcam/values-usdfprod-prompt-processing.yaml index ca5eec290a..72653c9f52 100644 --- a/applications/prompt-proto-service-lsstcomcam/values-usdfprod-prompt-processing.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values-usdfprod-prompt-processing.yaml @@ -19,41 +19,44 @@ prompt-proto-service: instrument: pipelines: - # BLOCK-T60 is optics alignment - # BLOCK-T75 is giant donuts - # BLOCK-T88 is optics alignment - # BLOCK-T215 is evening twilight flats - # BLOCK-T216 is morning twilight flats - # BLOCK-T219 is pretty picture - # BLOCK-T246 is instrument checkout - # BLOCK-T249 is AOS alignment - # BLOCK-T250 is TMA daytime checkout - main: >- - (survey="BLOCK-320")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/ApPipe.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/SingleFrame.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr.yaml] - (survey="BLOCK-T60")=[] - (survey="BLOCK-T75")=[] - (survey="BLOCK-T88")=[] - (survey="BLOCK-T215")=[] - (survey="BLOCK-T216")=[] - (survey="BLOCK-T219")=[] - (survey="BLOCK-T246")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr-cal.yaml] - (survey="BLOCK-T249")=[] - (survey="BLOCK-T250")=[] - (survey="")=[] - preprocessing: >- - (survey="BLOCK-320")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Preprocessing.yaml] - (survey="BLOCK-T60")=[] - (survey="BLOCK-T75")=[] - (survey="BLOCK-T88")=[] - (survey="BLOCK-T215")=[] - (survey="BLOCK-T216")=[] - (survey="BLOCK-T219")=[] - (survey="BLOCK-T246")=[] - (survey="BLOCK-T249")=[] - (survey="BLOCK-T250")=[] - (survey="")=[] + main: |- + - survey: BLOCK-320 # Science Validation + pipelines: + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/ApPipe.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/SingleFrame.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr.yaml + - survey: BLOCK-T60 # optics alignment + pipelines: [] + - survey: BLOCK-T75 # giant donuts + pipelines: [] + - survey: BLOCK-T88 # optics alignment + pipelines: [] + - survey: BLOCK-T215 # twilight flats + pipelines: [] + - survey: BLOCK-T216 # twilight flats + pipelines: [] + - survey: BLOCK-T219 # pretty pictures + pipeliens: [] + - survey: BLOCK-T246 # instrument checkout + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr-cal.yaml'] + - survey: BLOCK-T249 # AOS alignment + pipelines: [] + - survey: BLOCK-T250 # TMA daytime checkout + pipelines: [] + - {survey: "", pipelines: []} + preprocessing: |- + - survey: BLOCK-320 + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Preprocessing.yaml'] + - {survey: BLOCK-T60, pipelines: []} + - {survey: BLOCK-T75, pipelines: []} + - {survey: BLOCK-T88, pipelines: []} + - {survey: BLOCK-T215, pipelines: []} + - {survey: BLOCK-T216, pipelines: []} + - {survey: BLOCK-T219, pipelines: []} + - {survey: BLOCK-T246, pipelines: []} + - {survey: BLOCK-T249, pipelines: []} + - {survey: BLOCK-T250, pipelines: []} + - {survey: "", pipelines: []} calibRepo: s3://rubin-summit-users s3: diff --git a/applications/prompt-proto-service-lsstcomcam/values.yaml b/applications/prompt-proto-service-lsstcomcam/values.yaml index 13339b8fb8..4a46d6bb1a 100644 --- a/applications/prompt-proto-service-lsstcomcam/values.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values.yaml @@ -41,11 +41,11 @@ prompt-proto-service: # -- The "short" name of the instrument name: LSSTComCam pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument diff --git a/applications/prompt-proto-service-lsstcomcamsim/README.md b/applications/prompt-proto-service-lsstcomcamsim/README.md index 2dd70b63f0..9c1fa3e437 100644 --- a/applications/prompt-proto-service-lsstcomcamsim/README.md +++ b/applications/prompt-proto-service-lsstcomcamsim/README.md @@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear | | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | prompt-proto-service.instrument.name | string | `"LSSTComCamSim"` | The "short" name of the instrument | -| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | prompt-proto-service.instrument.preloadPadding | int | `50` | Number of arcseconds to pad the spatial region in preloading. | | prompt-proto-service.instrument.skymap | string | `"ops_rehersal_prep_2k_v1"` | Skymap to use with the instrument | | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/applications/prompt-proto-service-lsstcomcamsim/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-lsstcomcamsim/values-usdfdev-prompt-processing.yaml index 17d39cf975..194d937317 100644 --- a/applications/prompt-proto-service-lsstcomcamsim/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-proto-service-lsstcomcamsim/values-usdfdev-prompt-processing.yaml @@ -11,11 +11,15 @@ prompt-proto-service: instrument: pipelines: - main: >- - (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/ApPipe.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/SingleFrame.yaml, - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/Isr.yaml] - preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/Preprocessing.yaml] + main: |- + - survey: SURVEY + pipelines: + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/ApPipe.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/SingleFrame.yaml + - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/Isr.yaml + preprocessing: |- + - survey: SURVEY + pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCamSim/Preprocessing.yaml'] calibRepo: s3://rubin-pp-dev-users/central_repo_2 s3: diff --git a/applications/prompt-proto-service-lsstcomcamsim/values.yaml b/applications/prompt-proto-service-lsstcomcamsim/values.yaml index fc8dce6b9b..2b80a022a7 100644 --- a/applications/prompt-proto-service-lsstcomcamsim/values.yaml +++ b/applications/prompt-proto-service-lsstcomcamsim/values.yaml @@ -41,11 +41,11 @@ prompt-proto-service: # -- The "short" name of the instrument name: LSSTComCamSim pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument diff --git a/charts/prompt-proto-service/README.md b/charts/prompt-proto-service/README.md index 06c8d5eff9..41c08554c0 100644 --- a/charts/prompt-proto-service/README.md +++ b/charts/prompt-proto-service/README.md @@ -34,8 +34,8 @@ Event-driven processing of camera images | imagePullSecrets | list | `[]` | | | instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. | | instrument.name | string | None, must be set | The "short" name of the instrument | -| instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits' raws. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | -| instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. | +| instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. | +| instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. | | instrument.preloadPadding | int | `30` | Number of arcseconds to pad the spatial region in preloading. | | instrument.skymap | string | `""` | Skymap to use with the instrument | | knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). | diff --git a/charts/prompt-proto-service/templates/prompt-proto-service.yaml b/charts/prompt-proto-service/templates/prompt-proto-service.yaml index aca24d55ba..23eb47fd11 100644 --- a/charts/prompt-proto-service/templates/prompt-proto-service.yaml +++ b/charts/prompt-proto-service/templates/prompt-proto-service.yaml @@ -42,9 +42,11 @@ spec: - name: RUBIN_INSTRUMENT value: {{ .Values.instrument.name }} - name: PREPROCESSING_PIPELINES_CONFIG - value: {{ .Values.instrument.pipelines.preprocessing }} + value: |- + {{- .Values.instrument.pipelines.preprocessing | nindent 12 }} - name: MAIN_PIPELINES_CONFIG - value: {{ .Values.instrument.pipelines.main }} + value: |- + {{- .Values.instrument.pipelines.main | nindent 12 }} - name: SKYMAP value: {{ .Values.instrument.skymap }} - name: PRELOAD_PADDING diff --git a/charts/prompt-proto-service/values.yaml b/charts/prompt-proto-service/values.yaml index 04c55b7472..66fd0642e3 100644 --- a/charts/prompt-proto-service/values.yaml +++ b/charts/prompt-proto-service/values.yaml @@ -40,11 +40,11 @@ instrument: # @default -- None, must be set name: "" pipelines: - # -- Machine-readable string describing which pipeline(s) should be run for which visits' raws. - # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. + # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws. + # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. # @default -- None, must be set main: "" - # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. + # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. # @default -- None, must be set preprocessing: "" # -- Skymap to use with the instrument