Merge pull request #3878 from lsst-sqre/tickets/DM-47529

DM-47529: Migrate Prompt Processing PipelinesConfig to YAML
lsst-sqre · Nov 25, 2024 · 4f2c83c · 4f2c83c
2 parents e6b3f35 + d7b0880
commit 4f2c83c
Show file tree

Hide file tree

Showing 22 changed files with 139 additions and 113 deletions.
diff --git a/applications/prompt-proto-service-hsc-gpu/README.md b/applications/prompt-proto-service-hsc-gpu/README.md
@@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear |
 | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. |
 | prompt-proto-service.instrument.name | string | `"HSC"` | The "short" name of the instrument |
-| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
-| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. |
+| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
+| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. |
 | prompt-proto-service.instrument.preloadPadding | int | `42` | Number of arcseconds to pad the spatial region in preloading. |
 | prompt-proto-service.instrument.skymap | string | `"hsc_rings_v1"` | Skymap to use with the instrument |
 | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). |

diff --git a/applications/prompt-proto-service-hsc-gpu/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-hsc-gpu/values-usdfdev-prompt-processing.yaml
@@ -12,8 +12,12 @@ prompt-proto-service:
 
   instrument:
     pipelines:
-      main: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml]
-      preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml]
+      main: |-
+        - survey: SURVEY
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml']
+      preprocessing: |-
+        - survey: SURVEY
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml']
     calibRepo: s3://rubin-pp-dev-users/central_repo_2
 
   s3:

diff --git a/applications/prompt-proto-service-hsc-gpu/values.yaml b/applications/prompt-proto-service-hsc-gpu/values.yaml
@@ -38,11 +38,11 @@ prompt-proto-service:
     # -- The "short" name of the instrument
     name: HSC
     pipelines:
-      # -- Machine-readable string describing which pipeline(s) should be run for which visits.
-      # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
+      # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws.
+      # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
       # @default -- None, must be set
       main: ""
-      # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival.
+      # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival.
       # @default -- None, must be set
       preprocessing: ""
     # -- Skymap to use with the instrument

diff --git a/applications/prompt-proto-service-hsc/README.md b/applications/prompt-proto-service-hsc/README.md
@@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear |
 | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. |
 | prompt-proto-service.instrument.name | string | `"HSC"` | The "short" name of the instrument |
-| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
-| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. |
+| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
+| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. |
 | prompt-proto-service.instrument.preloadPadding | int | `42` | Number of arcseconds to pad the spatial region in preloading. |
 | prompt-proto-service.instrument.skymap | string | `"hsc_rings_v1"` | Skymap to use with the instrument |
 | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). |

diff --git a/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-hsc/values-usdfdev-prompt-processing.yaml
@@ -13,8 +13,12 @@ prompt-proto-service:
 
   instrument:
     pipelines:
-      main: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml]
-      preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml]
+      main: |-
+        - survey: SURVEY
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/ApPipe.yaml']
+      preprocessing: |-
+        - survey: SURVEY
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/HSC/Preprocessing.yaml']
     calibRepo: s3://rubin-pp-dev-users/central_repo_2
 
   s3:

diff --git a/applications/prompt-proto-service-hsc/values.yaml b/applications/prompt-proto-service-hsc/values.yaml
@@ -38,11 +38,11 @@ prompt-proto-service:
     # -- The "short" name of the instrument
     name: HSC
     pipelines:
-      # -- Machine-readable string describing which pipeline(s) should be run for which visits.
-      # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
+      # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws.
+      # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
       # @default -- None, must be set
       main: ""
-      # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival.
+      # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival.
       # @default -- None, must be set
       preprocessing: ""
     # -- Skymap to use with the instrument

diff --git a/applications/prompt-proto-service-latiss/README.md b/applications/prompt-proto-service-latiss/README.md
@@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear |
 | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. |
 | prompt-proto-service.instrument.name | string | `"LATISS"` | The "short" name of the instrument |
-| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
-| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. |
+| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
+| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. |
 | prompt-proto-service.instrument.preloadPadding | int | `30` | Number of arcseconds to pad the spatial region in preloading. |
 | prompt-proto-service.instrument.skymap | string | `"latiss_v1"` | Skymap to use with the instrument |
 | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). |

diff --git a/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfdev-prompt-processing.yaml
@@ -11,10 +11,15 @@ prompt-proto-service:
 
   instrument:
     pipelines:
-      main: >-
-        (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml,
-        ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml]
-      preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml]
+      main: |-
+        - survey: SURVEY
+          pipelines:
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml
+          # Fallback for an upload.py image without templates
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml
+      preprocessing: |-
+        - survey: SURVEY
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml']
     calibRepo: s3://rubin-pp-dev-users/central_repo_2
 
   s3:

diff --git a/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml b/applications/prompt-proto-service-latiss/values-usdfprod-prompt-processing.yaml
@@ -18,30 +18,30 @@ prompt-proto-service:
 
   instrument:
     pipelines:
-      # BLOCK-306 is photographic imaging
-      # BLOCK-T17 is daytime checkout
-      # BLOCK-271 is photon transfer curve calibrations
-      # BLOCK-295 is the daily calibration sequence as of May 27, 2024
-      main: >-
-        (survey="BLOCK-306")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml,
-        ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/SingleFrame.yaml,
-        ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml]
-        (survey="BLOCK-T17")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr-cal.yaml]
-        (survey="cwfs")=[]
-        (survey="cwfs-focus-sweep")=[]
-        (survey="spec-survey")=[]
-        (survey="BLOCK-271")=[]
-        (survey="BLOCK-295")=[]
-        (survey="")=[]
-      preprocessing: >-
-        (survey="BLOCK-306")=[${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml]
-        (survey="BLOCK-T17")=[]
-        (survey="cwfs")=[]
-        (survey="cwfs-focus-sweep")=[]
-        (survey="spec-survey")=[]
-        (survey="BLOCK-271")=[]
-        (survey="BLOCK-295")=[]
-        (survey="")=[]
+      main: |-
+        - survey: BLOCK-306  # photographic imaging
+          pipelines:
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/ApPipe.yaml
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/SingleFrame.yaml
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr.yaml
+        - survey: BLOCK-T17  # daytime checkout
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Isr-cal.yaml']
+        - {survey: cwfs, pipelines: []}
+        - {survey: cwfs-focus-sweep, pipelines: []}
+        - {survey: spec-survey, pipelines: []}
+        - {survey: BLOCK-271, pipelines: []}  # photon transfer curve calibrations
+        - {survey: BLOCK-295, pipelines: []}  # the daily calibration sequence as of May 27, 2024
+        - {survey: "", pipelines: []}
+      preprocessing: |-
+        - survey: BLOCK-306
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LATISS/Preprocessing.yaml']
+        - {survey: BLOCK-T17, pipelines: []}
+        - {survey: cwfs, pipelines: []}
+        - {survey: cwfs-focus-sweep, pipelines: []}
+        - {survey: spec-survey, pipelines: []}
+        - {survey: BLOCK-271, pipelines: []}
+        - {survey: BLOCK-295, pipelines: []}
+        - {survey: "", pipelines: []}
     calibRepo: s3://rubin-summit-users
 
   s3:

diff --git a/applications/prompt-proto-service-latiss/values.yaml b/applications/prompt-proto-service-latiss/values.yaml
@@ -39,11 +39,11 @@ prompt-proto-service:
     # -- The "short" name of the instrument
     name: LATISS
     pipelines:
-      # -- Machine-readable string describing which pipeline(s) should be run for which visits.
-      # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
+      # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws.
+      # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
       # @default -- None, must be set
       main: ""
-      # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival.
+      # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival.
       # @default -- None, must be set
       preprocessing: ""
     # -- Skymap to use with the instrument

diff --git a/applications/prompt-proto-service-lsstcam/README.md b/applications/prompt-proto-service-lsstcam/README.md
@@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear |
 | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. |
 | prompt-proto-service.instrument.name | string | `""` | The "short" name of the instrument |
-| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
-| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. |
+| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
+| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. |
 | prompt-proto-service.instrument.preloadPadding | int | `50` | Number of arcseconds to pad the spatial region in preloading. |
 | prompt-proto-service.instrument.skymap | string | `""` | Skymap to use with the instrument |
 | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). |

diff --git a/applications/prompt-proto-service-lsstcam/values.yaml b/applications/prompt-proto-service-lsstcam/values.yaml
@@ -38,11 +38,11 @@ prompt-proto-service:
     # -- The "short" name of the instrument
     name: ""
     pipelines:
-      # -- Machine-readable string describing which pipeline(s) should be run for which visits.
-      # Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
+      # -- YAML-formatted config describing which pipeline(s) should be run for which visits' raws.
+      # Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples.
       # @default -- None, must be set
       main: ""
-      # -- Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival.
+      # -- YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival.
       # @default -- None, must be set
       preprocessing: ""
     # -- Skymap to use with the instrument

diff --git a/applications/prompt-proto-service-lsstcomcam/README.md b/applications/prompt-proto-service-lsstcomcam/README.md
@@ -31,8 +31,8 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.imageNotifications.topic | string | None, must be set | Topic where raw image arrival notifications appear |
 | prompt-proto-service.instrument.calibRepo | string | None, must be set | URI to the shared repo used for calibrations, templates, and pipeline outputs. If `registry.centralRepoFile` is set, this URI points to a local redirect instead of the central repo itself. |
 | prompt-proto-service.instrument.name | string | `"LSSTComCam"` | The "short" name of the instrument |
-| prompt-proto-service.instrument.pipelines.main | string | None, must be set | Machine-readable string describing which pipeline(s) should be run for which visits. Notation is complex and still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
-| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | Machine-readable string describing which pipeline(s) should be run before which visits' raw arrival. |
+| prompt-proto-service.instrument.pipelines.main | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run for which visits' raws. Fields are still in flux; see [the source code](https://github.com/lsst-dm/prompt_processing/blob/main/python/activator/config.py) for examples. |
+| prompt-proto-service.instrument.pipelines.preprocessing | string | None, must be set | YAML-formatted config describing which pipeline(s) should be run before which visits' raw arrival. |
 | prompt-proto-service.instrument.preloadPadding | int | `50` | Number of arcseconds to pad the spatial region in preloading. |
 | prompt-proto-service.instrument.skymap | string | `"lsst_cells_v1"` | Skymap to use with the instrument |
 | prompt-proto-service.knative.cpuLimit | int | `1` | The maximum cpu cores for the full pod (see `containerConcurrency`). |

diff --git a/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml b/applications/prompt-proto-service-lsstcomcam/values-usdfdev-prompt-processing.yaml
@@ -11,10 +11,14 @@ prompt-proto-service:
 
   instrument:
     pipelines:
-      main: >-
-        (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/ApPipe.yaml,
-        ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr.yaml]
-      preprocessing: (survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Preprocessing.yaml]
+      main: |-
+        - survey: SURVEY
+          pipelines:
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/ApPipe.yaml
+          - ${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Isr.yaml
+      preprocessing: |-
+        - survey: SURVEY
+          pipelines: ['${PROMPT_PROCESSING_DIR}/pipelines/LSSTComCam/Preprocessing.yaml']
     calibRepo: s3://rubin-pp-dev-users/central_repo_2
 
   s3: