Merge pull request #56 from lsst/tickets/DM-45856

DM-45856: Add new verification pipelines for IsrTaskLSST to _ingredients.

pipelines/DECam/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.decam.DarkEnergyCamera
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/HSC/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.subaru.HyperSuprimeCam
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/LATISS/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.lsst.Latiss
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/LSSTCam-imSim/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.lsst.LsstCamImSim
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/LSSTCam/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.lsst.LsstCam
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/LSSTComCam/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.lsst.LsstComCam
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/LSSTComCamSim/verifyGain.yaml

@@ -1,4 +1,4 @@
 description: cp_verify gain-from-flat-pairs calibration verification
 instrument: lsst.obs.lsst.LsstComCamSim
 imports:
-  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGain.yaml
+  - location: $CP_VERIFY_DIR/pipelines/_ingredients/verifyGainFromFlatPairs.yaml

pipelines/_ingredients/verifyBfkLSST.yaml

@@ -0,0 +1,77 @@
+description: cp_verify BFK calibration verification
+# TODO DM-46445: This pipeline is a draft, but does not fully work
+# and will be fixed on that ticket.
+tasks:
+  verifyBfkIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "verifyBfkIsrExp"
+      connections.outputStatistics: "verifyBfkIsrStatistics"
+      crosstalk.doQuadraticCrosstalkCorrection: true
+      # TODO DM-46426: Add cpCtiLSST pipeline so this can be true.
+      doDeferredCharge: false
+      doBrighterFatter: true
+  verifyBfkChar:
+    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
+    config:
+      connections.exposure: "verifyBfkIsrExp"
+      connections.characterized: "verifyBfkIcExp"
+      connections.sourceCat: "verifyBfkIcSrc"
+      doApCorr: false
+      doDeblend: false
+      doMeasurePsf: false
+      doNormalizedCalibration: false
+  verifyUncorrectedBfkIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "verifyUncorrectedBfkIsrExp"
+      connections.outputStatistics: "verifyUncorrectedBfkIsrStatistics"
+      crosstalk.doQuadraticCrosstalkCorrection: true
+      # TODO DM-46426: Add cpCtiLSST pipeline so this can be true.
+      doDeferredCharge: false
+      doBrighterFatter: false
+  verifyUncorrectedBfkChar:
+    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
+    config:
+      connections.exposure: "verifyUncorrectedBfkIsrExp"
+      connections.characterized: "verifyUncorrectedBfkIcExp"
+      connections.sourceCat: "verifyUncorrectedBfkIcSrc"
+      connections.backgroundModel: "verifyUncorrectedBfkBackground"
+      connections.outputSchema: "verifyUncorrectedBfk_schema"
+      doApCorr: false
+      doDeblend: false
+      doMeasurePsf: false
+      doNormalizedCalibration: false
+  verifyBfkDet:
+    class: lsst.cp.verify.CpVerifyBfkTask
+    config:
+      connections.inputExp: "verifyBfkIcExp"
+      connections.inputCatalog: "verifyBfkIcSrc"
+      connections.uncorrectedCatalog: "verifyUncorrectedBfkIcSrc"
+      connections.outputStats: "verifyBfkDetStats"
+      connections.outputResults: "verifyBfkDetResults"
+  verifyBfkExp:
+    class: lsst.cp.verify.CpVerifyVisitExpMergeTask
+    config:
+      connections.inputStats: "verifyBfkDetStats"
+      connections.inputResults: "verifyBfkDetResults"
+      connections.outputStats: "verifyBfkExpStats"
+      connections.outputResults: "verifyBfkExpResults"
+  verifyBfk:
+    class: lsst.cp.verify.CpVerifyVisitRunMergeTask
+    config:
+      connections.inputStats: "verifyBfkExpStats"
+      connections.inputResults: "verifyBfkExpResults"
+      connections.outputStats: "verifyBfkStats"
+      connections.outputResults: "verifyBfkResults"
+contracts:
+  - verifyBfkChar.connections.exposure == verifyBfkIsr.connections.outputExposure
+  - verifyUncorrectedBfkChar.connections.exposure == verifyUncorrectedBfkIsr.connections.outputExposure
+  - verifyBfkDet.connections.inputCatalog == verifyBfkChar.connections.sourceCat
+  - verifyBfkDet.connections.uncorrectedCatalog == verifyUncorrectedBfkChar.connections.sourceCat
+  - verifyBfkExp.connections.inputStats == verifyBfkDet.connections.outputStats
+  - verifyBfkExp.connections.inputResults == verifyBfkDet.connections.outputResults
+  - verifyBfk.connections.inputStats == verifyBfkExp.connections.outputStats
+  - verifyBfk.connections.inputResults == verifyBfkExp.connections.outputResults

pipelines/_ingredients/verifyBiasLSST.yaml

@@ -0,0 +1,90 @@
+description: cp_verify BIAS calibration verification (LSST style).
+imports:
+  - location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml
+    include:
+      - analyzeBiasCore
+tasks:
+  verifyBiasIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "verifyBiasIsrExp"
+      connections.outputBin1Exposure: "verifyBiasIsrExpBin8"
+      connections.outputBin2Exposure: "verifyBiasIsrExpBin64"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = True
+        config.doApplyGains = True
+        config.doLinearize = True
+        config.doBias = True
+        config.doDefect = True
+        config.doInterpolate = True
+        config.isrStats.doCopyCalibDistributionStatistics = True
+        config.isrStats.doProjectionStatistics = True
+        config.isrStats.doBiasShiftStatistics = True
+        config.isrStats.doAmplifierCorrelationStatistics = True
+        config.doBinnedExposures = True
+        config.binFactor1 = 8
+        config.binFactor2 = 64
+  verifyBiasDet:
+    class: lsst.cp.verify.CpVerifyBiasTask
+    config:
+      connections.inputExp: "verifyBiasIsrExp"
+      connections.outputStats: "verifyBiasDetStats"
+      connections.outputResults: "verifyBiasDetResults"
+      connections.outputMatrix: "verifyBiasDetMatrix"
+      doVignette: false
+      useReadNoise: true
+      useIsrStatistics: true
+      hasMatrixCatalog: true
+  verifyBiasExp:
+    class: lsst.cp.verify.CpVerifyExpMergeTask
+    config:
+      connections.inputStats: "verifyBiasDetStats"
+      connections.inputResults: "verifyBiasDetResults"
+      connections.inputMatrix: "verifyBiasDetMatrix"
+      connections.outputStats: "verifyBiasExpStats"
+      connections.outputResults: "verifyBiasExpResults"
+      connections.outputMatrix: "verifyBiasExpMatrix"
+      hasInputResults: true
+      hasMatrixCatalog: true
+  verifyBias:
+    class: lsst.cp.verify.CpVerifyRunMergeTask
+    config:
+      connections.inputStats: "verifyBiasExpStats"
+      connections.inputResults: "verifyBiasExpResults"
+      connections.inputMatrix: "verifyBiasExpMatrix"
+      connections.outputStats: "verifyBiasStats"
+      connections.outputResults: "verifyBiasResults"
+      connections.outputMatrix: "verifyBiasMatrix"
+      hasInputResults: true
+      hasMatrixCatalog: true
+  analyzeBiasCore: lsst.analysis.tools.tasks.VerifyCalibAnalysisTask
+  mosaicExp8:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicExpTask
+    config:
+      connections.inputExps: 'verifyBiasIsrExpBin8'
+      connections.outputData: 'verifyBiasResidual8'
+      binning: 8
+  mosaicExp64:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicExpTask
+    config:
+      connections.inputExps: 'verifyBiasIsrExpBin64'
+      connections.outputData: 'verifyBiasResidual64'
+      binning: 64
+contracts:
+  - verifyBiasIsr.binFactor1 == mosaicExp8.binning
+  - verifyBiasIsr.binFactor2 == mosaicExp64.binning
+  - verifyBiasDet.connections.inputExp == verifyBiasIsr.connections.outputExposure
+  - verifyBiasExp.connections.inputStats == verifyBiasDet.connections.outputStats
+  - verifyBiasExp.connections.inputResults == verifyBiasDet.connections.outputResults
+  - verifyBiasExp.connections.inputMatrix == verifyBiasDet.connections.outputMatrix
+  - verifyBias.connections.inputStats == verifyBiasExp.connections.outputStats
+  - verifyBias.connections.inputResults == verifyBiasExp.connections.outputResults
+  - verifyBias.connections.inputMatrix == verifyBiasExp.connections.outputMatrix
+  - mosaicExp8.connections.inputExps == verifyBiasIsr.connections.outputBin1Exposure
+  - mosaicExp64.connections.inputExps == verifyBiasIsr.connections.outputBin2Exposure

pipelines/_ingredients/verifyDarkLSST.yaml

@@ -0,0 +1,51 @@
+description: cp_verify DARK calibration verification (LSST isr task)
+tasks:
+  verifyDarkIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "verifyDarkIsrExp"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = True
+        config.doApplyGains = True
+        config.doLinearize = True
+        config.doBias = True
+        config.doDark = True
+        config.doDefect = True
+        config.isrStats.doCopyCalibDistributionStatistics = True
+  verifyDarkDet:
+    class: lsst.cp.verify.CpVerifyDarkTask
+    config:
+      connections.inputExp: "verifyDarkIsrExp"
+      connections.outputStats: "verifyDarkDetStats"
+      connections.outputResults: "verifyDarkDetResults"
+      useIsrStatistics: true
+  verifyDarkExp:
+    class: lsst.cp.verify.CpVerifyExpMergeTask
+    config:
+      connections.inputStats: "verifyDarkDetStats"
+      connections.inputResults: "verifyDarkDetResults"
+      connections.outputStats: "verifyDarkExpStats"
+      connections.outputResults: "verifyDarkExpResults"
+      hasInputResults: true
+      hasMatrixCatalog: false
+  verifyDark:
+    class: lsst.cp.verify.CpVerifyRunMergeTask
+    config:
+      connections.inputStats: "verifyDarkExpStats"
+      connections.inputResults: "verifyDarkExpResults"
+      connections.outputStats: "verifyDarkStats"
+      connections.outputResults: "verifyDarkResults"
+      hasInputResults: true
+      hasMatrixCatalog: false
+contracts:
+  - verifyDarkDet.connections.inputExp == verifyDarkIsr.connections.outputExposure
+  - verifyDarkExp.connections.inputStats == verifyDarkDet.connections.outputStats
+  - verifyDarkExp.connections.inputResults == verifyDarkDet.connections.outputResults
+  - verifyDark.connections.inputStats == verifyDarkExp.connections.outputStats
+  - verifyDark.connections.inputResults == verifyDarkExp.connections.outputResults