BUG tie PSF section to objects in RNGs (#29)

* BUG tie PSF section to objects in RNGs

* Update des_tile.py

* REF clean this up more

* REF use logger

* turn down logger

* no logging

* REF set rng_num more completely

* TST add test for function to set rng num

* BUG dumb bug

* STY please the flake8

montara/catalogsampler.py

@@ -5,6 +5,10 @@
 
 from .utils import add_field
 
+import logging
+
+logger = logging.getLogger("pipeline")
+
 
 class CatalogSampler(object):
     """class for randomly sampling object properties from a catalog"""
@@ -64,7 +68,7 @@ def sample(self, rng):
         index = rand.randint(0, len(self.catalog_data))
         if self.replace is False:
             self.catalog_data = np.delete(self.catalog_data, index)
-        return self.catalog_data[index], self.dtype
+        return self.catalog_data[index], self.dtype, index
 
 
 class CatalogSamplerLoader(InputLoader):
@@ -107,6 +111,7 @@ def getKwargs(self, config, base, logger):
             safe = False
         if "cuts" in config:
             kwargs["cuts"] = config["cuts"]
+
         return kwargs, safe
 
 
@@ -117,18 +122,35 @@ def CatalogRow(config, base, name):
         catalog_sampler = galsim.config.GetInputObj(
             'catalog_sampler', config, base, name,
         )
-        catalog_row_data, dtype = catalog_sampler.sample(rng)
+        catalog_row_data, dtype, catalog_row_index = catalog_sampler.sample(rng)
         colnames = dtype.names
+        base['_catalog_row_data_catalog_row_ind'] = catalog_row_index
         base['_catalog_row_data'] = catalog_row_data
         base['_catalog_sampler_index'] = index
         base['_catalog_colnames'] = colnames
+        base['_catalog_used_rngnum'] = config.get("rng_num", None)
+    else:
+        if base['_catalog_used_rngnum'] != config.get("rng_num", None):
+            raise ValueError("Catalog sampler rng num changed from %s to %s!" % (
+                base['_catalog_used_rngnum'], config.get("rng_num", None)
+            ))
 
     return base['_catalog_row_data'], base['_catalog_colnames']
 
 
 def CatalogValue(config, base, value_type):
     row_data, colnames = CatalogRow(config, base, value_type)
     col = galsim.config.ParseValue(config, 'col', base, str)[0]
+
+    logger.log(
+        logging.DEBUG,
+        " sampling gal catalog band|index|col: %s %s %s" % (
+            base["eval_variables"]["sband"],
+            base['_catalog_row_data_catalog_row_ind'],
+            col,
+        ),
+    )
+
     try:
         return float(row_data[colnames.index(col)])
     except ValueError as e:

montara/des_tile.py

@@ -2,6 +2,7 @@
 import shutil
 from collections import OrderedDict
 import subprocess
+import logging
 
 import fitsio
 import galsim
@@ -106,11 +107,11 @@ def addNoise(
             if params["_zero_bkg"]:
                 assert bkg_image.array.mean() == 0, "bkg not zero when it should be!"
 
-            logger.error("adding bkg with mean %.2e from file %s" % (
+            logger.debug("adding bkg with mean %.2e from file %s" % (
                 (bkg_image.array).mean(), params["bkg_filename"]))
             im += bkg_image
         else:
-            logger.error("no bkg added")
+            logger.debug("no bkg added")
 
     def getBkg(self, config, base):
         params, safe = galsim.config.GetAllParams(
@@ -149,6 +150,22 @@ def getNoiseVariance(self, config, base, full=None):
         return var
 
 
+def _set_catalog_sampler_rng_num(cfg, rng_num):
+    "recursive function to set rng_num for catalog sampler"
+    if isinstance(cfg, dict):
+        for k in list(cfg.keys()):
+            cfg[k] = _set_catalog_sampler_rng_num(cfg[k], rng_num)
+
+        if cfg.get("type", None) == "catalog_sampler_value":
+            cfg["rng_num"] = rng_num
+
+    elif isinstance(cfg, list):
+        for i in range(len(cfg)):
+            cfg[i] = _set_catalog_sampler_rng_num(cfg[i], rng_num)
+
+    return cfg
+
+
 class DESTileBuilder(OutputBuilder):
     """Implements the DESTile custom output type.
 
@@ -294,9 +311,9 @@ def setup(self, config, base, file_num, logger):
                 ]
                 for i, is_rejectlisted in enumerate(is_rejectlisted_list):
                     if is_rejectlisted:
-                        logger.info("PSF for output file %s is rejectlisted" % (
+                        logger.debug("PSF for output file %s is rejectlisted" % (
                             output_file_names[i]))
-                    logger.info(
+                    logger.debug(
                         "%d/%d piff files for tile %s rejectlisted" % (
                             is_rejectlisted_list.count(True),
                             len(is_rejectlisted_list),
@@ -371,11 +388,11 @@ def setup(self, config, base, file_num, logger):
         base["eval_variables"]["fcoadd_ra_max_deg"] = tile_setup["coadd_ra_ranges_deg"][1]
         base["eval_variables"]["fcoadd_dec_min_deg"] = tile_setup["coadd_dec_ranges_deg"][0]
         base["eval_variables"]["fcoadd_dec_max_deg"] = tile_setup["coadd_dec_ranges_deg"][1]
-        logger.info(
+        logger.debug(
             "ra min/max for tile %s: %f,%f degrees" % (
                 tilename, base["eval_variables"]["fra_min_deg"],
                 base["eval_variables"]["fra_max_deg"]))
-        logger.info(
+        logger.debug(
             "dec min/max for tile %s: %f,%f degrees" % (
                 tilename, base["eval_variables"]["fdec_min_deg"],
                 base["eval_variables"]["fdec_max_deg"]))
@@ -543,7 +560,8 @@ def setup(self, config, base, file_num, logger):
                     base['rng'] = galsim.BaseDeviate(seed)
                     ngalaxies = galsim.config.ParseValue(nobjects, 'ngalaxies',
                                                          base, int)[0]
-                    logger.error("simulating %d galaxies" % ngalaxies)
+                logger.log(logging.CRITICAL, "simulating %d galaxies" % ngalaxies)
+
                 if nobjects.get("use_all_stars", True):
                     # Now the stars. In this case
                     # use all the stars in the star input catalog. We need
@@ -567,7 +585,8 @@ def setup(self, config, base, file_num, logger):
                     # If use_all_stars is False, and nstars is not specified, then
                     # set nstars to zero. No stars for you.
                     nstars = 0
-                logger.error("simulating %d stars" % nstars)
+                logger.log(logging.CRITICAL, "simulating %d stars" % nstars)
+
                 nobj = nstars + ngalaxies
                 # Save an object_type_list as a base_eval_variable, this can be used
                 # with MixedScene to specify whether to draw a galaxy or star.
@@ -607,7 +626,7 @@ def setup(self, config, base, file_num, logger):
             logger.critical("found non-integer nobj:", nobj)
             raise e
         base['image']['nobjects'] = nobj
-        logger.info(
+        logger.debug(
             'nobjects = %s',
             galsim.config.CleanConfig(base['image']['nobjects']))
 
@@ -674,12 +693,16 @@ def setup(self, config, base, file_num, logger):
                 base['gal']['rng_num'] = 1
             if 'star' in base:
                 base['star']['rng_num'] = 1
+            if base["psf"]["type"] in ["DES_Piff", "DES_SmoothPiff"]:
+                base["psf"] = _set_catalog_sampler_rng_num(base["psf"], 1)
             if 'stamp' in base:
                 base['stamp']['rng_num'] = 1
             if 'image_pos' in base['image']:
                 base['image']['image_pos']['rng_num'] = 1
             if 'world_pos' in base['image']:
                 base['image']['world_pos']['rng_num'] = 1
+            if "truth" in base["output"]:
+                base["output"]["truth"] = _set_catalog_sampler_rng_num(base["output"]["truth"], 1)
 
         logger.debug(
             'random_seed = %s',
@@ -710,7 +733,7 @@ def setup(self, config, base, file_num, logger):
                 else:
                     border = 0  # 0 pixels
 
-                logger.info(
+                logger.debug(
                     "generating gridded objects positions with dither %s and border %s",
                     config.get("dither_scale", 0.5), border,
                 )

montara/input_desstar.py

@@ -5,6 +5,10 @@
 import fitsio
 from .utils import add_field
 
+import logging
+
+logger = logging.getLogger("pipeline")
+
 
 class DESStarCatalog(object):
     _req_params = {'file_name': str}
@@ -51,6 +55,16 @@ def _GenerateFromStarCatalog(config, base, value_type):
     kwargs, safe = galsim.config.GetAllParams(config, base, req=req, opt=opt)
     col = kwargs['col']
     index = kwargs['index']
+
+    logger.log(
+        logging.DEBUG,
+        "sampling star catalog band|index|col: %s %s %s" % (
+            base["eval_variables"]["sband"],
+            index,
+            col,
+        ),
+    )
+
     val = star_input.get(index, col)
     return val, safe
 

montara/tests/test_set_rng_num.py

@@ -0,0 +1,42 @@
+import copy
+from montara.des_tile import _set_catalog_sampler_rng_num
+
+
+def test_set_catalog_sampler_rng_num():
+    cfg = {
+        "blah": {"foo": [{"type": "catalog_sampler_value"}]},
+        "blahblah": {"type": "blah"},
+        "bar": {"type": "catalog_sampler_value"},
+    }
+    new_cfg = _set_catalog_sampler_rng_num(copy.deepcopy(cfg), 1)
+    assert new_cfg == {
+        "blah": {
+            "foo": [{"type": "catalog_sampler_value", "rng_num": 1}]
+            },
+        "blahblah": {
+            "type": "blah"
+        },
+        "bar": {
+            "type": "catalog_sampler_value",
+            "rng_num": 1
+        }
+    }
+
+    cfg = {
+        "blah": {"foo": [{"type": "catalog_sampler_value"}]},
+        "blahblah": {"type": "blah"},
+        "bar": {"type": "catalog_sampler_value"},
+    }
+    cfg = _set_catalog_sampler_rng_num(cfg, 1)
+    assert cfg == {
+        "blah": {
+            "foo": [{"type": "catalog_sampler_value", "rng_num": 1}]
+            },
+        "blahblah": {
+            "type": "blah"
+        },
+        "bar": {
+            "type": "catalog_sampler_value",
+            "rng_num": 1
+        }
+    }