diff --git a/montara/catalogsampler.py b/montara/catalogsampler.py index 8e3ee45..870c3f3 100644 --- a/montara/catalogsampler.py +++ b/montara/catalogsampler.py @@ -5,6 +5,10 @@ from .utils import add_field +import logging + +logger = logging.getLogger("pipeline") + class CatalogSampler(object): """class for randomly sampling object properties from a catalog""" @@ -64,7 +68,7 @@ def sample(self, rng): index = rand.randint(0, len(self.catalog_data)) if self.replace is False: self.catalog_data = np.delete(self.catalog_data, index) - return self.catalog_data[index], self.dtype + return self.catalog_data[index], self.dtype, index class CatalogSamplerLoader(InputLoader): @@ -107,6 +111,7 @@ def getKwargs(self, config, base, logger): safe = False if "cuts" in config: kwargs["cuts"] = config["cuts"] + return kwargs, safe @@ -117,11 +122,18 @@ def CatalogRow(config, base, name): catalog_sampler = galsim.config.GetInputObj( 'catalog_sampler', config, base, name, ) - catalog_row_data, dtype = catalog_sampler.sample(rng) + catalog_row_data, dtype, catalog_row_index = catalog_sampler.sample(rng) colnames = dtype.names + base['_catalog_row_data_catalog_row_ind'] = catalog_row_index base['_catalog_row_data'] = catalog_row_data base['_catalog_sampler_index'] = index base['_catalog_colnames'] = colnames + base['_catalog_used_rngnum'] = config.get("rng_num", None) + else: + if base['_catalog_used_rngnum'] != config.get("rng_num", None): + raise ValueError("Catalog sampler rng num changed from %s to %s!" % ( + base['_catalog_used_rngnum'], config.get("rng_num", None) + )) return base['_catalog_row_data'], base['_catalog_colnames'] @@ -129,6 +141,16 @@ def CatalogRow(config, base, name): def CatalogValue(config, base, value_type): row_data, colnames = CatalogRow(config, base, value_type) col = galsim.config.ParseValue(config, 'col', base, str)[0] + + logger.log( + logging.DEBUG, + " sampling gal catalog band|index|col: %s %s %s" % ( + base["eval_variables"]["sband"], + base['_catalog_row_data_catalog_row_ind'], + col, + ), + ) + try: return float(row_data[colnames.index(col)]) except ValueError as e: diff --git a/montara/des_tile.py b/montara/des_tile.py index 92f097f..7964ef6 100644 --- a/montara/des_tile.py +++ b/montara/des_tile.py @@ -2,6 +2,7 @@ import shutil from collections import OrderedDict import subprocess +import logging import fitsio import galsim @@ -106,11 +107,11 @@ def addNoise( if params["_zero_bkg"]: assert bkg_image.array.mean() == 0, "bkg not zero when it should be!" - logger.error("adding bkg with mean %.2e from file %s" % ( + logger.debug("adding bkg with mean %.2e from file %s" % ( (bkg_image.array).mean(), params["bkg_filename"])) im += bkg_image else: - logger.error("no bkg added") + logger.debug("no bkg added") def getBkg(self, config, base): params, safe = galsim.config.GetAllParams( @@ -149,6 +150,22 @@ def getNoiseVariance(self, config, base, full=None): return var +def _set_catalog_sampler_rng_num(cfg, rng_num): + "recursive function to set rng_num for catalog sampler" + if isinstance(cfg, dict): + for k in list(cfg.keys()): + cfg[k] = _set_catalog_sampler_rng_num(cfg[k], rng_num) + + if cfg.get("type", None) == "catalog_sampler_value": + cfg["rng_num"] = rng_num + + elif isinstance(cfg, list): + for i in range(len(cfg)): + cfg[i] = _set_catalog_sampler_rng_num(cfg[i], rng_num) + + return cfg + + class DESTileBuilder(OutputBuilder): """Implements the DESTile custom output type. @@ -294,9 +311,9 @@ def setup(self, config, base, file_num, logger): ] for i, is_rejectlisted in enumerate(is_rejectlisted_list): if is_rejectlisted: - logger.info("PSF for output file %s is rejectlisted" % ( + logger.debug("PSF for output file %s is rejectlisted" % ( output_file_names[i])) - logger.info( + logger.debug( "%d/%d piff files for tile %s rejectlisted" % ( is_rejectlisted_list.count(True), len(is_rejectlisted_list), @@ -371,11 +388,11 @@ def setup(self, config, base, file_num, logger): base["eval_variables"]["fcoadd_ra_max_deg"] = tile_setup["coadd_ra_ranges_deg"][1] base["eval_variables"]["fcoadd_dec_min_deg"] = tile_setup["coadd_dec_ranges_deg"][0] base["eval_variables"]["fcoadd_dec_max_deg"] = tile_setup["coadd_dec_ranges_deg"][1] - logger.info( + logger.debug( "ra min/max for tile %s: %f,%f degrees" % ( tilename, base["eval_variables"]["fra_min_deg"], base["eval_variables"]["fra_max_deg"])) - logger.info( + logger.debug( "dec min/max for tile %s: %f,%f degrees" % ( tilename, base["eval_variables"]["fdec_min_deg"], base["eval_variables"]["fdec_max_deg"])) @@ -543,7 +560,8 @@ def setup(self, config, base, file_num, logger): base['rng'] = galsim.BaseDeviate(seed) ngalaxies = galsim.config.ParseValue(nobjects, 'ngalaxies', base, int)[0] - logger.error("simulating %d galaxies" % ngalaxies) + logger.log(logging.CRITICAL, "simulating %d galaxies" % ngalaxies) + if nobjects.get("use_all_stars", True): # Now the stars. In this case # use all the stars in the star input catalog. We need @@ -567,7 +585,8 @@ def setup(self, config, base, file_num, logger): # If use_all_stars is False, and nstars is not specified, then # set nstars to zero. No stars for you. nstars = 0 - logger.error("simulating %d stars" % nstars) + logger.log(logging.CRITICAL, "simulating %d stars" % nstars) + nobj = nstars + ngalaxies # Save an object_type_list as a base_eval_variable, this can be used # with MixedScene to specify whether to draw a galaxy or star. @@ -607,7 +626,7 @@ def setup(self, config, base, file_num, logger): logger.critical("found non-integer nobj:", nobj) raise e base['image']['nobjects'] = nobj - logger.info( + logger.debug( 'nobjects = %s', galsim.config.CleanConfig(base['image']['nobjects'])) @@ -674,12 +693,16 @@ def setup(self, config, base, file_num, logger): base['gal']['rng_num'] = 1 if 'star' in base: base['star']['rng_num'] = 1 + if base["psf"]["type"] in ["DES_Piff", "DES_SmoothPiff"]: + base["psf"] = _set_catalog_sampler_rng_num(base["psf"], 1) if 'stamp' in base: base['stamp']['rng_num'] = 1 if 'image_pos' in base['image']: base['image']['image_pos']['rng_num'] = 1 if 'world_pos' in base['image']: base['image']['world_pos']['rng_num'] = 1 + if "truth" in base["output"]: + base["output"]["truth"] = _set_catalog_sampler_rng_num(base["output"]["truth"], 1) logger.debug( 'random_seed = %s', @@ -710,7 +733,7 @@ def setup(self, config, base, file_num, logger): else: border = 0 # 0 pixels - logger.info( + logger.debug( "generating gridded objects positions with dither %s and border %s", config.get("dither_scale", 0.5), border, ) diff --git a/montara/input_desstar.py b/montara/input_desstar.py index 3ac114c..64b4dd0 100644 --- a/montara/input_desstar.py +++ b/montara/input_desstar.py @@ -5,6 +5,10 @@ import fitsio from .utils import add_field +import logging + +logger = logging.getLogger("pipeline") + class DESStarCatalog(object): _req_params = {'file_name': str} @@ -51,6 +55,16 @@ def _GenerateFromStarCatalog(config, base, value_type): kwargs, safe = galsim.config.GetAllParams(config, base, req=req, opt=opt) col = kwargs['col'] index = kwargs['index'] + + logger.log( + logging.DEBUG, + "sampling star catalog band|index|col: %s %s %s" % ( + base["eval_variables"]["sband"], + index, + col, + ), + ) + val = star_input.get(index, col) return val, safe diff --git a/montara/tests/test_set_rng_num.py b/montara/tests/test_set_rng_num.py new file mode 100644 index 0000000..9eb2466 --- /dev/null +++ b/montara/tests/test_set_rng_num.py @@ -0,0 +1,42 @@ +import copy +from montara.des_tile import _set_catalog_sampler_rng_num + + +def test_set_catalog_sampler_rng_num(): + cfg = { + "blah": {"foo": [{"type": "catalog_sampler_value"}]}, + "blahblah": {"type": "blah"}, + "bar": {"type": "catalog_sampler_value"}, + } + new_cfg = _set_catalog_sampler_rng_num(copy.deepcopy(cfg), 1) + assert new_cfg == { + "blah": { + "foo": [{"type": "catalog_sampler_value", "rng_num": 1}] + }, + "blahblah": { + "type": "blah" + }, + "bar": { + "type": "catalog_sampler_value", + "rng_num": 1 + } + } + + cfg = { + "blah": {"foo": [{"type": "catalog_sampler_value"}]}, + "blahblah": {"type": "blah"}, + "bar": {"type": "catalog_sampler_value"}, + } + cfg = _set_catalog_sampler_rng_num(cfg, 1) + assert cfg == { + "blah": { + "foo": [{"type": "catalog_sampler_value", "rng_num": 1}] + }, + "blahblah": { + "type": "blah" + }, + "bar": { + "type": "catalog_sampler_value", + "rng_num": 1 + } + }