Merge pull request #192 from esheldon/change-log-mom

CHANGES.md

@@ -1,4 +1,20 @@
-## v2.0.3 (unreleased)
+## v2.0.3
+
+### new features
+
+    - All moments codes now produce similar output dictionaries and flags.
+    - The `PrePSFGaussMom` class has been renamed to `PGaussMom`. The old name still
+      exists for backwards compatibility but will be deprecated and removed at
+      some future date.
+    - The moments codes and `gaussap.py` codes now use np.nan for missing values.
+    - The flag bits have been normalized across the code base to now all reside in
+      `ngmix.flags`. A new function to convert the flags to a descriptive string is
+      located there as well.
+    - A new function exists to convert raw moments to shape results:
+      `ngmix.moments.make_mom_result`
+    - The moments fitters now have a `kind` attribute to help downstream code
+      process them uniformly.
+    - Added slots for position moments in moments output for moments fitters. 
 
 ### bug fixes
 

ngmix/admom/admom.py

@@ -250,7 +250,7 @@ class AdmomFitter(object):
         on a T guess
     """
 
-    kind = "admom"
+    kind = "am"
 
     def __init__(self,
                  maxiter=DEFAULT_MAXITER,

ngmix/gmix/gmix.py

@@ -1178,19 +1178,8 @@ def get_weighted_moments_stats(ares):
         else:
             res[n] = ares[n]
 
-    mom = np.array([
-        res['sums'][5],
-        res['sums'][4],
-        res['sums'][2],
-        res['sums'][3],
-    ])
-    mom_cov = np.zeros((4, 4))
-    for inew, iold in enumerate([5, 4, 2, 3]):
-        for jnew, jold in enumerate([5, 4, 2, 3]):
-            mom_cov[inew, jnew] += res['sums_cov'][iold, jold]
-
     res.update(
-        moments.make_mom_result(mom, mom_cov)
+        moments.make_mom_result(res["sums"], res["sums_cov"])
     )
 
     return res

ngmix/moments.py

@@ -5,6 +5,15 @@
 import ngmix.flags
 from .util import get_ratio_error
 
+MOMENTS_NAME_MAP = {
+    "Mv": 0,
+    "Mu": 1,
+    "M1": 2,
+    "M2": 3,
+    "MT": 4,
+    "MF": 5,
+}
+
 
 def sigma_to_fwhm(sigma):
     """
@@ -346,7 +355,7 @@ def make_mom_result(mom, mom_cov):
     Parameters
     ----------
     mom : np.ndarray
-        The array of moments in the order [flux, <x**2 + y**2>, <x**2 - y**2>, 2*<xy>].
+        The array of moments in the order [Mv, Mu, M1, M2, MT, MF].
     mom_cov : np.ndarray
         The array of moment covariances.
 
@@ -355,11 +364,29 @@ def make_mom_result(mom, mom_cov):
     res : dict
         A dictionary of results.
     """
+    # for safety...
+    if len(mom) != 6:
+        raise ValueError(
+            "You must pass exactly 6 moments in the order [Mv, Mu, M1, M2, MT, MF] "
+            "for ngmix.moments.make_mom_result."
+        )
+    if mom_cov.shape != (6, 6):
+        raise ValueError(
+            "You must pass a 6x6 matrix for ngmix.moments.make_mom_result."
+        )
+
+    mv_ind = MOMENTS_NAME_MAP["Mv"]
+    mu_ind = MOMENTS_NAME_MAP["Mu"]
+    mf_ind = MOMENTS_NAME_MAP["MF"]
+    mt_ind = MOMENTS_NAME_MAP["MT"]
+    m1_ind = MOMENTS_NAME_MAP["M1"]
+    m2_ind = MOMENTS_NAME_MAP["M2"]
+
     # now finally build the outputs and their errors
     res = {}
     res["flags"] = 0
     res["flagstr"] = ""
-    res["flux"] = mom[0]
+    res["flux"] = mom[mf_ind]
     res["mom"] = mom
     res["mom_cov"] = mom_cov
     res["flux_flags"] = 0
@@ -377,22 +404,25 @@ def make_mom_result(mom, mom_cov):
     res["e"] = np.array([np.nan, np.nan])
     res["e_err"] = np.array([np.nan, np.nan])
     res["e_cov"] = np.diag([np.nan, np.nan])
-    res["mom_err"] = np.array([np.nan, np.nan, np.nan, np.nan])
+    res["mom_err"] = np.array([np.nan, np.nan, np.nan, np.nan, np.nan, np.nan])
 
     # handle flux-only
-    if np.diagonal(mom_cov)[0] > 0:
-        res["flux_err"] = np.sqrt(mom_cov[0, 0])
+    if mom_cov[mf_ind, mf_ind] > 0:
+        res["flux_err"] = np.sqrt(mom_cov[mf_ind, mf_ind])
         res["s2n"] = res["flux"] / res["flux_err"]
     else:
         res["flux_flags"] |= ngmix.flags.NONPOS_VAR
 
     # handle flux+T only
-    if np.all(np.diagonal(mom_cov)[0:2] > 0):
-        if mom[0] > 0:
-            res["T"] = mom[1] / mom[0]
+    if mom_cov[mf_ind, mf_ind] > 0 and mom_cov[mt_ind, mt_ind] > 0:
+        if mom[mf_ind] > 0:
+            res["T"] = mom[mt_ind] / mom[mf_ind]
             res["T_err"] = get_ratio_error(
-                mom[1], mom[0],
-                mom_cov[1, 1], mom_cov[0, 0], mom_cov[0, 1]
+                mom[mt_ind],
+                mom[mf_ind],
+                mom_cov[mt_ind, mt_ind],
+                mom_cov[mf_ind, mf_ind],
+                mom_cov[mt_ind, mf_ind],
             )
         else:
             # flux <= 0.0
@@ -407,26 +437,35 @@ def make_mom_result(mom, mom_cov):
         res["flags"] |= ngmix.flags.NONPOS_VAR
 
     if res["flags"] == 0:
-        if mom[0] > 0:
+        if res["flux"] > 0:
             if res["T"] > 0:
+                res["e1"] = mom[m1_ind] / mom[mt_ind]
+                res["e2"] = mom[m2_ind] / mom[mt_ind]
+                res["e"] = np.array([res["e1"], res["e2"]])
+
                 res["pars"] = np.array([
-                    0, 0,
-                    mom[2]/mom[0],
-                    mom[3]/mom[0],
-                    mom[1]/mom[0],
-                    mom[0],
+                    mom[mv_ind],
+                    mom[mu_ind],
+                    res["e1"],
+                    res["e2"],
+                    res["T"],
+                    res["flux"],
                 ])
-                res["e1"] = mom[2] / mom[1]
-                res["e2"] = mom[3] / mom[1]
-                res["e"] = np.array([res["e1"], res["e2"]])
+
                 e_err = np.zeros(2)
                 e_err[0] = get_ratio_error(
-                    mom[2], mom[1],
-                    mom_cov[2, 2], mom_cov[1, 1], mom_cov[1, 2]
+                    mom[m1_ind],
+                    mom[mt_ind],
+                    mom_cov[m1_ind, m1_ind],
+                    mom_cov[mt_ind, mt_ind],
+                    mom_cov[m1_ind, mt_ind],
                 )
                 e_err[1] = get_ratio_error(
-                    mom[3], mom[1],
-                    mom_cov[3, 3], mom_cov[1, 1], mom_cov[1, 3]
+                    mom[m2_ind],
+                    mom[mt_ind],
+                    mom_cov[m2_ind, m2_ind],
+                    mom_cov[mt_ind, mt_ind],
+                    mom_cov[m2_ind, mt_ind]
                 )
                 if np.all(np.isfinite(e_err)):
                     res["e_err"] = e_err

ngmix/prepsfmom.py

@@ -28,7 +28,8 @@ class PrePSFMom(object):
         whatever the Jacobian on the obs converts pixels units to. This is typically
         arcseconds.
     kernel : str
-        The kernel to use. Either `ksigma` or `gauss`.
+        The kernel to use. Either `ksigma` or `pgauss` or `gauss`. `gauss` and `pgauss`
+        are aliases for the same thing.
     pad_factor : int, optional
         The factor by which to pad the FFTs used for the image. Default is 4.
     """
@@ -117,7 +118,7 @@ def _meas(self, obs, psf_obs, return_kernels):
                 obs.jacobian.dvdrow, obs.jacobian.dvdcol,
                 obs.jacobian.dudrow, obs.jacobian.dudcol,
             )
-        elif self.kernel == "gauss":
+        elif self.kernel in ["gauss", "pgauss"]:
             kernels = _gauss_kernels(
                 target_dim,
                 self.fwhm,
@@ -202,7 +203,7 @@ class PGaussMom(PrePSFMom):
     kind = "pgauss"
 
     def __init__(self, fwhm, pad_factor=4):
-        super().__init__(fwhm, 'gauss', pad_factor=pad_factor)
+        super().__init__(fwhm, 'pgauss', pad_factor=pad_factor)
 
 
 # keep this here for API consistency
@@ -254,17 +255,22 @@ def _measure_moments_fft(kim, kpsf_im, tot_var, eff_pad_factor, kernels, drow, d
     # here we assume each Fourier mode is independent and sum the variances
     # the variance in each mode is simply the total variance over the input image
     # we need a factor of the padding to correct for something...
-    m_cov = np.zeros((4, 4))
+    m_cov = np.zeros((6, 6))
+    # TODO
+    # FIXME
+    # set these for real
+    m_cov[0, 0] = 1
+    m_cov[1, 1] = 1
     tot_var *= eff_pad_factor**2
     tot_var_df4 = tot_var * df4
-    kerns = [fkf / kpsf_im, fkr / kpsf_im, fkp / kpsf_im, fkc / kpsf_im]
+    kerns = [fkp / kpsf_im, fkc / kpsf_im, fkr / kpsf_im, fkf / kpsf_im]
     conj_kerns = [np.conj(k) for k in kerns]
-    for i in range(4):
-        for j in range(i, 4):
-            m_cov[i, j] = np.sum((kerns[i] * conj_kerns[j]).real) * tot_var_df4
+    for i in range(2, 6):
+        for j in range(i, 6):
+            m_cov[i, j] = np.sum((kerns[i-2] * conj_kerns[j-2]).real) * tot_var_df4
             m_cov[j, i] = m_cov[i, j]
 
-    mom = np.array([mf, mr, mp, mc])
+    mom = np.array([np.nan, np.nan, mp, mc, mr, mf])
 
     return mom, m_cov
 

ngmix/tests/test_leastsqbound.py

@@ -93,7 +93,7 @@ def test_leastsqbound_smoke(use_prior):
         ntry=2,
     )
 
-    allflags = np.zeros(ntrial)
+    allflags = np.zeros(ntrial, dtype=int)
     for i in range(ntrial):
         data = get_model_obs(
             rng=rng,
@@ -132,7 +132,7 @@ def test_leastsqbound_bounds(fracdev_bounds):
     psf_runner = ngmix.runners.PSFRunner(
         fitter=psf_fitter,
         guesser=psf_guesser,
-        ntry=2,
+        ntry=1,
     )
 
     prior = get_prior(
@@ -154,10 +154,10 @@ def test_leastsqbound_bounds(fracdev_bounds):
     runner = ngmix.runners.Runner(
         fitter=fitter,
         guesser=guesser,
-        ntry=2,
+        ntry=1,
     )
 
-    allflags = np.zeros(ntrial)
+    allflags = np.zeros(ntrial, dtype=int)
     for i in range(ntrial):
         data = get_model_obs(
             rng=rng,
@@ -254,7 +254,7 @@ def test_leastsqbound_bad_data(fit_model, psf_noise):
         guesser=guesser,
     )
 
-    allflags = np.zeros(ntrial)
+    allflags = np.zeros(ntrial, dtype=int)
     for i in range(ntrial):
         data = get_model_obs(
             rng=rng,
@@ -265,7 +265,8 @@ def test_leastsqbound_bad_data(fit_model, psf_noise):
         obs = data['obs']
 
         if psf_noise > 0.0:
-            obs.image = rng.normal(scale=1.e9, size=obs.image.shape)
+            with obs.writeable():
+                obs.image = rng.normal(scale=1.e9, size=obs.image.shape)
 
         try:
             res = bootstrap(obs=obs, runner=runner, psf_runner=psf_runner)

ngmix/tests/test_prepsfmom.py

@@ -364,7 +364,18 @@ def test_prepsfmom_mn_cov(
         np.std(res["e"][:, 1]), np.mean(res["e_err"][:, 1]), atol=0, rtol=2e-2)
 
     assert np.allclose(
-        res["mom_cov"], np.mean(res["mom_cov"], axis=0), atol=0, rtol=4e-1)
+        mom_cov[2:, 2:],
+        np.mean(res["mom_cov"][:, 2:, 2:], axis=0),
+        atol=2.5e-1,
+        rtol=0,
+    )
+
+    assert np.allclose(
+        np.diagonal(mom_cov[2:, 2:]),
+        np.diagonal(np.mean(res["mom_cov"][:, 2:, 2:], axis=0)),
+        atol=0,
+        rtol=2e-2,
+    )
 
 
 @pytest.mark.parametrize("cls,mom_fwhm,snr", [
@@ -478,28 +489,39 @@ def test_prepsfmom_mn_cov_nopsf(
         np.std(res["e"][:, 1]), np.mean(res["e_err"][:, 1]), atol=0, rtol=2e-2)
 
     assert np.allclose(
-        res["mom_cov"], np.mean(res["mom_cov"], axis=0), atol=0, rtol=4e-1)
+        mom_cov[2:, 2:],
+        np.mean(res["mom_cov"][:, 2:, 2:], axis=0),
+        atol=2.5e-1,
+        rtol=0,
+    )
+
+    assert np.allclose(
+        np.diagonal(mom_cov[2:, 2:]),
+        np.diagonal(np.mean(res["mom_cov"][:, 2:, 2:], axis=0)),
+        atol=0,
+        rtol=2e-2,
+    )
 
 
 def test_moments_make_mom_result_flags():
-    mom = np.ones(4)
-    mom_cov = np.diag(np.ones(4))
+    mom = np.ones(6)
+    mom_cov = np.diag(np.ones(6))
 
     # weird cov
-    for i in range(4):
+    for i in range(2, 6):
         _mom_cov = mom_cov.copy()
         _mom_cov[i, i] = -1
         res = make_mom_result(mom, _mom_cov)
         assert (res["flags"] & ngmix.flags.NONPOS_VAR) != 0
         assert ngmix.flags.NAME_MAP[ngmix.flags.NONPOS_VAR] in res["flagstr"]
-        if i == 0:
+        if i == 5:
             assert (res["flux_flags"] & ngmix.flags.NONPOS_VAR) != 0
             assert ngmix.flags.NAME_MAP[ngmix.flags.NONPOS_VAR] in res["flux_flagstr"]
         else:
             assert res["flux_flags"] == 0
             assert res["flux_flagstr"] == ""
 
-        if i < 2:
+        if i >= 4:
             assert (res["T_flags"] & ngmix.flags.NONPOS_VAR) != 0
             assert ngmix.flags.NAME_MAP[ngmix.flags.NONPOS_VAR] in res["T_flagstr"]
         else:
@@ -508,7 +530,7 @@ def test_moments_make_mom_result_flags():
 
     # neg flux
     _mom = mom.copy()
-    _mom[0] = -1
+    _mom[5] = -1
     res = make_mom_result(_mom, mom_cov)
     assert (res["flags"] & ngmix.flags.NONPOS_FLUX) != 0
     assert ngmix.flags.NAME_MAP[ngmix.flags.NONPOS_FLUX] in res["flagstr"]
@@ -519,7 +541,7 @@ def test_moments_make_mom_result_flags():
 
     # neg T
     _mom = mom.copy()
-    _mom[1] = -1
+    _mom[4] = -1
     res = make_mom_result(_mom, mom_cov)
     assert (res["flags"] & ngmix.flags.NONPOS_SIZE) != 0
     assert ngmix.flags.NAME_MAP[ngmix.flags.NONPOS_SIZE] in res["flagstr"]
@@ -531,8 +553,8 @@ def test_moments_make_mom_result_flags():
     # bad shape errs
     for i in [2, 3]:
         _mom_cov = mom_cov.copy()
-        _mom_cov[1, i] = np.nan
-        _mom_cov[i, 1] = np.nan
+        _mom_cov[4, i] = np.nan
+        _mom_cov[i, 4] = np.nan
         res = make_mom_result(mom, _mom_cov)
         assert (res["flags"] & ngmix.flags.NONPOS_SHAPE_VAR) != 0
         assert ngmix.flags.NAME_MAP[ngmix.flags.NONPOS_SHAPE_VAR] in res["flagstr"]