diff --git a/docs/examples/fftfit.md b/docs/examples/fftfit.md index 62e1866667..80254b1963 100644 --- a/docs/examples/fftfit.md +++ b/docs/examples/fftfit.md @@ -30,35 +30,24 @@ template = np.zeros(256) template[:16] = 1 plt.plot(np.linspace(0, 1, len(template), endpoint=False), template) up_template = fftfit.upsample(template, 16) -plt.plot( - np.linspace(0, 1, len(up_template), endpoint=False), up_template -) +plt.plot(np.linspace(0, 1, len(up_template), endpoint=False), up_template) plt.xlim(0, 1) ``` ```python -template = np.diff( - scipy.stats.vonmises(100).cdf(np.linspace(0, 2 * np.pi, 1024 + 1)) -) +template = np.diff(scipy.stats.vonmises(100).cdf(np.linspace(0, 2 * np.pi, 1024 + 1))) plt.plot(np.linspace(0, 1, len(template), endpoint=False), template) up_template = fftfit.upsample(template, 16) +plt.plot(np.linspace(0, 1, len(up_template), endpoint=False), up_template) plt.plot( - np.linspace(0, 1, len(up_template), endpoint=False), up_template -) -plt.plot( - np.linspace(0, 1, len(template), endpoint=False), - fftfit.shift(template, 0.25), + np.linspace(0, 1, len(template), endpoint=False), fftfit.shift(template, 0.25), ) plt.xlim(0, 1) ``` ```python if False: - template = np.diff( - scipy.stats.vonmises(10).cdf( - np.linspace(0, 2 * np.pi, 64 + 1) - ) - ) + template = np.diff(scipy.stats.vonmises(10).cdf(np.linspace(0, 2 * np.pi, 64 + 1))) profile = fftfit.shift(template, 0.25) else: template = np.random.randn(64) @@ -67,8 +56,7 @@ else: upsample = 8 if len(template) != len(profile): raise ValueError( - "Template is length %d but profile is length %d" - % (len(template), len(profile)) + "Template is length %d but profile is length %d" % (len(template), len(profile)) ) t_c = np.fft.rfft(template) p_c = np.fft.rfft(profile) @@ -87,11 +75,7 @@ plt.axvline(x) def gof(x): - return ( - -(ccf_c * np.exp(2.0j * np.pi * np.arange(len(ccf_c)) * x)) - .sum() - .real - ) + return -(ccf_c * np.exp(2.0j * np.pi * np.arange(len(ccf_c)) * x)).sum().real plt.plot(xs, [-2 * gof(x) / len(xs) for x in xs]) @@ -105,18 +89,11 @@ plt.axvline(x) ```python template = fftfit.upsample( - np.diff( - scipy.stats.vonmises(10).cdf( - np.linspace(0, 2 * np.pi, 16 + 1) - ) - ), - 2, + np.diff(scipy.stats.vonmises(10).cdf(np.linspace(0, 2 * np.pi, 16 + 1))), 2, ) for s in np.linspace(0, 1 / len(template), 33): profile = fftfit.shift(template, s) - print( - (s - fftfit.fftfit_basic(template, profile)) * len(template) - ) + print((s - fftfit.fftfit_basic(template, profile)) * len(template)) ``` ```python @@ -126,12 +103,7 @@ a_c[-1] = 0 a_c[0] = 0 xs = np.linspace(0, 1, len(a), endpoint=False) a_m = ( - ( - a_c[:, None] - * np.exp( - 2.0j * np.pi * xs[None, :] * np.arange(len(a_c))[:, None] - ) - ) + (a_c[:, None] * np.exp(2.0j * np.pi * xs[None, :] * np.arange(len(a_c))[:, None])) .sum(axis=0) .real * 2 @@ -231,9 +203,7 @@ print(x, gof(x)) print(r, gof(r)) print(-s / n, gof(-s / n)) -res = scipy.optimize.minimize_scalar( - gof, bracket=(l, x, r), method="brent", tol=1e-10 -) +res = scipy.optimize.minimize_scalar(gof, bracket=(l, x, r), method="brent", tol=1e-10) res ``` @@ -275,29 +245,30 @@ for i in range(10000): t = np.random.randn(n) t_c = np.fft.rfft(t) - r.append( - np.mean(np.abs(t_c[1:-1]) ** 2) - / (n * np.mean(np.abs(t) ** 2)) - ) + r.append(np.mean(np.abs(t_c[1:-1]) ** 2) / (n * np.mean(np.abs(t) ** 2))) np.mean(r) ``` ```python template = fftfit.vonmises_profile(1, 256) plt.plot(template) -plt.xlim(0,len(template)) +plt.xlim(0, len(template)) std = 1 shift = 0 scale = 1 -r = fftfit.fftfit_full(template, scale*fftfit.shift(template, shift), std=std) +r = fftfit.fftfit_full(template, scale * fftfit.shift(template, shift), std=std) r.shift, r.scale, r.offset, r.uncertainty, r.cov ``` +```python +fftfit.fftfit_full? +``` + ```python def gen_shift(): return fftfit.wrap( fftfit.fftfit_basic( - template, scale*template + std * np.random.randn(len(template)) + template, scale * template + std * np.random.randn(len(template)) ) ) @@ -312,43 +283,50 @@ np.std(shifts) ``` ```python -r.uncertainty/np.std(shifts) +r.uncertainty / np.std(shifts) ``` ```python snrs = {} -template = fftfit.vonmises_profile(200, 1024, 1 / 3) -plt.plot(np.linspace(0, 1, len(template), endpoint=False), template) +scale = 1e-3 +template = fftfit.vonmises_profile(100, 1024, 1 / 3) + 0.5*fftfit.vonmises_profile(50, 1024, 1 / 2) +plt.plot(np.linspace(0, 1, len(template), endpoint=False), scale*template) def gen_prof(std): - shift = np.random.uniform(0,1) + shift = np.random.uniform(0, 1) shift_template = fftfit.shift(template, shift) - return shift_template + std*np.random.standard_normal(len(template))/np.sqrt(len(template)) - + return scale*shift_template + scale*std * np.random.standard_normal(len(template)) / np.sqrt( + len(template) + ) + + plt.plot(np.linspace(0, 1, len(template), endpoint=False), gen_prof(0.01)) plt.xlim(0, 1) - + + def gen_shift(std): - shift = np.random.uniform(0,1) + shift = np.random.uniform(0, 1) shift_template = fftfit.shift(template, shift) - profile = shift_template + std*np.random.standard_normal(len(template))/np.sqrt(len(template)) - return fftfit.wrap( - fftfit.fftfit_basic( - template, profile - ) - shift + profile = scale*shift_template + scale*std * np.random.standard_normal(len(template)) / np.sqrt( + len(template) ) + return fftfit.wrap(fftfit.fftfit_basic(template, profile) - shift) + gen_shift(0.01) ``` ```python def gen_uncert(std): - return fftfit.fftfit_full(template, gen_prof(std), std=std).uncertainty + return fftfit.fftfit_full(template, gen_prof(std), std=scale*std/np.sqrt(len(template))).uncertainty + +def gen_uncert_estimate(std): + return fftfit.fftfit_full(template, gen_prof(std)).uncertainty ``` ```python -for s in np.geomspace(1,1e-4,9): +for s in np.geomspace(1, 1e-4, 9): if s not in snrs: snrs[s] = [] for i in range(1000): @@ -357,9 +335,26 @@ for s in np.geomspace(1,1e-4,9): ```python snr_list = sorted(snrs.keys()) -plt.loglog(snr_list, [np.std(snrs[s]) for s in snr_list], ".", label="measured std") -plt.loglog(snr_list, [gen_uncert(s) for s in snr_list], ".", label="computed std") +plt.loglog(snr_list, [np.std(snrs[s]) for s in snr_list], "o", label="measured std.") +plt.loglog(snr_list, [gen_uncert(s) for s in snr_list], ".", label="computed uncert.") +plt.loglog(snr_list, [gen_uncert_estimate(s) for s in snr_list], "+", label="computed uncert. w/estimate") plt.legend() +plt.xlabel("SNR (some strange units)") +plt.ylabel("Uncertainty or standard deviation (phase)") + +``` + +```python +p = 1-2*scipy.stats.norm.sf(1) +p +``` + +```python +scipy.stats.binom.isf(0.01, 100, p), scipy.stats.binom.isf(0.99, 100, p) +``` + +```python +scipy.stats.binom(16, p).ppf(0.99) ``` ```python diff --git a/src/pint/profile/__init__.py b/src/pint/profile/__init__.py index 2ae28399f5..34a9b21364 100644 --- a/src/pint/profile/__init__.py +++ b/src/pint/profile/__init__.py @@ -1 +1,23 @@ -pass + +import pint.profile.fftfit_aarchiba +import pint.profile.fftfit_nustar +import pint.profile.fftfit_presto + +def fftfit_full(template, profile, code="aarchiba"): + if code=="aarchiba": + return pint.profile.fftfit_aarchiba.fftfit_full(template, profile) + elif code=="nustar": + return pint.profile.fftfit_nustar.fftfit_full(template, profile) + elif code=="presto": + if pint.profile.fftfit_presto.presto is None: + raise ValueError("The PRESTO compiled code is not available") + return pint.profile.fftfit_presto.fftfit_full(template, profile) + else: + raise ValueError("Unrecognized FFTFIT implementation {}".format(code)) + +def fftfit_basic(template, profile, code="aarchiba"): + if code=="aarchiba": + return pint.profile.fftfit_aarchiba.fftfit_basic(template, profile) + else: + return fftfit_full(template, profile, code=code).shift + diff --git a/src/pint/profile/fftfit_aarchiba.py b/src/pint/profile/fftfit_aarchiba.py index b59b1b245c..67be9a0dc9 100644 --- a/src/pint/profile/fftfit_aarchiba.py +++ b/src/pint/profile/fftfit_aarchiba.py @@ -156,7 +156,7 @@ def gof(x): if compute_uncertainty: if std is None: resid = r.scale * shift(template, r.shift) + r.offset - profile - std = np.mean(resid ** 2) + std = np.sqrt(np.mean(resid ** 2)) J = np.zeros((2 * len(s_c) - 2, 2)) J[: len(s_c) - 1, 0] = ( diff --git a/src/pint/profile/fftfit_nustar.py b/src/pint/profile/fftfit_nustar.py index af5c8ed93b..0a58ad1288 100644 --- a/src/pint/profile/fftfit_nustar.py +++ b/src/pint/profile/fftfit_nustar.py @@ -1,5 +1,6 @@ # by mateobachetti # from https://github.com/NuSTAR/nustar-clock-utils/blob/master/nuclockutils/diagnostics/fftfit.py +from collections import namedtuple import numpy as np from scipy.optimize import minimize, brentq @@ -74,6 +75,11 @@ def chi_sq_alt(b, tau, P, S, theta, phi, ngood=20): return res +FFTFITResult = namedtuple( + "FFTFITResult", ["mean_amp", "std_amp", "mean_phase", "std_phase"] +) + + def fftfit(prof, template): """Align a template to a pulse profile. Parameters @@ -161,7 +167,7 @@ def func_to_minimize(tau): eb = sigma ** 2 / (2 * np.sum(S[good] ** 2)) - return b, np.sqrt(eb), normalize_phase_0d5(shift), np.sqrt(eshift) + return FFTFITResult(b, np.sqrt(eb), normalize_phase_0d5(shift), np.sqrt(eshift)) def normalize_phase_0d5(phase): @@ -187,3 +193,15 @@ def normalize_phase_0d5(phase): def fftfit_basic(template, profile): n, seb, shift, eshift = fftfit(profile, template) return shift + + +class FullResult: + pass + + +def fftfit_full(template, profile): + r = fftfit(profile, template) + ro = FullResult() + ro.shift = r.mean_phase + ro.uncertainty = r.std_phase + return ro diff --git a/src/pint/profile/fftfit_presto.py b/src/pint/profile/fftfit_presto.py new file mode 100644 index 0000000000..2f482ebeec --- /dev/null +++ b/src/pint/profile/fftfit_presto.py @@ -0,0 +1,28 @@ +import numpy as np +from numpy.fft import rfft +from pint.profile.fftfit_aarchiba import wrap + +try: + import presto.fftfit +except ImportError: + presto = None + +class FFTFITResult: + pass + +def fftfit_full(template, profile): + if len(template) != len(profile): + raise ValueError("template has length {} but profile has length {}".format(len(template),len(profile))) + if len(template) > 2**13: + raise ValueError("template has length {} which is too long".format(len(template))) + tc = rfft(template) + shift, eshift, snr, esnr, b, errb, ngood = presto.fftfit.fftfit(profile, np.abs(tc)[1:], -np.angle(tc)[1:]) + r = FFTFITResult() + # Need to add 1 to the shift for some reason + r.shift = wrap((shift + 1)/len(template)) + r.uncertainty = eshift/len(template) + return r + +def fftfit_basic(template, profile): + return fftfit_full(template, profile).shift + diff --git a/tests/test_fftfit.py b/tests/test_fftfit.py index 5361e29aeb..e6d02e8869 100644 --- a/tests/test_fftfit.py +++ b/tests/test_fftfit.py @@ -1,4 +1,5 @@ from functools import wraps +from itertools import product import numpy as np import pytest @@ -19,21 +20,70 @@ import pint.profile.fftfit_aarchiba as fftfit from pint.profile import fftfit_aarchiba from pint.profile import fftfit_nustar +from pint.profile import fftfit_presto +from pint.profile import fftfit_full, fftfit_basic -def assert_rms_close(a, b, rtol=1e-8, atol=1e-8): +fftfit_basics = [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] +fftfit_fulls = [fftfit_aarchiba.fftfit_full, fftfit_nustar.fftfit_full] + +if fftfit_presto.presto is not None: + fftfit_basics.append(fftfit_presto.fftfit_basic) + fftfit_fulls.append(fftfit_presto.fftfit_full) + +NO_PRESTO = fftfit_presto.presto is None + + +def assert_rms_close(a, b, rtol=1e-8, atol=1e-8, name=None): + __tracebackhide__ = True target(np.mean((a - b) ** 2), label="mean") - target((a - b).max(), label="max") - target(-(a - b).min(), label="min") + if name is not None: + target((a - b).max(), label="{} max".format(name)) + target(-(a - b).min(), label="{} min".format(name)) assert np.mean((a - b) ** 2) < rtol * (np.mean(a ** 2) + np.mean(b ** 2)) + atol -def assert_allclose_phase(a, b, atol=1e-8): - target(np.abs(fftfit.wrap(a - b)).max(), label="max phase") - target(np.abs(fftfit.wrap(a - b)).mean(), label="mean phase") +def assert_allclose_phase(a, b, atol=1e-8, name=None): + __tracebackhide__ = True + if name is not None: + target(np.abs(fftfit.wrap(a - b)).max(), label="{} max".format(name)) + target(np.abs(fftfit.wrap(a - b)).mean(), label="{} mean".format(name)) assert np.all(np.abs(fftfit.wrap(a - b)) <= atol) +ONE_SIGMA = 1 - 2 * scipy.stats.norm().sf(1) + + +def assert_happens_with_probability( + func, + p=ONE_SIGMA, + n=100, + p_lower=None, + p_upper=None, + fpp=0.05, +): + __tracebackhide__ = True + if p_lower is None: + p_lower = p + if p_upper is None: + p_upper = p + if p_lower > p_upper: + raise ValueError( + "Lower limit on probability {} is higher than upper limit {}".format( + p_lower, p_upper + ) + ) + + k = 0 + for i in range(n): + if func(): + k += 1 + low_k = scipy.stats.binom(n, p_lower).ppf(fpp / 2) + high_k = scipy.stats.binom(n, p_upper).isf(fpp / 2) + assert low_k <= k + assert k <= high_k + + @composite def powers_of_two(draw): return 2 ** draw(integers(4, 16)) @@ -73,7 +123,7 @@ def state(): return np.random.default_rng(0) -def randomized_test(tries=3, seed=0): +def randomized_test(tries=5, seed=0): if tries < 1: raise ValueError("Must carry out at least one try") @@ -136,51 +186,119 @@ def test_shift_invertible(s, template): @given(integers(0, 2 ** 20), floats(1, 1000), integers(5, 16), floats(0, 1)) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + ], ) -def test_fftfit_basic_integer_vonmises(fftfit_basic, i, kappa, profile_length, phase): +def test_fftfit_basic_integer_vonmises(code, i, kappa, profile_length, phase): + if code == "presto": + assume(profile_length <= 13) n = 2 ** profile_length template = fftfit.vonmises_profile(kappa, n, phase) + ( 1e-3 / n ) * np.random.default_rng(0).standard_normal(n) assume(sum(template > 0.5 * template.max()) > 1) s = i / len(template) - rs = fftfit_basic(template, fftfit.shift(template, s)) - assert_allclose_phase(i / len(template), rs) + rs = fftfit_basic(template, fftfit.shift(template, s), code=code) + assert_allclose_phase(s, rs, atol=1 / (32 * len(template)), name="shift") @given(integers(0, 2 ** 20), vonmises_templates_noisy()) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + ], ) -def test_fftfit_basic_integer(fftfit_basic, i, template): - assume(len(template) >= 32) +def test_fftfit_basic_integer(code, i, template): + if code != "aarchiba": + assume(len(template) >= 32) s = i / len(template) - rs = fftfit_basic(template, fftfit.shift(template, s)) - assert_allclose_phase(i / len(template), rs) + rs = fftfit_basic(template, fftfit.shift(template, s), code=code) + assert_allclose_phase(s, rs, name="shift") @given(integers(0, 2 ** 5), vonmises_templates_noisy()) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + ], ) -def test_fftfit_basic_integer_fraction(fftfit_basic, i, template): +def test_fftfit_basic_integer_fraction(code, i, template): s = i / len(template) / 2 ** 5 - rs = fftfit_basic(template, fftfit.shift(template, s)) - assert_allclose_phase(rs, s, atol=1e-4 / len(template)) + rs = fftfit_basic(template, fftfit.shift(template, s), code=code) + assert_allclose_phase(rs, s, atol=1e-4 / len(template), name="shift") @given(floats(0, 1), floats(1, 1000), powers_of_two()) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + ], ) -def test_fftfit_basic_subbin(fftfit_basic, s, kappa, n): - assume(n >= 32) +def test_fftfit_basic_subbin(code, s, kappa, n): + if code != "aarchiba": + assume(n >= 32) template = fftfit.vonmises_profile(kappa, n) + (1e-3 / n) * np.random.default_rng( 0 ).standard_normal(n) - rs = fftfit_basic(template, fftfit.shift(template, s / n)) - assert_allclose_phase(rs, s / n, atol=1e-4 / len(template)) + rs = fftfit_basic(template, fftfit.shift(template, s / n), code=code) + assert_allclose_phase(rs, s / n, atol=1e-4 / len(template), name="shift") @given( @@ -188,12 +306,29 @@ def test_fftfit_basic_subbin(fftfit_basic, s, kappa, n): one_of(vonmises_templates_noisy(), random_templates(), boxcar_templates()), ) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profile too symmetric"), + ], + ), + ], ) -def test_fftfit_basic_template(fftfit_basic, s, template): - assume(len(template) >= 32) - rs = fftfit_basic(template, fftfit.shift(template, s)) - assert_allclose_phase(rs, s, atol=1e-3 / len(template)) +def test_fftfit_basic_template(code, s, template): + if code != "aarchiba": + assume(len(template) >= 32) + rs = fftfit_basic(template, fftfit.shift(template, s), code=code) + assert_allclose_phase(rs, s, atol=1e-3 / len(template), name="shift") @given( @@ -201,11 +336,28 @@ def test_fftfit_basic_template(fftfit_basic, s, template): one_of(vonmises_templates(), random_templates(), boxcar_templates()), ) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profiles different lengths"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profiles different lengths"), + ], + ), + ], ) -def test_fftfit_basic_different_profiles(fftfit_basic, profile1, profile2): - assume(len(profile1) >= 32) - fftfit_basic(profile1, profile2) +def test_fftfit_basic_different_profiles(code, profile1, profile2): + if code != "aarchiba": + assume(len(profile1) >= 32) + fftfit_basic(profile1, profile2, code=code) @given( @@ -213,15 +365,33 @@ def test_fftfit_basic_different_profiles(fftfit_basic, profile1, profile2): one_of(vonmises_templates(), random_templates()), ) @pytest.mark.parametrize( - "fftfit_basic", [fftfit_aarchiba.fftfit_basic, fftfit_nustar.fftfit_basic] + "code", + [ + "aarchiba", + pytest.param( + "nustar", + marks=[ + pytest.mark.xfail(reason="profiles different lengths"), + ], + ), + pytest.param( + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + pytest.mark.xfail(reason="profiles different lengths"), + ], + ), + ], ) -def test_fftfit_basic_shift(fftfit_basic, profile1, profile2): - assume(len(profile1) >= 32) - s = fftfit.fftfit_basic(profile1, profile2) +def test_fftfit_shift_equivalence(code, profile1, profile2): + if code != "aarchiba": + assume(len(profile1) >= 32) + s = fftfit_basic(profile1, profile2, code=code) assert_allclose_phase( - fftfit.fftfit_basic(fftfit.shift(profile1, s), profile2), + fftfit_basic(fftfit.shift(profile1, s), profile2, code=code), 0, atol=1e-3 / min(len(profile1), len(profile2)), + name="shift", ) @@ -234,11 +404,14 @@ def test_fftfit_basic_shift(fftfit_basic, profile1, profile2): def test_fftfit_compute_scale(template, s, a, b): profile = a * fftfit.shift(template, s) + b r = fftfit.fftfit_full(template, profile) - assert_allclose_phase(s, r.shift, atol=1e-3 / len(template)) + assert_allclose_phase(s, r.shift, atol=1e-3 / len(template), name="shift") assert_allclose(b, r.offset, atol=a * 1e-8) assert_allclose(a, r.scale, atol=(1 + abs(b)) * 1e-8) assert_rms_close( - profile, r.scale * fftfit.shift(template, r.shift) + r.offset, atol=1e-7 + profile, + r.scale * fftfit.shift(template, r.shift) + r.offset, + atol=1e-7, + name="profile", ) @@ -259,38 +432,424 @@ def gen_shift(): ks, fpp = scipy.stats.kstest(values, scipy.stats.norm(0, r.uncertainty).cdf) +# could be hypothesized @pytest.mark.parametrize( "kappa,n,std,shift,scale,offset", [ (1, 256, 0.01, 0, 1, 0), - # (10, 64, 0.01, 1 / 3, 2, 0), - # (100, 1024, 0.02, 0.2, 0.1, 100), + (10, 64, 0.01, 1 / 3, 2e-3, 0), + (100, 1024, 0.02, 0.2, 1e4, 0), + (100, 2048, 0.01, 0.2, 1e4, -100), ], ) -@randomized_test() -def test_fftfit_uncertainty_estimate_std(kappa, n, std, shift, scale, offset, state): +def test_fftfit_uncertainty_scaling_invariance(kappa, n, std, shift, scale, offset): + state = np.random.default_rng(0) + template = fftfit.vonmises_profile(kappa, n) + profile = fftfit.shift(template, shift) + std * state.standard_normal(len(template)) + + r_1 = fftfit.fftfit_full(template, profile) + r_2 = fftfit.fftfit_full(template, scale * profile + offset) + + assert_allclose_phase(r_2.shift, r_1.shift, 1.0 / (32 * n)) + assert_allclose(r_2.uncertainty, r_1.uncertainty, rtol=1e-3) + assert_allclose(r_2.scale, scale * r_1.scale, rtol=1e-3) + assert_allclose(r_2.offset, offset + scale * r_1.offset, rtol=1e-3, atol=1e-6) + + +@pytest.mark.parametrize( + "kappa,n,std,shift,scale,offset,estimate", + [ + a + (b,) + for a, b in product( + [ + (1, 256, 0.01, 0, 1, 0), + (10, 64, 0.01, 1 / 3, 1e-6, 0), + (100, 1024, 0.002, 0.2, 1e4, 0), + (100, 1024, 0.02, 0.2, 1e4, 0), + ], + [False, True], + ) + ], +) +@randomized_test(tries=8) +def test_fftfit_uncertainty_estimate( + kappa, n, std, shift, scale, offset, estimate, state +): + """Check the noise level estimation works.""" + template = fftfit.vonmises_profile(kappa, n) + + def value_within_one_sigma(): + profile = ( + fftfit.shift(template, shift) + + offset + + std * state.standard_normal(len(template)) + ) + if estimate: + r = fftfit.fftfit_full(template, scale * profile) + else: + r = fftfit.fftfit_full(template, scale * profile, std=scale * std) + return np.abs(fftfit.wrap(r.shift - shift)) < r.uncertainty + + assert_happens_with_probability(value_within_one_sigma, ONE_SIGMA) + + +@pytest.mark.parametrize( + "kappa,n,std,shift,scale,offset,code", + [ + (1, 256, 0.01, 0, 1, 0, "aarchiba"), + (10, 64, 0.01, 1 / 3, 1e-6, 0, "aarchiba"), + (100, 1024, 0.002, 0.2, 1e4, 0, "aarchiba"), + (100, 1024, 0.02, 0.2, 1e4, 0, "aarchiba"), + (1000, 4096, 0.01, 0.7, 1e4, 0, "aarchiba"), + pytest.param( + 1, + 256, + 0.01, + 0, + 1, + 0, + "nustar", + ), + pytest.param( + 10, + 64, + 0.01, + 1 / 3, + 1e-6, + 0, + "nustar", + ), + pytest.param( + 100, + 1024, + 0.002, + 0.2, + 1e4, + 0, + "nustar", + ), + pytest.param( + 100, + 1024, + 0.02, + 0.2, + 1e4, + 0, + "nustar", + ), + pytest.param( + 1000, + 4096, + 0.01, + 0.7, + 1e4, + 0, + "nustar", + ), + pytest.param( + 1, + 256, + 0.01, + 0, + 1, + 0, + "presto", + marks=[ + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + ], + ), + pytest.param( + 10, + 64, + 0.01, + 1 / 3, + 1e-6, + 0, + "presto", + marks=[ + pytest.mark.xfail(reason="bug?"), + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + ], + ), + pytest.param( + 100, + 1024, + 0.002, + 0.2, + 1e4, + 0, + "presto", + marks=[ + pytest.mark.xfail(reason="bug?"), + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + ], + ), + pytest.param( + 100, + 1024, + 0.02, + 0.2, + 1e4, + 0, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available")], + ), + pytest.param( + 1000, + 4096, + 0.01, + 0.7, + 1e4, + 0, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available")], + ), + ], +) +@randomized_test(tries=8) +def test_fftfit_value(kappa, n, std, shift, scale, offset, code, state): + """Check if the returned values are okay with a noisy profile. + + Here we define "okay" as scattered about the right value and within + one sigma as defined by the uncertainty returned by the aarchiba version + of the code (this is presumably a trusted uncertainty). + """ template = fftfit.vonmises_profile(kappa, n) profile = ( - scale * fftfit.shift(template, shift) + fftfit.shift(template, shift) + offset - # + std * state.standard_normal((len(template),)) + + std * state.standard_normal(len(template)) ) - r = fftfit.fftfit_full(template, profile, std=std) + r_true = fftfit.fftfit_full(template, scale * profile, std=scale * std) + assert r_true.uncertainty < 0.1, "This uncertainty is too big for accuracy" - def gen_shift(): + def value_within_one_sigma(): profile = ( - scale * fftfit.shift(template, shift) + fftfit.shift(template, shift) + offset - + std * state.standard_normal((len(template),)) + + std * state.standard_normal(len(template)) ) - return fftfit.fftfit_basic(template, profile) - - values = np.array([gen_shift() for i in range(100)]) - c = np.sum(np.abs(fftfit.wrap(values - shift)) < r.uncertainty) - # breakpoint() - p = 1 - 2 * scipy.stats.norm().sf(1) - assert ( - scipy.stats.binom.isf(0.01, len(values), p) - <= c - <= scipy.stats.binom.isf(0.99, len(values), p) - ) + r = fftfit_full(template, scale * profile, code=code) + return np.abs(fftfit.wrap(r.shift - shift)) < r_true.uncertainty + + assert_happens_with_probability(value_within_one_sigma, ONE_SIGMA) + + +@pytest.mark.parametrize( + "kappa,n,std,shift,scale,offset,code", + [ + (1, 256, 0.01, 0, 1, 0, "aarchiba"), + (10, 64, 0.01, 1 / 3, 1e-6, 0, "aarchiba"), + (100, 1024, 0.002, 0.2, 1e4, 0, "aarchiba"), + (100, 1024, 0.02, 0.2, 1e4, 0, "aarchiba"), + (1000, 4096, 0.01, 0.7, 1e4, 0, "aarchiba"), + pytest.param( + 1, + 256, + 0.01, + 0, + 1, + 0, + "nustar", + marks=pytest.mark.xfail(reason="claimed uncertainty too big"), + ), + pytest.param( + 10, + 64, + 0.01, + 1 / 3, + 1e-6, + 0, + "nustar", + marks=pytest.mark.xfail(reason="bug?"), + ), + pytest.param( + 100, + 1024, + 0.002, + 0.2, + 1e4, + 0, + "nustar", + marks=pytest.mark.xfail(reason="bug?"), + ), + pytest.param( + 100, + 1024, + 0.02, + 0.2, + 1e4, + 0, + "nustar", + marks=pytest.mark.xfail(reason="bug?"), + ), + pytest.param( + 1000, + 4096, + 0.01, + 0.7, + 1e4, + 0, + "nustar", + marks=pytest.mark.xfail(reason="bug?"), + ), + pytest.param( + 1, + 256, + 0.01, + 0, + 1, + 0, + "presto", + marks=[ + pytest.mark.xfail(reason="bug?"), + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + ], + ), + pytest.param( + 10, + 64, + 0.01, + 1 / 3, + 1e-6, + 0, + "presto", + marks=[ + pytest.mark.xfail(reason="bug?"), + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + ], + ), + pytest.param( + 100, + 1024, + 0.002, + 0.2, + 1e4, + 0, + "presto", + marks=[ + pytest.mark.xfail(reason="bug?"), + pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available"), + ], + ), + pytest.param( + 100, + 1024, + 0.02, + 0.2, + 1e4, + 0, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available")], + ), + pytest.param( + 1000, + 4096, + 0.01, + 0.7, + 1e4, + 0, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO is not available")], + ), + ], +) +@randomized_test(tries=8) +def test_fftfit_value_vs_uncertainty(kappa, n, std, shift, scale, offset, code, state): + """Check if the scatter matches the claimed uncertainty.""" + template = fftfit.vonmises_profile(kappa, n) + + def value_within_one_sigma(): + profile = ( + fftfit.shift(template, shift) + + offset + + std * state.standard_normal(len(template)) + ) + r = fftfit_full(template, scale * profile, code=code) + assert r.uncertainty < 0.1, "This uncertainty is too big for accuracy" + return np.abs(fftfit.wrap(r.shift - shift)) < r.uncertainty + + assert_happens_with_probability(value_within_one_sigma, ONE_SIGMA) + + +@pytest.mark.parametrize( + "kappa1,kappa2,n,std,code", + [ + (1, 1.1, 256, 0.01, "aarchiba"), + (10, 11, 2048, 0.01, "aarchiba"), + (100, 110, 2048, 0.01, "aarchiba"), + (1.1, 1, 256, 0.01, "aarchiba"), + (11, 10, 2048, 0.01, "aarchiba"), + (110, 100, 2048, 0.01, "aarchiba"), + (1, 1.1, 256, 0.01, "nustar"), + (10, 11, 2048, 0.01, "nustar"), + (100, 110, 2048, 0.01, "nustar"), + (1.1, 1, 256, 0.01, "nustar"), + (11, 10, 2048, 0.01, "nustar"), + (110, 100, 2048, 0.01, "nustar"), + pytest.param( + 1, + 1.1, + 256, + 0.01, + "presto", + marks=[pytest.mark.xfail(reason="bug?"), pytest.mark.skipif(NO_PRESTO, reason="PRESTO not available")], + ), + pytest.param( + 10, + 11, + 2048, + 0.01, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO not available")], + ), + pytest.param( + 100, + 110, + 2048, + 0.01, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO not available")], + ), + pytest.param( + 1.1, + 1, + 256, + 0.01, + "presto", + marks=[pytest.mark.xfail(reason="bug?"), pytest.mark.skipif(NO_PRESTO, reason="PRESTO not available")], + ), + pytest.param( + 11, + 10, + 2048, + 0.01, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO not available")], + ), + pytest.param( + 110, + 100, + 2048, + 0.01, + "presto", + marks=[pytest.mark.skipif(NO_PRESTO, reason="PRESTO not available")], + ), + ], +) +@randomized_test(tries=8) +def test_fftfit_wrong_profile(kappa1, kappa2, n, std, code, state): + """Check that the uncertainty is okay or pessimistic if the template is wrong.""" + template = fftfit.vonmises_profile(kappa1, n) + wrong_template = fftfit.vonmises_profile(kappa2, n) + + def value_within_one_sigma(): + shift = state.uniform(0, 1) + profile = fftfit.shift(template, shift) + std * state.standard_normal( + len(template) + ) + r = fftfit_full(wrong_template, profile, code=code) + return np.abs(fftfit.wrap(r.shift - shift)) < r.uncertainty + + # Must be pessimistic + assert_happens_with_probability(value_within_one_sigma, ONE_SIGMA, p_upper=1)