From 988ea109621653e7e2de6a609e0bec79600c1494 Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Mon, 18 Mar 2024 10:20:39 +1100 Subject: [PATCH 1/7] Add ref. freq. to fitIcube outputs. --- RMtools_3D/do_fitIcube.py | 54 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/RMtools_3D/do_fitIcube.py b/RMtools_3D/do_fitIcube.py index 1ffe5e1..3c14d50 100755 --- a/RMtools_3D/do_fitIcube.py +++ b/RMtools_3D/do_fitIcube.py @@ -329,7 +329,8 @@ def savefits_mask(data, header, outDir, prefixOut): headMask = strip_fits_dims(header=header, minDim=2) headMask["DATAMAX"] = 1 headMask["DATAMIN"] = 0 - del headMask["BUNIT"] + if "BUNIT" in headMask: + del headMask["BUNIT"] mskArr = np.where(data > 0, 1.0, np.nan) MaskfitsFile = os.path.join(outDir, prefixOut + "mask.fits") @@ -348,17 +349,16 @@ def savefits_Coeffs(data, dataerr, header, polyOrd, outDir, prefixOut): prefixOut: prefix to use on the output name """ - headcoeff = strip_fits_dims(header=header, minDim=2) - headcoeff["BUNIT"] = "" - if "BTYPE" in headcoeff: - del headcoeff["BTYPE"] + header["BUNIT"] = "" + if "BTYPE" in header: + del header["BTYPE"] for i in range(np.abs(polyOrd) + 1): outname = os.path.join(outDir, prefixOut + "coeff" + str(i) + ".fits") - pf.writeto(outname, data[i], headcoeff, overwrite=True) + pf.writeto(outname, data[i], header, overwrite=True) outname = os.path.join(outDir, prefixOut + "coeff" + str(i) + "err.fits") - pf.writeto(outname, dataerr[i], headcoeff, overwrite=True) + pf.writeto(outname, dataerr[i], header, overwrite=True) def savefits_model_I(data, header, outDir, prefixOut): @@ -422,10 +422,11 @@ def fit_spectra_I( outs["I"] = pixImodel.astype("float32") outs["coeffs"] = pixFitDict["p"].astype("float32") outs["coeffs_err"] = pixFitDict["perror"].astype("float32") - outs["chiSq"] = pixFitDict["chiSq"].astype("float32") - outs["chiSqRed"] = pixFitDict["chiSqRed"].astype("float32") + outs["chiSq"] = pixFitDict["chiSq"] + outs["chiSqRed"] = pixFitDict["chiSqRed"] outs["nIter"] = pixFitDict["nIter"] - outs["AIC"] = pixFitDict["AIC"].astype("float32") + outs["AIC"] = pixFitDict["AIC"] + outs["reference_frequency_Hz"] = pixFitDict["reference_frequency_Hz"] return outs @@ -501,6 +502,7 @@ def make_model_I( coeffs = np.array([mskArr] * 6) coeffs_error = np.array([mskArr] * 6) + reffreq = np.squeeze(np.array([mskArr])) datacube = np.squeeze(datacube) # Select only the spectra with emission srcData = np.rot90(datacube[:, mskSrc > 0]) @@ -547,36 +549,48 @@ def make_model_I( for _, an in enumerate(xy): i, x, y = an - modelIcube[:, x, y] = results[_]["I"] + modelIcube[:, x, y] = results[i]["I"] + reffreq[x, y] = results[i]["reference_frequency_Hz"] for k, j, l in zip( - range(len(coeffs)), results[_]["coeffs"], results[_]["coeffs_err"] + range(len(coeffs)), results[i]["coeffs"], results[i]["coeffs_err"] ): coeffs[5 - k, x, y] = j coeffs_error[5 - k, x, y] = l - header["HISTORY"] = "Stokes I model fitted by RM-Tools" + headcoeff["HISTORY"] = "Stokes I model fitted by RM-Tools" if polyOrd < 0: - header["HISTORY"] = ( + headcoeff["HISTORY"] = ( f"Fit model is dynamic order {fit_function}-polynomial, max order {-polyOrd}" ) else: - header["HISTORY"] = f"Fit model is {polyOrd}-order {fit_function}-polynomial" + headcoeff["HISTORY"] = f"Fit model is {polyOrd}-order {fit_function}-polynomial" - print("Saving mask image.") - savefits_mask(data=mskSrc, header=header, outDir=outDir, prefixOut=prefixOut) + if verbose: + print("Saving mask image.") + savefits_mask(data=mskSrc, header=headcoeff, outDir=outDir, prefixOut=prefixOut) - print("Saving model I coefficients.") + if verbose: + print("Saving model I coefficients.") savefits_Coeffs( data=coeffs, dataerr=coeffs_error, - header=header, + header=headcoeff, polyOrd=polyOrd, outDir=outDir, prefixOut=prefixOut, ) - print("Saving model I cube image. ") + head_freq = headcoeff.copy() + head_freq["BUNIT"] = "Hz" + if "BTYPE" in headcoeff: + del headcoeff["BTYPE"] + + outname = os.path.join(outDir, prefixOut + "reffreq.fits") + pf.writeto(outname, reffreq, head_freq, overwrite=True) + + if verbose: + print("Saving model I cube image. ") savefits_model_I(data=modelIcube, header=header, outDir=outDir, prefixOut=prefixOut) np.savetxt(os.path.join(outDir, prefixOut + "noise.dat"), rms_Arr) From ef6b375f7cb6e37dd261bc1538893851e9994de7 Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Mon, 18 Mar 2024 10:22:24 +1100 Subject: [PATCH 2/7] Undeprecated the Stokes I renormalize parameters function. Added renormalization of uncertainties. --- RMutils/util_misc.py | 120 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 14 deletions(-) diff --git a/RMutils/util_misc.py b/RMutils/util_misc.py index 1fdb837..dc16ec0 100644 --- a/RMutils/util_misc.py +++ b/RMutils/util_misc.py @@ -346,6 +346,7 @@ def fit_StokesI_model(freqArr, IArr, dIArr, polyOrd, fit_function="log"): 'nIter': Number of iterations used by fitter. 'reference_frequency_Hz': reference frequency for polynomial. 'dof': degrees of freedom in the fit. + 'pcov': covariance matrix of fit parameters """ # Frequency axis must be in GHz to avoid overflow errors @@ -361,6 +362,7 @@ def fit_StokesI_model(freqArr, IArr, dIArr, polyOrd, fit_function="log"): "AIC": 0, "reference_frequency_Hz": 1, "perror": None, + "pcov": None, } goodchan = np.logical_and( @@ -421,6 +423,7 @@ def fit_StokesI_model(freqArr, IArr, dIArr, polyOrd, fit_function="log"): fitDict["chiSqRed"] = mp.fnorm / fitDict["dof"] fitDict["nIter"] = mp.niter fitDict["perror"] = mp.perror + fitDict["pcov"] = mp.covar if mp.perror is None: fitDict["perror"] = np.zeros_like(fitDict["p"]) @@ -444,12 +447,6 @@ def calculate_StokesI_model(fitDict, freqArr_Hz): return IModArr -@deprecated( - deprecated_in="1.3", - removed_in="2.0", - current_version=__version__, - details="This function cannot current propagate errors. It may be reactivated if someone works through how to do that.", -) def renormalize_StokesI_model(fitDict, new_reference_frequency): """This functions adjusts the reference frequency for the Stokes I model and fixes the fit parameters such that the the model is the same. This is @@ -457,30 +454,125 @@ def renormalize_StokesI_model(fitDict, new_reference_frequency): reference frequency, and it may be desirable for users to know the exact reference frequency of the model. - This function is depreciated because it can't propagate the errors in - the fit parameters.""" - print( - "The renormalize_StokesI_model function is depreciated because it can't propagate errors.\n" - "If this message appears, it's been invoked (perhaps by legacy code?)" - ) + This function now includes the ability to transform the model parameter + errors to the new reference frequency. This feature uses a first order + approximation, that scales with the ratio of new to old reference frequencies. + Large changes in reference frequency may be outside the linear valid regime + of the first order approximation, and thus should be avoided. + """ # Renormalization ratio: x = new_reference_frequency / fitDict["reference_frequency_Hz"] (a, b, c, d, f, g) = fitDict["p"] newDict = fitDict.copy() + + # Modify fit parameters to new reference frequency. + # I have derived all these conversion equations analytically for the + # linear- and log-polynomial models. if fitDict["fit_function"] == "linear": new_parms = [a * x**5, b * x**4, c * x**3, d * x**2, f * x, g] elif fitDict["fit_function"] == "log": - lnx = np.log(x) + lnx = np.log10(x) new_parms = [ a, 5 * a * lnx + b, 10 * a * lnx**2 + 4 * b * lnx + c, 10 * a * lnx**3 + 6 * b * lnx**2 + 3 * c * lnx + d, 5 * a * lnx**4 + 4 * b * lnx**3 + 3 * c * lnx**2 + 2 * d * lnx + f, - g * np.exp(a * lnx**5 + b * lnx**4 + c * lnx**3 + d * lnx**2 + f * lnx), + g + * np.power(10, a * lnx**5 + b * lnx**4 + c * lnx**3 + d * lnx**2 + f * lnx), + ] + + # Modify fit parameter errors to new reference frequency. + # Note this implicitly makes a first-order approximation in the correletion + # structure between uncertainties + # The general equation for the transformation of uncertainties is: + # var(p) = sum_i,j((\partial p / \partial a_i) * (\partial p / \partial a_j) * cov(a_i,a_j)) + # where a_i are the initial parameters, p is a final parameter, + # and the partial derivatives are evaluated at the fit parameter values (and frequency ratio). + # The partial derivatives all come from the parameter conversion equations above. + + cov = fitDict["pcov"] + if fitDict["fit_function"] == "linear": + new_errors = [ + np.sqrt(x**10 * cov[0, 0]), + np.sqrt(x**8 * cov[1, 1]), + np.sqrt(x**6 * cov[2, 2]), + np.sqrt(x**4 * cov[3, 3]), + np.sqrt(x**2 * cov[4, 4]), + np.sqrt(cov[5, 5]), ] + elif fitDict["fit_function"] == "log": + g2 = new_parms[5] # Convenient shorthand for new value of g variable. + new_errors = [ + np.sqrt(cov[0, 0]), + np.sqrt(25 * lnx**2 * cov[0, 0] + 10 * lnx * cov[0, 1] + cov[1, 1]), + np.sqrt( + 100 * lnx**4 * cov[0, 0] + + 80 * lnx**3 * cov[0, 1] + + 20 * lnx**2 * cov[0, 2] + + 16 * lnx**2 * cov[1, 1] + + 8 * lnx * cov[1, 2] + + cov[2, 2] + ), + np.sqrt( + 100 * lnx**6 * cov[0, 0] + + 120 * lnx**5 * cov[0, 1] + + 60 * lnx**4 * cov[0, 2] + + 20 * lnx**3 * cov[0, 3] + + 36 * lnx**4 * cov[1, 1] + + 36 * lnx**3 * cov[1, 2] + + 12 * lnx**2 * cov[1, 3] + + 9 * lnx**2 * cov[2, 2] + + 6 * lnx * cov[2, 3] + + cov[3, 3] + ), + np.sqrt( + 25 * lnx**8 * cov[0, 0] + + 40 * lnx**7 * cov[0, 1] + + 30 * lnx**6 * cov[0, 2] + + 20 * lnx**5 * cov[0, 3] + + 10 * lnx**4 * cov[0, 4] + + 16 * lnx**6 * cov[0, 5] + + 24 * lnx**5 * cov[1, 2] + + 16 * lnx**4 * cov[1, 3] + + 8 * lnx**3 * cov[1, 4] + + 9 * lnx**4 * cov[2, 2] + + 12 * lnx**3 * cov[2, 3] + + 6 * lnx**2 * cov[2, 4] + + 4 * lnx**2 * cov[3, 3] + + 4 * lnx * cov[3, 4] + + cov[4, 4] + ), + g2 + * np.sqrt( + lnx**10 * cov[0, 0] + + 2 * lnx**9 * cov[0, 1] + + 2 * lnx**8 * cov[0, 2] + + 2 * lnx**7 * cov[0, 3] + + 2 * lnx**6 * cov[0, 4] + + 2 * lnx**5 / g * np.log(10) * cov[0, 5] + + lnx**8 * cov[1, 1] + + 2 * lnx**7 * cov[1, 2] + + 2 * lnx**6 * cov[1, 3] + + 2 * lnx**5 * cov[1, 4] + + 2 * lnx**4 / g * np.log(10) * cov[1, 5] + + lnx**6 * cov[2, 2] + + 2 * lnx**5 * cov[2, 3] + + 2 * lnx**4 * cov[2, 4] + + 2 * lnx**3 / g * np.log(10) * cov[2, 5] + + lnx**4 * cov[3, 3] + + 2 * lnx**3 * cov[3, 4] + + 2 * lnx**2 / g * np.log(10) * cov[3, 5] + + lnx**2 * cov[4, 4] + + 2 * lnx / g * np.log(10) * cov[4, 5] + + 1 / g**2 * cov[5, 5] + ), + ] + newDict["p"] = new_parms newDict["reference_frequency_Hz"] = new_reference_frequency + newDict["perror"] = new_errors + return newDict From 4104c0d7de1369579ba7c4ab31d1a1b935f9502c Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Mon, 18 Mar 2024 13:12:25 +1100 Subject: [PATCH 3/7] Change default Stokes I freq to mean of lambda^2, not mean of freq. --- RMutils/util_misc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RMutils/util_misc.py b/RMutils/util_misc.py index dc16ec0..00e16fb 100644 --- a/RMutils/util_misc.py +++ b/RMutils/util_misc.py @@ -371,7 +371,9 @@ def fit_StokesI_model(freqArr, IArr, dIArr, polyOrd, fit_function="log"): # The fitting code is susceptible to numeric overflows because the frequencies are large. # To prevent this, normalize by a reasonable characteristic frequency in order # to make all the numbers close to 1: - fitDict["reference_frequency_Hz"] = nanmean(freqArr[goodchan]) + # The reference frequency is the frequency corresponding to the mean of lambda^2 + # since this should be close to the polarization reference frequency. + fitDict["reference_frequency_Hz"] = 1 / np.sqrt(nanmean(1 / freqArr[goodchan] ** 2)) # negative orders indicate that the code should dynamically increase # order of polynomial so long as it improves the fit From 6dc2234fb9de22511948cac8393af8f9d67a7042 Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Mon, 18 Mar 2024 13:15:46 +1100 Subject: [PATCH 4/7] Stokes I model parameters now being shifted to polarization reference frequency. Updated test values. --- RMtools_1D/do_RMsynth_1D.py | 21 +++++++++++++-------- tests/RMsynth1D_referencevalues.json | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/RMtools_1D/do_RMsynth_1D.py b/RMtools_1D/do_RMsynth_1D.py index f5ea486..44baad1 100755 --- a/RMtools_1D/do_RMsynth_1D.py +++ b/RMtools_1D/do_RMsynth_1D.py @@ -351,10 +351,20 @@ def run_rmsynth( if verbose: log("> RM-synthesis completed in %.2f seconds." % cputime) - # Determine the Stokes I value at lam0Sq_m2 from the Stokes I model - # This will break if lam0Sq_m2==0. Using the mean frequency in this case. + # Convert Stokes I model to polarization reference frequency. If lambda^2_0 is + # non-zero, use that as polarization reference frequency and adapt Stokes I model. + # If lambda^2_0 is zero, make polarization reference frequency equal to + # Stokes I reference frequency. + + if lam0Sq_m2 == 0: # Rudnick-Cotton adapatation + freq0_Hz = fitDict["reference_frequency_Hz"] + else: # standard RM-synthesis + freq0_Hz = C / m.sqrt(lam0Sq_m2) + fitDict = renormalize_StokesI_model(fitDict, freq0_Hz) + + # Set Ifreq0 (Stokes I at reference frequency) from either supplied model + # (interpolated as required) or fit model, as appropriate. # Multiply the dirty FDF by Ifreq0 to recover the PI - freq0_Hz = C / m.sqrt(lam0Sq_m2) if lam0Sq_m2 > 0 else np.nanmean(freqArr_Hz) if modStokesI is None: Ifreq0 = calculate_StokesI_model(fitDict, freq0_Hz) elif modStokesI is not None: @@ -362,10 +372,6 @@ def run_rmsynth( Ifreq0 = modStokesI_interp(freq0_Hz) dirtyFDF *= Ifreq0 # FDF is in fracpol units initially, convert back to flux - # if modStokesI is None: - # #Need to renormalize the Stokes I parameters here to the actual reference frequency. - # fitDict=renormalize_StokesI_model(fitDict,freq0_Hz) - # Calculate the theoretical noise in the FDF !!Old formula only works for wariance weights! weightArr = np.where(np.isnan(weightArr), 0.0, weightArr) dFDFth = np.abs(Ifreq0) * np.sqrt( @@ -387,7 +393,6 @@ def run_rmsynth( mDict["polyCoefferr"] = ",".join( [str(x.astype(np.float32)) for x in fitDict["perror"]] ) - mDict["poly_reffreq"] = fitDict["reference_frequency_Hz"] mDict["polyOrd"] = fitDict["polyOrd"] mDict["IfitStat"] = fitDict["fitStatus"] mDict["IfitChiSqRed"] = fitDict["chiSqRed"] diff --git a/tests/RMsynth1D_referencevalues.json b/tests/RMsynth1D_referencevalues.json index ce0de63..61c891b 100644 --- a/tests/RMsynth1D_referencevalues.json +++ b/tests/RMsynth1D_referencevalues.json @@ -1 +1 @@ -{"dFDFcorMAD": 0.019958389922976494, "phiPeakPIfit_rm2": 200.29003676576465, "dPhiPeakPIfit_rm2": 0.248062934269283, "ampPeakPIfit": 0.6996765531831387, "ampPeakPIfitEff": 0.699619480830623, "dAmpPeakPIfit": 0.005892556709695188, "snrPIfit": 118.73904446807298, "indxPeakPIfit": 266.7633455885882, "peakFDFimagFit": -0.5593282099678656, "peakFDFrealFit": 0.4190415975286484, "polAngleFit_deg": 153.42004263949445, "dPolAngleFit_deg": 0.24126764607950102, "polAngle0Fit_deg": 48.26124570608431, "dPolAngle0Fit_deg": 1.372712376102901, "Ifreq0": 1.0, "polyCoeffs": "0.0,0.0,0.0,0.0,0.0,1.0", "polyCoefferr": "0.0,0.0,0.0,19.157488,0.6786377,0.08796824", "poly_reffreq": 944000000.2222222, "polyOrd": 2, "IfitStat": 4, "IfitChiSqRed": 0.0, "fit_function": "log", "lam0Sq_m2": 0.10327484831236765, "freq0_Hz": 932874912.6426204, "fwhmRMSF": 58.90951156616211, "dQU": 0.10000000149011612, "dFDFth": 0.005892556709695188, "units": "Jy/beam", "min_freq": 800000000.0, "max_freq": 1088000000.0, "N_channels": 288, "median_channel_width": 1003456.0, "fracPol": 0.699619480830623, "sigmaAddQ": 0.3932147299302305, "dSigmaAddMinusQ": 0.28588346072691073, "dSigmaAddPlusQ": 0.13642591925592168, "sigmaAddU": 0.22666017312932038, "dSigmaAddMinusU": 0.20936429555590333, "dSigmaAddPlusU": 0.20587598580813857} +{"dFDFcorMAD": 0.01995876058936119, "phiPeakPIfit_rm2": 200.29003676576465, "dPhiPeakPIfit_rm2": 0.24806386441280062, "ampPeakPIfit": 0.6996739296667636, "ampPeakPIfitEff": 0.6996168571002304, "dAmpPeakPIfit": 0.005892556709695188, "snrPIfit": 118.73859924259541, "indxPeakPIfit": 266.7633455885882, "peakFDFimagFit": -0.5593258116763724, "peakFDFrealFit": 0.41904044934377743, "polAngleFit_deg": 153.42006391662085, "dPolAngleFit_deg": 0.2412685507432214, "polAngle0Fit_deg": 48.26126698321059, "dPolAngle0Fit_deg": 1.37271752326252, "Ifreq0": 1.0, "polyCoeffs": "0.0,0.0,0.0,0.0,0.0,1.0", "polyCoefferr": "0.0,0.0,0.0,19.157488,0.66518927,0.088204324", "polyOrd": 2, "IfitStat": 4, "IfitChiSqRed": 0.0, "fit_function": "log", "lam0Sq_m2": 0.10327484831236765, "freq0_Hz": 932874912.6426204, "fwhmRMSF": 58.90951156616211, "dQU": 0.10000000149011612, "dFDFth": 0.005892556709695188, "units": "Jy/beam", "min_freq": 800000000.0, "max_freq": 1088000000.0, "N_channels": 288, "median_channel_width": 1003456.0, "fracPol": 0.6996168571002304, "sigmaAddQ": 0.3932188246076685, "dSigmaAddMinusQ": 0.2858783770995449, "dSigmaAddPlusQ": 0.13642421257417375, "sigmaAddU": 0.22665457086880808, "dSigmaAddMinusU": 0.20935913039582774, "dSigmaAddPlusU": 0.20587796377894235} From a32e788e64a0f894f79c24132389537ad2c200b9 Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Wed, 20 Mar 2024 15:49:24 +1100 Subject: [PATCH 5/7] Added new tool to perform Stokes I ref.freq. rescaling for 3D Stokes I products. --- RMtools_3D/do_fitIcube.py | 30 ++- RMtools_3D/rescale_I_model_3D.py | 328 +++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 RMtools_3D/rescale_I_model_3D.py diff --git a/RMtools_3D/do_fitIcube.py b/RMtools_3D/do_fitIcube.py index 3c14d50..c1c0298 100755 --- a/RMtools_3D/do_fitIcube.py +++ b/RMtools_3D/do_fitIcube.py @@ -35,7 +35,6 @@ # =============================================================================# import argparse -import multiprocessing as mp import os import sys import time @@ -43,7 +42,6 @@ import astropy.io.fits as pf import numpy as np -from tqdm.auto import tqdm from tqdm.contrib.concurrent import process_map from RMtools_3D.do_RMsynth_3D import readFitsCube @@ -426,6 +424,7 @@ def fit_spectra_I( outs["chiSqRed"] = pixFitDict["chiSqRed"] outs["nIter"] = pixFitDict["nIter"] outs["AIC"] = pixFitDict["AIC"] + outs["covar"] = pixFitDict["pcov"] outs["reference_frequency_Hz"] = pixFitDict["reference_frequency_Hz"] return outs @@ -503,7 +502,10 @@ def make_model_I( coeffs = np.array([mskArr] * 6) coeffs_error = np.array([mskArr] * 6) reffreq = np.squeeze(np.array([mskArr])) + + covars = np.array([[mskArr] * 6] * 6) datacube = np.squeeze(datacube) + # Select only the spectra with emission srcData = np.rot90(datacube[:, mskSrc > 0]) @@ -551,6 +553,7 @@ def make_model_I( modelIcube[:, x, y] = results[i]["I"] reffreq[x, y] = results[i]["reference_frequency_Hz"] + covars[:, :, x, y] = results[i]["covar"] for k, j, l in zip( range(len(coeffs)), results[i]["coeffs"], results[i]["coeffs_err"] @@ -581,6 +584,7 @@ def make_model_I( prefixOut=prefixOut, ) + # Save frequency map head_freq = headcoeff.copy() head_freq["BUNIT"] = "Hz" if "BTYPE" in headcoeff: @@ -589,6 +593,28 @@ def make_model_I( outname = os.path.join(outDir, prefixOut + "reffreq.fits") pf.writeto(outname, reffreq, head_freq, overwrite=True) + # Save covariance maps -- these are necessary if/when converting the model + # reference frequency. + # Structure will be a single file as a 4D cube, with the 3rd and 4th dimensions + # iterating over the two axes of the covariance matrix. + head_covar = headcoeff.copy() + head_covar["NAXIS"] = 4 + head_covar["NAXIS3"] = 6 + head_covar["NAXIS4"] = 6 + head_covar["CTYPE3"] = "INDEX" + head_covar["CTYPE4"] = "INDEX" + head_covar["CRVAL3"] = 0 + head_covar["CRVAL4"] = 0 + head_covar["CDELT3"] = 1 + head_covar["CDELT4"] = 1 + head_covar["CRPIX3"] = 1 + head_covar["CRPIX4"] = 1 + head_covar["CUNIT3"] = "" + head_covar["CUNIT4"] = "" + + outname = os.path.join(outDir, prefixOut + "covariance.fits") + pf.writeto(outname, covars, head_covar, overwrite=True) + if verbose: print("Saving model I cube image. ") savefits_model_I(data=modelIcube, header=header, outDir=outDir, prefixOut=prefixOut) diff --git a/RMtools_3D/rescale_I_model_3D.py b/RMtools_3D/rescale_I_model_3D.py new file mode 100644 index 0000000..2dbad87 --- /dev/null +++ b/RMtools_3D/rescale_I_model_3D.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# # +# NAME: rescale_I_model_3D.py # +# # +# PURPOSE: Convert a 3D Stokes I model to a new reference frequency # +# # +# CREATED: 19-Mar-2024 by Cameron Van Eck +# # +# =============================================================================# +# # +# The MIT License (MIT) # +# # +# Copyright (c) 2024 Cameron Van Eck # +# # +# Permission is hereby granted, free of charge, to any person obtaining a # +# copy of this software and associated documentation files (the "Software"), # +# to deal in the Software without restriction, including without limitation # +# the rights to use, copy, modify, merge, publish, distribute, sublicense, # +# and/or sell copies of the Software, and to permit persons to whom the # +# Software is furnished to do so, subject to the following conditions: # +# # +# The above copyright notice and this permission notice shall be included in # +# all copies or substantial portions of the Software. # +# # +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # +# DEALINGS IN THE SOFTWARE. # +# # +# =============================================================================# + + +# Steps: 1) Read in covariance matrix, coefficient maps, I ref.freq., lambda^2_0 +# 2a) Create fitDict, pixel-wise +# 2b) Invoke rescaling function pixel-wise +# 3) Drizzle output fitDict value into arrays +# 4) Write new coefficient maps + +# Optional variations: user-specified reference frequency, unspecified +# (make uniform across all pixels) + +import argparse +import multiprocessing as mp +import os +from functools import partial + +import astropy.io.fits as pf +import numpy as np + +from RMutils.util_misc import renormalize_StokesI_model + + +def command_line(): + """Handle invocation from the command line, parsing inputs and running + everything.""" + + # Help string to be shown using the -h option + descStr = """ + Convert a Stokes I model to a new reference frequency. This changes the + model coefficients and their errors, but does not change (and does not + recalculate) the actual model spectrum. + The conversion of the coefficient uncertainties uses a first-order Taylor + approximation, so the uncertainties are only valid for relatively small + variations in reference frequency (depends a lot on the model, but 10% + seems to be a good rule of thumb). + The new reference frequency can either be from lambda^2_0 from an FDF cube, + which will create Stokes I maps that match the coresponding frequency, + a constant value given by the user, or unspecified (in which case the tool + will make all pixels have a common reference frequency at the mean input reference frequency). + The input files are assumed to be the products of do_fitIcube, with the usual filenames. + Outputs are new coefficient (and error) maps and reference frequency map. + """ + + # Parse the command line options + parser = argparse.ArgumentParser( + description=descStr, formatter_class=argparse.RawTextHelpFormatter + ) + + parser.add_argument( + "covar_file", + metavar="covariance.fits", + help="FITS cube Stokes I fit covariance matrices.", + ) + + parser.add_argument( + "-l", + dest="lambda2_0", + type=str, + default=None, + help="FDF cube from rmsynth3d. If given, will convert model to frequency matching the polarization products.", + ) + + parser.add_argument( + "-f", + dest="new_reffreq", + type=float, + default=None, + help="New reference frequency (in Hz). If given, forces all pixels to this frequency. Incompatible with -l", + ) + + parser.add_argument( + "-o", + dest="outname", + type=str, + default=None, + help="Output path+filename. If not given, defaults to input path+name (minus the covariance.fits).", + ) + parser.add_argument( + "-n", + dest="num_cores", + type=int, + default=1, + help="Number of cores to use for multiprocessing. Default is 1.", + ) + + parser.add_argument( + "-w", + dest="overwrite", + action="store_true", + help="Overwrite existing files? [False].", + ) + + args = parser.parse_args() + + if (args.new_reffreq is not None) and (args.lambda2_0 is not None): + raise Exception( + "Please do not set both -f and -l flags -- chose one or neither." + ) + + if not os.path.exists(args.covar_file): + raise Exception( + f"Cannot find covarance file at {args.covar_file}, please check filename/path." + ) + + if args.covar_file[-15:] != "covariance.fits": + raise Exception( + "Input covariance file name doesn't end in covariance.fits; this is required." + ) + + basename = args.covar_file[:-15] + + if args.outname is None: + args.outname = basename + + # Get data: + covar_map, old_reffreq_map, coeffs, header = read_data(basename) + + # Create new frequency map: + if args.lambda2_0 is not None: + FDF_header = pf.getheader(args.lambda2_0) + lam0Sq_m2 = FDF_header["LAMSQ0"] + freq0 = 2.997924538e8 / np.sqrt(lam0Sq_m2) + new_freq_map = np.ones_like(old_reffreq_map) * freq0 + elif args.new_reffreq is not None: + new_freq_map = np.ones_like(old_reffreq_map) * args.new_reffreq + else: + freq0 = np.nanmean(old_reffreq_map) + new_freq_map = np.ones_like(old_reffreq_map) * freq0 + + # Get fit function from header: + x = ["Fit model is" in card for card in header["HISTORY"]] + line = header["HISTORY"][np.where(x)[0][-1]] + fit_function = [x for x in line.split() if "polynomial" in x][0].split("-")[0] + + new_freq_map, new_coeffs, new_errors = rescale_I_model_3D( + covar_map, + old_reffreq_map, + new_freq_map, + coeffs, + fit_function, + num_cores=args.num_cores, + ) + + write_new_parameters( + new_freq_map, + new_coeffs, + new_errors, + args.outname, + header, + overwrite=args.overwrite, + ) + + +def read_data(basename): + """Reads the covariance matrix map, (current) reference frequency map, and + (current) coefficient+error maps. + Input: + basename (str): file path and name up to 'covariance.fits'/'reffreq.fits', etc. + """ + + if not ( + os.path.exists(basename + "coeff0.fits") + and os.path.exists(basename + "coeff0err.fits") + ): + raise Exception("Cannot find coeff0 map. At least coeff 0 map must exist.") + + covar_file = basename + "covariance.fits" + covar_map = pf.getdata(covar_file) + + freq_file = basename + "reffreq.fits" + old_reffreq_map, header = pf.getdata(freq_file, header=True) + # Grabs header from frequency map -- needed for fit function, and to use for + # writing out products. Better to have 2D map header to avoid fussing with + # extra axes. + + # Get coefficient maps (without knowing how many there are) + # Reverse index order to match RM-Tools internal ordering (highest to lowest polynomial order) + coeffs = np.zeros((6, *old_reffreq_map.shape)) + for i in range(6): + try: # Keep trying higher orders + data = pf.getdata(basename + f"coeff{i}.fits") + coeffs[5 - i] = data + except FileNotFoundError: + break # Once it runs out of valid coefficient maps, move on + + return covar_map, old_reffreq_map, coeffs, header + + +def rescale_I_pixel(data, fit_function): + covar, coeff, old_freq, new_freq = data + oldDict = {} # Initialize a fitDict, which contains the relevant fit information + oldDict["reference_frequency_Hz"] = old_freq + oldDict["p"] = coeff + oldDict["pcov"] = covar + oldDict["fit_function"] = fit_function + + newDict = renormalize_StokesI_model(oldDict, new_freq) + return newDict["p"], newDict["perror"] + + +def rescale_I_model_3D( + covar_map, old_reffreq_map, new_freq_map, coeffs, fit_function="log", num_cores=1 +): + """Rescale the Stokes I model parameters to a new reference frequency, for + an entire image (i.e., 3D pipeline products). + + Inputs: + covar_map (4D array): covariance matrix map, such as produced by do_fitIcube. + old_reffreq_map (2D array): map of current reference frequency, such as produced by do_fitIcube. + new_freq_map (2D array): map of new reference frequencies. + coeffs (3D array): model parameter map (going from highest to lowest order) + coeff_errors (3D array): model parameter uncertainties map (highest to lowest order) + + Returns: + new_freq_map (unchanged from input) + new_coeffs (3D array): maps of new model parameters (highest to lowest order) + new_errors (3D array): maps of new parameter uncertainties (highest to lowest) + """ + + # Initialize output arrays: + new_coeffs = np.zeros_like(coeffs) + new_errors = np.zeros_like(coeffs) + rs = old_reffreq_map.shape[ + 1 + ] # Get the length of a row, for array indexing later on. + + # Set up inputs for parallelization: + # Input order is: covariance matrix, coefficient vector, old frequency, new frequency + inputs = list( + zip( + np.reshape( + np.moveaxis(covar_map, (0, 1), (2, 3)), (old_reffreq_map.size, 6, 6) + ), + np.reshape(np.moveaxis(coeffs, 0, 2), (old_reffreq_map.size, 6)), + old_reffreq_map.flat, + new_freq_map.flat, + ) + ) + with mp.Pool(num_cores) as pool_: + results = pool_.map( + partial(rescale_I_pixel, fit_function=fit_function), inputs, chunksize=100 + ) + + for i, (p, perror) in enumerate(results): + new_coeffs[:, i // rs, i % rs] = p + new_errors[:, i // rs, i % rs] = perror + + return new_freq_map, new_coeffs, new_errors + + +def write_new_parameters( + new_freq_map, new_coeffs, new_errors, out_basename, header, overwrite=False +): + """Write out new parameter/uncertainty maps to FITS files. + Inputs: + new_freq_map (unchanged from input) + new_coeffs (3D array): maps of new model parameters (highest to lowest order) + new_errors (3D array): maps of new parameter uncertainties (highest to lowest) + out_basename (str): base path+name of the files to be written out + (will be postpended with 'newcoeff0.fits', etc.) + + Returns: (nothing) + Writes out coefficient maps (newcoeff0.fits, etc.) and + coefficient errors (newcoeff0err.fits, etc.) + """ + + out_header = header.copy() + out_header["HISTORY"] = "Stokes I model rescaled to new reference frequency." + out_header["REFFREQ"] = (new_freq_map[0, 0], "Hz") + if "BUNIT" in out_header: + del out_header["BUNIT"] + + # Work out highest order of polynomial: + max_order = np.sum(np.any(new_coeffs != 0.0, axis=(1, 2))) - 1 + + for i in range(max_order + 1): + pf.writeto( + out_basename + f"newcoeff{i}.fits", + new_coeffs[5 - i], + header=out_header, + overwrite=overwrite, + ) + pf.writeto( + out_basename + f"newcoeff{i}err.fits", + new_errors[5 - i], + header=out_header, + overwrite=overwrite, + ) + + +# -----------------------------------------------------------------------------# +if __name__ == "__main__": + command_line() diff --git a/setup.py b/setup.py index 5a1a91e..6d5b61d 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ "rmtools_extractregion=RMtools_3D.extract_region:main", "rmtools_bwdepol=RMtools_1D.rmtools_bwdepol:main", "rmtools_bwpredict=RMtools_1D.rmtools_bwpredict:main", + "rmools_3DIrescale=RMtools_3D.rescale_I_model_3D:main", ], }, install_requires=REQUIRED, From 3359c255e3ffc1626039637f227992e13a915050 Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Wed, 20 Mar 2024 16:01:05 +1100 Subject: [PATCH 6/7] Fixed dumb typo. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d5b61d..60fc5f7 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ "rmtools_extractregion=RMtools_3D.extract_region:main", "rmtools_bwdepol=RMtools_1D.rmtools_bwdepol:main", "rmtools_bwpredict=RMtools_1D.rmtools_bwpredict:main", - "rmools_3DIrescale=RMtools_3D.rescale_I_model_3D:main", + "rmtools_3DIrescale=RMtools_3D.rescale_I_model_3D:main", ], }, install_requires=REQUIRED, From d8de7257b9e3a021449403eecb0082a563819588 Mon Sep 17 00:00:00 2001 From: Cameron Van Eck Date: Thu, 21 Mar 2024 09:23:21 +1100 Subject: [PATCH 7/7] Added warning for large shifts in reference freq. --- RMutils/util_misc.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/RMutils/util_misc.py b/RMutils/util_misc.py index 00e16fb..f185f67 100644 --- a/RMutils/util_misc.py +++ b/RMutils/util_misc.py @@ -72,6 +72,7 @@ import re import sys import traceback +import warnings import numpy as np import numpy.ma as ma @@ -464,6 +465,14 @@ def renormalize_StokesI_model(fitDict, new_reference_frequency): """ # Renormalization ratio: x = new_reference_frequency / fitDict["reference_frequency_Hz"] + + # Check if ratio is within zone of probable accuracy (approx. 10%, from empirical tests) + if (x < 0.9) or (x > 1.1): + warnings.warn( + "New Stokes I reference frequency more than 10% different than original, uncertainties may be unreliable", + UserWarning, + ) + (a, b, c, d, f, g) = fitDict["p"] newDict = fitDict.copy()