From 9867229fa5c524a37ffded8bd94f5c2a25da97ce Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Sat, 9 Nov 2024 16:02:08 -0800 Subject: [PATCH 1/5] save the offsets in the psf file --- py/desispec/scripts/trace_shifts.py | 25 +++++++++++------ py/desispec/trace_shifts.py | 42 ++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/py/desispec/scripts/trace_shifts.py b/py/desispec/scripts/trace_shifts.py index 58ace0d5d..2b9d2b7fa 100644 --- a/py/desispec/scripts/trace_shifts.py +++ b/py/desispec/scripts/trace_shifts.py @@ -198,11 +198,13 @@ def fit_trace_shifts(image, args): x_for_dx,y_for_dx,dx,ex,fiber_for_dx,wave_for_dx = compute_dx_from_cross_dispersion_profiles(xcoef,ycoef,wavemin,wavemax, image=image, fibers=fibers, width=args.width, deg=args.degxy,image_rebin=args.ccd_rows_rebin) if internal_wavelength_calib : # measure y shifts - x_for_dy,y_for_dy,dy,ey,fiber_for_dy,wave_for_dy = compute_dy_using_boxcar_extraction(tset, image=image, fibers=fibers, width=args.width, continuum_subtract=continuum_subtract) + x_for_dy,y_for_dy,dy,ey,fiber_for_dy,wave_for_dy,dwave,dwave_err = compute_dy_using_boxcar_extraction(tset, image=image, fibers=fibers, width=args.width, continuum_subtract=continuum_subtract) mdy = np.median(dy) log.info("Subtract median(dy)={}".format(mdy)) dy -= mdy # remove median, because this is an internal calibration - + internal_offset_info = {'wave':wave_for_dy, + 'fiber':fiber_for_dy, 'dwave':dwave, + 'dwave_err':dwave_err} else : # duplicate dx results with zero shift to avoid write special case code below x_for_dy = x_for_dx.copy() @@ -211,7 +213,8 @@ def fit_trace_shifts(image, args): ey = 1.e-6*np.ones(ex.shape) fiber_for_dy = fiber_for_dx.copy() wave_for_dy = wave_for_dx.copy() - + internal_offset_info = None + degxx=args.degxx degxy=args.degxy degyx=args.degyx @@ -341,11 +344,16 @@ def fit_trace_shifts(image, args): # the psf is used only to convolve the input spectrum # the traceset of the psf is not used here psf = read_specter_psf(args.psf) - tset.y_vs_wave_traceset._coeff = shift_ycoef_using_external_spectrum(psf=psf,xytraceset=tset, + (tset.y_vs_wave_traceset._coeff, + (wave_external, dwave_external, dwave_err_external)) = shift_ycoef_using_external_spectrum(psf=psf,xytraceset=tset, image=image,fibers=fibers, spectrum_filename=spectrum_filename, degyy=args.degyy,width=7) - + external_offset_info = {'wave': wave_external, + 'dwave': dwave_external, + 'dwave_err':dwave_err_external} + else: + external_offset_info = None x = np.zeros(x0.shape) y = np.zeros(x0.shape) for s in range(tset.nspec) : @@ -361,7 +369,7 @@ def fit_trace_shifts(image, args): tset.meta["MINDY"]=np.min(dy) tset.meta["MAXDY"]=np.max(dy) - return tset + return tset, internal_offset_info, external_offset_info def main(args=None) : @@ -382,10 +390,11 @@ def main(args=None) : log.critical(f"Entire {os.path.basename(args.image)} image is masked; can't fit traceshifts") sys.exit(1) - tset = fit_trace_shifts(image=image, args=args) + tset, internal_offset_info, external_offset_info = fit_trace_shifts(image=image, args=args) tset.meta['IN_PSF'] = shorten_filename(args.psf) tset.meta['IN_IMAGE'] = shorten_filename(args.image) if args.outpsf is not None : - write_traces_in_psf(args.psf,args.outpsf,tset) + write_traces_in_psf(args.psf,args.outpsf,tset, internal_offset_info=internal_offset_info, + external_offset_info=external_offset_info) log.info("wrote modified PSF in %s"%args.outpsf) diff --git a/py/desispec/trace_shifts.py b/py/desispec/trace_shifts.py index 738a280f7..6c15e3a89 100644 --- a/py/desispec/trace_shifts.py +++ b/py/desispec/trace_shifts.py @@ -25,7 +25,8 @@ from desispec.interpolation import resample_flux from desispec.qproc.qextract import qproc_boxcar_extraction -def write_traces_in_psf(input_psf_filename,output_psf_filename,xytraceset) : +def write_traces_in_psf(input_psf_filename,output_psf_filename,xytraceset, internal_offset_info=None, + external_offset_info=None) : """ Writes traces in a PSF. @@ -33,6 +34,8 @@ def write_traces_in_psf(input_psf_filename,output_psf_filename,xytraceset) : input_psf_filename : Path to input fits file which has to contain XTRACE and YTRACE HDUs output_psf_filename : Path to output fits file which has to contain XTRACE and YTRACE HDUs xytraceset : xytraceset + internal_offset_info: dictionary of internal offsets in wavelength + external_offset_info: dictionary of external offsets in wavelength """ xcoef=xytraceset.x_vs_wave_traceset._coeff @@ -104,7 +107,24 @@ def write_traces_in_psf(input_psf_filename,output_psf_filename,xytraceset) : if (xytraceset.meta is not None) and ("PSF" in psf_fits): for k in xytraceset.meta.keys() : psf_fits["PSF"].header[k] = xytraceset.meta[k] - + if internal_offset_info is not None: + data = {} + dwave,dwave_err,fiber,wave=[internal_offset_info[_] for _ in ['dwave','dwave_err','fiber','wave']] + data = np.rec.fromarrays((fiber, wave, dwave, dwave_err), + dtype=np.dtype([('fiber','i4'), + ('wave','f4'), + ('dwave','f4'), + ('dwave_err','f4')])) + psf_fits.append(pyfits.BinTableHDU(data, name='INTOFF')) + if external_offset_info is not None: + data = {} + dwave,dwave_err,wave=[external_offset_info[_] for _ in ['dwave','dwave_err','wave']] + data = np.rec.fromarrays((wave, dwave, dwave_err), + dtype=np.dtype([ + ('wave','f4'), + ('dwave','f4'), + ('dwave_err','f4')])) + psf_fits.append( pyfits.BinTableHDU(data, name='EXTOFF')) tmpfile = get_tempfilename(output_psf_filename) psf_fits.writeto(tmpfile, overwrite=True) @@ -346,7 +366,9 @@ def compute_dy_from_spectral_cross_correlations_of_frame(flux, ivar, wave , xcoe ey=np.array([]) fiber_for_dy=np.array([]) wave_for_dy=np.array([]) - + dwave_list = np.array([]) + dwave_err_list = np.array([]) + nfibers = flux.shape[0] for fiber in range(nfibers) : @@ -375,7 +397,8 @@ def compute_dy_from_spectral_cross_correlations_of_frame(flux, ivar, wave , xcoe if err > 1 : continue - + dwave_list = np.append(dwave_list, dwave) + dwave_err_list = np.append(dwave_err_list, err) rw = legx(block_wave,wavemin,wavemax) tx = legval(rw,xcoef[fiber]) ty = legval(rw,ycoef[fiber]) @@ -392,7 +415,7 @@ def compute_dy_from_spectral_cross_correlations_of_frame(flux, ivar, wave , xcoe fiber_for_dy=np.append(fiber_for_dy,fiber) wave_for_dy=np.append(wave_for_dy,block_wave) - return x_for_dy,y_for_dy,dy,ey,fiber_for_dy,wave_for_dy + return x_for_dy,y_for_dy,dy,ey,fiber_for_dy,wave_for_dy, dwave_list, dwave_err_list def compute_dy_using_boxcar_extraction(xytraceset, image, fibers, width=7, degyy=2, continuum_subtract = False): @@ -444,7 +467,7 @@ def compute_dy_using_boxcar_extraction(xytraceset, image, fibers, width=7, degyy wavemax = xytraceset.wavemax xcoef = xytraceset.x_vs_wave_traceset._coeff ycoef = xytraceset.y_vs_wave_traceset._coeff - + return compute_dy_from_spectral_cross_correlations_of_frame(flux=flux, ivar=ivar, wave=wave, xcoef=xcoef, ycoef=ycoef, wavemin=wavemin, wavemax=wavemax, reference_flux = mflux , n_wavelength_bins = degyy+4) @numba.jit @@ -834,6 +857,8 @@ def shift_ycoef_using_external_spectrum(psf, xytraceset, image, fibers, dy = np.array([]) ey = np.array([]) wave_for_dy = np.array([]) + dwave_list = np.array([]) + dwave_err_list = np.array([]) fiber_for_psf_evaluation = flux.shape[0] //2 wavelength_bins = np.linspace(wave[0], wave[-1], n_wavelength_bins+1) for b in range(n_wavelength_bins) : @@ -857,6 +882,8 @@ def shift_ycoef_using_external_spectrum(psf, xytraceset, image, fibers, dy = np.append(dy, -dwave * dydw) ey = np.append(ey, err*dydw) wave_for_dy = np.append(wave_for_dy,bin_wave) + dwave_list = np.append(dwave_list, dwave) + dwave_err_list = np.append(dwave_err_list, err) y_for_dy=np.append(y_for_dy,y) log.info(f"wave = {bin_wave}A , y={y}, measured dwave = {dwave} +- {err} A") @@ -895,7 +922,8 @@ def shift_ycoef_using_external_spectrum(psf, xytraceset, image, fibers, for fiber in range(ycoef.shape[0]) : ycoef[fiber] += dycoef - return ycoef + return ycoef, (wave_for_dy, dwave_list, dwave_err_list) + # end of routines for cross-correlation method for trace shifts From e53624ad7904d17f843d325962e91b7594a1c912 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 19 Nov 2024 16:07:02 -0800 Subject: [PATCH 2/5] fixes following the review update QA code expand comments uppercase the column names --- py/desispec/qa/qa_quicklook.py | 2 +- py/desispec/scripts/trace_shifts.py | 18 +++++++++++++----- py/desispec/trace_shifts.py | 20 ++++++++++---------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/py/desispec/qa/qa_quicklook.py b/py/desispec/qa/qa_quicklook.py index 0eeaf353a..6b6984d2e 100644 --- a/py/desispec/qa/qa_quicklook.py +++ b/py/desispec/qa/qa_quicklook.py @@ -341,7 +341,7 @@ def run_qa(self,image,inputs): from desispec.trace_shifts import compute_dy_using_boxcar_extraction as compute_dy fibers=np.arange(500) #RS: setting nfibers to 500 for now ox,oy,odx,oex,of,ol=compute_dx(xcoef,ycoef,wavemin,wavemax,image,fibers=fibers) - x_for_dy,y_for_dy,ody,ey,fiber_for_dy,wave_for_dy=compute_dy(psftrace,image,fibers) + x_for_dy,y_for_dy,ody,ey,fiber_for_dy,wave_for_dy,dwave,dwave_err=compute_dy(psftrace,image,fibers) # return average shifts in x and y dx=np.mean(odx) diff --git a/py/desispec/scripts/trace_shifts.py b/py/desispec/scripts/trace_shifts.py index 2b9d2b7fa..f07307487 100644 --- a/py/desispec/scripts/trace_shifts.py +++ b/py/desispec/scripts/trace_shifts.py @@ -94,7 +94,15 @@ def read_specter_psf(filename) : def fit_trace_shifts(image, args): - + """ + Perform the fitting of shifts of spectral traces + This consists of two steps, one is internal, by + cross-correlating spectra to themselves, and then + cross-correlating to external (ususally sky) spectrum + + Return updated traceset and two dictionaies with offset information + to be written in the PSF file + """ global psfs log=get_logger() @@ -168,6 +176,7 @@ def fit_trace_shifts(image, args): nfibers = args.nfibers # FOR DEBUGGING fibers=np.arange(nfibers) + internal_offset_info = None if lines is not None : @@ -213,7 +222,6 @@ def fit_trace_shifts(image, args): ey = 1.e-6*np.ones(ex.shape) fiber_for_dy = fiber_for_dx.copy() wave_for_dy = wave_for_dx.copy() - internal_offset_info = None degxx=args.degxx degxy=args.degxy @@ -345,10 +353,10 @@ def fit_trace_shifts(image, args): # the traceset of the psf is not used here psf = read_specter_psf(args.psf) (tset.y_vs_wave_traceset._coeff, - (wave_external, dwave_external, dwave_err_external)) = shift_ycoef_using_external_spectrum(psf=psf,xytraceset=tset, - image=image,fibers=fibers, + (wave_external, dwave_external, dwave_err_external)) = shift_ycoef_using_external_spectrum(psf=psf, xytraceset=tset, + image=image, fibers=fibers, spectrum_filename=spectrum_filename, - degyy=args.degyy,width=7) + degyy=args.degyy, width=7) external_offset_info = {'wave': wave_external, 'dwave': dwave_external, 'dwave_err':dwave_err_external} diff --git a/py/desispec/trace_shifts.py b/py/desispec/trace_shifts.py index 6c15e3a89..0ea2923cc 100644 --- a/py/desispec/trace_shifts.py +++ b/py/desispec/trace_shifts.py @@ -34,8 +34,8 @@ def write_traces_in_psf(input_psf_filename,output_psf_filename,xytraceset, inter input_psf_filename : Path to input fits file which has to contain XTRACE and YTRACE HDUs output_psf_filename : Path to output fits file which has to contain XTRACE and YTRACE HDUs xytraceset : xytraceset - internal_offset_info: dictionary of internal offsets in wavelength - external_offset_info: dictionary of external offsets in wavelength + internal_offset_info: dictionary of internal offsets (i.e. fiber vs 'median fiber') in wavelength + external_offset_info: dictionary of external offsets (i.e. 'median fiber' vs external spectrum) in wavelength """ xcoef=xytraceset.x_vs_wave_traceset._coeff @@ -109,21 +109,21 @@ def write_traces_in_psf(input_psf_filename,output_psf_filename,xytraceset, inter psf_fits["PSF"].header[k] = xytraceset.meta[k] if internal_offset_info is not None: data = {} - dwave,dwave_err,fiber,wave=[internal_offset_info[_] for _ in ['dwave','dwave_err','fiber','wave']] + dwave, dwave_err, fiber, wave=[internal_offset_info[_] for _ in ['dwave','dwave_err','fiber','wave']] data = np.rec.fromarrays((fiber, wave, dwave, dwave_err), - dtype=np.dtype([('fiber','i4'), - ('wave','f4'), - ('dwave','f4'), - ('dwave_err','f4')])) + dtype=np.dtype([('FIBER','i4'), + ('WAVE','f4'), + ('DWAVE','f4'), + ('DWAVE_ERR','f4')])) psf_fits.append(pyfits.BinTableHDU(data, name='INTOFF')) if external_offset_info is not None: data = {} dwave,dwave_err,wave=[external_offset_info[_] for _ in ['dwave','dwave_err','wave']] data = np.rec.fromarrays((wave, dwave, dwave_err), dtype=np.dtype([ - ('wave','f4'), - ('dwave','f4'), - ('dwave_err','f4')])) + ('WAVE','f4'), + ('DWAVE','f4'), + ('DWAVE_ERR','f4')])) psf_fits.append( pyfits.BinTableHDU(data, name='EXTOFF')) tmpfile = get_tempfilename(output_psf_filename) From c4a5a8f80503b0654831e7dec9b82c7cee6dc27a Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 19 Nov 2024 16:29:54 -0800 Subject: [PATCH 3/5] update comments --- py/desispec/scripts/trace_shifts.py | 2 +- py/desispec/trace_shifts.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/py/desispec/scripts/trace_shifts.py b/py/desispec/scripts/trace_shifts.py index f07307487..496c2d503 100644 --- a/py/desispec/scripts/trace_shifts.py +++ b/py/desispec/scripts/trace_shifts.py @@ -353,7 +353,7 @@ def fit_trace_shifts(image, args): # the traceset of the psf is not used here psf = read_specter_psf(args.psf) (tset.y_vs_wave_traceset._coeff, - (wave_external, dwave_external, dwave_err_external)) = shift_ycoef_using_external_spectrum(psf=psf, xytraceset=tset, + wave_external, dwave_external, dwave_err_external) = shift_ycoef_using_external_spectrum(psf=psf, xytraceset=tset, image=image, fibers=fibers, spectrum_filename=spectrum_filename, degyy=args.degyy, width=7) diff --git a/py/desispec/trace_shifts.py b/py/desispec/trace_shifts.py index 0ea2923cc..7059b5a79 100644 --- a/py/desispec/trace_shifts.py +++ b/py/desispec/trace_shifts.py @@ -356,6 +356,8 @@ def compute_dy_from_spectral_cross_correlations_of_frame(flux, ivar, wave , xcoe ey : 1D array of uncertainties on dy fiber : 1D array of fiber ID (first fiber = 0) wave : 1D array of wavelength + dwave : 1D array of wavelength offsets + dwave_err: 1D array of wavelength offset uncertainties """ log=get_logger() @@ -823,6 +825,9 @@ def shift_ycoef_using_external_spectrum(psf, xytraceset, image, fibers, Returns: ycoef : 2D np.array of same shape as input, with modified Legendre coefficients for each fiber to convert wavelength to YCCD + wave: : 1D array of wavelengths for which offsets are computed + dwave: : 1D array of wavelength offsets + dwave_err: 1D array of wavelength offset uncertainties """ log = get_logger() @@ -922,7 +927,7 @@ def shift_ycoef_using_external_spectrum(psf, xytraceset, image, fibers, for fiber in range(ycoef.shape[0]) : ycoef[fiber] += dycoef - return ycoef, (wave_for_dy, dwave_list, dwave_err_list) + return ycoef, wave_for_dy, dwave_list, dwave_err_list From 6ef8b81c49d3a93aa98d8f5b4390d8e6656655ce Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 19 Nov 2024 16:39:03 -0800 Subject: [PATCH 4/5] reformat --- py/desispec/scripts/trace_shifts.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/py/desispec/scripts/trace_shifts.py b/py/desispec/scripts/trace_shifts.py index 496c2d503..71336ea37 100644 --- a/py/desispec/scripts/trace_shifts.py +++ b/py/desispec/scripts/trace_shifts.py @@ -211,9 +211,10 @@ def fit_trace_shifts(image, args): mdy = np.median(dy) log.info("Subtract median(dy)={}".format(mdy)) dy -= mdy # remove median, because this is an internal calibration - internal_offset_info = {'wave':wave_for_dy, - 'fiber':fiber_for_dy, 'dwave':dwave, - 'dwave_err':dwave_err} + internal_offset_info = dict(wave=wave_for_dy, + fiber=fiber_for_dy, + dwave=dwave, + dwave_err=dwave_err) else : # duplicate dx results with zero shift to avoid write special case code below x_for_dy = x_for_dx.copy() @@ -357,9 +358,9 @@ def fit_trace_shifts(image, args): image=image, fibers=fibers, spectrum_filename=spectrum_filename, degyy=args.degyy, width=7) - external_offset_info = {'wave': wave_external, - 'dwave': dwave_external, - 'dwave_err':dwave_err_external} + external_offset_info = dict(wave=wave_external, + dwave=dwave_external, + dwave_err=dwave_err_external) else: external_offset_info = None x = np.zeros(x0.shape) From 31c313d1d61fb7509108181fc802809947ae14df Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Thu, 21 Nov 2024 03:32:27 -0800 Subject: [PATCH 5/5] write_traces_in_psf was needlessly imported in qproc --- py/desispec/scripts/qproc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/desispec/scripts/qproc.py b/py/desispec/scripts/qproc.py index 06c011c32..228d060af 100644 --- a/py/desispec/scripts/qproc.py +++ b/py/desispec/scripts/qproc.py @@ -17,7 +17,6 @@ from desispec.io.fluxcalibration import read_average_flux_calibration from desispec.io.xytraceset import read_xytraceset,write_xytraceset import desispec.scripts.trace_shifts as trace_shifts_script -from desispec.trace_shifts import write_traces_in_psf from desispec.calibfinder import CalibFinder from desispec.qproc.qframe import QFrame from desispec.qproc.io import read_qframe,write_qframe