Bug Fix SiPM Simple Calibration + Optimization

Project.toml

@@ -61,7 +61,7 @@ Distributions = "0.25.87"
 FillArrays = "1.4.1"
 Format = "1.2, 1.3"
 ForwardDiff = "0.10.26"
-GaussianMixtures = "0.3.11"
+GaussianMixtures = "0.3.12"
 IntervalSets = "0.7"
 InverseFunctions = "0.1.8"
 IrrationalConstants = "0.1.1, 0.2"

ext/LegendSpecFitsRecipesBaseExt.jl

@@ -154,6 +154,36 @@
     end
 end
 
+
+@recipe function f(report:: NamedTuple{(:wl, :min_obj, :gain, :res_1pe, :pos_1pe, :threshold, :a_grid_wl_sg, :obj, :report_simple, :report_fit)})
+    xlabel := "Window Length ($(unit(first(report.a_grid_wl_sg))))"
+    ylabel := "Objective \n σ  * √(threshold) \n ______________ \n gain * √(pos_1pe)"
+    grid := :true
+    gridcolor := :black
+    gridalpha := 0.2
+    gridlinewidth := 0.5
+    ylims := (0, 1.5 * maximum(Measurements.value.(report.obj)))
+    @series begin
+        seriestype := :scatter
+        label := "Obj"
+        ustrip.(report.a_grid_wl_sg), ustrip.(report.obj)
+    end
+    @series begin
+        seriestype := :hline
+        label := "Min. Obj $(report.min_obj) (WL: $(report.wl))"
+        color := :red
+        linewidth := 2.5
+        [ustrip(Measurements.value(report.min_obj))]
+    end
+    @series begin
+        seriestype := :hspan
+        label := ""
+        color := :red
+        alpha := 0.1
+        ustrip.([Measurements.value(report.min_obj)-Measurements.uncertainty(report.min_obj), Measurements.value(report.min_obj)+Measurements.uncertainty(report.min_obj)])
+    end
+end
+
 @recipe function f(report::NamedTuple{(:v, :h, :f_fit, :f_components, :gof)}; show_label=true, show_fit=true, show_components=true, show_residuals=true, f_fit_x_step_scaling=1/100, _subplot=1, x_label="Energy (keV)")
     f_fit_x_step = ustrip(value(report.v.σ)) * f_fit_x_step_scaling
     bin_centers = collect(report.h.edges[1])[1:end-1] .+ diff(collect(report.h.edges[1]))[1]/2 
@@ -509,14 +539,14 @@
         pps = report.peakpos
     end
     xlims := (0, last(first(h.edges)))
-    min_y = minimum(h.weights) == 0.0 ? 1e-3*maximum(h.weights) : 0.8*minimum(h.weights)
+    min_y = minimum(h.weights) == 0.0 ? 1e1 : 0.8*minimum(h.weights)
     ylims --> (min_y, maximum(h.weights)*1.1)
     @series begin
         seriestype := :stepbins
         label := "amps"
         h
     end
-    y_vline = min_y:1:maximum(h.weights)*1.1
+    y_vline = [min_y, maximum(h.weights)*1.1]
     for (i, p) in enumerate(pps)
         @series begin
             seriestype := :line
@@ -544,7 +574,7 @@
     ylabel := "Counts / $(round_wo_units(report_sipm.bin_width * 1e3, digits=2))E-3 P.E."
     xlims := (first(first(report_sipm.h_cal.edges)), last(first(report_sipm.h_cal.edges)))
     xticks := (ceil(first(first(report_sipm.h_cal.edges)))-0.5:0.5:last(first(report_sipm.h_cal.edges)))
-    min_y = minimum(report_sipm.h_cal.weights) == 0.0 ? 1e-3*maximum(report_sipm.h_cal.weights) : 0.8*minimum(report_sipm.h_cal.weights)
+    min_y = minimum(report_sipm.h_cal.weights) == 0.0 ? 1e1 : 0.8*minimum(report_sipm.h_cal.weights)
     ylims := (min_y, maximum(report_sipm.h_cal.weights)*1.1)
     bin_centers = collect(report_sipm.h_cal.edges[1])[1:end-1] .+ diff(collect(report_sipm.h_cal.edges[1]))[1]/2 
     @series begin

src/LegendSpecFits.jl

@@ -81,6 +81,7 @@ include("specfit_functions.jl")
 include("calfunc.jl")
 include("sipm_simple_calibration.jl")
 include("sipmfit.jl")
+include("sipm_filter_optimization.jl")
 abstract type UncertTag end
 ForwardDiff.:(≺)(::Type{<:ForwardDiff.Tag}, ::Type{UncertTag}) = true
 ForwardDiff.:(≺)(::Type{UncertTag}, ::Type{<:ForwardDiff.Tag}) = false

src/sipm_filter_optimization.jl

@@ -0,0 +1,139 @@
+
+"""
+    fit_sipm_wl(trig_max_grid::VectorOfVectors{<:Real}, e_grid_wl::StepRangeLen)
+
+Fit the SiPM spectrum for different window lengths and return the optimal window length.
+
+# Arguments
+- `trig_max_grid`: grid of trigger maxima for different window lengths
+- `e_grid_wl`: range of window lengths to sweep through
+
+# Returns
+- `result`: optimal window length and corresponding gain, resolution and position of 1pe peak
+- `report`: report with all window lengths and corresponding gains, resolutions and positions of 1pe peaks
+"""
+function fit_sipm_wl(trig_max_grid::VectorOfVectors{<:Real}, e_grid_wl::StepRangeLen, thresholds::Vector{<:Real}=zeros(length(e_grid_wl));
+    min_pe_peak::Int=1, max_pe_peak::Int=5, n_fwhm_noise_cut::Real=2.0, peakfinder_threshold::Real=5.0, initial_max_amp::Real = 50.0, initial_max_bin_width_quantile::Real=0.9999, 
+    peakfinder_rtol::Real=0.1, peakfinder_α::Real=0.1, peakfinder_σ::Real=-1.0, 
+    min_pe_fit::Real=0.6, max_pe_fit::Real=3.5, Δpe_peak_assignment::Real=0.3)
+
+    gain_wl    = Vector{Measurement{Float64}}(undef, length(e_grid_wl))
+    res_1pe_wl = Vector{Measurement{Float64}}(undef, length(e_grid_wl))
+    pos_1pe_wl = Vector{Measurement{Float64}}(undef, length(e_grid_wl))
+    success    = falses(length(e_grid_wl))  
+    reports_simple     = Vector{NamedTuple}(undef, length(e_grid_wl))
+    reports_fit        = Vector{NamedTuple}(undef, length(e_grid_wl))
+
+    # for each window lenght, calculate gain, resolution and position of 1pe peak
+    Threads.@threads for w in eachindex(e_grid_wl)
+        wl = e_grid_wl[w]
+        trig_max = filter(isfinite, collect(trig_max_grid[w]))
+        threshold = thresholds[w]
+        try
+            result_simple, report_simple = sipm_simple_calibration(trig_max; initial_min_amp=threshold, initial_max_amp=initial_max_amp, initial_max_bin_width_quantile=initial_max_bin_width_quantile,
+                                            min_pe_peak=min_pe_peak, max_pe_peak=max_pe_peak, n_fwhm_noise_cut=n_fwhm_noise_cut, peakfinder_threshold=peakfinder_threshold, 
+                                            peakfinder_rtol=peakfinder_rtol, peakfinder_α=peakfinder_α, peakfinder_σ=peakfinder_σ)
+
+            result_fit, report_fit = fit_sipm_spectrum(result_simple.pe_simple_cal, min_pe_fit, max_pe_fit; f_uncal=result_simple.f_simple_uncal, Δpe_peak_assignment=Δpe_peak_assignment)
+
+            # gain_wl[w] = minimum(result_simple.peakpos) - ifelse(threshold == 0.0, result_simple.noisepeakpos, threshold)
+            gain_wl[w] = minimum(result_simple.peakpos) - result_simple.noisepeakpos
+            res_1pe_wl[w] = first(result_fit.resolutions)
+            pos_1pe_wl[w] = first(result_fit.positions)
+            reports_simple[w] = report_simple
+            reports_fit[w] = report_fit
+            success[w] = true
+        catch e
+            @warn "Failed to process wl: $wl: $e"
+        end
+    end
+
+    thrs = if all(thresholds .== 0.0) ones(length(e_grid_wl)) else thresholds end
+    obj = sqrt.(res_1pe_wl[success]) .* sqrt.(thrs[success]) ./ gain_wl[success]
+    wls = collect(e_grid_wl)[success]
+
+    if isempty(obj)
+        @error "No valid gain found"
+        throw(ErrorException("No valid gain found, could not determine optimal window length"))
+    end
+    min_obj, min_obj_idx = findmin(obj)  
+    wl_min_obj = wls[min_obj_idx]
+    min_res1pe = res_1pe_wl[success][min_obj_idx]
+    min_gain   = gain_wl[success][min_obj_idx]
+    min_pos1pe = pos_1pe_wl[success][min_obj_idx]
+    min_threshold = thresholds[success][min_obj_idx]
+    min_report_simple = reports_simple[success][min_obj_idx]
+    min_report_fit = reports_fit[success][min_obj_idx]
+
+    # generate result and report
+    result = (
+        wl = measurement(wl_min_obj, step(e_grid_wl)),
+        obj = min_obj,
+        res_1pe = min_res1pe,
+        gain = min_gain,
+        pos_1pe = min_pos1pe,
+        threshold = min_threshold
+    )
+    report = (
+        wl = result.wl,
+        min_obj = result.obj,
+        gain = gain_wl,
+        res_1pe = res_1pe_wl,
+        pos_1pe = pos_1pe_wl,
+        threshold = thresholds[success],
+        a_grid_wl_sg = wls,
+        obj = obj,
+        report_simple = min_report_simple,
+        report_fit = min_report_fit,
+    )
+    return result, report
+end
+export fit_sipm_wl
+
+
+
+"""
+    fit_sipm_threshold(thresholds::Vector{<:Real}, min_cut::Real=minimum(thresholds), max_cut::Real=maximum(thresholds); n_bins::Int=-1, relative_cut::Real=0.2, fit_thresholds::Bool=true, uncertainty::Bool=true)
+
+Fit the SiPM threshold spectrum and return the optimal threshold.
+
+# Arguments
+- `thresholds`: vector of thresholds
+- `min_cut`: minimum threshold
+- `max_cut`: maximum threshold
+- `n_bins`: number of bins for histogram
+- `relative_cut`: relative cut for threshold
+- `fit_thresholds`: fit thresholds
+- `uncertainty`: calculate uncertainty
+
+# Returns
+- `result`: optimal threshold and corresponding gain, resolution and position of 1pe peak
+- `report`: report with all thresholds and corresponding gains, resolutions and positions of 1pe peaks
+"""
+function fit_sipm_threshold(thresholds::Vector{<:Real}, min_cut::Real=minimum(thresholds), max_cut::Real=maximum(thresholds); n_bins::Int=-1, relative_cut::Real=0.2, fit_thresholds::Bool=true, uncertainty::Bool=true)
+    # cut out thresholds
+    filter!(in(min_cut .. max_cut), thresholds)
+
+    # get bin_width
+    h = if n_bins < 1
+        fit(Histogram, thresholds, min_cut:get_friedman_diaconis_bin_width(thresholds):max_cut)
+    else
+        fit(Histogram, thresholds, n_bins)
+    end
+    # get simple thresholds
+    result_simple = (μ_simple = mean(thresholds), σ_simple = std(thresholds))
+
+    # fit histogram
+    result_trig, report_trig = if fit_thresholds
+        # generate cuts for thresholds
+        cuts_thres = cut_single_peak(thresholds, min_cut, max_cut; n_bins=n_bins, relative_cut=relative_cut)
+        # fit histogram
+        fit_binned_trunc_gauss(h, cuts_thres; uncertainty=uncertainty)
+    else
+        (μ = result_simple.μ_simple, σ = result_simple.σ_simple), h
+    end
+    # get simple std and mu values
+    result = merge(result_trig, result_simple)
+    return result, report_trig
+end
+export fit_sipm_threshold

src/sipm_simple_calibration.jl

@@ -27,34 +27,75 @@
 export sipm_simple_calibration
 
 function sipm_simple_calibration(pe_uncal::Vector{<:Real};
-                                kwargs...)
+    min_pe_peak::Int=1, max_pe_peak::Int=5, relative_cut_noise_cut::Real=0.5, n_fwhm_noise_cut::Real=5.0,
+    initial_min_amp::Real=0.0, initial_max_amp::Real=50.0, initial_max_bin_width_quantile::Real=0.9, 
+    peakfinder_σ::Real=-1.0, peakfinder_threshold::Real=10.0, peakfinder_rtol::Real=0.1, peakfinder_α::Real=0.05
+)
+
+    # Initial peak search
+    cuts_1pe = cut_single_peak(pe_uncal, initial_min_amp, initial_max_amp, relative_cut=relative_cut_noise_cut)
 
-    h_uncal, peakpos = find_peaks(pe_uncal; kwargs...)
+    bin_width_cut_min = cuts_1pe.max+n_fwhm_noise_cut*(cuts_1pe.high - cuts_1pe.max)
+    bin_width_cut = get_friedman_diaconis_bin_width(filter(in(bin_width_cut_min..quantile(pe_uncal, initial_max_bin_width_quantile)), pe_uncal))
+    peakpos = []
+    for bin_width_scale in exp10.(range(0, stop=-3, length=50))
+        @debug "Using bin width: $(bin_width_cut)"
+
+        bin_width_cut_scaled = bin_width_cut * bin_width_scale
+        h_uncal_cut = fit(Histogram, pe_uncal, bin_width_cut_min:bin_width_cut_scaled:initial_max_amp)
+        if peakfinder_σ <= 0.0
+            peakfinder_σ = round(Int, 2*(cuts_1pe.high - cuts_1pe.max) / bin_width_cut_scaled / 2.355)
+        end
+        @debug "Peakfinder σ: $(peakfinder_σ)"
+        try
+            c, h_deconv, peakpos, threshold = RadiationSpectra.determine_calibration_constant_through_peak_ratios(h_uncal_cut, collect(range(min_pe_peak, max_pe_peak, step=1)),
+                min_n_peaks = 2, max_n_peaks = max_pe_peak, threshold=peakfinder_threshold, rtol=peakfinder_rtol, α=peakfinder_α, σ=peakfinder_σ)
+        catch e
+            @warn "Failed to find peaks with bin width scale $(bin_width_scale): $(e)"
+            continue
+        else
+            @debug "Found peaks with bin width scale $(bin_width_scale)"
+            if !isempty(peakpos)
+                break
+            end
+        end
+    end
+
+    if isempty(peakpos) || length(peakpos) < 2
+        throw(ErrorException("Failed to find peaks"))
+    end
 
     # simple calibration
     sort!(peakpos)
+    @debug "Found $(min_pe_peak) PE Peak positions: $(peakpos[1])"
+    @debug "Found $(min_pe_peak+1) PE Peak positions: $(peakpos[2])"
     gain = peakpos[2] - peakpos[1]
+    @debug "Calculated gain: $(round(gain, digits=2))"
     c = 1/gain
-    offset = - (peakpos[1] * c - 1) 
+    offset = - (peakpos[1] * c - min_pe_peak)
+    @debug "Calculated offset: $(round(offset, digits=2))"
 
     f_simple_calib = x -> x .* c .+ offset
     f_simple_uncal = x -> (x .- offset) ./ c
-    pe_simple_cal = pe_uncal .* c .+ offset
-    peakpos_cal = peakpos .* c .+ offset
 
-    bin_width_cal = get_friedman_diaconis_bin_width(filter(in(0.5..1.5), pe_simple_cal))
-    bin_width_uncal = get_friedman_diaconis_bin_width(filter(in( (0.5 - offset) / c .. (1.5 - offset) / c), pe_simple_cal))
+    pe_simple_cal = f_simple_calib.(pe_uncal)
+    peakpos_cal = f_simple_calib.(peakpos)
+
+    bin_width_cal = get_friedman_diaconis_bin_width(filter(in(0.5..min_pe_peak), pe_simple_cal))
+    bin_width_uncal = f_simple_uncal(bin_width_cal) - f_simple_uncal(0.0)
 
-    h_calsimple = fit(Histogram, pe_simple_cal, 0.0:bin_width_cal:6.0)
-    h_uncal = fit(Histogram, pe_uncal, 0.0:bin_width_uncal:(6.0 - offset) / c)
+    h_calsimple = fit(Histogram, pe_simple_cal, 0.0:bin_width_cal:max_pe_peak + 1)
+    h_uncal = fit(Histogram, pe_uncal, 0.0:bin_width_uncal:f_simple_uncal(max_pe_peak + 1))
 
     result = (
         pe_simple_cal = pe_simple_cal,
         peakpos = peakpos,
         f_simple_calib = f_simple_calib,
         f_simple_uncal = f_simple_uncal,
         c = c,
-        offset = offset
+        offset = offset,
+        noisepeakpos = cuts_1pe.max,
+        noisepeakwidth = cuts_1pe.high - cuts_1pe.low
     )
     report = (
         peakpos = peakpos,
@@ -63,66 +104,4 @@
         h_calsimple = h_calsimple
     )
     return result, report
-end
-
-
-function find_peaks(
-    amps::Vector{<:Real}; initial_min_amp::Real=1.0, initial_max_quantile::Real=0.99, 
-    peakfinder_σ::Real=2.0, peakfinder_threshold::Real=10.0
-)
-    # Start with a big window where the noise peak is included
-    min_amp = initial_min_amp
-    max_quantile = initial_max_quantile
-    max_amp = quantile(amps, max_quantile)
-    bin_width = get_friedman_diaconis_bin_width(filter(in(quantile(amps, 0.01)..quantile(amps, 0.9)), amps))
-
-    # Initial peak search
-    h_uncal = fit(Histogram, amps, min_amp:bin_width:max_amp)
-    h_decon, peakpos = peakfinder(h_uncal, σ=peakfinder_σ, backgroundRemove=true, threshold=peakfinder_threshold)
-
-    # Ensure at least 2 peaks
-    num_peaks = length(peakpos)
-    if num_peaks == 0
-        error("No peaks found.")
-    end
-
-    # Determine the 1 p.e. peak position based on the assumption that it is the highest
-    peakpos_idxs = StatsBase.binindex.(Ref(h_decon), peakpos)
-    cts_peakpos = h_decon.weights[peakpos_idxs]
-    first_pe_peak_pos = peakpos[argmax(cts_peakpos)]
-
-    # Remove all peaks with x vals < x pos of 1p.e. peak
-    filter!(x -> x >= first_pe_peak_pos, peakpos)
-    num_peaks = length(peakpos)
-
-    # while less than two peaks found, or second peakpos smaller than 1 pe peakpos, or gain smaller than 1 (peaks too close)
-    while num_peaks < 2 || peakpos[2] <= first_pe_peak_pos || (peakpos[2] - peakpos[1]) <= 1.0
-        # Adjust σ and recheck peaks
-        if peakfinder_σ < 10.0
-            println("Increasing peakfinder_σ: ", peakfinder_σ)
-            peakfinder_σ += 0.5
-        else
-            # If σ can't increase further, reduce threshold
-            println("Adjusting peakfinder_threshold: ", peakfinder_threshold)
-            peakfinder_threshold -= 1.0
-            peakfinder_σ = 2.0  # Reset σ for new threshold
-
-            # Safety check to avoid lowering threshold too much
-            if peakfinder_threshold < 2.0
-                error("Unable to find two distinct peaks within reasonable quantile range.")
-            end
-        end
-
-        # Find peaks with updated parameters
-        h_decon, peakpos = peakfinder(h_uncal, σ=peakfinder_σ, backgroundRemove=true, threshold=peakfinder_threshold)
-        filter!(x -> x >= first_pe_peak_pos, peakpos)
-        num_peaks = length(peakpos)
-
-        # Safety check to avoid infinite loops
-        if peakfinder_σ >= 10.0 && peakfinder_threshold < 2.0
-            error("Unable to find two peaks within reasonable quantile range.")
-        end
-    end
-
-    return h_decon, peakpos
 end

src/sipmfit.jl

@@ -24,7 +24,7 @@
 - `report`: a tuple with the fit report which can be plotted via a recipe
 """
 function fit_sipm_spectrum(pe_cal::Vector{<:Real}, min_pe::Real=0.5, max_pe::Real=3.5;
-    n_mixtures::Int=ceil(Int, (max_pe - min_pe) * 4), nIter::Int=50, nInit::Int=50, 
+    n_mixtures::Int=ceil(Int, (max_pe - min_pe) * 4), nIter::Int=25, nInit::Int=50, 
     method::Symbol=:kmeans, kind=:diag, Δpe_peak_assignment::Real=0.3, f_uncal::Function=identity, uncertainty::Bool=true)
 
     # first filter peak positions out of amplitude vector
@@ -104,10 +104,10 @@
 
     # get pe_pos
     get_pe_pos = pe -> let sel = in.(μ, (-Δpe_peak_assignment..Δpe_peak_assignment) .+ pe)
-        dot(view(μ,sel), view(w,sel)) / sum(view(w,sel))
+        dot(view(μ, sel), view(w, sel)) / sum(view(w, sel))
     end
     get_pe_res = pe -> let sel = in.(μ, (-Δpe_peak_assignment..Δpe_peak_assignment) .+ pe)
-        dot(view(σ,sel), view(w,sel)) / sum(view(w,sel))
+        sqrt(dot(view(σ, sel).^2, view(w, sel).^2))
     end 
     n_pos_mixtures = [count(in.(μ, (-Δpe_peak_assignment..Δpe_peak_assignment) .+ pe)) for pe in pes]