Skip to content

stopping criteria update #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

stopping criteria update #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/LMTR_alg.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,11 @@ function LMTR(

if verbose > 0
#! format: off
@info @sprintf "%6s %8s %8s %8s %7s %7s %8s %7s %7s %7s %7s %1s" "outer" "inner" "f(x)" "h(x)" "√ξ1" "√ξ" "ρ" "Δ" "‖x‖" "‖s‖" "1/ν" "TR"
@info @sprintf "%6s %8s %8s %8s %7s %7s %8s %7s %7s %7s %7s %1s" "outer" "inner" "f(x)" "h(x)" "√(ξ1/ν)" "√(ξ/ν)" "ρ" "Δ" "‖x‖" "‖s‖" "1/ν" "TR"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Compute $\xi_1 / \nu$ once and for all instead of recomputing it many times.

The other solvers don't compute or output $\sqrt{\xi / \nu}$; only $\sqrt{\xi}$. We could think in another PR if what we really want is $\sqrt{\xi / \nu}$.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was stuck on this when I was rebasing on Friday. It doesn't make sense to me to only do $\sqrt{\xi}$. I believe i had changed them all in this but I guess we could revert.

#! format: on
end

local ξ1
local ξ1, ξ
k = 0

Fk = residual(nls, xk)
Expand All @@ -132,6 +132,7 @@ function LMTR(

σmax = opnorm(Jk)
νInv = (1 + θ) * σmax^2 # ‖J'J‖ = ‖J‖²
ν = 1 / νInv

mν∇fk = -∇fk / νInv

Expand Down Expand Up @@ -179,18 +180,18 @@ function LMTR(
ξ1 > 0 || error("LMTR: first prox-gradient step should produce a decrease but ξ1 = $(ξ1)")

if ξ1 ≥ 0 && k == 1
ϵ_increment = ϵr * sqrt(ξ1)
ϵ_increment = ϵr * sqrt(ξ1/ν)
ϵ += ϵ_increment # make stopping test absolute and relative
ϵ_subsolver += ϵ_increment
end

if sqrt(ξ1) < ϵ
if sqrt(ξ1/ν) < ϵ
# the current xk is approximately first-order stationary
optimal = true
continue
end

subsolver_options.ϵa = k == 1 ? 1.0e-5 : max(ϵ_subsolver, min(1.0e-1, ξ1 / 10))
subsolver_options.ϵa = k == 1 ? 1.0e-5 : max(ϵ_subsolver, min(1.0e-1, ξ1 / ν / 10))
∆_effective = min(β * χ(s), Δk)
treats_bounds ?
set_bounds!(ψ, max.(-∆_effective, l_bound - xk), min.(∆_effective, u_bound - xk)) :
Expand Down Expand Up @@ -230,7 +231,7 @@ function LMTR(

if (verbose > 0) && (k % ptf == 0)
#! format: off
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8.1e %7.1e %7.1e %7.1e %7.1e %1s" k iter fk hk sqrt(ξ1) sqrt(ξ) ρk ∆_effective χ(xk) sNorm νInv TR_stat
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8.1e %7.1e %7.1e %7.1e %7.1e %1s" k iter fk hk sqrt(ξ1/ν) sqrt(ξ/ν) ρk ∆_effective χ(xk) sNorm νInv TR_stat
#! format: on
end

Expand Down Expand Up @@ -270,9 +271,9 @@ function LMTR(
@info @sprintf "%6d %8s %8.1e %8.1e" k "" fk hk
elseif optimal
#! format: off
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8s %7.1e %7.1e %7.1e %7.1e" k 1 fk hk sqrt(ξ1) sqrt(ξ1) "" Δk χ(xk) χ(s) νInv
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8s %7.1e %7.1e %7.1e %7.1e" k 1 fk hk sqrt(ξ1/ν) sqrt(ξ/ν) "" Δk χ(xk) χ(s) νInv
#! format: on
@info "LMTR: terminating with √ξ1 = $(sqrt(ξ1))"
@info "LMTR: terminating with √(ξ1/ν) = $(sqrt(ξ1/ν))"
end
end

Expand All @@ -290,7 +291,7 @@ function LMTR(
set_status!(stats, status)
set_solution!(stats, xk)
set_objective!(stats, fk + hk)
set_residuals!(stats, zero(eltype(xk)), ξ1 ≥ 0 ? sqrt(ξ1) : ξ1)
set_residuals!(stats, zero(eltype(xk)), ξ1 ≥ 0 ? sqrt(ξ1/ν) : ξ1)
set_iter!(stats, k)
set_time!(stats, elapsed_time)
set_solver_specific!(stats, :Fhist, Fobj_hist[1:k])
Expand Down
19 changes: 10 additions & 9 deletions src/LM_alg.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ function LM(

xkn = similar(xk)

local ξ1
local ξ1, ξ
k = 0
Fobj_hist = zeros(maxIter)
Hobj_hist = zeros(maxIter)
Expand All @@ -110,7 +110,7 @@ function LM(

if verbose > 0
#! format: off
@info @sprintf "%6s %8s %8s %8s %7s %7s %8s %7s %7s %7s %7s %1s" "outer" "inner" "f(x)" "h(x)" "√ξ1" "√ξ" "ρ" "σ" "‖x‖" "‖s‖" "‖Jₖ‖²" "reg"
@info @sprintf "%6s %8s %8s %8s %7s %7s %8s %7s %7s %7s %7s %1s" "outer" "inner" "f(x)" "h(x)" "√(ξ1/ν)" "√(ξ/ν)" "ρ" "σ" "‖x‖" "‖s‖" "‖Jₖ‖²" "reg"
#! format: on
end

Expand All @@ -125,6 +125,7 @@ function LM(

σmax = opnorm(Jk)
νInv = (1 + θ) * (σmax^2 + σk) # ‖J'J + σₖ I‖ = ‖J‖² + σₖ
ν = 1 / νInv
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we bother doing this? Why not just set ν directly?


s = zero(xk)

Expand Down Expand Up @@ -178,18 +179,18 @@ function LM(
ξ1 > 0 || error("LM: first prox-gradient step should produce a decrease but ξ1 = $(ξ1)")

if ξ1 ≥ 0 && k == 1
ϵ_increment = ϵr * sqrt(ξ1)
ϵ_increment = ϵr * sqrt(ξ1/ν)
ϵ += ϵ_increment # make stopping test absolute and relative
ϵ_subsolver += ϵ_increment
end

if sqrt(ξ1) < ϵ
if sqrt(ξ1/ν) < ϵ
# the current xk is approximately first-order stationary
optimal = true
continue
end

subsolver_options.ϵa = k == 1 ? 1.0e-1 : max(ϵ_subsolver, min(1.0e-2, ξ1 / 10))
subsolver_options.ϵa = k == 1 ? 1.0e-1 : max(ϵ_subsolver, min(1.0e-2, ξ1 / ν / 10))
subsolver_options.ν = ν
@debug "setting inner stopping tolerance to" subsolver_options.optTol
s, iter, _ = with_logger(subsolver_logger) do
Expand Down Expand Up @@ -220,7 +221,7 @@ function LM(

if (verbose > 0) && (k % ptf == 0)
#! format: off
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8.1e %7.1e %7.1e %7.1e %7.1e %1s" k iter fk hk sqrt(ξ1) sqrt(ξ) ρk σk norm(xk) norm(s) νInv σ_stat
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8.1e %7.1e %7.1e %7.1e %7.1e %1s" k iter fk hk sqrt(ξ1/ν) sqrt(ξ/ν) ρk σk norm(xk) norm(s) νInv σ_stat
#! format: off
end

Expand Down Expand Up @@ -260,9 +261,9 @@ function LM(
@info @sprintf "%6d %8s %8.1e %8.1e" k "" fk hk
elseif optimal
#! format: off
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8s %7.1e %7.1e %7.1e %7.1e" k 1 fk hk sqrt(ξ1) sqrt(ξ1) "" σk norm(xk) norm(s) νInv
@info @sprintf "%6d %8d %8.1e %8.1e %7.1e %7.1e %8s %7.1e %7.1e %7.1e %7.1e" k 1 fk hk sqrt(ξ1/ν) sqrt(ξ/ν) "" σk norm(xk) norm(s) νInv
#! format: on
@info "LM: terminating with √ξ1 = $(sqrt(ξ1))"
@info "LM: terminating with √(ξ1/ν) = $(sqrt(ξ1/ν))"
end
end
status = if optimal
Expand All @@ -279,7 +280,7 @@ function LM(
set_status!(stats, status)
set_solution!(stats, xk)
set_objective!(stats, fk + hk)
set_residuals!(stats, zero(eltype(xk)), ξ1 ≥ 0 ? sqrt(ξ1) : ξ1)
set_residuals!(stats, zero(eltype(xk)), ξ1 ≥ 0 ? sqrt(ξ1/ν) : ξ1)
set_iter!(stats, k)
set_time!(stats, elapsed_time)
set_solver_specific!(stats, :Fhist, Fobj_hist[1:k])
Expand Down
1 change: 0 additions & 1 deletion src/R2_alg.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,6 @@ function R2!(
#! format: off
σ_stat = (η2 ≤ ρk < Inf) ? "↘" : (ρk < η1 ? "↗" : "=")
@info @sprintf "%6d %8.1e %8.1e %7.1e %8.1e %7.1e %7.1e %7.1e %1s" k fk hk sqrt_ξ_νInv ρk σk norm(xk) norm(s) σ_stat

#! format: on
end

Expand Down
2 changes: 1 addition & 1 deletion src/TRDH_alg.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ function TRDH(
set_radius!(ψ, Δ_effective)
end

# model with diagonal hessian
# model with diagonal hessian
φ(d) = ∇fk' * d + (d' * (Dk.d .* d)) / 2
mk(d) = φ(d) + ψ(d)

Expand Down
6 changes: 2 additions & 4 deletions src/TR_alg.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ function TR(
#! format: on
end

local ξ1
local ξ1, ξ
k = 0

fk = obj(f, xk)
Expand All @@ -133,8 +133,6 @@ function TR(

λmax = opnorm(Bk)
νInv = (1 + θ) * λmax
ν = 1 / (νInv + 1 / (Δk * α))
sqrt_ξ1_νInv = one(R)

optimal = false
tired = k ≥ maxIter || elapsed_time > maxTime
Expand Down Expand Up @@ -179,7 +177,7 @@ function TR(
continue
end

subsolver_options.ϵa = k == 1 ? 1.0e-5 : max(ϵ_subsolver, min(1e-2, sqrt_ξ1_νInv))
subsolver_options.ϵa = k == 1 ? 1.0e-5 : max(ϵ_subsolver, min(1e-2, sqrt_ξ1_νInv) * ξ1)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@geoffroyleconte I reinstated this because it disappeared in ac255073a980ee340c02d7dff568dc0f2a07d265. It's not clear which produces the best results.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was intended, and this choice is mentioned in the section numerical results of indef-pg.

∆_effective = min(β * χ(s), Δk)
(has_bounds(f) || subsolver == TRDH) ?
set_bounds!(ψ, max.(-∆_effective, l_bound - xk), min.(∆_effective, u_bound - xk)) :
Expand Down