diff --git a/benchmarks/profiles/data-profile.jl b/benchmarks/profiles/data-profile.jl new file mode 100644 index 00000000..4799606d --- /dev/null +++ b/benchmarks/profiles/data-profile.jl @@ -0,0 +1,86 @@ +using LaTeXStrings +using PrettyTables, LaTeXStrings +using Random +using LinearAlgebra +using ProximalOperators +using Plots +using BenchmarkProfiles +using NLPModels, + NLPModelsModifiers, + RegularizedProblems, + RegularizedOptimization, + ShiftedProximalOperators, + SolverBenchmark +using Printf + +function data_prof_nnmf( + solvers, + solver_names, + nb_prob::Int, + random_seed::Int; + measured::Symbol = :obj, # set to :grad to eval grad +) + + m, n, k = 100, 50, 5 + λ = 1.0e-1 + h = NormL1(λ) + n_solvers = length(solvers) + objdecs = Vector{Float64}[] + + current_seed = random_seed + nb_data_min = Int(10^16) # min number of data for each solver on every pb + for i=1:nb_prob + Random.seed!(current_seed) + model, nls_model, A, selected = nnmf_model(m, n, k) + f = LSR1Model(model) + @info "pb $i" + for (solver, name) in zip(solvers, solver_names) + @info "pb: $i solver: $name" + args = name == "R2" ? () : (NormLinf(1.0),) + solver_out = solver(f, h, selected) + objdec = solver_out.solver_specific[:Fhist] + solver_out.solver_specific[:Hhist] + measured == :grad && unique!(objdec) + nb_data_min = min(nb_data_min, length(objdec)) + push!(objdecs, objdec) + reset!(f) + end + end + + data = zeros(nb_data_min, nb_prob, n_solvers) + idx_objdec = 1 + for i=1:nb_prob + for idx_solv in 1:n_solvers + objdec = objdecs[idx_objdec] + data[:, i, idx_solv] .= objdec[1:nb_data_min] + idx_objdec += 1 + end + end + + data_profile(PlotsBackend(), data, ones(nb_prob), solver_names, legend=:topleft, τ= 1.0e-2) +end + +ν = 1.0 +ϵ = 1.0e-4 +ϵi = 1.0e-4 +ϵri = 1.0e-4 +maxIter = 200 +maxIter_inner = 100 +function TR_R2(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, verbose = 0, maxIter = maxIter) + sub_opt = ROSolverOptions(ϵa = ϵi, ϵr = ϵri, maxIter = maxIter_inner) + TR(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, subsolver_options = sub_opt, selected = selected) +end + +function TR_TRDH(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, verbose = 0, maxIter = maxIter) + sub_opt = ROSolverOptions(ϵa = ϵi, ϵr = ϵri, maxIter = maxIter_inner) + TR(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, subsolver_options = sub_opt, selected = selected, subsolver = TRDH) +end + +data_prof_nnmf( + [TR_R2, TR_TRDH], + ["TR-R2", "TR-TRDH"], + 5, + 1234; + measured = :grad, # set to :grad to eval grad +) \ No newline at end of file diff --git a/benchmarks/profiles/perf-profile.jl b/benchmarks/profiles/perf-profile.jl new file mode 100644 index 00000000..10ff53e0 --- /dev/null +++ b/benchmarks/profiles/perf-profile.jl @@ -0,0 +1,125 @@ +using LaTeXStrings +using PrettyTables, LaTeXStrings +using Random +using LinearAlgebra +using ProximalOperators +using Plots +using BenchmarkProfiles +using NLPModels, + NLPModelsModifiers, + RegularizedProblems, + RegularizedOptimization, + ShiftedProximalOperators, + SolverBenchmark +using Printf + +function benchmark_prof( + pb::Symbol, #:nnmf, :bpdn + solvers, + solver_names, + nb_prob::Int, + random_seed::Int; + measured::Symbol = :obj, # set to :grad to eval grad +) + + if pb == :nnmf + m, n, k = 100, 50, 5 + λ = 1.0e-1 + elseif pb == :bpdn + compound = 1 + else + error("Problem not supported") + end + n_solvers = length(solvers) + data = zeros(nb_prob, n_solvers) + + current_seed = random_seed + nb_data_min = Int(10^16) # min number of data for each solver on every pb + for i=1:nb_prob + current_seed = random_seed + i - 1 + Random.seed!(current_seed) + if pb == :nnmf + model, nls_model, A, selected = nnmf_model(m, n, k) + h = NormL0(λ) + elseif pb == :bpdn + model, nls_model, sol = bpdn_model(compound, bounds = false) + selected = 1:length(sol) + λ = norm(grad(model, zeros(model.meta.nvar)), Inf) / 10 + reset!(model) + h = NormL0(λ) + end + f = LSR1Model(model) + @info "pb $i" + for (j, solver, name) in zip(1:n_solvers, solvers, solver_names) + solver_out = solver(f, h, selected) + @info "pb: $i solver: $name status = $(solver_out.status) obj = $(solver_out.objective)" + if solver_out.status == :first_order + data[i, j] = neval_grad(f) + else + data[i, j] = +Inf + end + reset!(f) + end + end + + performance_profile(PlotsBackend(), data, solver_names, legend=:bottomright, title = String(measured)) +end + +ν = 1.0 +ϵ = 1.0e-5 +ϵi = 1.0e-3 +ϵri = 1.0e-6 +maxIter = 2000 +maxIter_inner = 100 +function TR_R2(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, verbose = 0, maxIter = maxIter) + sub_opt = ROSolverOptions(ϵa = ϵi, ϵr = ϵri, maxIter = maxIter_inner) + TR(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, subsolver_options = sub_opt, selected = selected) +end + +function TR_TRDH_PSB(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, verbose = 0, maxIter = maxIter) + sub_opt = ROSolverOptions(ϵa = ϵi, ϵr = ϵri, maxIter = maxIter_inner, psb = true, reduce_TR = false) + TR(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, subsolver_options = sub_opt, selected = selected, subsolver = TRDH) +end + +function TR_TRDH_Andrei(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, verbose = 0, maxIter = maxIter) + sub_opt = ROSolverOptions(ϵa = ϵi, ϵr = ϵri, maxIter = maxIter_inner, spectral = false, psb = false, reduce_TR = false) + TR(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, subsolver_options = sub_opt, selected = selected, subsolver = TRDH) +end + +function TR_TRDH_Spec(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, verbose = 0, maxIter = maxIter) + sub_opt = ROSolverOptions(ϵa = ϵi, ϵr = ϵri, maxIter = maxIter_inner, spectral = true, reduce_TR = false) + TR(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, subsolver_options = sub_opt, selected = selected, subsolver = TRDH) +end + +function TRDH_Spec(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, maxIter = maxIter, spectral = true, reduce_TR = false) + TRDH(f, h, NormLinf(1.0), opt, x0 = f.meta.x0, selected = selected) +end + +function R2_None(f, h, selected; ϵ = ϵ, ϵi = ϵi, ϵri = ϵri, maxIter = maxIter, maxIter_inner = maxIter_inner) + opt = ROSolverOptions(ν = ν, ϵa = ϵ, ϵr = ϵ, maxIter = maxIter, spectral = true, reduce_TR = false) + R2(f, h, opt; x0 = f.meta.x0, selected = selected) +end + + +# benchmark_prof( +# :nnmf, +# [TRDH_Spec, TR_R2, TR_TRDH_PSB, TR_TRDH_Andrei, TR_TRDH_Spec], +# ["TRDH_Spec", "TR-R2", "TR-TRDH-PSB", "TR_TRDH_Andrei", "TR_TRDH_Spec"], +# 50, +# 1234; +# measured = :grad, # set to :grad to eval grad +# ) + +benchmark_prof( + :bpdn, + [R2_None, TRDH_Spec, TR_R2, TR_TRDH_PSB], + ["R2", "TRDH-Spec", "TR-R2", "TR-TRDH-PSB"], + 50, + 1234; + measured = :grad, # set to :grad to eval grad +) \ No newline at end of file