Add demo SVM

.github/workflows/demos.yml

@@ -84,6 +84,11 @@ jobs:
           pkg_path = dirname(Base.find_package("RegularizedOptimization"))
           include(joinpath(pkg_path, "..", "examples", "demo-nnmf-constr.jl"))
         shell: julia --project="examples" --color=yes {0}
+      - name: Run SVM demo
+        run: |
+          pkg_path = dirname(Base.find_package("RegularizedOptimization"))
+          include(joinpath(pkg_path, "..", "examples", "demo-svm.jl"))
+        shell: julia --project="examples" --color=yes {0}
       - name: Upload results
         uses: actions/upload-artifact@v3
         with:

Project.toml

@@ -22,7 +22,7 @@ NLPModelsModifiers = "0.6"
 ProximalOperators = "0.15"
 RegularizedProblems = "0.1"
 ShiftedProximalOperators = "0.1"
-SolverCore = "0.3.0"
+SolverCore = "0.3"
 TSVD = "0.4"
 julia = "^1.6.0"
 

examples/Project.toml

@@ -1,20 +1,23 @@
 [deps]
 ADNLPModels = "54578032-b7ea-4c30-94aa-7cbd1cce6c9a"
 DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa"
+MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458"
 NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6"
 NLPModelsModifiers = "e01155f1-5c6f-4375-a9d8-616dd036575f"
 PGFPlots = "3b7a836e-365b-5785-a47d-02c71176b4aa"
 ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537"
 RegularizedOptimization = "196f2941-2d58-45ba-9f13-43a2532b2fa8"
 RegularizedProblems = "ea076b23-609f-44d2-bb12-a4ae45328278"
+ShiftedProximalOperators = "d4fd37fa-580c-4e43-9b30-361c21aae263"
 
 [compat]
-ADNLPModels = "0.4"
+ADNLPModels = "0.6"
 DifferentialEquations = "7"
+MLDatasets = "0.7"
 NLPModels = "0.19"
 NLPModelsModifiers = "0.6"
 PGFPlots = "3"
 ProximalOperators = "0.15"
 RegularizedOptimization = "0.1"
 RegularizedProblems = "0.1"
-
+ShiftedProximalOperators = "0.1"

examples/demo-svm.jl

@@ -0,0 +1,70 @@
+using Random
+using LinearAlgebra
+using MLDatasets
+using ProximalOperators, ShiftedProximalOperators, RegularizedProblems
+using NLPModels, NLPModelsModifiers
+using RegularizedOptimization
+
+include("plot-utils-svm.jl")
+
+Random.seed!(1234)
+
+function demo_solver(nlp_tr, nls_tr, sol_tr, nlp_test, nls_test, sol_test, h, χ, suffix="l0-linf")
+    options = ROSolverOptions(ν=1.0, β=1e16, ϵa=1e-4, ϵr=1e-4, verbose=10, σmin=1e-5)
+    suboptions = ROSolverOptions(maxIter = 100)
+    acc = vec -> length(findall(x -> x < 1, vec)) / length(vec) * 100
+
+    @info "using R2 to solve with" h
+    reset!(nlp_tr)
+    R2_out = R2(nlp_tr, h, options, x0=nlp_tr.meta.x0)
+    nr2 = neval_obj(nlp_tr)
+    ngr2 = neval_grad(nlp_tr)
+    r2train = residual(nls_tr, R2_out.solution) #||e - tanh(b * <A, x>)||^2, b ∈ {-1,1}^n
+    r2test = residual(nls_test, R2_out.solution)
+    @show acc(r2train), acc(r2test)
+    r2dec = plot_svm(R2_out, R2_out.solution, "r2-$(suffix)")
+
+    @info "using TR to solve with" h χ
+    reset!(nlp_tr)
+    TR_out = TR(nlp_tr, h, χ, options, x0=nlp_tr.meta.x0, subsolver_options = suboptions)
+    trtrain = residual(nls_tr, TR_out.solution)
+    trtest = residual(nls_test, TR_out.solution)
+    ntr = neval_obj(nlp_tr)
+    ngtr = neval_grad(nlp_tr)
+    @show acc(trtrain), acc(trtest)
+    trdec = plot_svm(TR_out, TR_out.solution, "tr-$(suffix)")
+
+    @info " using LMTR to solve with" h χ
+    reset!(nls_tr)
+    LMTR_out = LMTR(nls_tr, h, χ, options, x0=nls_tr.meta.x0, subsolver_options = suboptions)
+    lmtrtrain = residual(nls_tr, LMTR_out.solution)
+    lmtrtest = residual(nls_test, LMTR_out.solution)
+    nlmtr = neval_residual(nls_tr)
+    nglmtr = neval_jtprod_residual(nls_tr) + neval_jprod_residual(nls_tr)
+    @show acc(lmtrtrain), acc(lmtrtest)
+    lmtrdec = plot_svm(LMTR_out, LMTR_out.solution, "lmtr-$(suffix)")
+
+    @info " using LMTR to solve with" h χ
+    reset!(nls_tr)
+    LM_out = LM(nls_tr, h, options, x0=nls_tr.meta.x0, subsolver_options = suboptions)
+    lmtrain = residual(nls_tr, LM_out.solution)
+    lmtest = residual(nls_test, LM_out.solution)
+    nlm = neval_residual(nls_tr)
+    nglm = neval_jtprod_residual(nls_tr) + neval_jprod_residual(nls_tr)
+    @show acc(lmtrain), acc(lmtest)
+    lmdec = plot_svm(LM_out, LM_out.solution, "lm-$(suffix)")
+end
+
+function demo_svm()
+    nlp_train, nls_train, sol_train = RegularizedProblems.svm_train_model()
+    nlp_test, nls_test, sol_test = RegularizedProblems.svm_test_model()
+    nlp_train = LSR1Model(nlp_train)
+    λ = 1e-1
+    h = RootNormLhalf(λ)
+    χ = NormLinf(1.0)
+
+    demo_solver(nlp_train, nls_train, sol_train, nlp_test, nls_test, sol_test, h, χ, "lhalf-linf")
+end
+
+demo_svm()
+

examples/plot-utils-svm.jl

@@ -0,0 +1,31 @@
+import PGFPlots
+
+function plot_svm(outstruct, sol, name="tr-qr")
+    Comp_pg = outstruct.solver_specific[:SubsolverCounter]
+    objdec = outstruct.solver_specific[:Fhist] + outstruct.solver_specific[:Hhist]
+    x = outstruct.solution
+    a = PGFPlots.Axis(
+        [
+            PGFPlots.Plots.MatrixPlot(reshape(x, 28, 28);
+            colormap = PGFPlots.ColorMaps.GrayMap())#, zmin = -700, zmax = 700)#, legendentry="computed"),
+        ],
+    )
+    PGFPlots.save("svm-$(name).pdf", a)
+
+    b = PGFPlots.Axis(
+        PGFPlots.Plots.Linear(1:length(Comp_pg), Comp_pg, mark="none"),
+        xlabel="outer iterations",
+        ylabel="inner iterations",
+        ymode="log",
+    )
+    PGFPlots.save("svm-inner-outer-$(name).pdf", b)
+
+    c = PGFPlots.Axis(
+        PGFPlots.Plots.Linear(1:length(objdec), objdec, mark="none"),
+        xlabel="\$ k^{th}\$  \$ \\nabla f \$ Call",
+        ylabel="Objective Value",
+        ymode="log",
+    )
+    PGFPlots.save("svm-objdec-$(name).pdf", c)
+    return objdec
+end