From 3d219da19cf9e2091563df7cbb07bfc656e3cc20 Mon Sep 17 00:00:00 2001
From: Alexis Montoison <alexis.montoison@polymtl.ca>
Date: Fri, 7 Jun 2024 11:25:37 -0400
Subject: [PATCH 1/9] Full support of sparse AD for NLS models

---
 src/ad_api.jl               | 19 ++++++++++++++++
 src/sparse_hessian.jl       | 41 +++++++++++++++++++++++++++++++++
 test/runtests.jl            |  1 +
 test/sparse_hessian_nls.jl  | 45 +++++++++++++++++++++++++++++++++++++
 test/sparse_jacobian.jl     |  4 +++-
 test/sparse_jacobian_nls.jl |  4 +++-
 6 files changed, 112 insertions(+), 2 deletions(-)
 create mode 100644 test/sparse_hessian_nls.jl

diff --git a/src/ad_api.jl b/src/ad_api.jl
index c9867b25..0c938b5b 100644
--- a/src/ad_api.jl
+++ b/src/ad_api.jl
@@ -289,6 +289,25 @@ function NLPModels.hess_coord!(
   return vals
 end
 
+function NLPModels.hess_structure_residuals!(
+  b::ADBackend,
+  nls::AbstractADNLSModel,
+  rows::AbstractVector{<:Integer},
+  cols::AbstractVector{<:Integer},
+)
+  nothing
+end
+
+function NLPModels.hess_coord_residuals!(
+  b::ADBackend,
+  nls::AbstractADNLSModel,
+  x::AbstractVector,
+  v::AbstractVector,
+  vals::AbstractVector,
+)
+  nothing
+end
+
 function NLPModels.hprod!(
   b::ADBackend,
   nlp::ADModel,
diff --git a/src/sparse_hessian.jl b/src/sparse_hessian.jl
index 4140762b..91d7696a 100644
--- a/src/sparse_hessian.jl
+++ b/src/sparse_hessian.jl
@@ -296,3 +296,44 @@ function NLPModels.hess_coord!(
   sparse_hess_coord!(ℓ, b, x, obj_weight, b.y, vals)
   return vals
 end
+
+function NLPModels.hess_structure_residuals!(
+  b::Union{SparseADHessian, SparseReverseADHessian},
+  nls::AbstractADNLSModel,
+  rows::AbstractVector{<:Integer},
+  cols::AbstractVector{<:Integer},
+)
+  function objective(x)
+    F = get_F(nls, b)
+    Fx = F(x)
+    return dot(Fx, Fx) / 2
+  end
+
+  H = compute_hessian_sparsity(objective, nls.meta.nvar, nothing, 0)
+  trilH = tril(H)
+  rowval = trilH.rowval
+  colptr = trilH.colptr
+  rows .= rowval
+  for i = 1:(nls.meta.nvar)
+    for j = colptr[i]:(colptr[i + 1] - 1)
+      cols[j] = i
+    end
+  end
+  return rows, cols
+end
+
+function NLPModels.hess_coord_residuals!(
+  b::Union{SparseADHessian, SparseReverseADHessian},
+  nls::AbstractADNLSModel,
+  x::AbstractVector,
+  v::AbstractVector,
+  vals::AbstractVector,
+)
+  function objective(x)
+    F = get_F(nls, b)
+    Fx = F(x)
+    return dot(Fx, Fx) / 2
+  end
+
+  sparse_hess_coord!(objective, b, x, 1.0, v, vals)
+end
diff --git a/test/runtests.jl b/test/runtests.jl
index a69dc634..b98044b0 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -29,6 +29,7 @@ end
 
 @testset "Basic Hessian derivative test" begin
   include("sparse_hessian.jl")
+  include("sparse_hessian_nls.jl")
 end
 
 for problem in NLPModelsTest.nlp_problems ∪ ["GENROSE"]
diff --git a/test/sparse_hessian_nls.jl b/test/sparse_hessian_nls.jl
new file mode 100644
index 00000000..69f791d3
--- /dev/null
+++ b/test/sparse_hessian_nls.jl
@@ -0,0 +1,45 @@
+list_sparse_hess_backend = (
+  (ADNLPModels.SparseADHessian, Dict()),
+  (ADNLPModels.ForwardDiffADHessian, Dict()),
+)
+
+dt = (Float32, Float64)
+
+@testset "Basic Hessian of residual derivative with backend=$(backend) and T=$(T)" for T in dt,
+  (backend, kw) in list_sparse_hess_backend
+
+  F!(Fx, x) = begin
+    Fx[1] = x[1] - 1
+    Fx[2] = 10 * (x[2] - x[1]^2)
+    Fx[3] = x[2] + 1
+    Fx
+  end
+  x0 = T[-1.2; 1.0]
+  nvar = 2
+  nequ = 3
+  nls = ADNLPModels.ADNLSModel!(F!, x0, 3, hessian_residual_backend = backend; kw...)
+
+  x = rand(T, nvar)
+  v = rand(T, nequ)
+  rows, cols = zeros(Int, nls.nls_meta.nnzh), zeros(Int, nls.nls_meta.nnzh)
+  vals = zeros(T, nls.nls_meta.nnzh)
+  hess_structure_residual!(nls, rows, cols)
+  hess_coord_residual!(nls, x, v, vals)
+  @test eltype(vals) == T
+  H = sparse(rows, cols, vals, nvar, nvar)
+  # @test H == []
+
+  # Test also the implementation of the backends
+  b = nls.adbackend.hessian_residual_backend
+  obj_weight = 0.5
+  @test nls.nls_meta.nnzh == ADNLPModels.get_nln_nnzh(b, nvar)
+  ADNLPModels.hess_structure_residual!(b, nls, rows, cols)
+  ADNLPModels.hess_coord_residual!(b, nls, x, y, obj_weight, vals)
+  @test eltype(vals) == T
+  H = sparse(rows, cols, vals, nvar, nvar)
+  # @test H == []
+
+  nls = ADNLPModels.ADNLSModel!(F!, x0, 3, matrix_free = true; kw...)
+  @test nls.adbackend.hessian_backend isa ADNLPModels.EmptyADbackend
+  @test nls.adbackend.hessian_residual_backend isa ADNLPModels.EmptyADbackend
+end
diff --git a/test/sparse_jacobian.jl b/test/sparse_jacobian.jl
index 20b4d05e..0a522eb1 100644
--- a/test/sparse_jacobian.jl
+++ b/test/sparse_jacobian.jl
@@ -1,8 +1,10 @@
 list_sparse_jac_backend = (
-  (ADNLPModels.SparseADJacobian, Dict()), # default
+  (ADNLPModels.SparseADJacobian, Dict()),
   (ADNLPModels.ForwardDiffADJacobian, Dict()),
 )
+
 dt = (Float32, Float64)
+
 @testset "Basic Jacobian derivative with backend=$(backend) and T=$(T)" for T in dt,
   (backend, kw) in list_sparse_jac_backend
 
diff --git a/test/sparse_jacobian_nls.jl b/test/sparse_jacobian_nls.jl
index 23df0dc3..0144c50f 100644
--- a/test/sparse_jacobian_nls.jl
+++ b/test/sparse_jacobian_nls.jl
@@ -1,8 +1,10 @@
 list_sparse_jac_backend = (
-  (ADNLPModels.SparseADJacobian, Dict()), # default
+  (ADNLPModels.SparseADJacobian, Dict()),
   (ADNLPModels.ForwardDiffADJacobian, Dict()),
 )
+
 dt = (Float32, Float64)
+
 @testset "Basic Jacobian of residual derivative with backend=$(backend) and T=$(T)" for T in dt,
   (backend, kw) in list_sparse_jac_backend
 

From 963c27fc87a30ca58bd0950dc6407d5876d32706 Mon Sep 17 00:00:00 2001
From: tmigot <tangi.migot@gmail.com>
Date: Tue, 18 Jun 2024 23:08:34 +0200
Subject: [PATCH 2/9] up test

---
 test/sparse_hessian_nls.jl | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/test/sparse_hessian_nls.jl b/test/sparse_hessian_nls.jl
index 69f791d3..8420e89b 100644
--- a/test/sparse_hessian_nls.jl
+++ b/test/sparse_hessian_nls.jl
@@ -26,18 +26,17 @@ dt = (Float32, Float64)
   hess_structure_residual!(nls, rows, cols)
   hess_coord_residual!(nls, x, v, vals)
   @test eltype(vals) == T
-  H = sparse(rows, cols, vals, nvar, nvar)
-  # @test H == []
+  H = Symmetric(sparse(rows, cols, vals, nvar, nvar), :L)
+  @test H == [-20 * v[2] 0; 0 0]
 
   # Test also the implementation of the backends
   b = nls.adbackend.hessian_residual_backend
-  obj_weight = 0.5
   @test nls.nls_meta.nnzh == ADNLPModels.get_nln_nnzh(b, nvar)
   ADNLPModels.hess_structure_residual!(b, nls, rows, cols)
-  ADNLPModels.hess_coord_residual!(b, nls, x, y, obj_weight, vals)
+  ADNLPModels.hess_coord_residual!(b, nls, x, v, vals)
   @test eltype(vals) == T
-  H = sparse(rows, cols, vals, nvar, nvar)
-  # @test H == []
+  H = Symmetric(sparse(rows, cols, vals, nvar, nvar), :L)
+  @test H == [-20 * v[2] 0; 0 0]
 
   nls = ADNLPModels.ADNLSModel!(F!, x0, 3, matrix_free = true; kw...)
   @test nls.adbackend.hessian_backend isa ADNLPModels.EmptyADbackend

From f302b0311965b8beafab9c0921d4901af966535c Mon Sep 17 00:00:00 2001
From: tmigot <tangi.migot@gmail.com>
Date: Tue, 18 Jun 2024 23:09:37 +0200
Subject: [PATCH 3/9] Update dense hessian residual

---
 src/ad_api.jl         | 26 ++++++++++++++++++++++----
 src/nls.jl            | 17 ++---------------
 src/sparse_hessian.jl |  4 ++--
 3 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/src/ad_api.jl b/src/ad_api.jl
index 0c938b5b..e27a7779 100644
--- a/src/ad_api.jl
+++ b/src/ad_api.jl
@@ -289,23 +289,41 @@ function NLPModels.hess_coord!(
   return vals
 end
 
-function NLPModels.hess_structure_residuals!(
+function NLPModels.hess_structure_residual!(
   b::ADBackend,
   nls::AbstractADNLSModel,
   rows::AbstractVector{<:Integer},
   cols::AbstractVector{<:Integer},
 )
-  nothing
+  n = nls.meta.nvar
+  pos = 0
+  for j = 1:n
+    for i = j:n
+      pos += 1
+      rows[pos] = i
+      cols[pos] = j
+    end
+  end
+  return rows, cols
 end
 
-function NLPModels.hess_coord_residuals!(
+function NLPModels.hess_coord_residual!(
   b::ADBackend,
   nls::AbstractADNLSModel,
   x::AbstractVector,
   v::AbstractVector,
   vals::AbstractVector,
 )
-  nothing
+  F = get_F(nls, b)
+  Hx = hessian(b, x -> dot(F(x), v), x)
+  k = 1
+  for j = 1:(nls.meta.nvar)
+    for i = j:(nls.meta.nvar)
+      vals[k] = Hx[i, j]
+      k += 1
+    end
+  end
+  return vals
 end
 
 function NLPModels.hprod!(
diff --git a/src/nls.jl b/src/nls.jl
index c7c81e7b..8d70168e 100644
--- a/src/nls.jl
+++ b/src/nls.jl
@@ -864,11 +864,7 @@ function NLPModels.hess_structure_residual!(
   cols::AbstractVector{<:Integer},
 )
   @lencheck nls.nls_meta.nnzh rows cols
-  n = nls.meta.nvar
-  I = ((i, j) for i = 1:n, j = 1:n if i ≥ j)
-  rows .= getindex.(I, 1)
-  cols .= getindex.(I, 2)
-  return rows, cols
+  return hess_structure_residual!(nls.adbackend.hessian_residual_backend, nls, rows, cols)
 end
 
 function NLPModels.hess_coord_residual!(
@@ -881,16 +877,7 @@ function NLPModels.hess_coord_residual!(
   @lencheck nls.nls_meta.nequ v
   @lencheck nls.nls_meta.nnzh vals
   increment!(nls, :neval_hess_residual)
-  F = get_F(nls, nls.adbackend.hessian_residual_backend)
-  Hx = hessian(nls.adbackend.hessian_residual_backend, x -> dot(F(x), v), x)
-  k = 1
-  for j = 1:(nls.meta.nvar)
-    for i = j:(nls.meta.nvar)
-      vals[k] = Hx[i, j]
-      k += 1
-    end
-  end
-  return vals
+  return hess_coord_residual!(nls.adbackend.hessian_residual_backend, nls, x, v, vals)
 end
 
 function NLPModels.jth_hess_residual(nls::ADNLSModel, x::AbstractVector, i::Int)
diff --git a/src/sparse_hessian.jl b/src/sparse_hessian.jl
index 91d7696a..2a5d1118 100644
--- a/src/sparse_hessian.jl
+++ b/src/sparse_hessian.jl
@@ -297,7 +297,7 @@ function NLPModels.hess_coord!(
   return vals
 end
 
-function NLPModels.hess_structure_residuals!(
+function NLPModels.hess_structure_residual!(
   b::Union{SparseADHessian, SparseReverseADHessian},
   nls::AbstractADNLSModel,
   rows::AbstractVector{<:Integer},
@@ -322,7 +322,7 @@ function NLPModels.hess_structure_residuals!(
   return rows, cols
 end
 
-function NLPModels.hess_coord_residuals!(
+function NLPModels.hess_coord_residual!(
   b::Union{SparseADHessian, SparseReverseADHessian},
   nls::AbstractADNLSModel,
   x::AbstractVector,

From 34c2fd81b7e2b776d14deb7cf8af99b8f4deecc7 Mon Sep 17 00:00:00 2001
From: Alexis Montoison <alexis.montoison@polymtl.ca>
Date: Tue, 18 Jun 2024 17:27:41 -0400
Subject: [PATCH 4/9] Test SparseReverseADHessian

---
 test/sparse_hessian.jl     | 4 +++-
 test/sparse_hessian_nls.jl | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/sparse_hessian.jl b/test/sparse_hessian.jl
index 7c5c0529..15327e97 100644
--- a/test/sparse_hessian.jl
+++ b/test/sparse_hessian.jl
@@ -1,5 +1,7 @@
 list_sparse_hess_backend =
-  ((ADNLPModels.SparseADHessian, Dict()), (ADNLPModels.ForwardDiffADHessian, Dict()))
+  ((ADNLPModels.SparseADHessian, Dict()),
+   (ADNLPModels.SparseReverseADHessian, Dict()),
+   (ADNLPModels.ForwardDiffADHessian, Dict()))
 
 dt = (Float32, Float64)
 
diff --git a/test/sparse_hessian_nls.jl b/test/sparse_hessian_nls.jl
index 8420e89b..4cc056ee 100644
--- a/test/sparse_hessian_nls.jl
+++ b/test/sparse_hessian_nls.jl
@@ -1,5 +1,6 @@
 list_sparse_hess_backend = (
   (ADNLPModels.SparseADHessian, Dict()),
+  (ADNLPModels.SparseReverseADHessian, Dict()),
   (ADNLPModels.ForwardDiffADHessian, Dict()),
 )
 

From afe470cc3443d6ad3955ff33910ca9b100641172 Mon Sep 17 00:00:00 2001
From: Alexis Montoison <alexis.montoison@polymtl.ca>
Date: Wed, 19 Jun 2024 01:09:57 -0400
Subject: [PATCH 5/9] Add get_residual_nnzh

---
 src/ad_api.jl         | 16 ++++++++++++++++
 src/nls.jl            | 18 ++++++++++++------
 src/sparse_hessian.jl | 19 +++++--------------
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/src/ad_api.jl b/src/ad_api.jl
index e27a7779..740d4f74 100644
--- a/src/ad_api.jl
+++ b/src/ad_api.jl
@@ -118,6 +118,22 @@ function get_nln_nnzh(nlp::AbstractNLPModel, nvar)
   nlp.meta.nnzh
 end
 
+"""
+    get_residual_nnzh(b::ADModelBackend, nvar)
+
+Return `get_nln_nnzh(b.hessian_residual_backend, nvar)`.
+"""
+function get_residual_nnzh(b::ADModelBackend, nvar)
+  get_nln_nnzh(b.hessian_residual_backend, nvar)
+end
+
+function get_residual_nnzh(
+  b::ADModelBackend{GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS},
+  nvar) where {GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS <: AbstractNLPModel}
+  nls = b.hessian_residual_backend
+  nls.nls_meta.nnzh
+end
+
 throw_error(b) =
   throw(ArgumentError("The AD backend $b is not loaded. Please load the corresponding AD package."))
 gradient(b::ADBackend, ::Any, ::Any) = throw_error(b)
diff --git a/src/nls.jl b/src/nls.jl
index 8d70168e..b3accac8 100644
--- a/src/nls.jl
+++ b/src/nls.jl
@@ -165,8 +165,9 @@ function ADNLSModel!(
 
   meta = NLPModelMeta{T, S}(nvar, x0 = x0, nnzh = nnzh, name = name, minimize = minimize)
   nls_nnzj = get_residual_nnzj(adbackend, nvar, nequ)
+  nls_nnzh = get_residual_nnzh(adbackend, nvar)
   nls_meta =
-    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = div(nvar * (nvar + 1), 2), lin = linequ)
+    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = nls_nnzh, lin = linequ)
   return ADNLSModel(meta, nls_meta, NLSCounters(), adbackend, F!, (cx, x) -> cx)
 end
 
@@ -210,8 +211,9 @@ function ADNLSModel!(
     minimize = minimize,
   )
   nls_nnzj = get_residual_nnzj(adbackend, nvar, nequ)
+  nls_nnzh = get_residual_nnzh(adbackend, nvar)
   nls_meta =
-    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = div(nvar * (nvar + 1), 2), lin = linequ)
+    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = nls_nnzh, lin = linequ)
   return ADNLSModel(meta, nls_meta, NLSCounters(), adbackend, F!, (cx, x) -> cx)
 end
 
@@ -272,8 +274,9 @@ function ADNLSModel!(
     minimize = minimize,
   )
   nls_nnzj = get_residual_nnzj(adbackend, nvar, nequ)
+  nls_nnzh = get_residual_nnzh(adbackend, nvar)
   nls_meta =
-    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = div(nvar * (nvar + 1), 2), lin = linequ)
+    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = nls_nnzh, lin = linequ)
   return ADNLSModel(meta, nls_meta, NLSCounters(), adbackend, F!, c!)
 end
 
@@ -422,8 +425,9 @@ function ADNLSModel!(
     minimize = minimize,
   )
   nls_nnzj = get_residual_nnzj(adbackend, nvar, nequ)
+  nls_nnzh = get_residual_nnzh(adbackend, nvar)
   nls_meta =
-    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = div(nvar * (nvar + 1), 2), lin = linequ)
+    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = nls_nnzh, lin = linequ)
   return ADNLSModel(meta, nls_meta, NLSCounters(), adbackend, F!, clinrows, clincols, clinvals, c!)
 end
 
@@ -639,8 +643,9 @@ function ADNLSModel!(
     minimize = minimize,
   )
   nls_nnzj = get_residual_nnzj(adbackend, nvar, nequ)
+  nls_nnzh = get_residual_nnzh(adbackend, nvar)
   nls_meta =
-    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = div(nvar * (nvar + 1), 2), lin = linequ)
+    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = nls_nnzh, lin = linequ)
   return ADNLSModel(meta, nls_meta, NLSCounters(), adbackend, F!, c!)
 end
 
@@ -744,8 +749,9 @@ function ADNLSModel!(
     minimize = minimize,
   )
   nls_nnzj = get_residual_nnzj(adbackend, nvar, nequ)
+  nls_nnzh = get_residual_nnzh(adbackend, nvar)
   nls_meta =
-    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = div(nvar * (nvar + 1), 2), lin = linequ)
+    NLSMeta{T, S}(nequ, nvar, nnzj = nls_nnzj, nnzh = nls_nnzh, lin = linequ)
   return ADNLSModel(meta, nls_meta, NLSCounters(), adbackend, F!, clinrows, clincols, clinvals, c!)
 end
 
diff --git a/src/sparse_hessian.jl b/src/sparse_hessian.jl
index 2a5d1118..bf1b2736 100644
--- a/src/sparse_hessian.jl
+++ b/src/sparse_hessian.jl
@@ -303,13 +303,7 @@ function NLPModels.hess_structure_residual!(
   rows::AbstractVector{<:Integer},
   cols::AbstractVector{<:Integer},
 )
-  function objective(x)
-    F = get_F(nls, b)
-    Fx = F(x)
-    return dot(Fx, Fx) / 2
-  end
-
-  H = compute_hessian_sparsity(objective, nls.meta.nvar, nothing, 0)
+  H = compute_hessian_sparsity(x -> 0, nls.meta.nvar, nls.F!, nls.nls_meta.nequ)
   trilH = tril(H)
   rowval = trilH.rowval
   colptr = trilH.colptr
@@ -329,11 +323,8 @@ function NLPModels.hess_coord_residual!(
   v::AbstractVector,
   vals::AbstractVector,
 )
-  function objective(x)
-    F = get_F(nls, b)
-    Fx = F(x)
-    return dot(Fx, Fx) / 2
-  end
-
-  sparse_hess_coord!(objective, b, x, 1.0, v, vals)
+  b.y .= 0
+  Fx = similar(b.y, nls.nls_meta.nequ)
+  Ψ(x) = dot(nls.F!(Fx, x), v)
+  sparse_hess_coord!(Ψ, b, x, 1.0, b.y, vals)
 end

From 23da6954e900575239d0b961974ec122c8b5b2b0 Mon Sep 17 00:00:00 2001
From: Alexis Montoison <alexis.montoison@polymtl.ca>
Date: Fri, 21 Jun 2024 21:33:42 -0400
Subject: [PATCH 6/9] Update predefined_backend.jl

---
 src/predefined_backend.jl | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/predefined_backend.jl b/src/predefined_backend.jl
index 6e2d0ffd..740fde7e 100644
--- a/src/predefined_backend.jl
+++ b/src/predefined_backend.jl
@@ -3,14 +3,14 @@ default_backend = Dict(
   :hprod_backend => ForwardDiffADHvprod,
   :jprod_backend => ForwardDiffADJprod,
   :jtprod_backend => ForwardDiffADJtprod,
-  :jacobian_backend => SparseADJacobian, # ForwardDiffADJacobian
-  :hessian_backend => SparseADHessian, # ForwardDiffADHessian
+  :jacobian_backend => SparseADJacobian,
+  :hessian_backend => SparseADHessian,
   :ghjvprod_backend => ForwardDiffADGHjvprod,
   :hprod_residual_backend => ForwardDiffADHvprod,
   :jprod_residual_backend => ForwardDiffADJprod,
   :jtprod_residual_backend => ForwardDiffADJtprod,
-  :jacobian_residual_backend => SparseADJacobian, # ForwardDiffADJacobian,
-  :hessian_residual_backend => ForwardDiffADHessian,
+  :jacobian_residual_backend => SparseADJacobian,
+  :hessian_residual_backend => SparseADHessian,
 )
 
 optimized = Dict(
@@ -18,14 +18,14 @@ optimized = Dict(
   :hprod_backend => ReverseDiffADHvprod,
   :jprod_backend => ForwardDiffADJprod,
   :jtprod_backend => ReverseDiffADJtprod,
-  :jacobian_backend => SparseADJacobian, # ForwardDiffADJacobian
-  :hessian_backend => SparseReverseADHessian, # ForwardDiffADHessian,
+  :jacobian_backend => SparseADJacobian,
+  :hessian_backend => SparseReverseADHessian,
   :ghjvprod_backend => ForwardDiffADGHjvprod,
   :hprod_residual_backend => ReverseDiffADHvprod,
   :jprod_residual_backend => ForwardDiffADJprod,
   :jtprod_residual_backend => ReverseDiffADJtprod,
-  :jacobian_residual_backend => SparseADJacobian, # ForwardDiffADJacobian
-  :hessian_residual_backend => ForwardDiffADHessian,
+  :jacobian_residual_backend => SparseADJacobian,
+  :hessian_residual_backend => SparseReverseADHessian,
 )
 
 generic = Dict(

From 3e7c3d8a7e62d6c9af0864cf7fea4f7b69a5c01f Mon Sep 17 00:00:00 2001
From: Alexis Montoison <alexis.montoison@polymtl.ca>
Date: Fri, 21 Jun 2024 21:34:27 -0400
Subject: [PATCH 7/9] Update sparse_hessian.jl for NLS

---
 src/ad_api.jl         | 18 +++++----------
 src/sparse_hessian.jl | 51 +++++++++++++++----------------------------
 2 files changed, 23 insertions(+), 46 deletions(-)

diff --git a/src/ad_api.jl b/src/ad_api.jl
index 740d4f74..c9e62252 100644
--- a/src/ad_api.jl
+++ b/src/ad_api.jl
@@ -82,19 +82,15 @@ end
 
 """
     get_residual_nnzj(b::ADModelBackend, nvar, nequ)
+    get_residual_nnzj(nls::AbstractNLSModel, nvar, nequ)
 
-Return `get_nln_nnzj(b.jacobian_residual_backend, nvar, nequ)`.
+Return the number of nonzeros elements in the residual Jacobians.
 """
 function get_residual_nnzj(b::ADModelBackend, nvar, nequ)
   get_nln_nnzj(b.jacobian_residual_backend, nvar, nequ)
 end
 
-function get_residual_nnzj(
-  b::ADModelBackend{GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS},
-  nvar,
-  nequ,
-) where {GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS <: AbstractNLPModel, HBLS}
-  nls = b.jacobian_residual_backend
+function get_residual_nnzj(nls::AbstractNLSModel, nvar, nequ)
   nls.nls_meta.nnzj
 end
 
@@ -120,17 +116,15 @@ end
 
 """
     get_residual_nnzh(b::ADModelBackend, nvar)
+    get_residual_nnzh(nls::AbstractNLSModel, nvar)
 
-Return `get_nln_nnzh(b.hessian_residual_backend, nvar)`.
+Return the number of nonzeros elements in the residual Hessians.
 """
 function get_residual_nnzh(b::ADModelBackend, nvar)
   get_nln_nnzh(b.hessian_residual_backend, nvar)
 end
 
-function get_residual_nnzh(
-  b::ADModelBackend{GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS},
-  nvar) where {GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS <: AbstractNLPModel}
-  nls = b.hessian_residual_backend
+function get_residual_nnzh(nls::AbstractNLSModel, nvar)
   nls.nls_meta.nnzh
 end
 
diff --git a/src/sparse_hessian.jl b/src/sparse_hessian.jl
index bf1b2736..8d899666 100644
--- a/src/sparse_hessian.jl
+++ b/src/sparse_hessian.jl
@@ -180,8 +180,16 @@ function NLPModels.hess_structure!(
   return rows, cols
 end
 
+function NLPModels.hess_structure_residual!(
+  b::Union{SparseADHessian, SparseReverseADHessian},
+  nls::AbstractADNLSModel,
+  rows::AbstractVector{<:Integer},
+  cols::AbstractVector{<:Integer},
+)
+  return hess_structure!(b, nls, rows, cols)
+end
+
 function sparse_hess_coord!(
-  ℓ::Function,
   b::SparseADHessian{Tag, GT, S, T},
   x::AbstractVector,
   obj_weight,
@@ -219,7 +227,6 @@ function sparse_hess_coord!(
 end
 
 function sparse_hess_coord!(
-  ℓ::Function,
   b::SparseReverseADHessian{T, S, Tagf, F, Tagψ, P},
   x::AbstractVector,
   obj_weight,
@@ -265,8 +272,7 @@ function NLPModels.hess_coord!(
   obj_weight::Real,
   vals::AbstractVector,
 )
-  ℓ = get_lag(nlp, b, obj_weight, y)
-  sparse_hess_coord!(ℓ, b, x, obj_weight, y, vals)
+  sparse_hess_coord!(b, x, obj_weight, y, vals)
 end
 
 function NLPModels.hess_coord!(
@@ -277,8 +283,7 @@ function NLPModels.hess_coord!(
   vals::AbstractVector,
 )
   b.y .= 0
-  ℓ = get_lag(nlp, b, obj_weight, b.y)
-  sparse_hess_coord!(ℓ, b, x, obj_weight, b.y, vals)
+  sparse_hess_coord!(b, x, obj_weight, b.y, vals)
 end
 
 function NLPModels.hess_coord!(
@@ -286,36 +291,16 @@ function NLPModels.hess_coord!(
   nlp::ADModel,
   x::AbstractVector,
   j::Integer,
-  vals::AbstractVector{T},
-) where {T}
+  vals::AbstractVector,
+)
   for (w, k) in enumerate(nlp.meta.nln)
     b.y[w] = k == j ? 1 : 0
   end
-  obj_weight = zero(T)
-  ℓ = get_lag(nlp, b, obj_weight, b.y)
-  sparse_hess_coord!(ℓ, b, x, obj_weight, b.y, vals)
+  obj_weight = zero(eltype(x))
+  sparse_hess_coord!(b, x, obj_weight, b.y, vals)
   return vals
 end
 
-function NLPModels.hess_structure_residual!(
-  b::Union{SparseADHessian, SparseReverseADHessian},
-  nls::AbstractADNLSModel,
-  rows::AbstractVector{<:Integer},
-  cols::AbstractVector{<:Integer},
-)
-  H = compute_hessian_sparsity(x -> 0, nls.meta.nvar, nls.F!, nls.nls_meta.nequ)
-  trilH = tril(H)
-  rowval = trilH.rowval
-  colptr = trilH.colptr
-  rows .= rowval
-  for i = 1:(nls.meta.nvar)
-    for j = colptr[i]:(colptr[i + 1] - 1)
-      cols[j] = i
-    end
-  end
-  return rows, cols
-end
-
 function NLPModels.hess_coord_residual!(
   b::Union{SparseADHessian, SparseReverseADHessian},
   nls::AbstractADNLSModel,
@@ -323,8 +308,6 @@ function NLPModels.hess_coord_residual!(
   v::AbstractVector,
   vals::AbstractVector,
 )
-  b.y .= 0
-  Fx = similar(b.y, nls.nls_meta.nequ)
-  Ψ(x) = dot(nls.F!(Fx, x), v)
-  sparse_hess_coord!(Ψ, b, x, 1.0, b.y, vals)
+  obj_weight = zero(eltype(x))
+  sparse_hess_coord!(b, x, obj_weight, v, vals)
 end

From 040c194bcbc5c79ada3ed840a1d6031153a20489 Mon Sep 17 00:00:00 2001
From: Alexis Montoison <alexis.montoison@polymtl.ca>
Date: Fri, 21 Jun 2024 22:12:31 -0400
Subject: [PATCH 8/9] Fix the dispatch

---
 src/ad_api.jl | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/ad_api.jl b/src/ad_api.jl
index c9e62252..a142f93a 100644
--- a/src/ad_api.jl
+++ b/src/ad_api.jl
@@ -94,6 +94,15 @@ function get_residual_nnzj(nls::AbstractNLSModel, nvar, nequ)
   nls.nls_meta.nnzj
 end
 
+function get_residual_nnzj(
+  b::ADModelBackend{GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS},
+  nvar,
+  nequ,
+) where {GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS <: AbstractNLPModel, HBLS}
+  nls = b.jacobian_residual_backend
+  nls.nls_meta.nnzj
+end
+
 """
     get_nln_nnzh(::ADBackend, nvar)
     get_nln_nnzh(b::ADModelBackend, nvar)
@@ -128,6 +137,13 @@ function get_residual_nnzh(nls::AbstractNLSModel, nvar)
   nls.nls_meta.nnzh
 end
 
+function get_residual_nnzh(
+  b::ADModelBackend{GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS},
+  nvar) where {GB, HvB, JvB, JtvB, JB, HB, GHJ, HvBLS, JvBLS, JtvBLS, JBLS, HBLS <: AbstractNLPModel}
+  nls = b.hessian_residual_backend
+  nls.nls_meta.nnzh
+end
+
 throw_error(b) =
   throw(ArgumentError("The AD backend $b is not loaded. Please load the corresponding AD package."))
 gradient(b::ADBackend, ::Any, ::Any) = throw_error(b)

From f79f076d6d2928fcc2c4bde34e40f02da860d228 Mon Sep 17 00:00:00 2001
From: tmigot <tangi.migot@gmail.com>
Date: Sat, 22 Jun 2024 16:38:22 +0200
Subject: [PATCH 9/9] Fix jth_hess_residual

---
 src/nls.jl | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/nls.jl b/src/nls.jl
index b3accac8..0edcfef2 100644
--- a/src/nls.jl
+++ b/src/nls.jl
@@ -886,13 +886,6 @@ function NLPModels.hess_coord_residual!(
   return hess_coord_residual!(nls.adbackend.hessian_residual_backend, nls, x, v, vals)
 end
 
-function NLPModels.jth_hess_residual(nls::ADNLSModel, x::AbstractVector, i::Int)
-  @lencheck nls.meta.nvar x
-  increment!(nls, :neval_jhess_residual)
-  F = get_F(nls, nls.adbackend.hessian_residual_backend)
-  return Symmetric(hessian(nls.adbackend.hessian_residual_backend, x -> F(x)[i], x), :L)
-end
-
 function NLPModels.hprod_residual!(
   nls::ADNLSModel,
   x::AbstractVector,