From 866fa817c53474c513477630aa7403c3f5139b1a Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 12 Jul 2023 01:41:50 +0200
Subject: [PATCH 01/22] started linesearch implementation

---
 Project.toml          | 1 +
 src/NonlinearSolve.jl | 1 +
 src/raphson.jl        | 9 +++++----
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Project.toml b/Project.toml
index 34ce7598d..afb76fd5d 100644
--- a/Project.toml
+++ b/Project.toml
@@ -9,6 +9,7 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl
index 6a93a63e4..2c643bf89 100644
--- a/src/NonlinearSolve.jl
+++ b/src/NonlinearSolve.jl
@@ -15,6 +15,7 @@ import ArrayInterface
 import LinearSolve
 using DiffEqBase
 using SparseDiffTools
+using LineSearches
 
 @reexport using SciMLBase
 using SciMLBase: NLStats
diff --git a/src/raphson.jl b/src/raphson.jl
index 437af65c3..492f9039b 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -48,19 +48,20 @@ for large-scale and numerically-difficult nonlinear systems.
     Currently, the linear solver and chunk size choice only applies to in-place defined
     `NonlinearProblem`s. That is expected to change in the future.
 """
-struct NewtonRaphson{CS, AD, FDT, L, P, ST, CJ} <:
+struct NewtonRaphson{CS, AD, FDT, L, P, ST, CJ, LS} <:
        AbstractNewtonAlgorithm{CS, AD, FDT, ST, CJ}
     linsolve::L
     precs::P
+    linesearch::LS
 end
 
 function NewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
     standardtag = Val{true}(), concrete_jac = nothing,
-    diff_type = Val{:forward}, linsolve = nothing, precs = DEFAULT_PRECS)
+    diff_type = Val{:forward}, linesearch = nothing, linsolve = nothing, precs = DEFAULT_PRECS)
     NewtonRaphson{_unwrap_val(chunk_size), _unwrap_val(autodiff), diff_type,
         typeof(linsolve), typeof(precs), _unwrap_val(standardtag),
-        _unwrap_val(concrete_jac)}(linsolve,
-        precs)
+        _unwrap_val(concrete_jac), typeof(linesearch)}(linsolve,
+        precs, linesearch)
 end
 
 mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType,

From 19838428c58de9d896b42fc882c58aa1cb03ac84 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 13 Jul 2023 02:47:02 +0200
Subject: [PATCH 02/22] added objective functions

---
 src/raphson.jl | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 492f9039b..eda9eedff 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -34,6 +34,7 @@ for large-scale and numerically-difficult nonlinear systems.
   - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
     `Val{:forward}` for forward finite differences. For more details on the choices, see the
     [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+  - `linesearch`: the line search algorithm used. Defaults to no  line search being used.
   - `linsolve`: the [LinearSolve.jl](https://github.com/SciML/LinearSolve.jl) used for the
     linear solves within the Newton method. Defaults to `nothing`, which means it uses the
     LinearSolve.jl default algorithm choice. For more information on available algorithm
@@ -57,7 +58,7 @@ end
 
 function NewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
     standardtag = Val{true}(), concrete_jac = nothing,
-    diff_type = Val{:forward}, linesearch = nothing, linsolve = nothing, precs = DEFAULT_PRECS)
+    diff_type = Val{:forward}, linesearch = LineSearches.Static(), linsolve = nothing, precs = DEFAULT_PRECS)
     NewtonRaphson{_unwrap_val(chunk_size), _unwrap_val(autodiff), diff_type,
         typeof(linsolve), typeof(precs), _unwrap_val(standardtag),
         _unwrap_val(concrete_jac), typeof(linesearch)}(linsolve,
@@ -166,8 +167,19 @@ function perform_step!(cache::NewtonRaphsonCache{true})
     linres = dolinsolve(alg.precs, linsolve, A = J, b = _vec(fu), linu = _vec(du1),
         p = p, reltol = cache.abstol)
     cache.linsolve = linres.cache
-    @. u = u - du1
-    f(fu, u, p)
+    function f!(ulin)
+        f(fu, ulin, p)
+        return dot(fu, fu) / 2
+    end
+
+    function g!(gvec, ulin)
+        mul!(gvec, J, fu)
+    end
+
+    function fg!()
+
+    end
+
 
     if cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true

From c54601fa02206361b1716c3e2e4faef6982218a9 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 14 Jul 2023 04:12:42 +0200
Subject: [PATCH 03/22] modified objective and added helper func

---
 src/raphson.jl | 19 ++++++-------------
 src/utils.jl   | 10 ++++++++++
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index eda9eedff..5b62f1127 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -153,6 +153,10 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     end
     uf, linsolve, J, du1, jac_config = jacobian_caches(alg, f, u, p, Val(iip))
 
+    function fo(x)
+        dot(value_f(f, x, iip), value_f(f, x, iip)) / 2
+    end
+
     return NewtonRaphsonCache{iip}(f, alg, u, fu, p, uf, linsolve, J, du1, jac_config,
         false, maxiters, internalnorm,
         ReturnCode.Default, abstol, prob, NLStats(1, 0, 0, 0, 0))
@@ -167,19 +171,8 @@ function perform_step!(cache::NewtonRaphsonCache{true})
     linres = dolinsolve(alg.precs, linsolve, A = J, b = _vec(fu), linu = _vec(du1),
         p = p, reltol = cache.abstol)
     cache.linsolve = linres.cache
-    function f!(ulin)
-        f(fu, ulin, p)
-        return dot(fu, fu) / 2
-    end
-
-    function g!(gvec, ulin)
-        mul!(gvec, J, fu)
-    end
-
-    function fg!()
-
-    end
-
+    @. u = u - du1
+    f(fu, u, p)
 
     if cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true
diff --git a/src/utils.jl b/src/utils.jl
index 9eb7cc6f1..e58cc17ca 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -163,3 +163,13 @@ function rfunc(r::R, c2::R, M::R, γ1::R, γ2::R, β::R) where {R <: Real} # R-f
         return (1 - γ1 - β) * (exp(r - c2) + β / (1 - γ1 - β))
     end
 end
+
+function value_f(f::F, x, iip::Bool) where {F}
+    if iip
+        fu = similar(x)
+        f(fu, x)
+    else
+        fu = f(x)
+    end
+    fu
+end

From 879e11886cf2c29359a530db74d0a58a5e391769 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 16 Jul 2023 16:58:14 +0200
Subject: [PATCH 04/22] reversing changes

---
 src/raphson.jl |  4 ----
 src/utils.jl   | 10 +---------
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 5b62f1127..57cbe04cb 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -153,10 +153,6 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     end
     uf, linsolve, J, du1, jac_config = jacobian_caches(alg, f, u, p, Val(iip))
 
-    function fo(x)
-        dot(value_f(f, x, iip), value_f(f, x, iip)) / 2
-    end
-
     return NewtonRaphsonCache{iip}(f, alg, u, fu, p, uf, linsolve, J, du1, jac_config,
         false, maxiters, internalnorm,
         ReturnCode.Default, abstol, prob, NLStats(1, 0, 0, 0, 0))
diff --git a/src/utils.jl b/src/utils.jl
index e58cc17ca..5a55fdeba 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -164,12 +164,4 @@ function rfunc(r::R, c2::R, M::R, γ1::R, γ2::R, β::R) where {R <: Real} # R-f
     end
 end
 
-function value_f(f::F, x, iip::Bool) where {F}
-    if iip
-        fu = similar(x)
-        f(fu, x)
-    else
-        fu = f(x)
-    end
-    fu
-end
+

From 58774f1d3d84f5a42f2812e1ecb7a33ec0641fdf Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 16 Jul 2023 20:02:11 +0200
Subject: [PATCH 05/22] modified cache and objective func final dispatch

---
 src/raphson.jl | 47 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 57cbe04cb..05f34d41a 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -73,6 +73,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
     u::uType
     fu::resType
     p::pType
+    α::Real
     uf::ufType
     linsolve::L
     J::jType
@@ -99,7 +100,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
         tolType,
         probType, ufType, L, jType, JC}
         new{iip, fType, algType, uType, duType, resType, pType, INType, tolType,
-            probType, ufType, L, jType, JC}(f, alg, u, fu, p,
+            probType, ufType, L, jType, JC}(f, alg, u, fu, p, α,
             uf, linsolve, J, du1, jac_config,
             force_stop, maxiters, internalnorm,
             retcode, abstol, prob, stats)
@@ -145,6 +146,7 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     end
     f = prob.f
     p = prob.p
+    α = 1.0 #line search coefficient
     if iip
         fu = zero(u)
         f(fu, u, p)
@@ -153,7 +155,7 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     end
     uf, linsolve, J, du1, jac_config = jacobian_caches(alg, f, u, p, Val(iip))
 
-    return NewtonRaphsonCache{iip}(f, alg, u, fu, p, uf, linsolve, J, du1, jac_config,
+    return NewtonRaphsonCache{iip}(f, alg, u, fu, p, α, uf, linsolve, J, du1, jac_config,
         false, maxiters, internalnorm,
         ReturnCode.Default, abstol, prob, NLStats(1, 0, 0, 0, 0))
 end
@@ -167,8 +169,10 @@ function perform_step!(cache::NewtonRaphsonCache{true})
     linres = dolinsolve(alg.precs, linsolve, A = J, b = _vec(fu), linu = _vec(du1),
         p = p, reltol = cache.abstol)
     cache.linsolve = linres.cache
-    @. u = u - du1
-    f(fu, u, p)
+    # @. u = u - du1
+    # f(fu, u, p)
+    ## Line Search ##
+    perform_linesearch!(cache)
 
     if cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true
@@ -195,6 +199,41 @@ function perform_step!(cache::NewtonRaphsonCache{false})
     return nothing
 end
 
+function objective_linesearch!(cache::NewtonRaphsonCache) ## returns the objective functions required in LS
+    @unpack f = cache
+
+    function fo(x)
+        return dot(value_f(cache, x), value_f(cache, x))
+    end
+
+    function g!(grad, x)
+        grad = simple_jacobian(f, x)' * f(x)
+        return grad
+    end
+
+    function fg!(grad, x)
+        g!(grad, x)
+        fo(x)
+    end
+    return fo, g!, fg!
+end
+
+function perform_linesearch(cache::NewtonRaphsonCache)
+    @unpack u, fu, du1, alg, α = cache
+    fo, g!, fg! = objective_linesearch!(cache)
+    ϕ(α) = fo(u .- α .* du1)
+    ###
+
+    # other functions to be defined
+
+
+    if alg.linesearch isa Static
+
+    else
+        α, fu = alg.linesearch(ϕ, dϕ, ϕdϕ)
+    end
+end
+
 function SciMLBase.solve!(cache::NewtonRaphsonCache)
     while !cache.force_stop && cache.stats.nsteps < cache.maxiters
         perform_step!(cache)

From 4b04e2abbc98992ead1ffa4b7457b3ca8b073dc3 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 16 Jul 2023 20:02:30 +0200
Subject: [PATCH 06/22] Update utils.jl

added objective func value output
---
 src/utils.jl | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/utils.jl b/src/utils.jl
index 5a55fdeba..fef81dffb 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -164,4 +164,22 @@ function rfunc(r::R, c2::R, M::R, γ1::R, γ2::R, β::R) where {R <: Real} # R-f
     end
 end
 
+function value_f(cache::NewtonRaphsonCache, x)
+    iip = get_iip(cache)
+    @unpack f, p = cache
+    if iip
+        res = zero(x)
+        f(res, x, p)
+    else
+        res = f(x, p)
+    end
+    res
+end
+
+get_iip(cache::NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType,
+    INType, tolType,
+    probType, ufType, L, jType, JC}) where {iip, fType, algType, uType, duType, resType, pType,
+    INType, tolType,
+    probType, ufType, L, jType, JC} = iip
+
 

From 72eb6b8dae7234f98b04ce026e6e26fc0e7ad3ea Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 16 Jul 2023 20:02:43 +0200
Subject: [PATCH 07/22] Update jacobian.jl

simple jacobian calculation
---
 src/jacobian.jl | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/jacobian.jl b/src/jacobian.jl
index cb696400e..55f1be3b7 100644
--- a/src/jacobian.jl
+++ b/src/jacobian.jl
@@ -149,3 +149,11 @@ function jacobian_autodiff(f, x::AbstractArray, nonlinfun, alg)
             jac_prototype = jac_prototype, chunksize = chunk_size),
         num_of_chunks)
 end
+
+function simple_jacobian(f, x::Number)
+    ForwardDiff.derivative(f, x)
+end
+
+function simple_jacobian(f, x::AbstractArray{<:Number})
+    ForwardDiff.jacobian(f, x)
+end

From 95c89a49a3b05295719ce1a0d5312b9f5357d195 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 16 Jul 2023 20:03:23 +0200
Subject: [PATCH 08/22] objective correction

---
 src/raphson.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 05f34d41a..5fd98b216 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -203,7 +203,7 @@ function objective_linesearch!(cache::NewtonRaphsonCache) ## returns the objecti
     @unpack f = cache
 
     function fo(x)
-        return dot(value_f(cache, x), value_f(cache, x))
+        return dot(value_f(cache, x), value_f(cache, x)) / 2
     end
 
     function g!(grad, x)

From 929552d7058ea6e0ef5ae4965851ed3ab91b7a63 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Mon, 17 Jul 2023 01:22:57 +0200
Subject: [PATCH 09/22] quick fixes

---
 src/raphson.jl | 38 +++++++++++++++++++++++++++++---------
 src/utils.jl   | 16 ----------------
 2 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 5fd98b216..5ae9e1b14 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -88,7 +88,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
     stats::NLStats
 
     function NewtonRaphsonCache{iip}(f::fType, alg::algType, u::uType, fu::resType,
-        p::pType, uf::ufType, linsolve::L, J::jType,
+        p::pType, α::Real, uf::ufType, linsolve::L, J::jType,
         du1::duType,
         jac_config::JC, force_stop::Bool, maxiters::Int,
         internalnorm::INType,
@@ -172,7 +172,7 @@ function perform_step!(cache::NewtonRaphsonCache{true})
     # @. u = u - du1
     # f(fu, u, p)
     ## Line Search ##
-    perform_linesearch!(cache)
+    #perform_linesearch!(cache)
 
     if cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true
@@ -199,20 +199,20 @@ function perform_step!(cache::NewtonRaphsonCache{false})
     return nothing
 end
 
-function objective_linesearch!(cache::NewtonRaphsonCache) ## returns the objective functions required in LS
-    @unpack f = cache
+function objective_linesearch(cache::NewtonRaphsonCache) ## returns the objective functions required in LS
+    @unpack f, p = cache
 
     function fo(x)
         return dot(value_f(cache, x), value_f(cache, x)) / 2
     end
 
-    function g!(grad, x)
-        grad = simple_jacobian(f, x)' * f(x)
-        return grad
+    function g!(gvec, x)
+        mul!(gvec, simple_jacobian(f, x, p)', value_f(cache, x)) 
+        gvec
     end
 
-    function fg!(grad, x)
-        g!(grad, x)
+    function fg!(gvec, x)
+        g!(gvec, x)
         fo(x)
     end
     return fo, g!, fg!
@@ -269,3 +269,23 @@ function SciMLBase.reinit!(cache::NewtonRaphsonCache{iip}, u0 = cache.u; p = cac
     cache.retcode = ReturnCode.Default
     return cache
 end
+
+## some helper func ###
+
+function value_f(cache::NewtonRaphsonCache, x)
+    iip = get_iip(cache)
+    @unpack f, p = cache
+    if iip
+        res = zero(x)
+        f(res, x, p)
+    else
+        res = f(x, p)
+    end
+    res
+end
+
+get_iip(cache::NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType,
+    INType, tolType,
+    probType, ufType, L, jType, JC}) where {iip, fType, algType, uType, duType, resType, pType,
+    INType, tolType,
+    probType, ufType, L, jType, JC} = iip
diff --git a/src/utils.jl b/src/utils.jl
index fef81dffb..506429366 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -164,22 +164,6 @@ function rfunc(r::R, c2::R, M::R, γ1::R, γ2::R, β::R) where {R <: Real} # R-f
     end
 end
 
-function value_f(cache::NewtonRaphsonCache, x)
-    iip = get_iip(cache)
-    @unpack f, p = cache
-    if iip
-        res = zero(x)
-        f(res, x, p)
-    else
-        res = f(x, p)
-    end
-    res
-end
 
-get_iip(cache::NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType,
-    INType, tolType,
-    probType, ufType, L, jType, JC}) where {iip, fType, algType, uType, duType, resType, pType,
-    INType, tolType,
-    probType, ufType, L, jType, JC} = iip
 
 

From f6920f7eb7278688f2454d55fe252584d3d6a481 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Mon, 17 Jul 2023 01:23:16 +0200
Subject: [PATCH 10/22] improve simple_jacobian

---
 src/jacobian.jl | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/jacobian.jl b/src/jacobian.jl
index 55f1be3b7..0a5c645ee 100644
--- a/src/jacobian.jl
+++ b/src/jacobian.jl
@@ -150,10 +150,12 @@ function jacobian_autodiff(f, x::AbstractArray, nonlinfun, alg)
         num_of_chunks)
 end
 
-function simple_jacobian(f, x::Number)
-    ForwardDiff.derivative(f, x)
+function simple_jacobian(f, x::Number, p)
+    g = Base.Fix2(f, p)
+    ForwardDiff.derivative(g, x)
 end
 
-function simple_jacobian(f, x::AbstractArray{<:Number})
-    ForwardDiff.jacobian(f, x)
+function simple_jacobian(f, x::AbstractArray{<:Number}, p)
+    g = Base.Fix2(f, p)
+    ForwardDiff.jacobian(g, x)
 end

From 4982846aa90b5560c77b7fddb23e1c08e58219cb Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 18 Jul 2023 00:55:47 +0200
Subject: [PATCH 11/22] added linesearch functions

---
 src/raphson.jl | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 5ae9e1b14..d8b9c8059 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -74,6 +74,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
     fu::resType
     p::pType
     α::Real
+    gvec::uType
     uf::ufType
     linsolve::L
     J::jType
@@ -88,7 +89,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
     stats::NLStats
 
     function NewtonRaphsonCache{iip}(f::fType, alg::algType, u::uType, fu::resType,
-        p::pType, α::Real, uf::ufType, linsolve::L, J::jType,
+        p::pType, α::Real, gvec::uType, uf::ufType, linsolve::L, J::jType,
         du1::duType,
         jac_config::JC, force_stop::Bool, maxiters::Int,
         internalnorm::INType,
@@ -100,7 +101,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
         tolType,
         probType, ufType, L, jType, JC}
         new{iip, fType, algType, uType, duType, resType, pType, INType, tolType,
-            probType, ufType, L, jType, JC}(f, alg, u, fu, p, α,
+            probType, ufType, L, jType, JC}(f, alg, u, fu, p, α, gvec,
             uf, linsolve, J, du1, jac_config,
             force_stop, maxiters, internalnorm,
             retcode, abstol, prob, stats)
@@ -144,6 +145,7 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     else
         u = deepcopy(prob.u0)
     end
+    gvec = zero(u)
     f = prob.f
     p = prob.p
     α = 1.0 #line search coefficient
@@ -155,7 +157,7 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     end
     uf, linsolve, J, du1, jac_config = jacobian_caches(alg, f, u, p, Val(iip))
 
-    return NewtonRaphsonCache{iip}(f, alg, u, fu, p, α, uf, linsolve, J, du1, jac_config,
+    return NewtonRaphsonCache{iip}(f, alg, u, fu, p, α, gvec, uf, linsolve, J, du1, jac_config,
         false, maxiters, internalnorm,
         ReturnCode.Default, abstol, prob, NLStats(1, 0, 0, 0, 0))
 end
@@ -218,13 +220,19 @@ function objective_linesearch(cache::NewtonRaphsonCache) ## returns the objectiv
     return fo, g!, fg!
 end
 
-function perform_linesearch(cache::NewtonRaphsonCache)
-    @unpack u, fu, du1, alg, α = cache
+function perform_linesearch!(cache::NewtonRaphsonCache)
+    @unpack u, fu, du1, alg, α, gvec = cache
     fo, g!, fg! = objective_linesearch!(cache)
     ϕ(α) = fo(u .- α .* du1)
-    ###
 
-    # other functions to be defined
+    function dϕ(α)
+        g!(gvec, u .- α .* du1)
+        return dot(gvec, du1)
+    end
+
+    function ϕdϕ(α)
+        return (fg!(gvec, u .- α .* du1), dot(gvec, s))
+    end
 
 
     if alg.linesearch isa Static

From b5d9f3ed0a392cd1493f43a93ac798333853537e Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 19 Jul 2023 02:52:44 +0200
Subject: [PATCH 12/22] Update raphson.jl

1. step variable du1 for out of place functions.
2. added linesearch func
---
 src/raphson.jl | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index d8b9c8059..28502bafb 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -130,7 +130,7 @@ function jacobian_caches(alg::NewtonRaphson, f, u, p, ::Val{true})
 end
 
 function jacobian_caches(alg::NewtonRaphson, f, u, p, ::Val{false})
-    JacobianWrapper(f, p), nothing, nothing, nothing, nothing
+    JacobianWrapper(f, p), nothing, nothing, zero(u), nothing
 end
 
 function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson,
@@ -189,7 +189,9 @@ end
 function perform_step!(cache::NewtonRaphsonCache{false})
     @unpack u, fu, f, p = cache
     J = jacobian(cache, f)
-    cache.u = u - J \ fu
+    cache.du1 = J \ fu
+    perform_linesearch!(cache) ## need some edits here
+    cache.u = u - cache.α * cache.du1
     cache.fu = f(cache.u, p)
     if iszero(cache.fu) || cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true
@@ -222,7 +224,7 @@ end
 
 function perform_linesearch!(cache::NewtonRaphsonCache)
     @unpack u, fu, du1, alg, α, gvec = cache
-    fo, g!, fg! = objective_linesearch!(cache)
+    fo, g!, fg! = objective_linesearch(cache)
     ϕ(α) = fo(u .- α .* du1)
 
     function dϕ(α)
@@ -231,15 +233,9 @@ function perform_linesearch!(cache::NewtonRaphsonCache)
     end
 
     function ϕdϕ(α)
-        return (fg!(gvec, u .- α .* du1), dot(gvec, s))
-    end
-
-
-    if alg.linesearch isa Static
-
-    else
-        α, fu = alg.linesearch(ϕ, dϕ, ϕdϕ)
+        return (fg!(gvec, u .- α .* du1), dot(gvec, du1))
     end
+    cache.α, fu = alg.linesearch(ϕ, dϕ, ϕdϕ, 1.0, fo(u), dot(du1, gvec))
 end
 
 function SciMLBase.solve!(cache::NewtonRaphsonCache)

From e1a2da989572eaf82124a2c866a431f1eebe27b5 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 19 Jul 2023 02:53:45 +0200
Subject: [PATCH 13/22] export

1. export LineSearches

Alternative:
export LineSearches: ... ?
---
 src/NonlinearSolve.jl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl
index 2c643bf89..9e0b1c0b2 100644
--- a/src/NonlinearSolve.jl
+++ b/src/NonlinearSolve.jl
@@ -66,7 +66,10 @@ PrecompileTools.@compile_workload begin
 end
 
 export RadiusUpdateSchemes
-
+export LineSearches
 export NewtonRaphson, TrustRegion, LevenbergMarquardt
 
+## test exports - to be removed later ##
+export get_iip, value_f, objective_linesearch, simple_jacobian
+##
 end # module

From 2593f9ad5b831b46a884d2ddfb011e624488ce69 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 20 Jul 2023 02:21:15 +0200
Subject: [PATCH 14/22] modified jac calc

---
 src/jacobian.jl | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/jacobian.jl b/src/jacobian.jl
index 0a5c645ee..74b18f591 100644
--- a/src/jacobian.jl
+++ b/src/jacobian.jl
@@ -150,12 +150,18 @@ function jacobian_autodiff(f, x::AbstractArray, nonlinfun, alg)
         num_of_chunks)
 end
 
-function simple_jacobian(f, x::Number, p)
+function simple_jacobian(cache, x::Number)
+    @unpack f, p = cache
     g = Base.Fix2(f, p)
     ForwardDiff.derivative(g, x)
 end
 
-function simple_jacobian(f, x::AbstractArray{<:Number}, p)
-    g = Base.Fix2(f, p)
-    ForwardDiff.jacobian(g, x)
+function simple_jacobian(cache, x::AbstractArray{<:Number})
+    @unpack f, fu, p, prob = cache
+    if !get_iip(prob)
+        g = Base.Fix2(f, p)
+        return ForwardDiff.jacobian(g, x)
+    else
+        return ForwardDiff.jacobian((fu, x) -> f(fu, x, p), fu, x)
+    end
 end

From b604e5bbb6d8f1bce54ae540bddca1ae3bdd5754 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 20 Jul 2023 02:21:34 +0200
Subject: [PATCH 15/22] removed test exports

---
 src/NonlinearSolve.jl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl
index 9e0b1c0b2..fd63a8de2 100644
--- a/src/NonlinearSolve.jl
+++ b/src/NonlinearSolve.jl
@@ -69,7 +69,5 @@ export RadiusUpdateSchemes
 export LineSearches
 export NewtonRaphson, TrustRegion, LevenbergMarquardt
 
-## test exports - to be removed later ##
-export get_iip, value_f, objective_linesearch, simple_jacobian
-##
+
 end # module

From 1591dff79be76fb9a7e9819ca145511a50ab84ee Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 20 Jul 2023 02:21:49 +0200
Subject: [PATCH 16/22] get_iip improvement

---
 src/utils.jl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/utils.jl b/src/utils.jl
index 506429366..097ded84b 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -164,6 +164,8 @@ function rfunc(r::R, c2::R, M::R, γ1::R, γ2::R, β::R) where {R <: Real} # R-f
     end
 end
 
-
+function get_iip(prob::NonlinearProblem{uType, iip}) where {uType, iip}
+    iip
+end
 
 

From c74c78d8f3559fe172a328e32d295e85ad022fcc Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 20 Jul 2023 02:22:28 +0200
Subject: [PATCH 17/22] Update raphson.jl

1. modified cache
2. final tweaks
---
 src/raphson.jl | 60 ++++++++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 28502bafb..51d3651b4 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -65,7 +65,7 @@ function NewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
         precs, linesearch)
 end
 
-mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType,
+mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType, foType, goType,
     INType, tolType,
     probType, ufType, L, jType, JC}
     f::fType
@@ -74,7 +74,8 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
     fu::resType
     p::pType
     α::Real
-    gvec::uType
+    f_o::foType # stores value of objective
+    g_o::goType # stores grad of objective at x
     uf::ufType
     linsolve::L
     J::jType
@@ -89,7 +90,7 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
     stats::NLStats
 
     function NewtonRaphsonCache{iip}(f::fType, alg::algType, u::uType, fu::resType,
-        p::pType, α::Real, gvec::uType, uf::ufType, linsolve::L, J::jType,
+        p::pType, α::Real, f_o::foType, g_o::goType, uf::ufType, linsolve::L, J::jType,
         du1::duType,
         jac_config::JC, force_stop::Bool, maxiters::Int,
         internalnorm::INType,
@@ -97,11 +98,11 @@ mutable struct NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, p
         prob::probType,
         stats::NLStats) where {
         iip, fType, algType, uType,
-        duType, resType, pType, INType,
+        duType, resType, pType, foType, goType, INType,
         tolType,
         probType, ufType, L, jType, JC}
-        new{iip, fType, algType, uType, duType, resType, pType, INType, tolType,
-            probType, ufType, L, jType, JC}(f, alg, u, fu, p, α, gvec,
+        new{iip, fType, algType, uType, duType, resType, pType, foType, goType, INType, tolType,
+            probType, ufType, L, jType, JC}(f, alg, u, fu, p, α, f_o, g_o,
             uf, linsolve, J, du1, jac_config,
             force_stop, maxiters, internalnorm,
             retcode, abstol, prob, stats)
@@ -145,10 +146,11 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     else
         u = deepcopy(prob.u0)
     end
-    gvec = zero(u)
     f = prob.f
     p = prob.p
     α = 1.0 #line search coefficient
+    f_o = 0.0
+    g_o = zero(u)
     if iip
         fu = zero(u)
         f(fu, u, p)
@@ -157,7 +159,7 @@ function SciMLBase.__init(prob::NonlinearProblem{uType, iip}, alg::NewtonRaphson
     end
     uf, linsolve, J, du1, jac_config = jacobian_caches(alg, f, u, p, Val(iip))
 
-    return NewtonRaphsonCache{iip}(f, alg, u, fu, p, α, gvec, uf, linsolve, J, du1, jac_config,
+    return NewtonRaphsonCache{iip}(f, alg, u, fu, p, α, f_o, g_o, uf, linsolve, J, du1, jac_config,
         false, maxiters, internalnorm,
         ReturnCode.Default, abstol, prob, NLStats(1, 0, 0, 0, 0))
 end
@@ -166,15 +168,13 @@ function perform_step!(cache::NewtonRaphsonCache{true})
     @unpack u, fu, f, p, alg = cache
     @unpack J, linsolve, du1 = cache
     jacobian!(J, cache)
-
     # u = u - J \ fu
     linres = dolinsolve(alg.precs, linsolve, A = J, b = _vec(fu), linu = _vec(du1),
         p = p, reltol = cache.abstol)
     cache.linsolve = linres.cache
-    # @. u = u - du1
-    # f(fu, u, p)
-    ## Line Search ##
-    #perform_linesearch!(cache)
+    perform_linesearch!(cache)
+    @. cache.u = cache.u - cache.α * cache.du1
+    f(cache.fu, cache.u, p)
 
     if cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true
@@ -190,8 +190,8 @@ function perform_step!(cache::NewtonRaphsonCache{false})
     @unpack u, fu, f, p = cache
     J = jacobian(cache, f)
     cache.du1 = J \ fu
-    perform_linesearch!(cache) ## need some edits here
-    cache.u = u - cache.α * cache.du1
+    perform_linesearch!(cache) 
+    cache.u = cache.u - cache.α * cache.du1
     cache.fu = f(cache.u, p)
     if iszero(cache.fu) || cache.internalnorm(cache.fu) < cache.abstol
         cache.force_stop = true
@@ -210,36 +210,39 @@ function objective_linesearch(cache::NewtonRaphsonCache) ## returns the objectiv
         return dot(value_f(cache, x), value_f(cache, x)) / 2
     end
 
-    function g!(gvec, x)
-        mul!(gvec, simple_jacobian(f, x, p)', value_f(cache, x)) 
-        gvec
+    function g!(g_o, x)
+        mul!(g_o, simple_jacobian(cache, x)', value_f(cache, x)) 
+        g_o
     end
 
-    function fg!(gvec, x)
-        g!(gvec, x)
+    function fg!(g_o, x)
+        g!(g_o, x)
         fo(x)
     end
     return fo, g!, fg!
 end
 
 function perform_linesearch!(cache::NewtonRaphsonCache)
-    @unpack u, fu, du1, alg, α, gvec = cache
+    @unpack u, fu, du1, alg, α, g_o = cache
     fo, g!, fg! = objective_linesearch(cache)
     ϕ(α) = fo(u .- α .* du1)
 
     function dϕ(α)
-        g!(gvec, u .- α .* du1)
-        return dot(gvec, du1)
+        g!(g_o, u .- α .* du1)
+        return dot(g_o, du1)
     end
 
     function ϕdϕ(α)
-        return (fg!(gvec, u .- α .* du1), dot(gvec, du1))
+        return (fg!(g_o, u .- α .* du1), dot(g_o, du1))
     end
-    cache.α, fu = alg.linesearch(ϕ, dϕ, ϕdϕ, 1.0, fo(u), dot(du1, gvec))
+    cache.f_o = fo(u)
+    @unpack f_o = cache
+    cache.α, cache.f_o = alg.linesearch(ϕ, dϕ, ϕdϕ, 1.0, f_o, dot(du1, g_o))
 end
 
 function SciMLBase.solve!(cache::NewtonRaphsonCache)
     while !cache.force_stop && cache.stats.nsteps < cache.maxiters
+        @show "yes"
         perform_step!(cache)
         cache.stats.nsteps += 1
     end
@@ -277,7 +280,7 @@ end
 ## some helper func ###
 
 function value_f(cache::NewtonRaphsonCache, x)
-    iip = get_iip(cache)
+    iip = get_iip(cache.prob)
     @unpack f, p = cache
     if iip
         res = zero(x)
@@ -288,8 +291,3 @@ function value_f(cache::NewtonRaphsonCache, x)
     res
 end
 
-get_iip(cache::NewtonRaphsonCache{iip, fType, algType, uType, duType, resType, pType,
-    INType, tolType,
-    probType, ufType, L, jType, JC}) where {iip, fType, algType, uType, duType, resType, pType,
-    INType, tolType,
-    probType, ufType, L, jType, JC} = iip

From ad4ea572937c3d637b9dc13733944170f0480fc6 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 20 Jul 2023 14:42:17 +0200
Subject: [PATCH 18/22] --

---
 src/raphson.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 51d3651b4..cb4065adf 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -242,7 +242,6 @@ end
 
 function SciMLBase.solve!(cache::NewtonRaphsonCache)
     while !cache.force_stop && cache.stats.nsteps < cache.maxiters
-        @show "yes"
         perform_step!(cache)
         cache.stats.nsteps += 1
     end

From 87525561950f1af258db9e38d730947c02373415 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 15 Sep 2023 02:42:39 +0200
Subject: [PATCH 19/22] isinplace use

---
 src/raphson.jl | 2 +-
 src/utils.jl   | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 111a61a7a..81a05e34b 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -258,7 +258,7 @@ end
 ## some helper func ###
 
 function value_f(cache::NewtonRaphsonCache, x)
-    iip = get_iip(cache.prob)
+    iip = SciMLBase.isinplace(cache.prob)
     @unpack f, p = cache
     if iip
         res = zero(x)
diff --git a/src/utils.jl b/src/utils.jl
index 98a6cfcb4..eacf06727 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -158,8 +158,5 @@ function rfunc(r::R, c2::R, M::R, γ1::R, γ2::R, β::R) where {R <: Real} # R-f
     end
 end
 
-function get_iip(prob::NonlinearProblem{uType, iip}) where {uType, iip}
-    iip
-end
 
 

From ff524b0b40836c263f39e4e89287d0252858b1d6 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 15 Sep 2023 03:35:52 +0200
Subject: [PATCH 20/22] isinplace use

---
 src/jacobian.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/jacobian.jl b/src/jacobian.jl
index b9b73f608..1574e4b13 100644
--- a/src/jacobian.jl
+++ b/src/jacobian.jl
@@ -192,7 +192,7 @@ end
 
 function simple_jacobian(cache, x::AbstractArray{<:Number})
     @unpack f, fu, p, prob = cache
-    if !get_iip(prob)
+    if !SciMLBase.isinplace(prob)
         g = Base.Fix2(f, p)
         return ForwardDiff.jacobian(g, x)
     else

From 7673c85932b170b8a278595a23fe878c54439d7a Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 17 Sep 2023 02:43:17 +0200
Subject: [PATCH 21/22] minor sign fix

---
 src/raphson.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index 81a05e34b..d8b52d46f 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -208,11 +208,11 @@ function perform_linesearch!(cache::NewtonRaphsonCache)
 
     function dϕ(α)
         g!(g_o, u .- α .* du1)
-        return dot(g_o, du1)
+        return dot(g_o, -du1)
     end
 
     function ϕdϕ(α)
-        return (fg!(g_o, u .- α .* du1), dot(g_o, du1))
+        return (fg!(g_o, u .- α .* du1), dot(g_o, -du1))
     end
     cache.f_o = fo(u)
     @unpack f_o = cache

From 137d8fd13d8a6343a3b131cffdbcf4b614cdafae Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 17 Sep 2023 03:19:37 +0200
Subject: [PATCH 22/22] more minor fixes

---
 src/raphson.jl | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/raphson.jl b/src/raphson.jl
index d8b52d46f..a082e6bb1 100644
--- a/src/raphson.jl
+++ b/src/raphson.jl
@@ -202,8 +202,11 @@ function objective_linesearch(cache::NewtonRaphsonCache) ## returns the objectiv
 end
 
 function perform_linesearch!(cache::NewtonRaphsonCache)
-    @unpack u, fu, du1, alg, α, g_o = cache
+    @unpack u, fu, du1, alg, α = cache
     fo, g!, fg! = objective_linesearch(cache)
+    cache.f_o = fo(u)
+    g!(cache.g_o, u)
+    @unpack f_o, g_o = cache
     ϕ(α) = fo(u .- α .* du1)
 
     function dϕ(α)
@@ -214,9 +217,7 @@ function perform_linesearch!(cache::NewtonRaphsonCache)
     function ϕdϕ(α)
         return (fg!(g_o, u .- α .* du1), dot(g_o, -du1))
     end
-    cache.f_o = fo(u)
-    @unpack f_o = cache
-    cache.α, cache.f_o = alg.linesearch(ϕ, dϕ, ϕdϕ, 1.0, f_o, dot(du1, g_o))
+    cache.α, cache.f_o = alg.linesearch(ϕ, dϕ, ϕdϕ, 1.0, f_o, dot(du1, -g_o))
 end
 
 function SciMLBase.solve!(cache::NewtonRaphsonCache)