From b387a94cf891d4fc597ee6c82b6c4f1188751216 Mon Sep 17 00:00:00 2001 From: "Documenter.jl" Date: Wed, 31 Jul 2024 13:30:45 +0000 Subject: [PATCH] build based on eee30d1 --- previews/PR255/.documenter-siteinfo.json | 2 +- previews/PR255/assets/Manifest.toml | 40 ++++---- .../{bfc414c0.svg => 87b7500c.svg} | 88 ++++++++--------- previews/PR255/getting_started/8d491187.svg | 40 -------- previews/PR255/getting_started/9e467392.svg | 40 ++++++++ previews/PR255/getting_started/b1538f7c.svg | 58 ----------- previews/PR255/getting_started/ee01fe0c.svg | 58 +++++++++++ previews/PR255/getting_started/index.html | 76 +++++++------- previews/PR255/index.html | 20 ++-- .../PR255/methods/collocation_loss/index.html | 2 +- .../optimization_based_methods/index.html | 2 +- .../methods/recommended_methods/index.html | 2 +- .../ensemble/{71dd5f68.svg => 5df84959.svg} | 98 +++++++++---------- previews/PR255/tutorials/ensemble/index.html | 4 +- .../generalized_likelihood/index.html | 30 +++--- .../tutorials/global_optimization/index.html | 8 +- .../stochastic_evaluations/index.html | 90 ++++++++--------- 17 files changed, 329 insertions(+), 329 deletions(-) rename previews/PR255/getting_started/{bfc414c0.svg => 87b7500c.svg} (94%) delete mode 100644 previews/PR255/getting_started/8d491187.svg create mode 100644 previews/PR255/getting_started/9e467392.svg delete mode 100644 previews/PR255/getting_started/b1538f7c.svg create mode 100644 previews/PR255/getting_started/ee01fe0c.svg rename previews/PR255/tutorials/ensemble/{71dd5f68.svg => 5df84959.svg} (98%) diff --git a/previews/PR255/.documenter-siteinfo.json b/previews/PR255/.documenter-siteinfo.json index 4da21e4..ae08f87 100644 --- a/previews/PR255/.documenter-siteinfo.json +++ b/previews/PR255/.documenter-siteinfo.json @@ -1 +1 @@ -{"documenter":{"julia_version":"1.10.4","generation_timestamp":"2024-07-29T19:38:30","documenter_version":"1.5.0"}} \ No newline at end of file +{"documenter":{"julia_version":"1.10.4","generation_timestamp":"2024-07-31T13:30:38","documenter_version":"1.5.0"}} \ No newline at end of file diff --git a/previews/PR255/assets/Manifest.toml b/previews/PR255/assets/Manifest.toml index 2c1ba08..6623929 100644 --- a/previews/PR255/assets/Manifest.toml +++ b/previews/PR255/assets/Manifest.toml @@ -84,9 +84,9 @@ version = "0.4.0" [[deps.ArrayInterface]] deps = ["Adapt", "LinearAlgebra"] -git-tree-sha1 = "8c5b39db37c1d0340bf3b14895fba160c2d6cbb5" +git-tree-sha1 = "f54c23a5d304fb87110de62bace7777d59088c34" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "7.14.0" +version = "7.15.0" [deps.ArrayInterface.extensions] ArrayInterfaceBandedMatricesExt = "BandedMatrices" @@ -97,7 +97,7 @@ version = "7.14.0" ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore" ArrayInterfaceReverseDiffExt = "ReverseDiff" ArrayInterfaceSparseArraysExt = "SparseArrays" - ArrayInterfaceStaticArraysExt = "StaticArrays" + ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore" ArrayInterfaceTrackerExt = "Tracker" [deps.ArrayInterface.weakdeps] @@ -109,7 +109,7 @@ version = "7.14.0" GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" [[deps.ArrayLayouts]] @@ -1368,10 +1368,10 @@ weakdeps = ["ChainRulesCore", "SparseArrays", "Statistics"] LinearMapsStatisticsExt = "Statistics" [[deps.LinearSolve]] -deps = ["ArrayInterface", "ChainRulesCore", "ConcreteStructs", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "LazyArrays", "Libdl", "LinearAlgebra", "MKL_jll", "Markdown", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "StaticArraysCore", "UnPack"] -git-tree-sha1 = "b2e2dba60642e07c062eb3143770d7e234316772" +deps = ["ArrayInterface", "ChainRulesCore", "ConcreteStructs", "CpuId", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "LazyArrays", "Libdl", "LinearAlgebra", "MKL_jll", "Markdown", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "StaticArraysCore", "UnPack"] +git-tree-sha1 = "bae39cec701960d14dc7d3c05491c1bfd922fa1d" uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" -version = "2.30.2" +version = "2.31.1" [deps.LinearSolve.extensions] LinearSolveBandedMatricesExt = "BandedMatrices" @@ -1574,10 +1574,10 @@ version = "1.2.0" uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.ModelingToolkit]] -deps = ["AbstractTrees", "ArrayInterface", "Combinatorics", "Compat", "ConstructionBase", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "DiffEqNoiseProcess", "DiffRules", "Distributed", "Distributions", "DocStringExtensions", "DomainSets", "DynamicQuantities", "ExprTools", "FindFirstFunctions", "ForwardDiff", "FunctionWrappersWrappers", "Graphs", "InteractiveUtils", "JuliaFormatter", "JumpProcesses", "LabelledArrays", "Latexify", "Libdl", "LinearAlgebra", "MLStyle", "NaNMath", "NonlinearSolve", "OrderedCollections", "PrecompileTools", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLBase", "SciMLStructures", "Serialization", "Setfield", "SimpleNonlinearSolve", "SparseArrays", "SpecialFunctions", "StaticArrays", "SymbolicIndexingInterface", "SymbolicUtils", "Symbolics", "URIs", "UnPack", "Unitful"] -git-tree-sha1 = "568a7c80b2e3ff43c6c6c92212deeeba7ff4e639" +deps = ["AbstractTrees", "ArrayInterface", "Combinatorics", "Compat", "ConstructionBase", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "DiffEqNoiseProcess", "DiffRules", "Distributed", "Distributions", "DocStringExtensions", "DomainSets", "DynamicQuantities", "ExprTools", "Expronicon", "FindFirstFunctions", "ForwardDiff", "FunctionWrappersWrappers", "Graphs", "InteractiveUtils", "JuliaFormatter", "JumpProcesses", "LabelledArrays", "Latexify", "Libdl", "LinearAlgebra", "MLStyle", "NaNMath", "NonlinearSolve", "OrderedCollections", "PrecompileTools", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLBase", "SciMLStructures", "Serialization", "Setfield", "SimpleNonlinearSolve", "SparseArrays", "SpecialFunctions", "StaticArrays", "SymbolicIndexingInterface", "SymbolicUtils", "Symbolics", "URIs", "UnPack", "Unitful"] +git-tree-sha1 = "8f8176851819a573cb08d65166c876faa1c3877c" uuid = "961ee093-0014-501f-94e3-6117800e7a78" -version = "9.26.0" +version = "9.28.0" [deps.ModelingToolkit.extensions] MTKBifurcationKitExt = "BifurcationKit" @@ -2192,9 +2192,9 @@ version = "0.6.43" [[deps.SciMLBase]] deps = ["ADTypes", "Accessors", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "Expronicon", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "SciMLStructures", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"] -git-tree-sha1 = "380a059a9fd18a56d98e50ed98d40e1c1202e662" +git-tree-sha1 = "b316ed5e5e71a6414b0c0e0c9f334afcc701ebf8" uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" -version = "2.46.0" +version = "2.48.0" [deps.SciMLBase.extensions] SciMLBaseChainRulesCoreExt = "ChainRulesCore" @@ -2335,9 +2335,9 @@ version = "0.1.2" [[deps.SparseMatrixColorings]] deps = ["ADTypes", "Compat", "DocStringExtensions", "LinearAlgebra", "Random", "SparseArrays"] -git-tree-sha1 = "277e10c002cd780a752bded3b95a8cbc791d646b" +git-tree-sha1 = "ad048e784b816e4de5553a13f1daf148412f3dbd" uuid = "0a514795-09f3-496d-8182-132a7b665d35" -version = "0.3.5" +version = "0.3.6" [[deps.Sparspak]] deps = ["Libdl", "LinearAlgebra", "Logging", "OffsetArrays", "Printf", "SparseArrays", "Test"] @@ -2423,9 +2423,9 @@ weakdeps = ["ChainRulesCore", "InverseFunctions"] [[deps.SteadyStateDiffEq]] deps = ["ConcreteStructs", "DiffEqBase", "DiffEqCallbacks", "LinearAlgebra", "Reexport", "SciMLBase"] -git-tree-sha1 = "1158cfdf0da5b0eacdfcfba7c16b174a37bdf6c7" +git-tree-sha1 = "ea7119de1cdd294c9b0d8e8ce6a120711c7904c8" uuid = "9672c7b4-1e72-59bd-8a11-6ac3964bc41f" -version = "2.2.0" +version = "2.3.0" [[deps.StochasticDiffEq]] deps = ["Adapt", "ArrayInterface", "DataStructures", "DiffEqBase", "DiffEqNoiseProcess", "DocStringExtensions", "FiniteDiff", "ForwardDiff", "JumpProcesses", "LevyArea", "LinearAlgebra", "Logging", "MuladdMacro", "NLsolve", "OrdinaryDiffEq", "Random", "RandomNumbers", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "SparseArrays", "SparseDiffTools", "StaticArrays", "UnPack"] @@ -2487,9 +2487,9 @@ version = "0.1.0" [[deps.SymbolicIndexingInterface]] deps = ["Accessors", "ArrayInterface", "RuntimeGeneratedFunctions", "StaticArraysCore"] -git-tree-sha1 = "9c490ee01823dc443da25bf9225827e3cdd2d7e9" +git-tree-sha1 = "2dd32da03adaf43fd91494e38ef3df0ab2e6c20e" uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5" -version = "0.3.26" +version = "0.3.27" [[deps.SymbolicLimits]] deps = ["SymbolicUtils"] @@ -2505,9 +2505,9 @@ version = "2.1.2" [[deps.Symbolics]] deps = ["ADTypes", "ArrayInterface", "Bijections", "CommonWorldInvalidations", "ConstructionBase", "DataStructures", "DiffRules", "Distributions", "DocStringExtensions", "DomainSets", "DynamicPolynomials", "ForwardDiff", "IfElse", "LaTeXStrings", "LambertW", "Latexify", "Libdl", "LinearAlgebra", "LogExpFunctions", "MacroTools", "Markdown", "NaNMath", "PrecompileTools", "RecipesBase", "Reexport", "Requires", "RuntimeGeneratedFunctions", "SciMLBase", "Setfield", "SparseArrays", "SpecialFunctions", "StaticArraysCore", "SymbolicIndexingInterface", "SymbolicLimits", "SymbolicUtils", "TermInterface"] -git-tree-sha1 = "44356a85991385883acd0f41ef66c6f770d9a2e7" +git-tree-sha1 = "d92fc9253d0070db98fd1bb410cccce9c1096922" uuid = "0c5d862f-8b57-4792-8d23-62f2024744c7" -version = "5.34.0" +version = "5.35.0" [deps.Symbolics.extensions] SymbolicsGroebnerExt = "Groebner" diff --git a/previews/PR255/getting_started/bfc414c0.svg b/previews/PR255/getting_started/87b7500c.svg similarity index 94% rename from previews/PR255/getting_started/bfc414c0.svg rename to previews/PR255/getting_started/87b7500c.svg index ef6e9cc..be61eef 100644 --- a/previews/PR255/getting_started/bfc414c0.svg +++ b/previews/PR255/getting_started/87b7500c.svg @@ -1,58 +1,58 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/previews/PR255/getting_started/8d491187.svg b/previews/PR255/getting_started/8d491187.svg deleted file mode 100644 index 47f0021..0000000 --- a/previews/PR255/getting_started/8d491187.svg +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/previews/PR255/getting_started/9e467392.svg b/previews/PR255/getting_started/9e467392.svg new file mode 100644 index 0000000..3a7ed2e --- /dev/null +++ b/previews/PR255/getting_started/9e467392.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/previews/PR255/getting_started/b1538f7c.svg b/previews/PR255/getting_started/b1538f7c.svg deleted file mode 100644 index caab351..0000000 --- a/previews/PR255/getting_started/b1538f7c.svg +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/previews/PR255/getting_started/ee01fe0c.svg b/previews/PR255/getting_started/ee01fe0c.svg new file mode 100644 index 0000000..c35b9a4 --- /dev/null +++ b/previews/PR255/getting_started/ee01fe0c.svg @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/previews/PR255/getting_started/index.html b/previews/PR255/getting_started/index.html index cdd4971..db73879 100644 --- a/previews/PR255/getting_started/index.html +++ b/previews/PR255/getting_started/index.html @@ -19,27 +19,27 @@ using RecursiveArrayTools # for VectorOfArray randomized = VectorOfArray([(sol(t[i]) + 0.01randn(2)) for i in 1:length(t)]) data = convert(Array, randomized)
2×200 Matrix{Float64}:
- 0.990495  1.02897   1.06291   1.09549   …  0.980557  1.007    1.05354
- 0.996392  0.890618  0.832509  0.738023     1.10216   1.00498  0.922369

Here, we used VectorOfArray from RecursiveArrayTools.jl to turn the result of an ODE into a matrix.

If we plot the solution with the parameter at a=1.42, we get the following:

newprob = remake(prob, p = [1.42])
+ 0.995641  1.02016   1.0483    1.1061    …  0.989842  0.989437  1.03649
+ 0.991172  0.889352  0.829449  0.753901     1.11038   1.02194   0.902183

Here, we used VectorOfArray from RecursiveArrayTools.jl to turn the result of an ODE into a matrix.

If we plot the solution with the parameter at a=1.42, we get the following:

newprob = remake(prob, p = [1.42])
 newsol = solve(newprob, Tsit5())
 plot(sol)
-plot!(newsol)
Example block output

Notice that after one period, this solution begins to drift very far off: this problem is sensitive to the choice of a.

To build the objective function for Optim.jl, we simply call the build_loss_objective function:

cost_function = build_loss_objective(prob, Tsit5(), L2Loss(t, data),
+plot!(newsol)
Example block output

Notice that after one period, this solution begins to drift very far off: this problem is sensitive to the choice of a.

To build the objective function for Optim.jl, we simply call the build_loss_objective function:

cost_function = build_loss_objective(prob, Tsit5(), L2Loss(t, data),
     Optimization.AutoForwardDiff(),
     maxiters = 10000, verbose = false)
(::SciMLBase.OptimizationFunction{true, ADTypes.AutoForwardDiff{nothing, Nothing}, DiffEqParamEstim.var"#29#30"{Nothing, typeof(DiffEqParamEstim.STANDARD_PROB_GENERATOR), Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, @NamedTuple{maxiters::Int64, verbose::Bool}}, SciMLBase.ODEProblem{Vector{Float64}, Tuple{Float64, Float64}, true, Vector{Float64}, SciMLBase.ODEFunction{true, SciMLBase.AutoSpecialize, typeof(Main.f), LinearAlgebra.UniformScaling{Bool}, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing, Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, @NamedTuple{}}, SciMLBase.StandardODEProblem}, OrdinaryDiffEq.Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, L2Loss{Vector{Float64}, Matrix{Float64}, Nothing, Nothing, Nothing}, Nothing, Tuple{}}, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED_NO_TIME), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing}) (generic function with 1 method)

This objective function internally is calling the ODE solver to get solutions to test against the data. The keyword arguments are passed directly to the solver. Note that we set maxiters in a way that causes the differential equation solvers to error more quickly when in bad regions of the parameter space, speeding up the process. If the integrator stops early (due to divergence), then those parameters are given an infinite loss, and thus this is a quick way to avoid bad parameters. We set verbose=false because this divergence can get noisy. The Optimization.AutoForwardDiff() is a choice of automatic differentiation, i.e., how the gradients are calculated. For more information on this choice, see the automatic differentiation choice API.

Note

A good rule of thumb is to use Optimization.AutoForwardDiff() for less than 100 parameters + states, and Optimization.AutoZygote() for more.

Before optimizing, let's visualize our cost function by plotting it for a range of parameter values:

vals = 0.0:0.1:10.0
 plot(vals, [cost_function(i) for i in vals], yscale = :log10,
     xaxis = "Parameter", yaxis = "Cost", title = "1-Parameter Cost Function",
-    lw = 3)
Example block output

Here we see that there is a very well-defined minimum in our cost function at the real parameter (because this is where the solution almost exactly fits the dataset).

Now we can use the BFGS algorithm to optimize the parameter starting at a=1.42. We do this by creating an optimization problem and solving that with BFGS():

optprob = Optimization.OptimizationProblem(cost_function, [1.42])
+    lw = 3)
Example block output

Here we see that there is a very well-defined minimum in our cost function at the real parameter (because this is where the solution almost exactly fits the dataset).

Now we can use the BFGS algorithm to optimize the parameter starting at a=1.42. We do this by creating an optimization problem and solving that with BFGS():

optprob = Optimization.OptimizationProblem(cost_function, [1.42])
 optsol = solve(optprob, BFGS())
retcode: Failure
 u: 1-element Vector{Float64}:
- 1.5003851756631508

Now let's see how well the fit performed:

newprob = remake(prob, p = optsol.u)
+ 1.500317250823813

Now let's see how well the fit performed:

newprob = remake(prob, p = optsol.u)
 newsol = solve(newprob, Tsit5())
 plot(sol)
-plot!(newsol)
Example block output

Note that some algorithms may be sensitive to the initial condition. For more details on using Optim.jl, see the documentation for Optim.jl.

Adding Bounds Constraints

We can improve our solution by noting that the Lotka-Volterra equation requires that the parameters are positive. Thus, following the Optimization.jl documentation we can add box constraints to ensure the optimizer only checks between 0.0 and 3.0 which improves the efficiency of our algorithm. We pass the lb and ub keyword arguments to the OptimizationProblem to pass these bounds to the optimizer:

lower = [0.0]
+plot!(newsol)
Example block output

Note that some algorithms may be sensitive to the initial condition. For more details on using Optim.jl, see the documentation for Optim.jl.

Adding Bounds Constraints

We can improve our solution by noting that the Lotka-Volterra equation requires that the parameters are positive. Thus, following the Optimization.jl documentation we can add box constraints to ensure the optimizer only checks between 0.0 and 3.0 which improves the efficiency of our algorithm. We pass the lb and ub keyword arguments to the OptimizationProblem to pass these bounds to the optimizer:

lower = [0.0]
 upper = [3.0]
 optprob = Optimization.OptimizationProblem(cost_function, [1.42], lb = lower, ub = upper)
 result = solve(optprob, BFGS())
retcode: Success
 u: 1-element Vector{Float64}:
- 1.5003858374155614

Estimating Multiple Parameters Simultaneously

Lastly, we can use the same tools to estimate multiple parameters simultaneously. Let's use the Lotka-Volterra equation with all parameters free:

function f2(du, u, p, t)
+ 1.5003178696330517

Estimating Multiple Parameters Simultaneously

Lastly, we can use the same tools to estimate multiple parameters simultaneously. Let's use the Lotka-Volterra equation with all parameters free:

function f2(du, u, p, t)
     du[1] = dx = p[1] * u[1] - p[2] * u[1] * u[2]
     du[2] = dy = -p[3] * u[2] + p[4] * u[1] * u[2]
 end
@@ -55,23 +55,23 @@
     Optimization.AutoForwardDiff(),
     maxiters = 10000, verbose = false)
 optprob = Optimization.OptimizationProblem(cost_function, [1.3, 0.8, 2.8, 1.2])
-result_bfgs = solve(optprob, BFGS())
retcode: Success
+result_bfgs = solve(optprob, BFGS())
retcode: Failure
 u: 4-element Vector{Float64}:
- 1.5004832987377994
- 1.0005235965950205
- 2.9988107121596133
- 1.0001662519648715

Alternative Cost Functions for Increased Robustness

The build_loss_objective with L2Loss is the most naive approach for parameter estimation. There are many others.

We can also use First-Differences in L2Loss by passing the kwarg differ_weight which decides the contribution of the differencing loss to the total loss.

cost_function = build_loss_objective(prob, Tsit5(),
+ 1.500602505493551
+ 1.0014332354955537
+ 2.9985514873944066
+ 0.9999801298915162

Alternative Cost Functions for Increased Robustness

The build_loss_objective with L2Loss is the most naive approach for parameter estimation. There are many others.

We can also use First-Differences in L2Loss by passing the kwarg differ_weight which decides the contribution of the differencing loss to the total loss.

cost_function = build_loss_objective(prob, Tsit5(),
     L2Loss(t, data, differ_weight = 0.3,
         data_weight = 0.7),
     Optimization.AutoForwardDiff(),
     maxiters = 10000, verbose = false)
 optprob = Optimization.OptimizationProblem(cost_function, [1.3, 0.8, 2.8, 1.2])
-result_bfgs = solve(optprob, BFGS())
retcode: Success
+result_bfgs = solve(optprob, BFGS())
retcode: Failure
 u: 4-element Vector{Float64}:
- 1.499804533457457
- 1.000582561460681
- 3.000949060870289
- 1.000890327850507

We can also use Multiple Shooting method by creating a multiple_shooting_objective

function ms_f1(du, u, p, t)
+ 1.5002625602501085
+ 1.001348829046728
+ 2.99948512945006
+ 1.0002930724466745

We can also use Multiple Shooting method by creating a multiple_shooting_objective

function ms_f1(du, u, p, t)
     du[1] = p[1] * u[1] - p[2] * u[1] * u[2]
     du[2] = -3.0 * u[2] + u[1] * u[2]
 end
@@ -93,26 +93,26 @@
     ub = last.(bound))
 optsol_ms = solve(optprob, BBO_adaptive_de_rand_1_bin_radiuslimited(), maxiters = 10_000)
retcode: Failure
 u: 18-element Vector{Float64}:
- 0.9067134894069822
- 1.1975211677247901
- 3.2012934188792497
- 0.30840903319075674
- 3.072613330018484
- 4.5839371429971285
- 1.1764852680826663
- 0.806958640284053
- 4.758897296690512
- 0.39223765966858437
- 1.5221515437506419
- 3.8248413814441675
- 1.4734415415743443
- 0.4863822757569302
- 6.467041444566654
- 0.7494860852515315
- 1.732410387379509
- 1.112789250132989
optsol_ms.u[(end - 1):end]
2-element Vector{Float64}:
- 1.732410387379509
- 1.112789250132989

Here as our model had 2 parameters, we look at the last 2 indexes of result to get our parameter values and the rest of the values are the initial values of the shorter timespans as described in the reference section. We can also use a gradient-based optimizer with the multiple shooting objective.

optsol_ms = solve(optprob, BFGS())
+ 0.973105307404053
+ 1.1550530027604726
+ 3.344129145821192
+ 0.27257511266073325
+ 3.023349504229528
+ 4.400697374569361
+ 1.2652744207915037
+ 0.7576837041215199
+ 4.833027658576654
+ 0.3984235242046755
+ 1.2001449639139266
+ 3.5341599023280503
+ 1.4074490888503062
+ 0.3290992860520703
+ 6.352020718791216
+ 0.9176420085478082
+ 1.5744664439772516
+ 0.9973772497376149
optsol_ms.u[(end - 1):end]
2-element Vector{Float64}:
+ 1.5744664439772516
+ 0.9973772497376149

Here as our model had 2 parameters, we look at the last 2 indexes of result to get our parameter values and the rest of the values are the initial values of the shorter timespans as described in the reference section. We can also use a gradient-based optimizer with the multiple shooting objective.

optsol_ms = solve(optprob, BFGS())
 optsol_ms.u[(end - 1):end]
2-element Vector{Float64}:
  1.5000000000037266
  1.000000000000063

The objective function for the Two Stage method can be created and passed to an optimizer as

two_stage_obj = two_stage_objective(ms_prob, t, data, Optimization.AutoForwardDiff())
@@ -122,4 +122,4 @@
  1.5035938533664905
  0.99257311537469
  2.8
- 1.2

The default kernel used in the method is Epanechnikov, available others are Uniform, Triangular, Quartic, Triweight, Tricube, Gaussian, Cosine, Logistic and Sigmoid, this can be passed by the kernel keyword argument. loss_func keyword argument can be used to pass the loss function (cost function) you want to use and passing a valid adtype argument enables Auto Differentiation.

Conclusion

There are many more choices for how to improve the robustness of a parameter estimation. With all of these tools, one likely should never do the simple “solve it with p and check the L2 loss”. Instead, we should use these tricks to improve the loss landscape and increase the ability for optimizers to find globally the best parameters.

+ 1.2

The default kernel used in the method is Epanechnikov, available others are Uniform, Triangular, Quartic, Triweight, Tricube, Gaussian, Cosine, Logistic and Sigmoid, this can be passed by the kernel keyword argument. loss_func keyword argument can be used to pass the loss function (cost function) you want to use and passing a valid adtype argument enables Auto Differentiation.

Conclusion

There are many more choices for how to improve the robustness of a parameter estimation. With all of these tools, one likely should never do the simple “solve it with p and check the L2 loss”. Instead, we should use these tricks to improve the loss landscape and increase the ability for optimizers to find globally the best parameters.

diff --git a/previews/PR255/index.html b/previews/PR255/index.html index f59fdb3..5c17fe6 100644 --- a/previews/PR255/index.html +++ b/previews/PR255/index.html @@ -38,7 +38,7 @@ [79e6a3ab] Adapt v4.0.4 [66dad0bd] AliasTables v1.1.3 [ec485272] ArnoldiMethod v0.4.0 - [4fba245c] ArrayInterface v7.14.0 + [4fba245c] ArrayInterface v7.15.0 [4c555306] ArrayLayouts v1.10.2 [a9b6321e] Atomix v0.1.0 [aae01518] BandedMatrices v1.7.2 @@ -173,7 +173,7 @@ [2d8b4e74] LevyArea v1.0.0 [d3d80556] LineSearches v7.2.0 [7a12625a] LinearMaps v3.11.3 - [7ed4a6bd] LinearSolve v2.30.2 + [7ed4a6bd] LinearSolve v2.31.1 [2ab3a3ac] LogExpFunctions v0.3.28 [e6f89c97] LoggingExtras v1.0.3 [bdcacae8] LoopVectorization v0.12.171 @@ -191,7 +191,7 @@ [739be429] MbedTLS v1.1.9 [442fdcdd] Measures v0.3.2 [e1d29d7a] Missings v1.2.0 - [961ee093] ModelingToolkit v9.26.0 + [961ee093] ModelingToolkit v9.28.0 [46d2c3a1] MuladdMacro v0.2.4 [102ac46a] MultivariatePolynomials v0.5.6 [d8a4904e] MutableArithmetics v1.4.5 @@ -254,7 +254,7 @@ [7e49a35a] RuntimeGeneratedFunctions v0.5.13 [94e857df] SIMDTypes v0.1.0 [476501e8] SLEEFPirates v0.6.43 - [0bca4576] SciMLBase v2.46.0 + [0bca4576] SciMLBase v2.48.0 [c0aeaf25] SciMLOperators v0.3.8 [1ed8b502] SciMLSensitivity v7.64.0 [53ae85a6] SciMLStructures v1.4.1 @@ -269,7 +269,7 @@ [a2af1166] SortingAlgorithms v1.2.1 [47a9eef4] SparseDiffTools v2.19.0 [dc90abb0] SparseInverseSubset v0.1.2 - [0a514795] SparseMatrixColorings v0.3.5 + [0a514795] SparseMatrixColorings v0.3.6 [e56a9233] Sparspak v0.3.9 [d4ead438] SpatialIndexing v0.1.6 [276daf66] SpecialFunctions v2.4.0 @@ -280,17 +280,17 @@ [82ae8749] StatsAPI v1.7.0 [2913bbd2] StatsBase v0.34.3 [4c63d2b9] StatsFuns v1.3.1 - [9672c7b4] SteadyStateDiffEq v2.2.0 + [9672c7b4] SteadyStateDiffEq v2.3.0 [789caeaf] StochasticDiffEq v6.67.0 [7792a7ef] StrideArraysCore v0.5.7 [09ab397b] StructArrays v0.6.18 [53d494c1] StructIO v0.3.0 [c3572dad] Sundials v4.24.0 ⌅ [4297ee4d] SymbolicAnalysis v0.1.0 - [2efcf032] SymbolicIndexingInterface v0.3.26 + [2efcf032] SymbolicIndexingInterface v0.3.27 ⌃ [19f23fe9] SymbolicLimits v0.2.1 ⌅ [d1185830] SymbolicUtils v2.1.2 - [0c5d862f] Symbolics v5.34.0 + [0c5d862f] Symbolics v5.35.0 [3783bdb8] TableTraits v1.0.1 [bd369af6] Tables v1.12.0 [62fd8b95] TensorCore v0.1.1 @@ -339,7 +339,7 @@ [aacddb02] JpegTurbo_jll v3.0.3+0 [c1c5ebd0] LAME_jll v3.100.2+0 ⌅ [88015f11] LERC_jll v3.0.0+1 - [dad2f222] LLVMExtra_jll v0.0.30+0 +⌅ [dad2f222] LLVMExtra_jll v0.0.30+0 [1d63c593] LLVMOpenMP_jll v15.0.7+0 [dd4b983a] LZO_jll v2.10.2+0 [81d17ec3] L_BFGS_B_jll v3.0.1+0 @@ -459,4 +459,4 @@ [8e850b90] libblastrampoline_jll v5.8.0+1 [8e850ede] nghttp2_jll v1.52.0+1 [3f19e933] p7zip_jll v17.4.0+2 -Info Packages marked with ⌃ and ⌅ have new versions available. Those with ⌃ may be upgradable, but those with ⌅ are restricted by compatibility constraints from upgrading. To see why use `status --outdated -m`

You can also download the manifest file and the project file.

+Info Packages marked with ⌃ and ⌅ have new versions available. Those with ⌃ may be upgradable, but those with ⌅ are restricted by compatibility constraints from upgrading. To see why use `status --outdated -m`

You can also download the manifest file and the project file.

diff --git a/previews/PR255/methods/collocation_loss/index.html b/previews/PR255/methods/collocation_loss/index.html index 550c644..23d1ba7 100644 --- a/previews/PR255/methods/collocation_loss/index.html +++ b/previews/PR255/methods/collocation_loss/index.html @@ -2,4 +2,4 @@ Two Stage method (Non-Parametric Collocation) · DiffEqParamEstim.jl
GitHub

Two Stage method (Non-Parametric Collocation)

The two-stage method is a collocation method for estimating parameters without requiring repeated solving of the differential equation. It does so by determining a smoothed estimated trajectory of the data (local quadratic polynomial fit by least squares) and optimizing the derivative function and the data's timepoints to match the derivatives of the smoothed trajectory. This method has less accuracy than other methods but is much faster, and is a good method to try first to get in the general “good parameter” region, to then finish using one of the other methods.

function two_stage_objective(prob::DEProblem, tpoints, data, adtype = SciMLBase.NoAD(), ;
         kernel = :Epanechnikov,
         loss_func = L2DistLoss)
-end
+end diff --git a/previews/PR255/methods/optimization_based_methods/index.html b/previews/PR255/methods/optimization_based_methods/index.html index b6f7409..7b9ed4b 100644 --- a/previews/PR255/methods/optimization_based_methods/index.html +++ b/previews/PR255/methods/optimization_based_methods/index.html @@ -29,4 +29,4 @@ build_loss_objective(prob, alg, loss, Optimization.AutoForwardDiff()) multiple_shooting_objective(prob, alg, loss, Optimization.AutoForwardDiff())

The Problem Generator Function

The argument prob_generator allows one to specify a function for generating new problems from a given parameter set. By default, this just builds a new problem which fixes the element types in a way that's autodifferentiation compatible and adds the new parameter vector p. For example, the code for this is:

prob_generator = (prob, p) -> remake(prob, u0 = convert.(eltype(p), prob.u0), p = p)

Then the new problem with these new values is returned.

One can use this to change the meaning of the parameters using this function. For example, if one instead wanted to optimize the initial conditions for a function without parameters, you could change this to:

prob_generator = (prob, p) -> remake(prob.f, u0 = p)

which simply uses p as the initial condition in the initial value problem.

Using the Objectives for MAP estimates

You can also add a prior option to build_loss_objective and multiple_shooting_objective that essentially turns it into MAP by multiplying the log-likelihood (the cost) by the prior. The option is available as the keyword argument priors, it can take in either an array of univariate distributions for each of the parameters or a multivariate distribution.

ms_obj = multiple_shooting_objective(ms_prob, Tsit5(), L2Loss(t, data); priors = priors,
     discontinuity_weight = 1.0, abstol = 1e-12,
-    reltol = 1e-12)
+ reltol = 1e-12) diff --git a/previews/PR255/methods/recommended_methods/index.html b/previews/PR255/methods/recommended_methods/index.html index 0d0278b..bd875c6 100644 --- a/previews/PR255/methods/recommended_methods/index.html +++ b/previews/PR255/methods/recommended_methods/index.html @@ -1,2 +1,2 @@ -Recommended Methods · DiffEqParamEstim.jl
GitHub

Recommended Methods

The recommended method is to use build_loss_objective with the optimizer of your choice. This method can thus be paired with global optimizers from packages like BlackBoxOptim.jl or NLopt.jl which can be much less prone to finding local minima than local optimization methods. Also, it allows the user to define the cost function in the way they choose as a function loss(sol). This package can thus fit using any cost function on the solution, making it applicable to fitting non-temporal data and other types of problems. Also, build_loss_objective works for all the DEProblem types, allowing it to optimize parameters on ODEs, SDEs, DDEs, DAEs, etc.

However, this method requires repeated solution of the differential equation. If the data is temporal data, the most efficient method is the two_stage_objective which does not require repeated solutions but is not as accurate. Usage of the two_stage_objective should have a post-processing step which refines using a method like build_loss_objective.

+Recommended Methods · DiffEqParamEstim.jl
GitHub

Recommended Methods

The recommended method is to use build_loss_objective with the optimizer of your choice. This method can thus be paired with global optimizers from packages like BlackBoxOptim.jl or NLopt.jl which can be much less prone to finding local minima than local optimization methods. Also, it allows the user to define the cost function in the way they choose as a function loss(sol). This package can thus fit using any cost function on the solution, making it applicable to fitting non-temporal data and other types of problems. Also, build_loss_objective works for all the DEProblem types, allowing it to optimize parameters on ODEs, SDEs, DDEs, DAEs, etc.

However, this method requires repeated solution of the differential equation. If the data is temporal data, the most efficient method is the two_stage_objective which does not require repeated solutions but is not as accurate. Usage of the two_stage_objective should have a post-processing step which refines using a method like build_loss_objective.

diff --git a/previews/PR255/tutorials/ensemble/71dd5f68.svg b/previews/PR255/tutorials/ensemble/5df84959.svg similarity index 98% rename from previews/PR255/tutorials/ensemble/71dd5f68.svg rename to previews/PR255/tutorials/ensemble/5df84959.svg index 4e58fe5..f63d07b 100644 --- a/previews/PR255/tutorials/ensemble/71dd5f68.svg +++ b/previews/PR255/tutorials/ensemble/5df84959.svg @@ -1,64 +1,64 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/previews/PR255/tutorials/ensemble/index.html b/previews/PR255/tutorials/ensemble/index.html index c6695dd..840f364 100644 --- a/previews/PR255/tutorials/ensemble/index.html +++ b/previews/PR255/tutorials/ensemble/index.html @@ -33,7 +33,7 @@ end enprob = EnsembleProblem(prob, prob_func = prob_func)
EnsembleProblem with problem ODEProblem

We can check this does what we want by solving it:

# Check above does what we want
 sim = solve(enprob, Tsit5(), trajectories = N)
-plot(sim)
Example block output

trajectories=N means “run N times”, and each time it runs the problem returned by the prob_func, which is always the same problem but with the ith initial condition.

Now let's generate a dataset from that. Let's get data points at every t=0.1 using saveat, and then convert the solution into an array.

# Generate a dataset from these runs
+plot(sim)
Example block output

trajectories=N means “run N times”, and each time it runs the problem returned by the prob_func, which is always the same problem but with the ith initial condition.

Now let's generate a dataset from that. Let's get data points at every t=0.1 using saveat, and then convert the solution into an array.

# Generate a dataset from these runs
 data_times = 0.0:0.1:10.0
 sim = solve(enprob, Tsit5(), trajectories = N, saveat = data_times)
 data = Array(sim)
2×101×10 Array{Float64, 3}:
@@ -114,4 +114,4 @@
  1.0012380201889803
result
retcode: Success
 u: 2-element Vector{Float64}:
  1.5007432843504753
- 1.0012380201889803

if you suspect error is the problem. However, if you're having problems it's most likely not the ODE solver tolerance and mostly because parameter inference is a very hard optimization problem.

+ 1.0012380201889803

if you suspect error is the problem. However, if you're having problems it's most likely not the ODE solver tolerance and mostly because parameter inference is a very hard optimization problem.

diff --git a/previews/PR255/tutorials/generalized_likelihood/index.html b/previews/PR255/tutorials/generalized_likelihood/index.html index 5bd915d..66dcf6a 100644 --- a/previews/PR255/tutorials/generalized_likelihood/index.html +++ b/previews/PR255/tutorials/generalized_likelihood/index.html @@ -59,33 +59,33 @@ end aggregate_data = convert(Array, VectorOfArray([generate_data(sol, t) for i in 1:100]))
2×200×100 Array{Float64, 3}:
 [:, :, 1] =
- 1.01044   1.01337   1.07125   1.10158   …  0.993435  1.00388  1.02639
- 0.995789  0.895965  0.812485  0.740351     1.11517   1.00868  0.897977
+ 0.985469  1.03376   1.0703    1.10204   …  0.98504  1.00742  1.02871
+ 0.998483  0.904478  0.827411  0.736858     1.09236  1.00663  0.907831
 
 [:, :, 2] =
- 0.990734  1.02005   1.07041   1.0938    …  0.96803  1.0215    1.01404
- 1.00216   0.905863  0.821143  0.730541     1.11059  0.994963  0.923292
+ 0.987888  1.02427   1.07428   1.11605   …  1.00596  0.996985  1.03311
+ 0.988412  0.910128  0.815703  0.727621     1.09021  1.00251   0.917243
 
 [:, :, 3] =
- 1.01306  1.01916   1.06881   1.09624   …  0.97406  1.00676   1.03541
- 1.00137  0.908959  0.834093  0.766254     1.12894  0.983946  0.902838
+ 0.993197  1.01852   1.05668   1.11537   …  0.974558  1.00872  1.02388
+ 1.01401   0.918949  0.822691  0.750467     1.10279   1.01058  0.906354
 
 ;;; … 
 
 [:, :, 98] =
- 1.02104   1.02408   1.06503   1.10175  …  0.978314  1.00726  1.04105
- 0.989929  0.910275  0.817341  0.75297     1.09782   1.01881  0.907543
+ 1.00098  1.0608    1.06559   1.10904   …  0.994176  1.00535   1.03126
+ 1.00135  0.900048  0.813974  0.734276     1.10396   0.989067  0.923723
 
 [:, :, 99] =
- 0.998721  1.02412  1.05138   1.10349   …  0.981765  1.00243   1.01262
- 1.00063   0.91419  0.802698  0.741612     1.09697   0.993304  0.915651
+ 1.00559  1.03912   1.07992   1.09014   …  0.977933  1.00131   1.0478
+ 1.02754  0.898959  0.808503  0.752867     1.0993    0.993573  0.900708
 
 [:, :, 100] =
- 1.00089  1.04925  1.09179   1.09719   …  0.983729  1.00562   1.03783
- 1.00488  0.90369  0.809864  0.734865     1.1051    0.989528  0.92366

here, with t we measure the solution at 200 evenly spaced points. Thus, aggregate_data is a 2x200x100 matrix where aggregate_data[i,j,k] is the ith component at time j of the kth dataset. What we first want to do is get a matrix of distributions where distributions[i,j] is the likelihood of component i at take j. We can do this via fit_mle on a chosen distributional form. For simplicity, we choose the Normal distribution. aggregate_data[i,j,:] is the array of points at the given component and time, and thus we find the distribution parameters which fits best at each time point via:

using Distributions
+ 1.00523  1.01572   1.05502   1.10446   …  0.987696  1.00406  1.04943
+ 1.00317  0.905559  0.818562  0.728218     1.10046   1.01512  0.917458

here, with t we measure the solution at 200 evenly spaced points. Thus, aggregate_data is a 2x200x100 matrix where aggregate_data[i,j,k] is the ith component at time j of the kth dataset. What we first want to do is get a matrix of distributions where distributions[i,j] is the likelihood of component i at take j. We can do this via fit_mle on a chosen distributional form. For simplicity, we choose the Normal distribution. aggregate_data[i,j,:] is the array of points at the given component and time, and thus we find the distribution parameters which fits best at each time point via:

using Distributions
 distributions = [fit_mle(Normal, aggregate_data[i, j, :]) for i in 1:2, j in 1:200]
2×200 Matrix{Distributions.Normal{Float64}}:
- Distributions.Normal{Float64}(μ=1.00063, σ=0.00927246)  …  Distributions.Normal{Float64}(μ=1.03251, σ=0.0107988)
- Distributions.Normal{Float64}(μ=1.00052, σ=0.0115162)      Distributions.Normal{Float64}(μ=0.906705, σ=0.00911113)

Notice for example that we have:

distributions[1, 1]
Distributions.Normal{Float64}(μ=1.000630367714539, σ=0.009272457853681223)

that is, it fits the distribution to have its mean just about where our original solution was, and the variance is about how much noise we added to the dataset. This is a good check to see that the distributions we are trying to fit our parameters to makes sense.

Note that in this case the Normal distribution was a good choice, and often it's a nice go-to choice, but one should experiment with other choices of distributions as well. For example, a TDist can be an interesting way to incorporate robustness to outliers since low degrees of free T-distributions act like Normal distributions but with longer tails (though fit_mle does not work with a T-distribution, you can get the means/variances and build appropriate distribution objects yourself).

Once we have the matrix of distributions, we can build the objective function corresponding to that distribution fit:

obj = build_loss_objective(prob1, Tsit5(), LogLikeLoss(t, distributions),
+ Distributions.Normal{Float64}(μ=1.00065, σ=0.00940338)  …  Distributions.Normal{Float64}(μ=1.03253, σ=0.00991387)
+ Distributions.Normal{Float64}(μ=1.00006, σ=0.0105478)      Distributions.Normal{Float64}(μ=0.907443, σ=0.010799)

Notice for example that we have:

distributions[1, 1]
Distributions.Normal{Float64}(μ=1.000652480677555, σ=0.009403379415734027)

that is, it fits the distribution to have its mean just about where our original solution was, and the variance is about how much noise we added to the dataset. This is a good check to see that the distributions we are trying to fit our parameters to makes sense.

Note that in this case the Normal distribution was a good choice, and often it's a nice go-to choice, but one should experiment with other choices of distributions as well. For example, a TDist can be an interesting way to incorporate robustness to outliers since low degrees of free T-distributions act like Normal distributions but with longer tails (though fit_mle does not work with a T-distribution, you can get the means/variances and build appropriate distribution objects yourself).

Once we have the matrix of distributions, we can build the objective function corresponding to that distribution fit:

obj = build_loss_objective(prob1, Tsit5(), LogLikeLoss(t, distributions),
     maxiters = 10000, verbose = false)
(::SciMLBase.OptimizationFunction{true, SciMLBase.NoAD, DiffEqParamEstim.var"#29#30"{Nothing, typeof(DiffEqParamEstim.STANDARD_PROB_GENERATOR), Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, @NamedTuple{maxiters::Int64, verbose::Bool}}, SciMLBase.ODEProblem{Vector{Float64}, Tuple{Float64, Float64}, true, Vector{Float64}, SciMLBase.ODEFunction{true, SciMLBase.AutoSpecialize, Main.var"#1#2", LinearAlgebra.UniformScaling{Bool}, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing, Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, @NamedTuple{}}, SciMLBase.StandardODEProblem}, OrdinaryDiffEq.Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, LogLikeLoss{Vector{Float64}, Matrix{Distributions.Normal{Float64}}}, Nothing, Tuple{}}, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED_NO_TIME), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing}) (generic function with 1 method)

First, let's use the objective function to plot the likelihood landscape:

using Plots;
 plotly();
 prange = 0.5:0.1:5.0
@@ -97,4 +97,4 @@
 optprob = OptimizationProblem(obj, [2.0, 2.0], lb = first.(bound1), ub = last.(bound1))
OptimizationProblem. In-place: true
 u0: 2-element Vector{Float64}:
  2.0
- 2.0

This shows that it found the true parameters as the best fit to the likelihood.

+ 2.0

This shows that it found the true parameters as the best fit to the likelihood.

diff --git a/previews/PR255/tutorials/global_optimization/index.html b/previews/PR255/tutorials/global_optimization/index.html index e2fbb86..adc22bb 100644 --- a/previews/PR255/tutorials/global_optimization/index.html +++ b/previews/PR255/tutorials/global_optimization/index.html @@ -21,20 +21,20 @@ optprob = Optimization.OptimizationProblem(obj, [1.3]) res = solve(optprob, opt)
retcode: Success
 u: 1-element Vector{Float64}:
- 1.4999858084029238

For a modified evolutionary algorithm, we can use:

opt = Opt(:GN_ESCH, 1)
+ 1.5000577870176366

For a modified evolutionary algorithm, we can use:

opt = Opt(:GN_ESCH, 1)
 lower_bounds!(opt, [0.0])
 upper_bounds!(opt, [5.0])
 xtol_rel!(opt, 1e-3)
 maxeval!(opt, 100000)
 res = solve(optprob, opt)
retcode: MaxIters
 u: 1-element Vector{Float64}:
- 1.5016035440804236

We can even use things like the Improved Stochastic Ranking Evolution Strategy (and add constraints if needed). Let's use this through OptimizationMOI:

optprob = Optimization.OptimizationProblem(obj, [0.2], lb = [-1.0], ub = [5.0])
+ 1.4997128897225644

We can even use things like the Improved Stochastic Ranking Evolution Strategy (and add constraints if needed). Let's use this through OptimizationMOI:

optprob = Optimization.OptimizationProblem(obj, [0.2], lb = [-1.0], ub = [5.0])
 res = solve(optprob,
     OptimizationMOI.MOI.OptimizerWithAttributes(NLopt.Optimizer,
         "algorithm" => :GN_ISRES,
         "xtol_rel" => 1e-3,
         "maxeval" => 10000))
retcode: Success
 u: 1-element Vector{Float64}:
- 1.5019376992339233

which is very robust to the initial condition. We can also directly use the NLopt interface as below. The fastest result comes from the following algorithm choice:

opt = Opt(:LN_BOBYQA, 1)
+ 1.5085157995254712

which is very robust to the initial condition. We can also directly use the NLopt interface as below. The fastest result comes from the following algorithm choice:

opt = Opt(:LN_BOBYQA, 1)
 min_objective!(opt, obj)
-(minf,minx,ret) = NLopt.optimize(opt,[1.3])

For more information, see the NLopt documentation for more details. And give IPOPT or MOSEK a try!

+(minf,minx,ret) = NLopt.optimize(opt,[1.3])

For more information, see the NLopt documentation for more details. And give IPOPT or MOSEK a try!

diff --git a/previews/PR255/tutorials/stochastic_evaluations/index.html b/previews/PR255/tutorials/stochastic_evaluations/index.html index b8f972e..22d60c9 100644 --- a/previews/PR255/tutorials/stochastic_evaluations/index.html +++ b/previews/PR255/tutorials/stochastic_evaluations/index.html @@ -29,36 +29,36 @@ 0.03571484690132414 0.04105769139296714 ⋮ - 9.844792025774144 - 9.865441232221434 - 9.886429927592697 - 9.907774299801213 - 9.929491081134323 - 9.951597402039063 - 9.974110645282579 - 9.997048223587646 + 9.844792377230828 + 9.86544146942802 + 9.886430048100591 + 9.90777430969068 + 9.929490967026235 + 9.951597168113992 + 9.974110291330684 + 9.997047738654722 10.0 u: 565-element Vector{Vector{Float64}}: [1.0, 1.0] - [1.0060115442765585, 0.9768818910050351] - [1.0072505794076894, 0.9723307871231138] - [1.008659115546982, 0.9672394842384375] - [1.0102621126120095, 0.9615478601411686] - [1.0120889322967688, 0.9551903525393541] - [1.0141736952022202, 0.9480952482038271] - [1.0165564182612838, 0.9401852926783991] - [1.0192841085155877, 0.9313766487186113] - [1.022412339550014, 0.9215800408207138] + [1.0060115563810934, 0.976881753881167] + [1.007250538318914, 0.9723306317598953] + [1.0086590041191632, 0.9672393453477035] + [1.0102621112803605, 0.9615477873249706] + [1.0120888034719602, 0.9551902997110864] + [1.0141735119641273, 0.9480953883629333] + [1.0165561413706403, 0.9401853574186084] + [1.0192837475099683, 0.9313767981036161] + [1.022412054132584, 0.9215800997689656] ⋮ - [0.9579214408371667, 1.2455792577473594] - [0.9634887677460341, 1.19422655238882] - [0.9702102074930504, 1.1443467019363838] - [0.9781195258190424, 1.0959294155067294] - [0.9872549054285944, 1.0489637441216524] - [0.9976608010157338, 1.0034392498823879] - [1.0093882545284805, 0.9593463501707781] - [1.022495653051659, 0.9166758164272144] - [1.0242657754422595, 0.9113430754091316]

Now let's generate a dataset from 10,000 solutions of the SDE

using RecursiveArrayTools # for VectorOfArray
+ [0.9579151702253104, 1.2455880887702]
+ [0.96348241868128, 1.1942348751649685]
+ [0.9702042263000273, 1.1443550084677003]
+ [0.978113284433389, 1.095937167700052]
+ [0.987248231291624, 1.048971056606304]
+ [0.9976537014171848, 1.0034465786748226]
+ [1.0093809708392778, 0.9593533618935145]
+ [1.0224877665482277, 0.91668258107838]
+ [1.024258202481948, 0.9113488583163608]

Now let's generate a dataset from 10,000 solutions of the SDE

using RecursiveArrayTools # for VectorOfArray
 t = collect(range(0, stop = 10, length = 200))
 function generate_data(t)
     sol = solve(prob, SRIW1())
@@ -67,40 +67,40 @@
 end
 aggregate_data = convert(Array, VectorOfArray([generate_data(t) for i in 1:10000]))
2×200×10000 Array{Float64, 3}:
 [:, :, 1] =
- 0.98493  1.03575   1.04048   1.10162   …  0.978762  0.987066  1.01471
- 1.0064   0.899053  0.826345  0.756438     1.10089   0.998392  0.926048
+ 0.992117  1.0183    1.07513   1.1082    …  0.983823  0.979353  1.02134
+ 1.00673   0.897654  0.810547  0.741044     1.13831   1.01465   0.911872
 
 [:, :, 2] =
- 1.01467   1.01874   1.07038   1.10101   …  0.97108  0.991031  1.04814
- 0.998377  0.897365  0.801722  0.759783     1.1145   1.01277   0.903992
+ 0.997364  1.0479    1.06867   1.09292   …  0.976889  1.00025   1.03111
+ 1.00963   0.895659  0.818484  0.740073     1.11793   0.982554  0.902846
 
 [:, :, 3] =
- 0.987335  1.02914   1.07015   1.08503   …  0.988006  0.998503  1.04518
- 0.999764  0.886994  0.803038  0.752725     1.11086   1.01242   0.927237
+ 0.996924  1.04478   1.08273   1.10528  …  0.957586  0.993258  1.0152
+ 0.998621  0.927975  0.822144  0.74092     1.11335   1.0158    0.899379
 
 ;;; … 
 
 [:, :, 9998] =
- 0.996597  1.02641   1.05742   1.10713   …  0.977379  0.992143  1.01784
- 1.00497   0.915137  0.808189  0.762448     1.10298   0.990452  0.923007
+ 0.999043  1.03916   1.06464   1.10584   …  0.971492  0.981449  1.02904
+ 0.999381  0.894432  0.838006  0.737231     1.09479   1.00463   0.918544
 
 [:, :, 9999] =
- 1.01405  1.01747   1.048     1.0991   …  0.966534  1.01008  1.02498
- 1.00126  0.893466  0.807735  0.75056     1.11794   1.00728  0.901078
+ 1.00879  1.01711   1.04228   1.09383   …  0.985817  0.965072  1.03121
+ 1.00164  0.889905  0.812752  0.757066     1.11249   0.997164  0.89963
 
 [:, :, 10000] =
- 0.997744  1.04964   1.06309   1.10447   …  0.981747  1.00402   1.02369
- 1.00319   0.906504  0.833853  0.749129     1.10119   0.999901  0.905394

Now let's estimate the parameters. Instead of using single runs from the SDE, we will use a EnsembleProblem. This means that it will solve the SDE N times to come up with an approximate probability distribution at each time point and use that in the likelihood estimate.

monte_prob = EnsembleProblem(prob)
EnsembleProblem with problem SDEProblem

We use Optim.jl for optimization below

obj = build_loss_objective(monte_prob, SOSRI(), L2Loss(t, aggregate_data),
+ 0.990254  1.02585   1.06607   1.09433   …  0.972687  0.98957  1.03109
+ 1.0157    0.918528  0.807686  0.737548     1.12438   1.01358  0.911261

Now let's estimate the parameters. Instead of using single runs from the SDE, we will use a EnsembleProblem. This means that it will solve the SDE N times to come up with an approximate probability distribution at each time point and use that in the likelihood estimate.

monte_prob = EnsembleProblem(prob)
EnsembleProblem with problem SDEProblem

We use Optim.jl for optimization below

obj = build_loss_objective(monte_prob, SOSRI(), L2Loss(t, aggregate_data),
     Optimization.AutoForwardDiff(),
     maxiters = 10000, verbose = false, trajectories = 1000)
 optprob = Optimization.OptimizationProblem(obj, [1.0, 0.5])
 result = solve(optprob, Optim.BFGS())
retcode: Success
 u: 2-element Vector{Float64}:
- 3.703152402378408
- 5.894698221199128

Parameter Estimation in case of SDE's with a regular L2Loss can have poor accuracy due to only fitting against the mean properties as mentioned in First Differencing.

result.original
 * Status: success
+ 3.702164405323527
+ 5.8938658022997625

Parameter Estimation in case of SDE's with a regular L2Loss can have poor accuracy due to only fitting against the mean properties as mentioned in First Differencing.

result.original
 * Status: success
 
  * Candidate solution
-    Final objective value:     2.523073e+03
+    Final objective value:     2.521710e+03
 
  * Found with
     Algorithm:     BFGS
@@ -113,7 +113,7 @@
     |g(x)|                 = 0.00e+00 ≤ 1.0e-08
 
  * Work counters
-    Seconds run:   225  (vs limit Inf)
+    Seconds run:   226  (vs limit Inf)
     Iterations:    1
     f(x) calls:    5
     ∇f(x) calls:   5
@@ -126,7 +126,7 @@
 result.original
 * Status: success
 
  * Candidate solution
-    Final objective value:     1.255862e+03
+    Final objective value:     1.254316e+03
 
  * Found with
     Algorithm:     BFGS
@@ -135,7 +135,7 @@
     |x - x'|               = 8.31e+00 ≰ 0.0e+00
     |x - x'|/|x'|          = 9.43e-01 ≰ 0.0e+00
     |f(x) - f(x')|         = 5.40e+02 ≰ 0.0e+00
-    |f(x) - f(x')|/|f(x')| = 4.30e-01 ≰ 0.0e+00
+    |f(x) - f(x')|/|f(x')| = 4.31e-01 ≰ 0.0e+00
     |g(x)|                 = 0.00e+00 ≤ 1.0e-08
 
  * Work counters
@@ -143,4 +143,4 @@
     Iterations:    1
     f(x) calls:    7
     ∇f(x) calls:   7
-

Here, we see that we successfully recovered the drift parameter, and got close to the original noise parameter after searching a two-orders-of-magnitude range.

println(result.u)
[5.016909833294367, 8.806894007990875]
+

Here, we see that we successfully recovered the drift parameter, and got close to the original noise parameter after searching a two-orders-of-magnitude range.

println(result.u)
[5.016000463910598, 8.805777426924722]