Merge branch 'main' into gd/shared_hessian

.github/workflows/CI.yml

@@ -27,10 +27,13 @@ jobs:
           - '1'
         group:
           - Core
+          - Benchmarks
           - NLPModels
         exclude:
           - version: '1.6'
             group: NLPModels
+          - version: '1.6'
+            group: Benchmarks
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
@@ -44,7 +47,7 @@ jobs:
           JULIA_SCT_TEST_GROUP: ${{ matrix.group }}
       - uses: julia-actions/julia-processcoverage@v1
         with:
-          directories: src,ext,test      
+          directories: src,ext,test,benchmark
       - uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -7,4 +7,5 @@
 /docs/build/
 /docs/src/index.md
 /benchmark/Manifest.toml
+/benchmark/SparseConnectivityTracerBenchmarks/Manifest.toml
 /references/*

CHANGELOG.md

@@ -0,0 +1,44 @@
+# SparseConnectivityTracer.jl
+
+## Version `v0.6.0`
+* ![BREAKING][badge-breaking] Remove `ConnectivityTracer` ([#140][pr-140])
+* ![BREAKING][badge-breaking] Remove legacy interface ([#140][pr-140])
+    * instead of `jacobian_pattern(f, x)`, use `jacobian_sparsity(f, x, TracerSparsityDetector())`
+    * instead of `hessian_pattern(f, x)`, use `hessian_sparsity(f, x, TracerSparsityDetector())`
+    * instead of `local_jacobian_pattern(f, x)`, use `jacobian_sparsity(f, x, TracerLocalSparsityDetector())`
+    * instead of `local_hessian_pattern(f, x)`, use `hessian_sparsity(f, x, TracerLocalSparsityDetector())`
+* ![Bugfix][badge-bugfix] Remove overloads on `similar` to reduce amount of invalidations  ([#132][pr-132])
+* ![Enhancement][badge-enhancement] Add array overloads ([#131][pr-131])
+* ![Enhancement][badge-enhancement] Generalize sparsity pattern representations ([#139][pr-139], [#119][pr-119])
+* ![Enhancement][badge-enhancement] Reduce allocations of new tracers ([#128][pr-128])
+* ![Enhancement][badge-enhancement] Reduce compile times ([#119][pr-119])
+
+[pr-140]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/140
+[pr-139]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/139
+[pr-132]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/132
+[pr-131]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/131
+[pr-128]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/128
+[pr-126]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/126
+[pr-119]: https://github.com/adrhill/SparseConnectivityTracer.jl/pull/119
+
+<!--
+# Badges
+![BREAKING][badge-breaking]
+![Deprecation][badge-deprecation]
+![Feature][badge-feature]
+![Enhancement][badge-enhancement]
+![Bugfix][badge-bugfix]
+![Experimental][badge-experimental]
+![Maintenance][badge-maintenance]
+![Documentation][badge-docs]
+-->
+
+[badge-breaking]: https://img.shields.io/badge/BREAKING-red.svg
+[badge-deprecation]: https://img.shields.io/badge/deprecation-orange.svg
+[badge-feature]: https://img.shields.io/badge/feature-green.svg
+[badge-enhancement]: https://img.shields.io/badge/enhancement-blue.svg
+[badge-bugfix]: https://img.shields.io/badge/bugfix-purple.svg
+[badge-security]: https://img.shields.io/badge/security-black.svg
+[badge-experimental]: https://img.shields.io/badge/experimental-lightgrey.svg
+[badge-maintenance]: https://img.shields.io/badge/maintenance-gray.svg
+[badge-docs]: https://img.shields.io/badge/docs-orange.svg

Project.toml

@@ -7,6 +7,8 @@ version = "0.6.0-DEV"
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -25,7 +27,10 @@ SparseConnectivityTracerSpecialFunctionsExt = "SpecialFunctions"
 ADTypes = "1"
 Compat = "3,4"
 DocStringExtensions = "0.9"
+FillArrays = "1"
+LinearAlgebra = "<0.0.1, 1"
 NNlib = "0.8, 0.9"
+Random = "<0.0.1, 1"
 Requires = "1.3"
 SparseArrays = "<0.0.1, 1"
 SpecialFunctions = "2.4"

README.md

@@ -8,9 +8,6 @@
 
 Fast Jacobian and Hessian sparsity detection via operator-overloading.
 
-> [!WARNING]
-> This package is in early development. Expect frequent breaking changes and refer to the stable documentation.
-
 ## Installation 
 To install this package, open the Julia REPL and run 
 
@@ -21,17 +18,19 @@ julia> ]add SparseConnectivityTracer
 ## Examples
 ### Jacobian
 
-For functions `y = f(x)` and `f!(y, x)`, the sparsity pattern of the Jacobian of $f$ can be obtained
-by computing a single forward-pass through `f`:
+For functions `y = f(x)` and `f!(y, x)`, the sparsity pattern of the Jacobian can be obtained
+by computing a single forward-pass through the function:
 
 ```julia-repl
 julia> using SparseConnectivityTracer
 
+julia> detector = TracerSparsityDetector();
+
 julia> x = rand(3);
 
 julia> f(x) = [x[1]^2, 2 * x[1] * x[2]^2, sin(x[3])];
 
-julia> jacobian_pattern(f, x)
+julia> jacobian_sparsity(f, x, detector)
 3×3 SparseArrays.SparseMatrixCSC{Bool, Int64} with 4 stored entries:
  1  ⋅  ⋅
  1  1  ⋅
@@ -43,11 +42,13 @@ As a larger example, let's compute the sparsity pattern from a convolutional lay
 ```julia-repl
 julia> using SparseConnectivityTracer, Flux
 
+julia> detector = TracerSparsityDetector();
+
 julia> x = rand(28, 28, 3, 1);
 
 julia> layer = Conv((3, 3), 3 => 2);
 
-julia> jacobian_pattern(layer, x)
+julia> jacobian_sparsity(layer, x, detector)
 1352×2352 SparseArrays.SparseMatrixCSC{Bool, Int64} with 36504 stored entries:
 ⎡⠙⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠻⣷⣤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠻⣷⣄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⎤
 ⎢⠀⠀⠙⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠙⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠻⣷⣤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⎥
@@ -64,7 +65,7 @@ julia> jacobian_pattern(layer, x)
 ⎣⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠙⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠛⢿⣦⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠻⣷⣄⎦
 ```
 
-The type of index set `S` that is internally used to keep track of connectivity can be specified via `jacobian_pattern(f, x, S)`, defaulting to `BitSet`. 
+The type of index set `S` that is internally used to keep track of connectivity can be specified via `jacobian_sparsity(f, x, S)`, defaulting to `BitSet`. 
 For high-dimensional functions, `Set{Int64}` can be more efficient .
 
 ### Hessian
@@ -77,7 +78,7 @@ julia> x = rand(5);
 
 julia> f(x) = x[1] + x[2]*x[3] + 1/x[4] + 1*x[5];
 
-julia> hessian_pattern(f, x)
+julia> hessian_sparsity(f, x, detector)
 5×5 SparseArrays.SparseMatrixCSC{Bool, Int64} with 3 stored entries:
  ⋅  ⋅  ⋅  ⋅  ⋅
  ⋅  ⋅  1  ⋅  ⋅
@@ -87,7 +88,7 @@ julia> hessian_pattern(f, x)
 
 julia> g(x) = f(x) + x[2]^x[5];
 
-julia> hessian_pattern(g, x)
+julia> hessian_sparsity(g, x, detector)
 5×5 SparseArrays.SparseMatrixCSC{Bool, Int64} with 7 stored entries:
  ⋅  ⋅  ⋅  ⋅  ⋅
  ⋅  1  1  ⋅  1
@@ -100,30 +101,40 @@ For more detailled examples, take a look at the [documentation](https://adrianhi
 
 ### Local tracing
 
-The functions `jacobian_pattern`, `hessian_pattern` and `connectivity_pattern` return conservative sparsity patterns over the entire input domain of `x`. 
-They are not compatible with functions that require information about the primal values of a computation (e.g. `iszero`, `>`, `==`).
+`TracerSparsityDetector` returns conservative sparsity patterns over the entire input domain of `x`. 
+It is not compatible with functions that require information about the primal values of a computation (e.g. `iszero`, `>`, `==`).
 
-To compute a less conservative sparsity pattern at an input point `x`, use `local_jacobian_pattern`, `local_hessian_pattern` and `local_connectivity_pattern` instead.
-Note that these patterns depend on the input `x`:
+To compute a less conservative sparsity pattern at an input point `x`, use `TracerLocalSparsityDetector` instead.
+Note that patterns computed with `TracerLocalSparsityDetector` depend on the input `x`:
 
 ```julia-repl
+julia> using SparseConnectivityTracer
+
+julia> detector = TracerLocalSparsityDetector();
+
 julia> f(x) = ifelse(x[2] < x[3], x[1] ^ x[2], x[3] * x[4]);
 
-julia> local_hessian_pattern(f, [1 2 3 4])
+julia> hessian_sparsity(f, [1 2 3 4], detector)
 4×4 SparseArrays.SparseMatrixCSC{Bool, Int64} with 4 stored entries:
  1  1  ⋅  ⋅
  1  1  ⋅  ⋅
  ⋅  ⋅  ⋅  ⋅
  ⋅  ⋅  ⋅  ⋅
 
-julia> local_hessian_pattern(f, [1 3 2 4])
+julia> hessian_sparsity(f, [1 3 2 4], detector)
 4×4 SparseArrays.SparseMatrixCSC{Bool, Int64} with 2 stored entries:
  ⋅  ⋅  ⋅  ⋅
  ⋅  ⋅  ⋅  ⋅
  ⋅  ⋅  ⋅  1
  ⋅  ⋅  1  ⋅
 ```
 
+## ADTypes.jl compatibility
+SparseConnectivityTracer uses [ADTypes.jl](https://github.com/SciML/ADTypes.jl)'s interface for [sparsity detection](https://sciml.github.io/ADTypes.jl/stable/#Sparsity-detector),
+making it compatible with [DifferentiationInterface.jl](https://github.com/gdalle/DifferentiationInterface.jl)'s [sparse automatic differentiation](https://gdalle.github.io/DifferentiationInterface.jl/DifferentiationInterface/stable/tutorial2/) functionality.
+
+In fact, the functions `jacobian_sparsity` and `hessian_sparsity` are re-exported from ADTypes.
+
 ## Related packages
 * [SparseDiffTools.jl](https://github.com/JuliaDiff/SparseDiffTools.jl): automatic sparsity detection via Symbolics.jl and Cassette.jl
 * [SparsityTracing.jl](https://github.com/PALEOtoolkit/SparsityTracing.jl): automatic Jacobian sparsity detection using an algorithm based on SparsLinC by Bischof et al. (1996)

benchmark/Project.toml

@@ -8,6 +8,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6"
 NLPModelsJuMP = "792afdf1-32c1-5681-94e0-d7bf7a5df49e"
 OptimizationProblems = "5049e819-d29b-5fba-b941-0eee7e64c1c6"
+PkgJogger = "10150987-6cc1-4b76-abee-b1c1cbd91c01"
 SimpleDiffEq = "05bca326-078c-5bf0-a5bf-ce7c7982d7fd"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"

benchmark/SparseConnectivityTracerBenchmarks/Project.toml

@@ -0,0 +1,15 @@
+name = "SparseConnectivityTracerBenchmarks"
+uuid = "fb1f6577-eb25-4e27-a243-7f62c22307d7"
+authors = ["Guillaume Dalle", "Adrian Hill"]
+version = "0.1.0"
+
+[deps]
+ADNLPModels = "54578032-b7ea-4c30-94aa-7cbd1cce6c9a"
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6"
+NLPModelsJuMP = "792afdf1-32c1-5681-94e0-d7bf7a5df49e"
+OptimizationProblems = "5049e819-d29b-5fba-b941-0eee7e64c1c6"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"

benchmark/SparseConnectivityTracerBenchmarks/src/SparseConnectivityTracerBenchmarks.jl

@@ -0,0 +1,27 @@
+module SparseConnectivityTracerBenchmarks
+
+module ODE
+    include("brusselator.jl")
+    export Brusselator!, brusselator_2d_loop!
+end
+
+module Optimization
+    using ADTypes: ADTypes
+    using SparseConnectivityTracer
+    import SparseConnectivityTracer as SCT
+
+    using ADNLPModels: ADNLPModels
+    using NLPModels: NLPModels, AbstractNLPModel
+    using NLPModelsJuMP: NLPModelsJuMP
+    using OptimizationProblems: OptimizationProblems
+
+    using LinearAlgebra
+    using SparseArrays
+
+    include("nlpmodels.jl")
+    export optimization_problem_names
+    export compute_jac_sparsity_sct, compute_hess_sparsity_sct
+    export compute_jac_and_hess_sparsity_sct, compute_jac_and_hess_sparsity_and_value_jump
+end
+
+end # module SparseConnectivityTracerBenchmarks

test/definitions/nlpmodels_definitions.jl → benchmark/SparseConnectivityTracerBenchmarks/src/nlpmodels.jl

@@ -1,18 +1,3 @@
-using ADTypes: ADTypes
-using SparseConnectivityTracer
-import SparseConnectivityTracer as SCT
-
-using ADNLPModels: ADNLPModels
-using NLPModels: NLPModels, AbstractNLPModel
-using NLPModelsJuMP: NLPModelsJuMP
-using OptimizationProblems: OptimizationProblems
-
-using Dates: now
-using LinearAlgebra
-using SparseArrays
-
-problem_names() = Symbol.(OptimizationProblems.meta[!, :name])
-
 #=
 Given an optimization problem `min f(x) s.t. c(x) <= 0`, we study
 
@@ -31,6 +16,8 @@ Package ecosystem overview: https://jso.dev/ecosystems/models/
   - OptimizationProblems.PureJuMP: spits out `JuMP.Model`
 =#
 
+optimization_problem_names() = Symbol.(OptimizationProblems.meta[!, :name])
+
 ## SCT
 
 #=

benchmark/bench_jogger.jl

@@ -0,0 +1,37 @@
+using Pkg
+Pkg.develop(; path=joinpath(@__DIR__, "SparseConnectivityTracerBenchmarks"))
+
+using BenchmarkTools
+using SparseConnectivityTracer
+using SparseConnectivityTracer: GradientTracer, HessianTracer
+using SparseConnectivityTracer: IndexSetGradientPattern, IndexSetHessianPattern
+using SparseConnectivityTracer: DuplicateVector, SortedVector, RecursiveSet
+
+SET_TYPES = (BitSet, Set{Int}, DuplicateVector{Int}, RecursiveSet{Int}, SortedVector{Int})
+
+include("jacobian.jl")
+include("hessian.jl")
+include("nlpmodels.jl")
+
+suite = BenchmarkGroup()
+
+suite["OptimizationProblems"] = optbench([:britgas])
+
+for S1 in SET_TYPES
+    S2 = Set{Tuple{Int,Int}}
+
+    PG = IndexSetGradientPattern{Int,S1}
+    PH = IndexSetHessianPattern{Int,S1,S2}
+
+    G = GradientTracer{PG}
+    H = HessianTracer{PH}
+
+    suite["Jacobian"]["Global"][nameof(S1)] = jacbench(TracerSparsityDetector(G, H))
+    suite["Jacobian"]["Local"][nameof(S1)] = jacbench(TracerLocalSparsityDetector(G, H))
+    suite["Hessian"]["Global"][(nameof(S1), nameof(S2))] = hessbench(
+        TracerSparsityDetector(G, H)
+    )
+    suite["Hessian"]["Local"][(nameof(S1), nameof(S2))] = hessbench(
+        TracerLocalSparsityDetector(G, H)
+    )
+end

benchmark/benchmarks.jl

@@ -1,34 +1,5 @@
-using BenchmarkTools
+using PkgJogger
 using SparseConnectivityTracer
-using SparseConnectivityTracer: GradientTracer, HessianTracer
-using SparseConnectivityTracer: DuplicateVector, SortedVector, RecursiveSet
-
-SET_TYPES = (BitSet, Set{Int}, DuplicateVector{Int}, RecursiveSet{Int}, SortedVector{Int})
-
-include("jacobian.jl")
-include("hessian.jl")
-include("nlpmodels.jl")
-
-SUITE = BenchmarkGroup()
-
-SUITE["OptimizationProblems"] = optbench([:britgas])
-
-for S1 in SET_TYPES
-    S2 = Set{Tuple{Int,Int}}
-
-    G = GradientTracer{S1}
-    H = HessianTracer{S1,S2,false}
-    H_shared = HessianTracer{S1,S2,true}
-
-    SUITE["Jacobian"]["Global"][nameof(S1)] = jacbench(TracerSparsityDetector(G, H))
-    SUITE["Jacobian"]["Local"][nameof(S1)] = jacbench(TracerLocalSparsityDetector(G, H))
-    SUITE["Hessian"]["Global"][(nameof(S1), nameof(S2))] = hessbench(
-        TracerSparsityDetector(G, H)
-    )
-    SUITE["Hessian"]["Global (shared)"][(nameof(S1), nameof(S2))] = hessbench(
-        TracerSparsityDetector(G, H_shared)
-    )
-    SUITE["Hessian"]["Local"][(nameof(S1), nameof(S2))] = hessbench(
-        TracerLocalSparsityDetector(G, H)
-    )
-end
+# Use PkgJogger.@jog to create the JogSparseConnectivityTracer module
+@jog SparseConnectivityTracer
+SUITE = JogSparseConnectivityTracer.suite()

benchmark/jacobian.jl

@@ -2,6 +2,7 @@ using BenchmarkTools
 
 using ADTypes: AbstractSparsityDetector, jacobian_sparsity
 using SparseConnectivityTracer
+using SparseConnectivityTracerBenchmarks.ODE: Brusselator!, brusselator_2d_loop!
 
 using SparseArrays: sprand
 using SimpleDiffEq: ODEProblem, solve, SimpleEuler
@@ -49,8 +50,6 @@ end
 
 ## Brusselator
 
-include("../test/definitions/brusselator_definition.jl")
-
 function jacbench_brusselator(method)
     suite = BenchmarkGroup()
     for N in (6, 24)