Add new backends with DifferentiationInterface.jl

.buildkite/pipeline.yml

@@ -8,5 +8,225 @@ steps:
       cuda: "*"
     command: |
       julia --color=yes --project -e 'using Pkg; Pkg.add("CUDA"); Pkg.add("NLPModels"); Pkg.add("NLPModelsTest"); Pkg.instantiate()'
-      julia --color=yes --project -e 'include("test/gpu.jl")'
+      julia --color=yes --project -e 'include("test/gpu/nvidia.jl")'
     timeout_in_minutes: 30
+
+  - label: "CPUs -- ForwardDiff.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("ForwardDiff")
+        Pkg.instantiate()
+        include("test/backend/ForwardDiff.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- ReverseDiff.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("ReverseDiff")
+        Pkg.instantiate()
+        include("test/backend/ReverseDiff.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- Enzyme.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("Enzyme")
+        Pkg.instantiate()
+        include("test/backend/Enzyme.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- Zygote.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("Zygote")
+        Pkg.instantiate()
+        include("test/backend/Zygote.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- Mooncake.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("Mooncake")
+        Pkg.instantiate()
+        include("test/backend/Mooncake.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- Diffractor.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("Diffractor")
+        Pkg.instantiate()
+        include("test/backend/Diffractor.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- Tracker.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("Tracker")
+        Pkg.instantiate()
+        include("test/backend/Tracker.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- Symbolics.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("Symbolics")
+        Pkg.instantiate()
+        include("test/backend/Symbolics.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- ChainRules.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("ChainRules")
+        Pkg.instantiate()
+        include("test/backend/ChainRules.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- FastDifferentiation.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("FastDifferentiation")
+        Pkg.instantiate()
+        include("test/backend/FastDifferentiation.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- FiniteDiff.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("FiniteDiff")
+        Pkg.instantiate()
+        include("test/backend/FiniteDiff.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- FiniteDifferences.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("FiniteDifferences")
+        Pkg.instantiate()
+        include("test/backend/FiniteDifferences.jl")'
+    timeout_in_minutes: 30
+
+  - label: "CPUs -- PolyesterForwardDiff.jl"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "juliaecosystem"
+      os: "linux"
+      arch: "x86_64"
+    command: |
+      julia --color=yes --project -e '
+        using Pkg
+        Pkg.add("OptimizationProblems")
+        Pkg.add("PolyesterForwardDiff")
+        Pkg.instantiate()
+        include("test/backend/PolyesterForwardDiff.jl")'

.github/workflows/CI.yml

@@ -28,7 +28,7 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v1
+      - uses: actions/cache@v4
         env:
           cache-name: cache-artifacts
         with:

Project.toml

@@ -4,21 +4,21 @@ version = "0.8.7"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6"
-Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
 
 [compat]
-ADTypes = "1.2.1"
-ForwardDiff = "0.9.0, 0.10.0"
-NLPModels = "0.18, 0.19, 0.20, 0.21"
-Requires = "1"
-ReverseDiff = "1"
+ADTypes = "1.9.0"
+DifferentiationInterface = "0.6.1"
+ForwardDiff = "0.10.36"
+NLPModels = "0.21.3"
+ReverseDiff = "1.15.3"
 SparseConnectivityTracer = "0.6.1"
 SparseMatrixColorings = "0.4.0"
-julia = "^1.6"
+julia = "1.6"

docs/Project.toml

@@ -16,10 +16,10 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 DataFrames = "1"
 Documenter = "1.0"
 ManualNLPModels = "0.1"
-NLPModels = "0.21"
-NLPModelsJuMP = "0.13"
-OptimizationProblems = "0.8"
+NLPModels = "0.21.3"
+NLPModelsJuMP = "0.13.2"
+OptimizationProblems = "0.9"
 Percival = "0.7"
 Plots = "1"
 SolverBenchmark = "0.6"
-Zygote = "0.6.62"
+Zygote = "0.6.70"

docs/src/backend.md

@@ -1,23 +1,25 @@
 # How to switch backend in ADNLPModels
 
 `ADNLPModels` allows the use of different backends to compute the derivatives required within NLPModel API.
-It uses `ForwardDiff.jl`, `ReverseDiff.jl`, and more via optional depencies.
+It uses `ForwardDiff.jl`, `ReverseDiff.jl`, and more via optional dependencies.
 
 The backend information is in a structure [`ADNLPModels.ADModelBackend`](@ref) in the attribute `adbackend` of a `ADNLPModel`, it can also be accessed with [`get_adbackend`](@ref).
 
 The functions used internally to define the NLPModel API and the possible backends are defined in the following table:
 
-| Functions | FowardDiff backends | ReverseDiff backends | Zygote backends | Enzyme backend | Sparse backend |
-| --------- | ------------------- | -------------------- | --------------- | -------------- | -------------- |
-| `gradient` and `gradient!` | `ForwardDiffADGradient`/`GenericForwardDiffADGradient` | `ReverseDiffADGradient`/`GenericReverseDiffADGradient` | `ZygoteADGradient` | `EnzymeADGradient` | -- |
-| `jacobian` | `ForwardDiffADJacobian` | `ReverseDiffADJacobian` | `ZygoteADJacobian` | -- | `SparseADJacobian` |
-| `hessian` | `ForwardDiffADHessian` | `ReverseDiffADHessian` | `ZygoteADHessian` | -- | `SparseADHessian`/`SparseReverseADHessian` |
-| `Jprod` | `ForwardDiffADJprod`/`GenericForwardDiffADJprod` | `ReverseDiffADJprod`/`GenericReverseDiffADJprod` | `ZygoteADJprod` | -- | -- |
-| `Jtprod` | `ForwardDiffADJtprod`/`GenericForwardDiffADJtprod` | `ReverseDiffADJtprod`/`GenericReverseDiffADJtprod` | `ZygoteADJtprod` | -- | -- |
-| `Hvprod` | `ForwardDiffADHvprod`/`GenericForwardDiffADHvprod` | `ReverseDiffADHvprod`/`GenericReverseDiffADHvprod` | -- | -- | -- |
-| `directional_second_derivative` | `ForwardDiffADGHjvprod` | -- | -- | -- | -- |
-
-The functions `hess_structure!`, `hess_coord!`, `jac_structure!` and `jac_coord!` defined in `ad.jl` are generic to all the backends for now.
+| package | ForwardDiff.jl | ReverseDiff.jl | Enzyme.jl | Zygote.jl | Mooncake.jl | Diffractor.jl | Tracker.jl | Symbolics.jl | ChainRules.jl | FastDifferentiation.jl | FiniteDiff.jl | FiniteDifferences.jl | PolyesterForwardDiff.jl |
+|--------|----------------|----------------|-----------|-----------|-------------|---------------|------------|--------------|----------------------|------------------------|---------------|----------------------|-------------------------|
+| $\nabla f(x)$               | `ForwardDiffADGradient`   | `ReverseDiffADGradient`     | `EnzymeADGradient`    | `ZygoteADGradient`    | `MooncakeADGradient`    | `DiffractorADGradient`    | `TrackerADGradient`    | `SymbolicsADGradient`    | `ChainRulesADGradient`           | `FastDifferentiationADGradient`   | `FiniteDiffADGradient`    | `FiniteDifferencesADGradient`   | `PolyesterForwardDiffADGradient`  |
+| $J_c(x)*v$                  | `ForwardDiffADJprod`      | `ReverseDiffADJprod`        | `EnzymeADJprod`       | `ZygoteADJprod`       | `MooncakeADJprod`       | `DiffractorADJprod`       | `TrackerADJprod`       | `SymbolicsADJprod`       | `ChainRulesADJprod`              | `FastDifferentiationADJprod`      | `FiniteDiffADJprod`       | `FiniteDifferencesADJprod`      | `PolyesterForwardDiffADJprod`     |
+| $J^T_c(x)*v$                | `ForwardDiffADJtprod`     | `ReverseDiffADJtprod`       | `EnzymeADJtprod`      | `ZygoteADJtprod`      | `MooncakeADJtprod`      | `DiffractorADJtprod`      | `TrackerADJtprod`      | `SymbolicsADJtprod`      | `ChainRulesADJtprod`             | `FastDifferentiationADJtprod`     | `FiniteDiffADJtprod`      | `FiniteDifferencesADJtprod`     | `PolyesterForwardDiffADJtprod`    |
+| $J_c(x)$                    | `ForwardDiffADJacobian`   | `ReverseDiffADJacobian`     | `EnzymeADJacobian`    | `ZygoteADJacobian`    | `MooncakeADJacobian`    | `DiffractorADJacobian`    | `TrackerADJacobian`    | `SymbolicsADJacobian`    | `ChainRulesADJacobian`           | `FastDifferentiationADJacobian`   | `FiniteDiffADJacobian`    | `FiniteDifferencesADJacobian`   | `PolyesterForwardDiffADJacobian`  |
+| $\nabla^2 \mathcal{L}(x)*v$ | `ForwardDiffADHvprod`     | `ReverseDiffADHvprod`       | `EnzymeADHvprod`      | `ZygoteADHvprod`      | `MooncakeADHvprod`      | `DiffractorADHvprod`      | `TrackerADHvprod`      | `SymbolicsADHvprod`      | `ChainRulesADHvprod`             | `FastDifferentiationADHvprod`     | `FiniteDiffADHvprod`       | `FiniteDifferencesADHvprod`     | `PolyesterForwardDiffADHvprod`    |
+| $\nabla^2 \mathcal{L}(x)$   | `ForwardDiffADHessian`    | `ReverseDiffADHessian`      | `EnzymeADHessian`     | `ZygoteADHessian`     | `MooncakeADHessian`     | `DiffractorADHessian`     | `TrackerADHessian`     | `SymbolicsADHessian`     | `ChainRulesADHessian`            | `FastDifferentiationADHessian`    | `FiniteDiffADHessian`      | `FiniteDifferencesADHessian`    | `PolyesterForwardDiffADHessian`   |
+
+$\mathcal{L}(x)$ denotes the Lagrangian $f(x) + \lambda^T c(x)$.
+Except for the backends based on `ForwardDiff.jl` and `ReverseDiff.jl`, all other backends require the associated AD package to be manually installed by the user to work.
+Note that the Jacobians and Hessians computed by the backends above are dense.
+The backends `SparseADJacobian`, `SparseADHessian`, and `SparseReverseADHessian` should be used instead if sparse Jacobians and Hessians are required.
 
 ```@example ex1
 using ADNLPModels

docs/src/predefined.md

@@ -40,7 +40,9 @@ ADNLPModels.predefined_backend[:optimized]
 
 The backend `:generic` focuses on backend that make no assumptions on the element type, see [Creating an ADNLPModels backend that supports multiple precisions](https://jso.dev/tutorials/generic-adnlpmodels/).
 
-It is possible to use these pre-defined backends using the keyword argument `backend` when instantiating the model.
+Each supported AD package also has its own symbol, such as `:Enzyme` or `:Zygote`, to easily switch between backends.
+
+It is possible to use these pre-defined backends by using the keyword argument `backend` when instantiating the model.
 
 ```@example ex1
 nlp = ADNLPModel!(f, x0, lvar, uvar, c!, lcon, ucon, backend = :optimized)

src/ADNLPModels.jl

@@ -4,14 +4,16 @@ module ADNLPModels
 using LinearAlgebra, SparseArrays
 
 # external
-using ADTypes: ADTypes, AbstractColoringAlgorithm, AbstractSparsityDetector
+import DifferentiationInterface
+using ADTypes: ADTypes, AbstractADType, AbstractColoringAlgorithm, AbstractSparsityDetector, AutoEnzyme, AutoZygote
+using ADTypes: AutoForwardDiff, AutoReverseDiff, AutoMooncake, AutoDiffractor, AutoTracker, AutoSymbolics
+using ADTypes: AutoChainRules, AutoFastDifferentiation, AutoFiniteDiff, AutoFiniteDifferences, AutoPolyesterForwardDiff
 using SparseConnectivityTracer: TracerSparsityDetector
 using SparseMatrixColorings
 using ForwardDiff, ReverseDiff
 
 # JSO
 using NLPModels
-using Requires
 
 abstract type AbstractADNLPModel{T, S} <: AbstractNLPModel{T, S} end
 abstract type AbstractADNLSModel{T, S} <: AbstractNLSModel{T, S} end
@@ -27,8 +29,7 @@ include("sparse_hessian.jl")
 
 include("forward.jl")
 include("reverse.jl")
-include("enzyme.jl")
-include("zygote.jl")
+include("di.jl")
 include("predefined_backend.jl")
 include("nlp.jl")