Make Flux scenarios private in DIT (#364)

* Relax Flux tests in DIT * Keep Flux scenarios private * Add Flux tests on pre-release
gdalle · Jul 17, 2024 · 37d404a · 37d404a
1 parent f4ba734
commit 37d404a
Show file tree

Hide file tree

Showing 6 changed files with 10 additions and 8 deletions.
diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml
@@ -84,8 +84,6 @@ jobs:
             group: Back/SecondOrder
           - version: 'pre'
             group: Down/Detector
-          - version: 'pre'
-            group: Down/Flux
 
     steps:
       - uses: actions/checkout@v4

diff --git a/DifferentiationInterface/test/Down/Flux/test.jl b/DifferentiationInterface/test/Down/Flux/test.jl
@@ -13,7 +13,7 @@ test_differentiation(
         AutoZygote(),
         # AutoEnzyme()  # TODO: fix
     ],
-    flux_scenarios();
+    DIT.flux_scenarios();
     isequal=DIT.flux_isequal,
     isapprox=DIT.flux_isapprox,
     rtol=1e-2,

diff --git a/DifferentiationInterfaceTest/docs/src/api.md b/DifferentiationInterfaceTest/docs/src/api.md
@@ -19,13 +19,14 @@ DifferentiationBenchmarkDataRow
 
 ## Pre-made scenario lists
 
+The precise contents of the scenario lists are not part of the API, only their existence.
+
 ```@docs
 default_scenarios
 sparse_scenarios
 component_scenarios
 gpu_scenarios
 static_scenarios
-flux_scenarios
 ```
 
 ## Scenario types

diff --git a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl
@@ -109,6 +109,5 @@ export DifferentiationBenchmarkDataRow
 export static_scenarios
 export component_scenarios
 export gpu_scenarios
-export flux_scenarios
 
 end
diff --git a/DifferentiationInterfaceTest/src/scenarios/extensions.jl b/DifferentiationInterfaceTest/src/scenarios/extensions.jl
@@ -35,6 +35,10 @@ Create a vector of [`Scenario`](@ref)s with neural networks from [Flux.jl](https
 
 !!! warning
     This function requires Flux.jl and FiniteDifferences.jl to be loaded (it is implemented in a package extension).
+
+!!! danger
+    These scenarios are still experimental and not part of the public API.
+    Their ground truth values are computed with finite differences, and thus subject to imprecision.
 """
 function flux_scenarios end
 

diff --git a/DifferentiationInterfaceTest/test/weird.jl b/DifferentiationInterfaceTest/test/weird.jl
@@ -25,10 +25,10 @@ test_differentiation(
 
 test_differentiation(
     AutoZygote(),
-    flux_scenarios();
+    DIT.flux_scenarios();
     isequal=DIT.flux_isequal,
     isapprox=DIT.flux_isapprox,
-    rtol=1e-2,
-    atol=1e-3,
+    rtol=5e-2,
+    atol=1e-2,
     logging=LOGGING,
 )