retry for safety

.github/workflows/ci-examples.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
       - name: Checkout JUDI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - id: set-matrix
         run: echo "matrix=$(ls examples/scripts/*.jl examples/machine-learning/*.jl | xargs -n 1 | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
@@ -52,7 +52,7 @@ jobs:
 
     steps:
       - name: Checkout JUDI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup julia
         uses: julia-actions/setup-julia@v1

.github/workflows/ci-judi.yml

@@ -33,10 +33,10 @@ jobs:
 
     steps:
       - name: Checkout JUDI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cache julia install
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-julia-pkgs
         with:

.github/workflows/ci-op.yml

@@ -62,10 +62,10 @@ jobs:
 
     steps:
       - name: Checkout JUDI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cache julia install
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-julia-pkgs
         with:

.github/workflows/deploy_doc.yaml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout master
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: julia-actions/setup-julia@latest
 

.github/workflows/docker-publish.yml

@@ -25,7 +25,7 @@ jobs:
         run: echo ${{ github.event_name }}
 
       - name: Checkout JUDI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/[email protected]

.github/workflows/flake8.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.9
       uses: actions/setup-python@v4

src/TimeModeling/Modeling/misfit_fg.jl

@@ -6,8 +6,7 @@ Dtypes = Union{<:judiVector, NTuple{N, <:judiVector} where N, Vector{<:judiVecto
 MTypes = Union{<:AbstractModel, NTuple{N, <:AbstractModel} where N, Vector{<:AbstractModel}}
 dmTypes = Union{dmType, NTuple{N, dmType} where N, Vector{dmType}}
 
-
-function multi_src_fg(model_full::AbstractModel, source::Dtypes, dObs::Dtypes, dm, options::JUDIOptions;
+function _multi_src_fg(model_full::AbstractModel, source::Dtypes, dObs::Dtypes, dm, options::JUDIOptions;
                       nlind::Bool=false, lin::Bool=false, misfit::Function=mse, illum::Bool=false,
                       data_precon=nothing, model_precon=LinearAlgebra.I)
     GC.gc(true)
@@ -92,6 +91,9 @@ function multi_src_fg(model_full::AbstractModel, source::Dtypes, dObs::Dtypes, d
     return fval, grad
 end
 
+multi_src_fg = retry(_multi_src_fg)
+
+
 # Find number of experiments
 """
     get_nexp(x)

src/TimeModeling/Modeling/time_modeling_serial.jl

@@ -5,10 +5,11 @@ GeomOrNot = Union{Geometry, Array, Nothing}
 ArrayOrNot = Union{Array, PyArray, PyObject, Nothing}
 PhysOrNot = Union{PhysicalParameter, Array, Nothing}
 
+
 # Setup time-domain linear or nonlinear foward and adjoint modeling and interface to devito
-function time_modeling(model_full::AbstractModel, srcGeometry::GeomOrNot, srcData::ArrayOrNot,
-                       recGeometry::GeomOrNot, recData::ArrayOrNot, dm::PhysOrNot,
-                       op::Symbol, options::JUDIOptions, fw::Bool, illum::Bool)
+function _time_modeling(model_full::AbstractModel, srcGeometry::GeomOrNot, srcData::ArrayOrNot,
+                        recGeometry::GeomOrNot, recData::ArrayOrNot, dm::PhysOrNot,
+                        op::Symbol, options::JUDIOptions, fw::Bool, illum::Bool)
     GC.gc(true)
     devito.clear_cache()
 
@@ -91,3 +92,5 @@ function save_to_disk(shot::judiVector{T}, srcGeometry::GeometryIC{T}, srcData::
     end
     return dout
 end
+
+time_modeling = retry(_time_modeling)

src/TimeModeling/Modeling/twri_objective.jl

@@ -53,18 +53,7 @@ end
 
 subsample(opt::TWRIOptions, srcnum::Int) = getindex(opt, srcnum)
 
-"""
-    twri_objective(model, source, dobs; options=Options(), optionswri=TWRIOptions())
-
-Evaluate the time domain Wavefield reconstruction inversion objective function. Returns a tuple with function value and
-gradient(s) w.r.t to m and/or y. `model` is a `Model` structure with the current velocity model and `source` and `dobs` are the wavelets and 
-observed data of type `judiVector`.
-
-Example
-=======
-    function_value, gradient_m, gradient_y = twri_objective(model, source, dobs; options=Options(), optionswri=TWRIOptions())
-"""
-function twri_objective(model_full::AbstractModel, source::judiVector, dObs::judiVector, y::Union{judiVector, Nothing},
+function _twri_objective(model_full::AbstractModel, source::judiVector, dObs::judiVector, y::Union{judiVector, Nothing},
                         options::JUDIOptions, optionswri::TWRIOptions)
     # Load full geometry for out-of-core geometry containers
     dObs.geometry = Geometry(dObs.geometry)
@@ -133,6 +122,7 @@ filter_out(obj, m, ::Nothing) = obj, m
 filter_out(obj, ::Nothing, y) = obj, y
 filter_out(obj, m, y) = obj, m, y
 
+twri = retry(_twri_objective)
 
 # Parallel
 """
@@ -152,7 +142,7 @@ function twri_objective(model::AbstractModel, source::judiVector, dObs::judiVect
     else
         arg_func = j -> (model, source[j], dObs[j], y[j], options[j], optionswri[j])
     end
-    results = run_and_reduce(twri_objective, pool, source.nsrc, arg_func)
+    results = run_and_reduce(twri, pool, source.nsrc, arg_func)
     # Collect and reduce gradients
     out = as_vec(results, Val(options.return_array))
     return out

test/test_gradients.jl

@@ -101,15 +101,13 @@ end
 # Test if lsrtm_objective produces the same value/gradient as is done by the correct algebra
 @testset "LSRTM gradient linear algebra test with $(nlayer) layers, tti $(tti), viscoacoustic $(viscoacoustic), freesurface $(fs)" begin
 	# Draw a random case to avoid long CI.
-	dft, optchk = rand([true, false], 2)
 	ic = rand(["isic", "fwi", "as"])
-	optchk = optchk && !dft
-    @timeit TIMEROUTPUT "LSRTM validity (IC=$(ic), checkpointing=$(optchk), dft=$(dft))" begin
+	printstyled("LSRTM validity with dft (IC=$(ic), checkpointing=$(optchk))\n", color=:blue)
+    @timeit TIMEROUTPUT "LSRTM validity with dft (IC=$(ic), checkpointing=$(optchk))" begin
 		ftol = fs ? 1f-3 : 5f-4
-		freq = dft ? [[2.5, 4.5],[3.5, 5.5],[10.0, 15.0], [30.0, 32.0]] : []
+		freq = [[2.5, 4.5],[3.5, 5.5],[10.0, 15.0], [30.0, 32.0]]
 		J.options.free_surface = fs
 		J.options.IC = ic
-		J.options.optimal_checkpointing = optchk
 		J.options.frequencies = freq
 
 		d_res = dobs0 + J*dm1 - dobs