Force differentiation w.r.t. all non-const pointer/array type paramet…

…ers.

We have an old problem that we need the user to differentiate w.r.t. pointer/array parameters. In those cases, the user does not provide an adjoint for the variables and we have to initialize them ourselves. And while it's clear how to deal with numerical types, e.g.
```
f_grad_0(double x, double y, double* _d_x) { // the user only wants the derivative w.r.t. x
    double _d_y = 0; // so we initialize _d_y ourselves
...
```
we cannot do the same with array/pointer types:
```
f_grad_0(double x, double* y, double* _d_x) { // the user only wants the derivative w.r.t. x
    double* _d_y = ???; // we don't know the size of y and, therefore, cannot initialize _d_y
...
```

It's worth noting that while the user is not directly interested in ``_d_y`` from the last example, ``_d_y`` can be used in the derivative internally. However, we can guarantee that ``_d_y`` will not be useful if ``y`` is const. Here's why this is the case:
In the reverse mode, "``y`` depends on ``x``" results in "``_d_x`` depends on ``_d_y``", e.g.
```
y = x;  -->  _d_x += _d_y;
```
Therefore, if ``y`` cannot be modified (i.e. is const), ``_d_y`` will not be used in the reverse pass.

This PR makes a compromise and forces differentiation w.r.t. non-const array/pointer parameters while allowing not differentiating w.r.t. const ones.

benchmark/AlgorithmicComplexity.cpp

@@ -11,7 +11,8 @@
 // FIXME: Make the benchmark work with a range of inputs. That's currently
 // problematic for reverse mode.
 
-// inline double gaus(double* x, double* p /*means*/, double sigma, int dim);
+// inline double gaus(const double* x, double* p /*means*/, double sigma, int
+// dim);
 static void BM_NumericGausP(benchmark::State& state) {
   using namespace numerical_diff;
   long double sum = 0;

benchmark/BenchmarkedFunctions.h

@@ -13,7 +13,7 @@ inline double sum(double* p, int dim) {
 ///\returns the gaussian distribution. We need always_inline to improve the
 /// performance of reverse mode.
 __attribute__((always_inline)) inline double
-gaus(double* x, double* p /*means*/, double sigma, int dim) {
+gaus(const double* x, double* p /*means*/, double sigma, int dim) {
   double t = 0;
   for (int i = 0; i < dim; i++)
     t += (x[i] - p[i]) * (x[i] - p[i]);

demos/Arrays.cpp

@@ -19,7 +19,7 @@
 // Necessary for clad to work include
 #include "clad/Differentiator/Differentiator.h"
 
-double weighted_avg(double* arr, double* weights) {
+double weighted_avg(double* arr, const double* weights) {
   return (arr[0] * weights[0] + arr[1] * weights[1] + arr[2] * weights[2]) / 3;
 }
 

include/clad/Differentiator/ReverseModeVisitor.h

@@ -16,6 +16,7 @@
 #include "clang/Sema/Sema.h"
 
 #include <llvm/ADT/ArrayRef.h>
+#include <llvm/ADT/SmallVector.h>
 
 #include <array>
 #include <limits>
@@ -43,6 +44,7 @@ namespace clad {
     // a separate namespace, as well as add getters/setters function of
     // several private/protected members of the visitor classes.
     friend class ErrorEstimationHandler;
+    llvm::SmallVector<const clang::ParmVarDecl*, 16> m_NonIndepParams;
     /// In addition to a sequence of forward-accumulated Stmts (m_Blocks), in
     /// the reverse mode we also accumulate Stmts for the reverse pass which
     /// will be executed on return.

lib/Differentiator/ReverseModeVisitor.cpp

@@ -39,6 +39,7 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/SaveAndRestore.h"
+#include <llvm/ADT/STLExtras.h>
 #include <llvm/Support/raw_ostream.h>
 
 #include <algorithm>
@@ -362,16 +363,22 @@ Expr* ReverseModeVisitor::getStdInitListSizeExpr(const Expr* E) {
     if (m_DiffReq.Mode == DiffMode::reverse && !m_ExternalSource) {
       // create derived variables for parameters which are not part of
       // independent variables (args).
-      for (ParmVarDecl* param : m_Derivative->parameters()) {
-        // derived variables are already created for independent variables.
-        if (m_Variables.count(param))
-          continue;
-        auto VDDerivedType =
-            getNonConstType(param->getType(), m_Context, m_Sema);
-        // We cannot initialize derived variable for pointer types because
-        // we do not know the correct size.
-        if (utils::isArrayOrPointerType(VDDerivedType))
+      for (const ParmVarDecl* param : m_NonIndepParams) {
+        QualType paramTy = param->getType();
+        if (utils::isArrayOrPointerType(paramTy)) {
+          // We cannot initialize derived variable for pointer types because
+          // we do not know the correct size.
+          if (!utils::GetValueType(paramTy).isConstQualified()) {
+            diag(DiagnosticsEngine::Error, param->getLocation(),
+                 "Non-differentiable non-const pointer and array parameters "
+                 "are not supported. Please differentiate w.r.t. '%0' or mark "
+                 "it const.",
+                 {param->getNameAsString()});
+            return;
+          }
           continue;
+        }
+        auto VDDerivedType = getNonConstType(paramTy, m_Context, m_Sema);
         auto* VDDerived =
             BuildGlobalVarDecl(VDDerivedType, "_d_" + param->getNameAsString(),
                                getZeroInit(VDDerivedType));
@@ -4417,10 +4424,11 @@ Expr* ReverseModeVisitor::getStdInitListSizeExpr(const Expr* E) {
         }
       }
 
-      if (!IsSelected)
-        continue;
-
       const ParmVarDecl* PVD = params[i];
+      if (!IsSelected) {
+        m_NonIndepParams.push_back(PVD);
+        continue;
+      }
       IdentifierInfo* II =
           CreateUniqueIdentifier("_d_" + PVD->getNameAsString());
       QualType dPVDTy = FnType->getParamType(p++);

test/Arrays/ArrayErrorsReverseMode.C

@@ -0,0 +1,20 @@
+// RUN: %cladclang %s -I%S/../../include -fsyntax-only -Xclang -verify 2>&1
+
+#include "clad/Differentiator/Differentiator.h"
+
+float func11(float* a, float b) { // expected-error {{Non-differentiable non-const pointer and array parameters are not supported. Please differentiate w.r.t. 'a' or mark it const.}}
+  float sum = 0;
+  sum += a[0] *= b;
+  return sum;
+}
+
+float func12(float a, float b[]) { // expected-error {{Non-differentiable non-const pointer and array parameters are not supported. Please differentiate w.r.t. 'b' or mark it const.}}
+  float sum = 0;
+  sum += a *= b[1];
+  return sum;
+}
+
+int main() {
+  clad::gradient(func11, "b");
+  clad::gradient(func12, "a");
+}

test/CUDA/GradientCuda.cu

@@ -19,7 +19,7 @@
 
 #define N 3
 
-__device__ __host__ double gauss(double* x, double* p, double sigma, int dim) {
+__device__ __host__ double gauss(const double* x, double* p, double sigma, int dim) {
    double t = 0;
    for (int i = 0; i< dim; i++)
        t += (x[i] - p[i]) * (x[i] - p[i]);
@@ -28,7 +28,7 @@ __device__ __host__ double gauss(double* x, double* p, double sigma, int dim) {
 }
 
 
-// CHECK: __attribute__((device)) __attribute__((host)) void gauss_grad_1(double *x, double *p, double sigma, int dim, double *_d_p) {
+// CHECK: __attribute__((device)) __attribute__((host)) void gauss_grad_1(const double *x, double *p, double sigma, int dim, double *_d_p) {
 //CHECK-NEXT:     double _d_sigma = 0.;
 //CHECK-NEXT:     int _d_dim = 0;
 //CHECK-NEXT:     int _d_i = 0;

test/CUDA/GradientKernels.cu

@@ -288,16 +288,16 @@ __global__ void add_kernel_7(double *a, double *b) {
 //CHECK-NEXT:    }
 //CHECK-NEXT:}
 
-__device__ double device_fn(double in, double val) {
+__device__ double device_fn(const double in, double val) {
   return in + val;
 }
 
-__global__ void kernel_with_device_call(double *out, double *in, double val) {
+__global__ void kernel_with_device_call(double *out, const double *in, double val) {
   int index = threadIdx.x;
   out[index] = device_fn(in[index], val);
 }
 
-// CHECK: void kernel_with_device_call_grad_0_2(double *out, double *in, double val, double *_d_out, double *_d_val) {
+// CHECK: void kernel_with_device_call_grad_0_2(double *out, const double *in, double val, double *_d_out, double *_d_val) {
 //CHECK-NEXT:    int _d_index = 0;
 //CHECK-NEXT:    int index0 = threadIdx.x;
 //CHECK-NEXT:    double _t0 = out[index0];
@@ -313,22 +313,22 @@ __global__ void kernel_with_device_call(double *out, double *in, double val) {
 //CHECK-NEXT:    }
 //CHECK-NEXT:}
 
-__device__ double device_fn_2(double *in, double val) {
+__device__ double device_fn_2(const double *in, double val) {
   int index = threadIdx.x + blockIdx.x * blockDim.x;
   return in[index] + val;
 }
 
-__global__ void kernel_with_device_call_2(double *out, double *in, double val) {
+__global__ void kernel_with_device_call_2(double *out, const double *in, double val) {
   int index = threadIdx.x;
   out[index] = device_fn_2(in, val);
 } 
 
-__global__ void dup_kernel_with_device_call_2(double *out, double *in, double val) {
+__global__ void dup_kernel_with_device_call_2(double *out, const double *in, double val) {
   int index = threadIdx.x;
   out[index] = device_fn_2(in, val);
 } 
 
-// CHECK: void kernel_with_device_call_2_grad_0_2(double *out, double *in, double val, double *_d_out, double *_d_val) {
+// CHECK: void kernel_with_device_call_2_grad_0_2(double *out, const double *in, double val, double *_d_out, double *_d_val) {
 //CHECK-NEXT:    int _d_index = 0;
 //CHECK-NEXT:    int index0 = threadIdx.x;
 //CHECK-NEXT:    double _t0 = out[index0];
@@ -343,7 +343,7 @@ __global__ void dup_kernel_with_device_call_2(double *out, double *in, double va
 //CHECK-NEXT:    }
 //CHECK-NEXT:}
 
-// CHECK: void kernel_with_device_call_2_grad_0_1(double *out, double *in, double val, double *_d_out, double *_d_in) {
+// CHECK: void kernel_with_device_call_2_grad_0_1(double *out, const double *in, double val, double *_d_out, double *_d_in) {
 //CHECK-NEXT:    double _d_val = 0.;
 //CHECK-NEXT:    int _d_index = 0;
 //CHECK-NEXT:    int index0 = threadIdx.x;
@@ -570,22 +570,22 @@ void launch_add_kernel_4(int *out, int *in, const int N) {
 //CHECK-NEXT:    cudaFree(_d_out_dev);
 //CHECK-NEXT:}
 
-// CHECK: __attribute__((device)) void device_fn_pullback_1(double in, double val, double _d_y, double *_d_in, double *_d_val) {
+// CHECK: __attribute__((device)) void device_fn_pullback_1(const double in, double val, double _d_y, double *_d_in, double *_d_val) {
 //CHECK-NEXT:    {
 //CHECK-NEXT:                *_d_in += _d_y;
 //CHECK-NEXT:                *_d_val += _d_y;
 //CHECK-NEXT:    }
 //CHECK-NEXT:}
 
-// CHECK: __attribute__((device)) void device_fn_2_pullback_0_1(double *in, double val, double _d_y, double *_d_val) {
+// CHECK: __attribute__((device)) void device_fn_2_pullback_0_1(const double *in, double val, double _d_y, double *_d_val) {
 //CHECK-NEXT:    unsigned int _t1 = blockIdx.x;
 //CHECK-NEXT:    unsigned int _t0 = blockDim.x;
 //CHECK-NEXT:    int _d_index = 0;
 //CHECK-NEXT:    int index0 = threadIdx.x + _t1 * _t0;
 //CHECK-NEXT:    *_d_val += _d_y;
 //CHECK-NEXT:}
 
-// CHECK: __attribute__((device)) void device_fn_2_pullback_0_1_3(double *in, double val, double _d_y, double *_d_in, double *_d_val) {
+// CHECK: __attribute__((device)) void device_fn_2_pullback_0_1_3(const double *in, double val, double _d_y, double *_d_in, double *_d_val) {
 //CHECK-NEXT:    unsigned int _t1 = blockIdx.x;
 //CHECK-NEXT:    unsigned int _t0 = blockDim.x;
 //CHECK-NEXT:    int _d_index = 0;

test/Gradient/FunctionCalls.C

@@ -542,22 +542,22 @@ double fn16(double x, double y) {
 //CHECK-NEXT:     }
 //CHECK-NEXT: }
 
-double add(double a, double* b) {
+double add(double a, const double* b) {
     return a + b[0];
 }
 
 
-//CHECK: void add_pullback(double a, double *b, double _d_y, double *_d_a);
+//CHECK: void add_pullback(double a, const double *b, double _d_y, double *_d_a);
 
-//CHECK: void add_pullback(double a, double *b, double _d_y, double *_d_a, double *_d_b);
+//CHECK: void add_pullback(double a, const double *b, double _d_y, double *_d_a, double *_d_b);
 
-double fn17 (double x, double* y) {
+double fn17 (double x, const double* y) {
     x = add(x, y);
     x = add(x, &x);
     return x;
 }
 
-//CHECK: void fn17_grad_0(double x, double *y, double *_d_x) {
+//CHECK: void fn17_grad_0(double x, const double *y, double *_d_x) {
 //CHECK-NEXT:     double _t0 = x;
 //CHECK-NEXT:     x = add(x, y);
 //CHECK-NEXT:     double _t1 = x;
@@ -1020,11 +1020,11 @@ double sq_defined_later(double x) {
 //CHECK-NEXT:         }
 //CHECK-NEXT: }
 
-//CHECK: void add_pullback(double a, double *b, double _d_y, double *_d_a) {
+//CHECK: void add_pullback(double a, const double *b, double _d_y, double *_d_a) {
 //CHECK-NEXT:     *_d_a += _d_y;
 //CHECK-NEXT: }
 
-//CHECK: void add_pullback(double a, double *b, double _d_y, double *_d_a, double *_d_b) {
+//CHECK: void add_pullback(double a, const double *b, double _d_y, double *_d_a, double *_d_b) {
 //CHECK-NEXT:     {
 //CHECK-NEXT:         *_d_a += _d_y;
 //CHECK-NEXT:         _d_b[0] += _d_y;

test/Gradient/Loops.C

@@ -363,15 +363,15 @@ double f_sum_squares(double *p, int n) {
 // CHECK-NEXT: }
 
 // log-likelihood of n-dimensional gaussian distribution with covariance sigma^2*I
-double f_log_gaus(double* x, double* p /*means*/, double n, double sigma) {
+double f_log_gaus(const double* x, double* p /*means*/, double n, double sigma) {
   double power = 0;
   for (int i = 0; i < n; i++)
     power += sq(x[i] - p[i]);
   power = -power/(2*sq(sigma));
   double gaus = 1./std::sqrt(std::pow(2*M_PI, n) * sigma) * std::exp(power);
   return std::log(gaus);
 }
-// CHECK: void f_log_gaus_grad_1(double *x, double *p, double n, double sigma, double *_d_p) {
+// CHECK: void f_log_gaus_grad_1(const double *x, double *p, double n, double sigma, double *_d_p) {
 // CHECK-NEXT:     double _d_n = 0.;
 // CHECK-NEXT:     double _d_sigma = 0.;
 // CHECK-NEXT:     int _d_i = 0;

test/ROOT/Hessian.C

@@ -14,7 +14,7 @@ namespace TMath {
   Double_t Sin(Double_t x) { return ::std::sin(x); }
 }
 
-Double_t TFormula_example(Double_t* x, Double_t* p) {
+Double_t TFormula_example(const Double_t* x, Double_t* p) {
   return x[0]*(p[0] + p[1] + p[2]) + TMath::Exp(-p[0]) + TMath::Abs(p[1]);
 }
 

test/ROOT/Interface.C

@@ -14,13 +14,13 @@ struct array_ref_interface {
   std::size_t size;
 };
 
-Double_t f(Double_t* x, Double_t* p) {
+Double_t f(const Double_t* x, Double_t* p) {
   return p[0] + x[0] * p[1];
 }
 
-void f_grad_1(Double_t* x, Double_t* p, Double_t *_d_p);
+void f_grad_1(const Double_t* x, Double_t* p, Double_t *_d_p);
 
-// CHECK: void f_grad_1(Double_t *x, Double_t *p, Double_t *_d_p) {
+// CHECK: void f_grad_1(const Double_t *x, Double_t *p, Double_t *_d_p) {
 // CHECK-NEXT:     {
 // CHECK-NEXT:         _d_p[0] += 1;
 // CHECK-NEXT:         _d_p[1] += x[0] * 1;

test/ROOT/TFormula.C

@@ -32,13 +32,13 @@ clad::ValueAndPushforward<Double_t, Double_t> Sin_pushforward(Double_t x, Double
 } // namespace custom_derivatives
 } // namespace clad
 
-Double_t TFormula_example(Double_t* x, Double_t* p) {
+Double_t TFormula_example(const Double_t* x, Double_t* p) {
   return x[0]*(p[0] + p[1] + p[2]) + TMath::Exp(-p[0]) + TMath::Abs(p[1]);
 }
 // _grad = { x[0] + (-1) * Exp_darg0(-p[0]), x[0] + Abs_darg0(p[1]), x[0] }
 
-void TFormula_example_grad_1(Double_t* x, Double_t* p, Double_t* _d_p);
-//CHECK:   void TFormula_example_grad_1(Double_t *x, Double_t *p, Double_t *_d_p) {
+void TFormula_example_grad_1(const Double_t* x, Double_t* p, Double_t* _d_p);
+//CHECK:   void TFormula_example_grad_1(const Double_t *x, Double_t *p, Double_t *_d_p) {
 //CHECK-NEXT:       {
 //CHECK-NEXT:           _d_p[0] += x[0] * 1;
 //CHECK-NEXT:           _d_p[1] += x[0] * 1;
@@ -52,28 +52,28 @@ void TFormula_example_grad_1(Double_t* x, Double_t* p, Double_t* _d_p);
 //CHECK-NEXT:       }
 //CHECK-NEXT:   }
 
-//CHECK:   Double_t TFormula_example_darg1_0(Double_t *x, Double_t *p) {
+//CHECK:   Double_t TFormula_example_darg1_0(const Double_t *x, Double_t *p) {
 //CHECK-NEXT:       {{double|Double_t}} _t0 = (p[0] + p[1] + p[2]);
 //CHECK-NEXT:       clad::ValueAndPushforward<Double_t, Double_t> _t1 = clad::custom_derivatives::TMath::Exp_pushforward(-p[0], -1.);
 //CHECK-NEXT:       clad::ValueAndPushforward<Double_t, Double_t> _t2 = clad::custom_derivatives::TMath::Abs_pushforward(p[1], 0.);
 //CHECK-NEXT:       return 0. * _t0 + x[0] * (1. + 0. + 0.) + _t1.pushforward + _t2.pushforward;
 //CHECK-NEXT:   }
 
-//CHECK:   Double_t TFormula_example_darg1_1(Double_t *x, Double_t *p) {
+//CHECK:   Double_t TFormula_example_darg1_1(const Double_t *x, Double_t *p) {
 //CHECK-NEXT:       {{double|Double_t}} _t0 = (p[0] + p[1] + p[2]);
 //CHECK-NEXT:       clad::ValueAndPushforward<Double_t, Double_t> _t1 = clad::custom_derivatives::TMath::Exp_pushforward(-p[0], -0.);
 //CHECK-NEXT:       clad::ValueAndPushforward<Double_t, Double_t> _t2 = clad::custom_derivatives::TMath::Abs_pushforward(p[1], 1.);
 //CHECK-NEXT:       return 0. * _t0 + x[0] * (0. + 1. + 0.) + _t1.pushforward + _t2.pushforward;
 //CHECK-NEXT:   }
 
-//CHECK:   Double_t TFormula_example_darg1_2(Double_t *x, Double_t *p) {
+//CHECK:   Double_t TFormula_example_darg1_2(const Double_t *x, Double_t *p) {
 //CHECK-NEXT:       {{double|Double_t}} _t0 = (p[0] + p[1] + p[2]);
 //CHECK-NEXT:       clad::ValueAndPushforward<Double_t, Double_t> _t1 = clad::custom_derivatives::TMath::Exp_pushforward(-p[0], -0.);
 //CHECK-NEXT:       clad::ValueAndPushforward<Double_t, Double_t> _t2 = clad::custom_derivatives::TMath::Abs_pushforward(p[1], 0.);
 //CHECK-NEXT:       return 0. * _t0 + x[0] * (0. + 0. + 1.) + _t1.pushforward + _t2.pushforward;
 //CHECK-NEXT:   }
 
-Double_t TFormula_hess1(Double_t *x, Double_t *p) {
+Double_t TFormula_hess1(const Double_t *x, Double_t *p) {
     return x[0] * std::sin(p[0]) - x[1] * std::cos(p[1]);
 }