Merge branch 'dev'

Version 6.0.3 ND

opennn/adaptive_moment_estimation.cpp

@@ -318,8 +318,6 @@ TrainingResults AdaptiveMomentEstimation::perform_training()
     LossIndexBackPropagation training_back_propagation(batch_size_training, loss_index_pointer);
     LossIndexBackPropagation selection_back_propagation(batch_size_selection, loss_index_pointer);
 
-    Index parameters_size = training_back_propagation.parameters.size();
-
     type training_error = type(0);
     type training_loss = type(0);
 
@@ -377,7 +375,6 @@ TrainingResults AdaptiveMomentEstimation::perform_training()
             training_loss += training_back_propagation.loss;
 
             update_parameters(training_back_propagation, optimization_data);
-
         }
 
         // Loss
@@ -580,6 +577,7 @@ void AdaptiveMomentEstimation::update_parameters(LossIndexBackPropagation& back_
     saxpby(&parameters_number, &a, back_propagation.gradient.data(), &incx, &b, optimization_data.gradient_exponential_decay.data(), &incy);
 
 #else
+
     optimization_data.gradient_exponential_decay.device(*thread_pool_device)
         = back_propagation.gradient * (type(1) - beta_1)
         + optimization_data.gradient_exponential_decay * beta_1;
@@ -590,11 +588,8 @@ void AdaptiveMomentEstimation::update_parameters(LossIndexBackPropagation& back_
         = back_propagation.gradient * back_propagation.gradient * (type(1) - beta_2)
         + optimization_data.square_gradient_exponential_decay * beta_2;
 
-    optimization_data.square_gradient_exponential_decay_square_root.device(*thread_pool_device)
-        = optimization_data.square_gradient_exponential_decay.sqrt() + epsilon;
-
     back_propagation.parameters.device(*thread_pool_device)
-        -= learning_rate * optimization_data.gradient_exponential_decay / optimization_data.square_gradient_exponential_decay_square_root;
+        -= learning_rate * optimization_data.gradient_exponential_decay / (optimization_data.square_gradient_exponential_decay.sqrt() + epsilon);
 
     optimization_data.iteration++;
 
@@ -834,7 +829,6 @@ void AdaptiveMomentEstimationData::set(AdaptiveMomentEstimation* new_adaptive_mo
     square_gradient_exponential_decay.resize(parameters_number);
     square_gradient_exponential_decay.setZero();
 
-    square_gradient_exponential_decay_square_root.resize(parameters_number);
     square_gradient_exponential_decay.setZero();
 }
 

opennn/adaptive_moment_estimation.h

@@ -192,10 +192,6 @@ struct AdaptiveMomentEstimationData : public OptimizationAlgorithmData
     Tensor<type, 1> gradient_exponential_decay;
     Tensor<type, 1> square_gradient_exponential_decay;
 
-
-    Tensor<type, 1> square_gradient_exponential_decay_square_root;
-
-
     Index iteration = 0;
 
     Index learning_rate_iteration = 0;

opennn/conjugate_gradient.cpp

@@ -354,7 +354,6 @@ type ConjugateGradient::calculate_PR_parameter(const Tensor<type, 1>& old_gradie
     }
 
     return PR_parameter;
-
 }
 
 
@@ -987,7 +986,7 @@ void ConjugateGradient::update_parameters(
     }
 
     optimization_data.training_slope.device(*thread_pool_device)
-            = (back_propagation.gradient).contract(optimization_data.training_direction, AT_B);
+            = back_propagation.gradient.contract(optimization_data.training_direction, AT_B);
 
     if(optimization_data.training_slope(0) >= type(0))
     {

opennn/cross_entropy_error.cpp

@@ -87,9 +87,13 @@ void CrossEntropyError::calculate_binary_error(const DataSetBatch& batch,
 
     const TensorMap<Tensor<type, 2>> targets(batch.targets_data, batch.targets_dimensions(0), batch.targets_dimensions(1));
 
+    Tensor<type, 2> binary_cross_entropy = - ((type(1)-targets)*((type(1)-outputs).log()));
+
+    std::replace_if(binary_cross_entropy.data(), binary_cross_entropy.data()+binary_cross_entropy.size(), [](type x){return isnan(x);}, 0);
+
     Tensor<type, 0> cross_entropy_error;
 
-    cross_entropy_error.device(*thread_pool_device) = -(targets*(outputs.log())).sum() - ((type(1)-targets)*((type(1)-outputs).log())).sum();
+    cross_entropy_error.device(*thread_pool_device) = -(targets*(outputs.log())).sum() + binary_cross_entropy.sum();
 
     back_propagation.error = cross_entropy_error()/static_cast<type>(batch_samples_number);
 }

opennn/gradient_descent.cpp

@@ -284,44 +284,37 @@ void GradientDescent::update_parameters(
 
     if(abs(optimization_data.learning_rate) > type(0))
     {
-        optimization_data.parameters_increment.device(*thread_pool_device)
-                = optimization_data.training_direction*optimization_data.learning_rate;
-
-        back_propagation.parameters.device(*thread_pool_device) += optimization_data.parameters_increment;
+        back_propagation.parameters.device(*thread_pool_device)
+                -= back_propagation.gradient*optimization_data.learning_rate;
     }
     else
     {
         const Index parameters_number = neural_network_pointer->get_parameters_number();
 
         for(Index i = 0; i < parameters_number; i++)
         {
-            if(abs(back_propagation.gradient(i)) < type(NUMERIC_LIMITS_MIN))
+            if(abs(back_propagation.gradient(i)) >= type(NUMERIC_LIMITS_MIN))
             {
-                optimization_data.parameters_increment(i) = type(0);
-            }
-            else if(back_propagation.gradient(i) > type(0))
-            {
-                back_propagation.parameters(i) -= numeric_limits<type>::epsilon();
-
-                optimization_data.parameters_increment(i) = -numeric_limits<type>::epsilon();
-            }
-            else if(back_propagation.gradient(i) < type(0))
-            {
-                back_propagation.parameters(i) += numeric_limits<type>::epsilon();
-
-                optimization_data.parameters_increment(i) = numeric_limits<type>::epsilon();
+                if(back_propagation.gradient(i) > type(0))
+                {
+                    back_propagation.parameters(i) -= numeric_limits<type>::epsilon();
+                }
+                else if(back_propagation.gradient(i) < type(0))
+                {
+                    back_propagation.parameters(i) += numeric_limits<type>::epsilon();
+                }
             }
         }
 
         optimization_data.learning_rate = optimization_data.old_learning_rate;
+
     }
 
     // Update parameters
 
     optimization_data.old_learning_rate = optimization_data.learning_rate;
 
     forward_propagation.neural_network_pointer->set_parameters(back_propagation.parameters);
-
 }
 
 

opennn/gradient_descent.h

@@ -179,8 +179,6 @@ struct GradientDescentData : public OptimizationAlgorithmData
 
         potential_parameters.resize(parameters_number);
 
-        parameters_increment.resize(parameters_number);
-
         // Optimization algorithm data
 
         training_direction.resize(parameters_number);
@@ -198,14 +196,6 @@ struct GradientDescentData : public OptimizationAlgorithmData
 
     GradientDescent* gradient_descent_pointer = nullptr;
 
-    // Neural network data
-
-//    Tensor<type, 1> potential_parameters;
-//    Tensor<type, 1> training_direction;
-//    type initial_learning_rate = type(0);
-
-    Tensor<type, 1> parameters_increment;
-
     // Optimization algorithm data
 
     Index epoch = 0;

opennn/statistics.cpp

@@ -315,7 +315,6 @@ Histogram::Histogram(const Tensor<type, 1>& data,
 
         corresponding_bin = int((value - data_minimum) / step);
 
-
         if(corresponding_bin >= number_of_bins)
             corresponding_bin = number_of_bins - 1;
 
@@ -1932,6 +1931,13 @@ Tensor<Descriptives, 1> descriptives(const Tensor<type, 2>& matrix,
             standard_deviation(i) = sqrt(variance);
         }
     }
+    else
+    {
+        for(Index i = 0; i < columns_indices_size; i++)
+        {
+            standard_deviation(i) = 0;
+        }
+    }
 
     for(Index i = 0; i < columns_indices_size; i++)
     {

opennn/stochastic_gradient_descent.cpp

@@ -271,8 +271,6 @@ void StochasticGradientDescent::set_maximum_time(const type& new_maximum_time)
 }
 
 
-/// Set hardware to use. Default: Multi-core.
-
 void StochasticGradientDescent::update_parameters(LossIndexBackPropagation& back_propagation,
                       StochasticGradientDescentData& optimization_data) const
 {
@@ -290,10 +288,7 @@ void StochasticGradientDescent::update_parameters(LossIndexBackPropagation& back
         }
         else
         {
-            optimization_data.nesterov_increment.device(*thread_pool_device)
-                    = optimization_data.parameters_increment*momentum - back_propagation.gradient*learning_rate;
-
-            back_propagation.parameters.device(*thread_pool_device) += optimization_data.nesterov_increment;
+            back_propagation.parameters.device(*thread_pool_device) += optimization_data.parameters_increment*momentum - back_propagation.gradient*learning_rate;;
         }
     }
     else

opennn/stochastic_gradient_descent.h

@@ -183,11 +183,9 @@ struct StochasticGradientDescentData : public OptimizationAlgorithmData
         const Index parameters_number = neural_network_pointer->get_parameters_number();
 
         parameters_increment.resize(parameters_number);
-        nesterov_increment.resize(parameters_number);
         last_parameters_increment.resize(parameters_number);
 
         parameters_increment.setZero();
-        nesterov_increment.setZero();
         last_parameters_increment.setZero();
     }
 
@@ -196,7 +194,6 @@ struct StochasticGradientDescentData : public OptimizationAlgorithmData
     Index iteration = 0;
 
     Tensor<type, 1> parameters_increment;
-    Tensor<type, 1> nesterov_increment;
     Tensor<type, 1> last_parameters_increment;
 };