Made progress on neural net for hard mode

Author: Paul Reesman

Cargo.toml

@@ -1,11 +1,12 @@
 [package]
 name = "evil_penguin"
-version = "0.8.1"
+version = "0.8.2"
 authors = ["Paul Reesman <[email protected]>"]
 edition = "2018"
 
 [dependencies]
 const-concat = { git = "https://github.com/reesmanp/const-concat" }
+rand = "0.7.3"
 
 [target.'cfg(target_os = "windows")'.dependencies.amethyst]
 version = "^0.15.0"

src/components/entities/coin.component.rs

@@ -8,6 +8,8 @@ use amethyst::{
 #[derive(Component, Default)]
 #[storage(VecStorage)]
 pub struct CoinComponent {
+    pub id: usize,
+    pub total_amount: usize,
     pub frame: usize,
     pub frames: usize,
     pub time_per_frame: f32,

src/main.rs

@@ -10,6 +10,7 @@ use amethyst::{
 };
 
 mod components;
+mod ml;
 mod states;
 mod systems;
 mod util;

src/ml/mod.rs

@@ -0,0 +1 @@
+pub mod neural_network;

src/ml/neural_network/mod.rs

@@ -0,0 +1,6 @@
+pub mod util;
+mod neural_network;
+mod neuron;
+
+pub use neural_network::NeuralNetwork;
+pub use neuron::Neuron;

src/ml/neural_network/neural_network.rs

@@ -0,0 +1,190 @@
+use crate::{
+    ml::neural_network::Neuron,
+    util::types::{
+        InstanceInput,
+        NeuralNetworkLayer,
+        ProblemSet
+    }
+};
+
+use rand::Rng;
+
+pub struct NeuralNetwork {
+    pub layers: Vec<NeuralNetworkLayer>,
+    test_instances: usize,
+    tests_correct: usize,
+    pub update_bias: bool,
+    pub mse: f32
+}
+
+impl NeuralNetwork {
+    pub fn new(layout: Vec<usize>, learning_rate: f32, feature_amount: usize) -> Self {
+        let mut layers = Vec::with_capacity(layout.len());
+        let mut rng = rand::thread_rng();
+        // The initial input is the features themselves
+        let mut next_input_size = feature_amount;
+
+        // Find the size of each layer
+        for node_amount in layout {
+            // Auto generate a bias [0.0, 1.0]
+            let bias = rng.gen();
+            let mut layer = Vec::with_capacity(node_amount);
+
+            // Instantiate the correct amount of neurons for this layer
+            for _ in 0..node_amount {
+                // Make sure the amount of weights equals the size of the input to this layer
+                let mut weights = Vec::with_capacity(next_input_size);
+
+                // Auto generate weights [0.0, 1.0]
+                for _ in 0..next_input_size {
+                    weights.push(rng.gen());
+                }
+
+                // Instantiate neuron
+                let node = Neuron::new(weights, bias, learning_rate);
+                layer.push(node);
+            }
+
+            // Add layer to network along with its associated generated bias value
+            layers.push(layer);
+            // The size of the next input is the size of this layers output
+            next_input_size = node_amount;
+        }
+
+        Self {
+            layers,
+            test_instances: 0,
+            tests_correct: 0,
+            update_bias: true,
+            mse: 0.0
+        }
+    }
+
+    fn feed_forward(&mut self, instance: InstanceInput) {
+        let mut this_input = instance;
+
+        for nodes in &mut self.layers {
+            let mut next_input = Vec::with_capacity(nodes.len());
+
+            for node in nodes {
+                node.calc_activity(this_input.clone());
+                node.calc_activation();
+
+                // The output of this node is an input to the next layer
+                next_input.push(node.activation);
+            }
+
+            // The outputs of this layer is the inputs of the next layer
+            this_input = next_input;
+        }
+    }
+
+    fn back_propagation(&mut self, expected_output: f32) {
+        let mut is_output_layer = true;
+        let mut error_sum_vec: Vec<f32>;
+        let mut next_error_sum_vec: Vec<f32> = vec![];
+
+        // For every layer, working backwards
+        for layer in &mut self.layers.iter_mut().rev() {
+            error_sum_vec = next_error_sum_vec;
+            next_error_sum_vec = Vec::with_capacity(layer.len());
+
+            // Make sure the vector has enough elements
+            for _ in 0..layer[0].get_weights().len() {
+                next_error_sum_vec.push(0.0);
+            }
+
+            for node in layer {
+                if is_output_layer {
+                    node.calc_delta_weights(expected_output - node.activation);
+                } else {
+                    node.calc_delta_weights(error_sum_vec.remove(0));
+                }
+
+                // Determine the appropriate error value for the next iteration
+                for index in 0..next_error_sum_vec.len() {
+                    next_error_sum_vec[index] += node.weights[index] * node.delta;
+                }
+
+                // Find the delta for the weight associated with the bias
+                node.calc_delta_bias()
+            }
+
+            is_output_layer = false;
+        }
+    }
+
+    /// All the weights need to be updated at the same time
+    /// So here we update the weights based off of the calculated deltas
+    fn update_weights(&mut self) {
+        for layer in &mut self.layers {
+            for node in layer {
+                node.update_weights();
+
+                if self.update_bias {
+                    node.update_bias();
+                }
+            }
+        }
+    }
+
+    // TODO: Figure out later
+    // pub fn train(&mut self, training_set: ProblemSet, epochs: usize, is_online: bool) {
+    //     for _ in 0..epochs {
+    //         for (training_instance, expected_output) in training_set.clone() {
+    //             // Feed Forward
+    //             self.feed_forward(training_instance);
+    //
+    //             // Back Propagation
+    //             self.back_propagation(expected_output);
+    //
+    //             // Update weights
+    //             if is_online {
+    //                 self.update_weights();
+    //             }
+    //         }
+    //
+    //         if !is_online {
+    //             self.update_weights();
+    //         }
+    //     }
+    // }
+
+    pub fn test(&mut self, test_instance: Vec<f32>, maybe_threshold: Option<f32>) -> Vec<f32> {
+        self.feed_forward(test_instance);
+
+        self.layers
+            .last()
+            .unwrap()
+            .iter()
+            .map(|neuron| neuron.activation)
+            .collect::<Vec<f32>>()
+    }
+
+    pub fn print(&self) {
+        for layer in &self.layers {
+            println!("********************");
+            for neuron in layer {
+                println!("Bias: {}", neuron.bias);
+                println!("{:?}", neuron.get_weights());
+            }
+            println!("********************");
+        }
+    }
+
+    pub fn accuracy(&self, is_classification: bool) -> Option<f32> {
+        if self.test_instances == 0 {
+            None
+        } else {
+            if is_classification {
+                Some((self.tests_correct as f32 / self.test_instances as f32) * 100.0)
+            } else {
+                Some(self.mse)
+            }
+        }
+    }
+
+    pub fn get_output(&self) -> Vec<f32> {
+        self.layers[self.layers.len() - 1].iter().map(|node| node.activation).collect::<Vec<f32>>()
+    }
+}

src/ml/neural_network/neuron.rs

@@ -0,0 +1,121 @@
+use crate::{
+    ml::neural_network::{
+        util::sigmoid::{
+            sigmoid,
+            sigmoid_delta
+        }
+    },
+    util::{
+        constants::{
+            DEFAULT_LEARNING_RATE,
+            DEFAULT_NUM_OF_WEIGHTS
+        },
+        types::{
+            InstanceInput,
+            ProblemSet,
+            Weights
+        }
+    }
+};
+
+use rand::prelude::*;
+
+pub struct Neuron {
+    inputs: Vec<f32>,
+
+    pub weights: Weights,
+    pub delta_weights: Weights,
+
+    pub bias: f32,
+    delta_bias: f32,
+
+    pub activity: f32,
+
+    learning_rate: f32,
+
+    pub activation: f32,
+    pub delta: f32
+}
+
+impl Neuron {
+    pub fn new(weights: Weights, bias: f32, learning_rate: f32) -> Self {
+        let mut delta_weights = weights.clone();
+        for index in 0..weights.len() {
+            delta_weights[index] = 0.0;
+        }
+
+        Self {
+            inputs: vec![],
+            weights,
+            delta_weights,
+            bias,
+            delta_bias: 0.0,
+            activity: 0.0,
+            activation: 0.0,
+            delta: 0.0,
+            learning_rate
+        }
+    }
+
+    pub fn calc_activity(&mut self, input_vec: InstanceInput) {
+        let mut activity_value = self.bias;
+
+        for (weight, value) in self.weights.iter().zip(input_vec.iter()) {
+            activity_value += weight * value;
+        }
+
+        self.activity = activity_value;
+        self.inputs = input_vec;
+    }
+
+    pub fn calc_activation(&mut self) {
+        self.activation = sigmoid(self.activity);
+    }
+
+    pub fn calc_delta_weights(&mut self, error_value: f32) {
+        self.delta = sigmoid_delta(
+            self.activation,
+            error_value
+        );
+
+        for index in 0..self.weights.len() {
+            self.delta_weights[index] = self.learning_rate
+                * self.delta
+                * self.inputs[index];
+        }
+    }
+
+    pub fn update_weights(&mut self) {
+        for index in 0..self.weights.len() {
+            self.weights[index] += self.delta_weights[index];
+            self.delta_weights[index] = 0.0;
+            self.delta = 0.0;
+        }
+    }
+
+    /// Assumes delta weights has been calculated first
+    pub fn calc_delta_bias(&mut self) {
+        self.delta_bias = self.learning_rate * self.delta;
+    }
+
+    pub fn update_bias(&mut self) {
+        self.bias += self.delta_bias;
+        self.delta_bias = 0.0;
+    }
+
+    pub fn get_weights(&self) -> &Weights {
+        &self.weights
+    }
+}
+
+impl Default for Neuron {
+    fn default() -> Self {
+        let mut rng = rand::thread_rng();
+        let mut weight_vec = Vec::with_capacity(DEFAULT_NUM_OF_WEIGHTS);
+        for _ in 0..DEFAULT_NUM_OF_WEIGHTS {
+            weight_vec.push(rng.gen());
+        }
+
+        Self::new(weight_vec, rng.gen(), DEFAULT_LEARNING_RATE)
+    }
+}

src/ml/neural_network/util/mod.rs

@@ -0,0 +1 @@
+pub mod sigmoid;

src/ml/neural_network/util/sigmoid.rs

@@ -0,0 +1,12 @@
+use std::f64::consts::E;
+
+pub fn sigmoid(activity_value: f32) -> f32 {
+    (1.0 / (1.0 + E.powf(-activity_value as f64))) as f32
+}
+
+/// error_value
+/// desired_output - actual_output ==> output_node
+/// Sum(outbound_weight * previous_sigmoid_delta) ==> hidden node
+pub fn sigmoid_delta(neuron_activation_value: f32, error_value: f32) -> f32 {
+    neuron_activation_value * (1.0 - neuron_activation_value) * error_value
+}

src/states/loading.state.rs

@@ -22,7 +22,7 @@ use crate::{
         }
     },
     systems::movement::Difficulty,
-    util::types::SpritesheetLoadingData
+    util::types::SpriteSheetLoadingData
 };
 
 use std::collections::HashMap;
@@ -102,7 +102,7 @@ impl LoadingState {
     fn load_sprite_sheet(
         &mut self,
         world: &mut World,
-        (sprite_name, sprite_sheet_path, ron_path): SpritesheetLoadingData
+        (sprite_name, sprite_sheet_path, ron_path): SpriteSheetLoadingData
     ) {
         let texture_handle = {
             let loader = world.read_resource::<Loader>();

src/states/menu/main_menu.state.rs

@@ -52,6 +52,8 @@ impl<'a, 'b> SimpleState for MainMenuState<'a, 'b> {
                 } else if Some(ui_event.target) == hard {
                     return Trans::Switch(Box::new(LoadingState::new(NextLoadingState::Run(Difficulty::Hard))))
                 }
+            } else if ui_event.event_type == UiEventType::HoverStart {
+                // TODO: change cursor
             }
         }