From aae3e29fac7695e4c159c6a8e3820202a949f76f Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Wed, 16 Aug 2023 15:39:07 +0200 Subject: [PATCH 01/17] Monte carlo block added (excluding configurability) --- src/block_monte_carlo.rs | 181 +++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 1 + src/port_buffer.rs | 10 +++ src/regressor.rs | 37 ++++++-- 5 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 src/block_monte_carlo.rs diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs new file mode 100644 index 00000000..c06215d8 --- /dev/null +++ b/src/block_monte_carlo.rs @@ -0,0 +1,181 @@ +use fasthash::Seed; +use std::any::Any; +use std::error::Error; + +use rand::distributions::{Distribution, Uniform}; + +use regressor::BlockTrait; + +use crate::block_helpers; +use crate::feature_buffer; +use crate::feature_buffer::FeatureBuffer; +use crate::graph; +use crate::port_buffer; +use crate::port_buffer::{MonteCarloStats, PortBuffer}; +use crate::regressor; +use crate::regressor::BlockCache; + +pub fn new_monte_carlo_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, + num_iterations: usize, + dropout_rate: f32, +) -> Result> { + let num_inputs = bg.get_num_output_values(vec![&input]); + assert_ne!(num_inputs, 0); + + let skip_index_generator = Uniform::from(0..num_inputs); + let number_of_inputs_to_skip = (dropout_rate * num_inputs as f32) as usize; + + let mut block = Box::new(BlockMonteCarlo { + num_iterations, + number_of_inputs_to_skip, + skip_index_generator, + output_offset: usize::MAX, + input_offset: usize::MAX, + num_inputs, + num_outputs: num_inputs, + }); + let mut block_outputs = bg.add_node(block, vec![input])?; + assert_eq!(block_outputs.len(), 1); + Ok(block_outputs.pop().unwrap()) +} + +pub struct BlockMonteCarlo { + num_iterations: usize, + number_of_inputs_to_skip: usize, + skip_index_generator: Uniform, + + pub num_inputs: usize, + pub num_outputs: usize, + pub input_offset: usize, + pub output_offset: usize, +} + +impl BlockTrait for BlockMonteCarlo { + fn as_any(&mut self) -> &mut dyn Any { + self + } + + fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { + assert_eq!(output.get_output_index(), 0); + self.num_outputs + } + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + assert_eq!(input.get_input_index(), 0); + assert_eq!(self.input_offset, usize::MAX); // We only allow a single call + self.input_offset = offset; + } + + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + assert_eq!(output.get_output_index(), 0); + assert_eq!(self.output_offset, usize::MAX); // We only allow a single call + self.output_offset = offset; + } + + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &FeatureBuffer, + pb: &mut PortBuffer, + update: bool, + ) { + block_helpers::forward_backward(further_blocks, fb, pb, update); + } + + fn forward( + &self, + further_blocks: &[Box], + fb: &FeatureBuffer, + pb: &mut PortBuffer, + ) { + unsafe { + let mut rng = rand::thread_rng(); + + for _ in 0..self.num_iterations { + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_outputs, + ); + + output_tape + .get_unchecked_mut(..) + .copy_from_slice(input_tape.get_unchecked(..)); + for _ in 0..self.number_of_inputs_to_skip { + let skip_index = self.skip_index_generator.sample(&mut rng); + *output_tape.get_unchecked_mut(skip_index) = 0.0; + } + block_helpers::forward(further_blocks, fb, pb); + } + + self.fill_stats(pb); + } + } + + fn forward_with_cache( + &self, + further_blocks: &[Box], + fb: &FeatureBuffer, + pb: &mut PortBuffer, + caches: &[BlockCache], + ) { + unsafe { + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_outputs, + ); + + let mut rng = rand::thread_rng(); + + for _ in 0..self.num_iterations { + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_outputs, + ); + + output_tape + .get_unchecked_mut(..) + .copy_from_slice(input_tape.get_unchecked(..)); + for _ in 0..self.number_of_inputs_to_skip { + let skip_index = self.skip_index_generator.sample(&mut rng); + *output_tape.get_unchecked_mut(skip_index) = 0.0; + } + block_helpers::forward_with_cache(further_blocks, fb, pb, caches); + } + + self.fill_stats(pb); + } + } +} + +impl BlockMonteCarlo { + fn fill_stats(&self, pb: &mut PortBuffer) { + let mean: f32 = pb.observations.iter().sum::() / self.num_iterations as f32; + let variance = pb + .observations + .iter() + .map(|prediction| { + let diff = mean - prediction; + diff * diff + }) + .sum::() + / self.num_iterations as f32; + let standard_deviation = variance.sqrt(); + + pb.monte_carlo_stats = Some(MonteCarloStats { + mean, + variance, + standard_deviation, + }); + } +} diff --git a/src/lib.rs b/src/lib.rs index d4fe09ca..13ea39c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ mod block_helpers; mod block_loss_functions; mod block_lr; mod block_misc; +mod block_monte_carlo; mod block_neural; mod block_normalize; mod block_relu; diff --git a/src/main.rs b/src/main.rs index b48108b0..76d37811 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,6 +31,7 @@ mod block_helpers; mod block_loss_functions; mod block_lr; mod block_misc; +mod block_monte_carlo; mod block_neural; mod block_normalize; mod block_relu; diff --git a/src/port_buffer.rs b/src/port_buffer.rs index 8d8e7bf0..07891e35 100644 --- a/src/port_buffer.rs +++ b/src/port_buffer.rs @@ -3,6 +3,14 @@ pub struct PortBuffer { pub tape: Vec, pub observations: Vec, pub tape_len: usize, + pub monte_carlo_stats: Option, +} + +#[derive(Clone, Debug)] +pub struct MonteCarloStats { + pub mean: f32, + pub variance: f32, + pub standard_deviation: f32, } impl PortBuffer { @@ -11,11 +19,13 @@ impl PortBuffer { tape: Default::default(), observations: Default::default(), tape_len, + monte_carlo_stats: None, } } pub fn reset(&mut self) { self.observations.truncate(0); self.tape.resize(self.tape_len, 0.0); + self.monte_carlo_stats = None; } } diff --git a/src/regressor.rs b/src/regressor.rs index 5e3ac986..2c25e2ef 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -9,6 +9,7 @@ use crate::block_helpers; use crate::block_loss_functions; use crate::block_lr; use crate::block_misc; +use crate::block_monte_carlo; use crate::block_neural; use crate::block_neural::InitType; use crate::block_normalize; @@ -175,7 +176,23 @@ impl Regressor { if mi.ffm_k > 0 { let mut block_ffm = block_ffm::new_ffm_block(&mut bg, mi).unwrap(); let mut triangle_ffm = block_misc::new_triangle_block(&mut bg, block_ffm).unwrap(); - output = block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); + if true { + let num_iterations = 5; + let dropout_rate = 0.01875; + + // TODO: make this configurable + let mut monte_carlo_ffm = block_monte_carlo::new_monte_carlo_block( + &mut bg, + triangle_ffm, + num_iterations, + dropout_rate, + ) + .unwrap(); + output = + block_misc::new_join_block(&mut bg, vec![output, monte_carlo_ffm]).unwrap(); + } else { + output = block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); + } } if !mi.nn_config.layers.is_empty() { @@ -391,9 +408,7 @@ impl Regressor { let further_blocks = &self.blocks_boxes[..]; block_helpers::forward(further_blocks, fb, pb); - assert_eq!(pb.observations.len(), 1); - - pb.observations.pop().unwrap() + self.select_prediction(pb) } pub fn predict_with_cache( @@ -407,10 +422,18 @@ impl Regressor { let further_blocks = &self.blocks_boxes[..]; block_helpers::forward_with_cache(further_blocks, fb, pb, caches); - assert_eq!(pb.observations.len(), 1); - let prediction_probability = pb.observations.pop().unwrap(); + self.select_prediction(pb) + } - return prediction_probability; + fn select_prediction(&self, pb: &mut port_buffer::PortBuffer) -> f32 { + match &pb.monte_carlo_stats { + Some(monte_carlo_stats) => monte_carlo_stats.mean, + None => { + assert_eq!(pb.observations.len(), 1); + + pb.observations.pop().unwrap() + } + } } pub fn setup_cache( From 76ec56715f12fc83f69a915ad960ffdb4b56e447 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Thu, 17 Aug 2023 15:19:21 +0200 Subject: [PATCH 02/17] MonteCarlo updates input on forward_backward step --- src/block_monte_carlo.rs | 75 ++++++++++++++++++++++++++++------------ src/regressor.rs | 3 -- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index c06215d8..479b0fd8 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -1,4 +1,3 @@ -use fasthash::Seed; use std::any::Any; use std::error::Error; @@ -7,10 +6,8 @@ use rand::distributions::{Distribution, Uniform}; use regressor::BlockTrait; use crate::block_helpers; -use crate::feature_buffer; use crate::feature_buffer::FeatureBuffer; use crate::graph; -use crate::port_buffer; use crate::port_buffer::{MonteCarloStats, PortBuffer}; use crate::regressor; use crate::regressor::BlockCache; @@ -34,7 +31,6 @@ pub fn new_monte_carlo_block( output_offset: usize::MAX, input_offset: usize::MAX, num_inputs, - num_outputs: num_inputs, }); let mut block_outputs = bg.add_node(block, vec![input])?; assert_eq!(block_outputs.len(), 1); @@ -47,7 +43,6 @@ pub struct BlockMonteCarlo { skip_index_generator: Uniform, pub num_inputs: usize, - pub num_outputs: usize, pub input_offset: usize, pub output_offset: usize, } @@ -59,7 +54,7 @@ impl BlockTrait for BlockMonteCarlo { fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert_eq!(output.get_output_index(), 0); - self.num_outputs + self.num_inputs } fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { @@ -81,7 +76,21 @@ impl BlockTrait for BlockMonteCarlo { pb: &mut PortBuffer, update: bool, ) { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward_backward(further_blocks, fb, pb, update); + + if update { + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_inputs, + ); + + input_tape.copy_from_slice(output_tape); + } } fn forward( @@ -91,20 +100,28 @@ impl BlockTrait for BlockMonteCarlo { pb: &mut PortBuffer, ) { unsafe { + if self.number_of_inputs_to_skip == 0 { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward(further_blocks, fb, pb); + return; + } + let mut rng = rand::thread_rng(); + let input_tape = pb + .tape + .get_unchecked_mut(self.input_offset..self.input_offset + self.num_inputs) + .to_vec(); for _ in 0..self.num_iterations { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, self.input_offset, self.num_inputs, self.output_offset, - self.num_outputs, + self.num_inputs, ); - output_tape - .get_unchecked_mut(..) - .copy_from_slice(input_tape.get_unchecked(..)); + output_tape.copy_from_slice(&input_tape); for _ in 0..self.number_of_inputs_to_skip { let skip_index = self.skip_index_generator.sample(&mut rng); *output_tape.get_unchecked_mut(skip_index) = 0.0; @@ -124,28 +141,28 @@ impl BlockTrait for BlockMonteCarlo { caches: &[BlockCache], ) { unsafe { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows( - &mut pb.tape, - self.input_offset, - self.num_inputs, - self.output_offset, - self.num_outputs, - ); + if self.number_of_inputs_to_skip == 0 { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward_with_cache(further_blocks, fb, pb, caches); + return; + } let mut rng = rand::thread_rng(); + let input_tape = pb + .tape + .get_unchecked_mut(self.input_offset..self.input_offset + self.num_inputs) + .to_vec(); for _ in 0..self.num_iterations { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, self.input_offset, self.num_inputs, self.output_offset, - self.num_outputs, + self.num_inputs, ); - output_tape - .get_unchecked_mut(..) - .copy_from_slice(input_tape.get_unchecked(..)); + output_tape.copy_from_slice(&input_tape); for _ in 0..self.number_of_inputs_to_skip { let skip_index = self.skip_index_generator.sample(&mut rng); *output_tape.get_unchecked_mut(skip_index) = 0.0; @@ -159,6 +176,18 @@ impl BlockTrait for BlockMonteCarlo { } impl BlockMonteCarlo { + fn copy_input_tape_to_output_tape(&self, pb: &mut PortBuffer) { + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_inputs, + ); + + output_tape.copy_from_slice(input_tape); + } + fn fill_stats(&self, pb: &mut PortBuffer) { let mean: f32 = pb.observations.iter().sum::() / self.num_iterations as f32; let variance = pb diff --git a/src/regressor.rs b/src/regressor.rs index 2c25e2ef..278494a6 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -341,9 +341,6 @@ impl Regressor { rg.tape_len = bg.get_tape_size(); rg.blocks_boxes = bg.take_blocks(); - /*for (i, block) in bg.blocks.into_iter().enumerate() { - rg.blocks_boxes.push(block); - }*/ rg } From 9648feb45b8ea9571b4f4bb0ac64a4345f7890c2 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Thu, 17 Aug 2023 15:31:42 +0200 Subject: [PATCH 03/17] Same order of methods applied in BlockMonteCarlo as in BlockTrait --- src/block_monte_carlo.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 479b0fd8..23a376c8 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -52,23 +52,6 @@ impl BlockTrait for BlockMonteCarlo { self } - fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { - assert_eq!(output.get_output_index(), 0); - self.num_inputs - } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { - assert_eq!(input.get_input_index(), 0); - assert_eq!(self.input_offset, usize::MAX); // We only allow a single call - self.input_offset = offset; - } - - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { - assert_eq!(output.get_output_index(), 0); - assert_eq!(self.output_offset, usize::MAX); // We only allow a single call - self.output_offset = offset; - } - fn forward_backward( &mut self, further_blocks: &mut [Box], @@ -173,6 +156,23 @@ impl BlockTrait for BlockMonteCarlo { self.fill_stats(pb); } } + + fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { + assert_eq!(output.get_output_index(), 0); + self.num_inputs + } + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + assert_eq!(input.get_input_index(), 0); + assert_eq!(self.input_offset, usize::MAX); // We only allow a single call + self.input_offset = offset; + } + + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + assert_eq!(output.get_output_index(), 0); + assert_eq!(self.output_offset, usize::MAX); // We only allow a single call + self.output_offset = offset; + } } impl BlockMonteCarlo { From ac76305087892433ce226b973b12e5bc1ce8f5e2 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Thu, 17 Aug 2023 16:23:31 +0200 Subject: [PATCH 04/17] Seed-ed random Xoshiro256Plus random number generator used in BlockMonteCarlo --- src/block_monte_carlo.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 23a376c8..a4b0dabe 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -2,6 +2,8 @@ use std::any::Any; use std::error::Error; use rand::distributions::{Distribution, Uniform}; +use rand_xoshiro::rand_core::SeedableRng; +use rand_xoshiro::Xoshiro256Plus; use regressor::BlockTrait; @@ -37,6 +39,10 @@ pub fn new_monte_carlo_block( Ok(block_outputs.pop().unwrap()) } +fn create_seed_from_input_tape(input_tape: &[f32]) -> u64 { + (input_tape.iter().sum::() * 1000.0) as u64 +} + pub struct BlockMonteCarlo { num_iterations: usize, number_of_inputs_to_skip: usize, @@ -89,12 +95,14 @@ impl BlockTrait for BlockMonteCarlo { return; } - let mut rng = rand::thread_rng(); - let input_tape = pb .tape .get_unchecked_mut(self.input_offset..self.input_offset + self.num_inputs) .to_vec(); + + let seed = create_seed_from_input_tape(&input_tape); + let mut rng = Xoshiro256Plus::seed_from_u64(seed); + for _ in 0..self.num_iterations { let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, @@ -130,12 +138,14 @@ impl BlockTrait for BlockMonteCarlo { return; } - let mut rng = rand::thread_rng(); - let input_tape = pb .tape .get_unchecked_mut(self.input_offset..self.input_offset + self.num_inputs) .to_vec(); + + let seed = create_seed_from_input_tape(&input_tape); + let mut rng = Xoshiro256Plus::seed_from_u64(seed); + for _ in 0..self.num_iterations { let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, From 357434cf28c9b4a525040660a94ac9e5d0cb96b1 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Thu, 17 Aug 2023 16:31:23 +0200 Subject: [PATCH 05/17] Configurability of FFM MonteCarlo added --- src/model_instance.rs | 7 +++++++ src/regressor.rs | 14 +++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/model_instance.rs b/src/model_instance.rs index c834bfce..4654638d 100644 --- a/src/model_instance.rs +++ b/src/model_instance.rs @@ -87,6 +87,11 @@ pub struct ModelInstance { #[serde(default = "default_f32_zero")] pub nn_power_t: f32, + #[serde(default = "default_u32_zero")] + pub ffm_mc_iteration_count: u32, + #[serde(default = "default_f32_zero")] + pub ffm_mc_dropout_rate: f32, + pub nn_config: NNConfig, #[serde(default = "default_optimizer_adagrad")] @@ -136,6 +141,8 @@ impl ModelInstance { ffm_init_width: 0.0, ffm_init_zero_band: 0.0, ffm_init_acc_gradient: 0.0, + ffm_mc_iteration_count: 0, + ffm_mc_dropout_rate: 0.0, nn_init_acc_gradient: 0.0, nn_learning_rate: 0.02, nn_power_t: 0.45, diff --git a/src/regressor.rs b/src/regressor.rs index 278494a6..baf34cff 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -176,22 +176,18 @@ impl Regressor { if mi.ffm_k > 0 { let mut block_ffm = block_ffm::new_ffm_block(&mut bg, mi).unwrap(); let mut triangle_ffm = block_misc::new_triangle_block(&mut bg, block_ffm).unwrap(); - if true { - let num_iterations = 5; - let dropout_rate = 0.01875; - - // TODO: make this configurable + if mi.ffm_mc_iteration_count == 0 || mi.ffm_mc_dropout_rate <= 0.0 { + output = block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); + } else { let mut monte_carlo_ffm = block_monte_carlo::new_monte_carlo_block( &mut bg, triangle_ffm, - num_iterations, - dropout_rate, + mi.ffm_mc_iteration_count as usize, + mi.ffm_mc_dropout_rate, ) .unwrap(); output = block_misc::new_join_block(&mut bg, vec![output, monte_carlo_ffm]).unwrap(); - } else { - output = block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); } } From e702de98eac9b86ec83d571f33ae1b059da60023 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Fri, 18 Aug 2023 15:33:45 +0200 Subject: [PATCH 06/17] Test added for MonteCarlo --- src/block_monte_carlo.rs | 58 ++++++++++++++++++++++++++++++++++++++++ src/port_buffer.rs | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index a4b0dabe..f00e6f9d 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -218,3 +218,61 @@ impl BlockMonteCarlo { }); } } + +mod tests { + use crate::assert_epsilon; + use crate::block_helpers::{slearn2, spredict2}; + use crate::block_misc::Observe; + use crate::block_monte_carlo::new_monte_carlo_block; + use crate::graph::BlockGraph; + use crate::model_instance::ModelInstance; + use crate::{block_misc, feature_buffer::FeatureBuffer}; + + fn fb_vec() -> FeatureBuffer { + FeatureBuffer { + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: Vec::new(), + ffm_buffer: Vec::new(), + } + } + + #[test] + fn test_monte_carlo_block() { + let mut mi = ModelInstance::new_empty().unwrap(); + let mut bg = BlockGraph::new(); + let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let triangle_block = + new_monte_carlo_block(&mut bg, observe_block_backward, 3, 0.3).unwrap(); + let observe_block_forward = + block_misc::new_observe_block(&mut bg, triangle_block, Observe::Forward, None).unwrap(); + block_misc::new_sink_block( + &mut bg, + observe_block_forward, + block_misc::SinkType::Untouched, + ) + .unwrap(); + bg.finalize(); + bg.allocate_and_init_weights(&mi); + + let mut pb = bg.new_port_buffer(); + let fb = fb_vec(); + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, [2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0]); + + spredict2(&mut bg, &fb, &mut pb, false); + let expected_observations = [2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 0.0, 2.0, 4.0, 4.0, 5.0]; + assert_eq!(pb.observations.len(), expected_observations.len()); + assert_eq!(pb.observations, expected_observations); + + assert_ne!(pb.monte_carlo_stats, None); + + let monte_carlo_stats = pb.monte_carlo_stats.unwrap(); + assert_epsilon!(monte_carlo_stats.mean, 11.333333); + assert_epsilon!(monte_carlo_stats.variance, 302.88885); + assert_epsilon!(monte_carlo_stats.standard_deviation, 17.403702); + } +} diff --git a/src/port_buffer.rs b/src/port_buffer.rs index 07891e35..85a71c10 100644 --- a/src/port_buffer.rs +++ b/src/port_buffer.rs @@ -6,7 +6,7 @@ pub struct PortBuffer { pub monte_carlo_stats: Option, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct MonteCarloStats { pub mean: f32, pub variance: f32, From d177241148280d5bbae59ac3ce7e32c9174d3d0e Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Sun, 20 Aug 2023 22:49:13 +0200 Subject: [PATCH 07/17] Prediction statistics exposed via FFI --- src/block_monte_carlo.rs | 19 ++++--- src/lib.rs | 109 ++++++++++++++++++++++++++++++++++++++- src/port_buffer.rs | 9 ++-- src/regressor.rs | 66 +++++++++++++++++++++--- 4 files changed, 183 insertions(+), 20 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index f00e6f9d..6e035611 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -10,7 +10,7 @@ use regressor::BlockTrait; use crate::block_helpers; use crate::feature_buffer::FeatureBuffer; use crate::graph; -use crate::port_buffer::{MonteCarloStats, PortBuffer}; +use crate::port_buffer::{PortBuffer, PredictionStats}; use crate::regressor; use crate::regressor::BlockCache; @@ -211,10 +211,11 @@ impl BlockMonteCarlo { / self.num_iterations as f32; let standard_deviation = variance.sqrt(); - pb.monte_carlo_stats = Some(MonteCarloStats { + pb.stats = Some(PredictionStats { mean, variance, standard_deviation, + count: self.num_iterations, }); } } @@ -264,15 +265,17 @@ mod tests { assert_eq!(pb.observations, [2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0]); spredict2(&mut bg, &fb, &mut pb, false); - let expected_observations = [2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 0.0, 2.0, 4.0, 4.0, 5.0]; + let expected_observations = [ + 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 0.0, 2.0, 4.0, 4.0, 5.0, + ]; assert_eq!(pb.observations.len(), expected_observations.len()); assert_eq!(pb.observations, expected_observations); - assert_ne!(pb.monte_carlo_stats, None); + assert_ne!(pb.stats, None); - let monte_carlo_stats = pb.monte_carlo_stats.unwrap(); - assert_epsilon!(monte_carlo_stats.mean, 11.333333); - assert_epsilon!(monte_carlo_stats.variance, 302.88885); - assert_epsilon!(monte_carlo_stats.standard_deviation, 17.403702); + let stats = pb.stats.unwrap(); + assert_epsilon!(stats.mean, 11.333333); + assert_epsilon!(stats.variance, 302.88885); + assert_epsilon!(stats.standard_deviation, 17.403702); } } diff --git a/src/lib.rs b/src/lib.rs index 13ea39c5..8957de86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ extern crate intel_mkl_src; use crate::feature_buffer::FeatureBufferTranslator; use crate::multithread_helpers::BoxedRegressorTrait; use crate::parser::VowpalParser; -use crate::port_buffer::PortBuffer; +use crate::port_buffer::{PortBuffer, PredictionStats}; use crate::regressor::BlockCache; use shellwords; use std::ffi::CStr; @@ -81,6 +81,29 @@ impl Predictor { .predict(&self.feature_buffer_translator.feature_buffer, &mut self.pb) } + unsafe fn predict_and_stats(&mut self, input_buffer: &str) -> Result { + let mut buffered_input = Cursor::new(input_buffer); + let reading_result = self.vw_parser.next_vowpal(&mut buffered_input); + let buffer = match reading_result { + Ok([]) => { + log::error!("Reading result for prediction and stats returns EOF"); + return Err(EOF_ERROR_CODE); + } // EOF + Ok(buffer2) => buffer2, + Err(e) => { + log::error!( + "Reading result for prediction and stats returns error {}", + e + ); + return Err(EXCEPTION_ERROR_CODE); + } + }; + self.feature_buffer_translator.translate(buffer, 0); + Ok(self + .regressor + .predict_and_stats(&self.feature_buffer_translator.feature_buffer, &mut self.pb)) + } + unsafe fn predict_with_cache(&mut self, input_buffer: &str) -> f32 { let mut buffered_input = Cursor::new(&input_buffer); let reading_result = self @@ -110,6 +133,38 @@ impl Predictor { ) } + unsafe fn predict_and_stats_with_cache( + &mut self, + input_buffer: &str, + ) -> Result { + let mut buffered_input = Cursor::new(&input_buffer); + let reading_result = self + .vw_parser + .next_vowpal_with_cache(&mut buffered_input, self.cache.input_buffer_size); + + let buffer = match reading_result { + Ok([]) => { + log::error!("Reading result for prediction and stats with cache returns EOF"); + return Err(EOF_ERROR_CODE); + } // EOF + Ok(buffer2) => buffer2, + Err(e) => { + log::error!( + "Reading result for prediction and stats with cache returns error {}", + e + ); + return Err(EXCEPTION_ERROR_CODE); + } + }; + + self.feature_buffer_translator.translate(buffer, 0); + Ok(self.regressor.predict_and_stats_with_cache( + &self.feature_buffer_translator.feature_buffer, + &mut self.pb, + self.cache.blocks.as_slice(), + )) + } + unsafe fn setup_cache(&mut self, input_buffer: &str) -> f32 { let mut buffered_input = Cursor::new(input_buffer); let reading_result = self.vw_parser.next_vowpal_with_size(&mut buffered_input); @@ -201,6 +256,24 @@ pub unsafe extern "C" fn fw_predict(ptr: *mut FfiPredictor, input_buffer: *const predictor.predict(str_buffer) } +#[no_mangle] +pub unsafe extern "C" fn fw_predict_and_stats( + ptr: *mut FfiPredictor, + input_buffer: *const c_char, + stats: *mut f32, + stats_size: usize, +) -> f32 { + let str_buffer = c_char_to_str(input_buffer); + let predictor: &mut Predictor = from_ptr(ptr); + match predictor.predict_and_stats(str_buffer) { + Ok(prediction_stats) => { + fill_prediction_stats(&prediction_stats, stats, stats_size); + prediction_stats.mean + } + Err(e) => e, + } +} + #[no_mangle] pub unsafe extern "C" fn fw_predict_with_cache( ptr: *mut FfiPredictor, @@ -211,6 +284,40 @@ pub unsafe extern "C" fn fw_predict_with_cache( predictor.predict_with_cache(str_buffer) } +#[no_mangle] +pub unsafe extern "C" fn fw_predict_with_cache_and_stats( + ptr: *mut FfiPredictor, + input_buffer: *const c_char, + stats: *mut f32, + stats_size: usize, +) -> f32 { + let str_buffer = c_char_to_str(input_buffer); + let predictor: &mut Predictor = from_ptr(ptr); + match predictor.predict_and_stats_with_cache(str_buffer) { + Ok(prediction_stats) => { + fill_prediction_stats(&prediction_stats, stats, stats_size); + prediction_stats.mean + } + Err(e) => e, + } +} + +unsafe fn fill_prediction_stats( + prediction_stats: &PredictionStats, + stats: *mut f32, + stats_size: usize, +) { + if stats_size < 4 { + log::error!("Stats buffer is too small, expecting 4, was {}", stats_size); + return; + } + let stats_slice = std::slice::from_raw_parts_mut(stats, stats_size); + *stats_slice.get_unchecked_mut(0) = prediction_stats.mean; + *stats_slice.get_unchecked_mut(1) = prediction_stats.variance; + *stats_slice.get_unchecked_mut(2) = prediction_stats.standard_deviation; + *stats_slice.get_unchecked_mut(3) = prediction_stats.count as f32; +} + #[no_mangle] pub unsafe extern "C" fn fw_setup_cache( ptr: *mut FfiPredictor, diff --git a/src/port_buffer.rs b/src/port_buffer.rs index 85a71c10..3bab79b2 100644 --- a/src/port_buffer.rs +++ b/src/port_buffer.rs @@ -3,14 +3,15 @@ pub struct PortBuffer { pub tape: Vec, pub observations: Vec, pub tape_len: usize, - pub monte_carlo_stats: Option, + pub stats: Option, } #[derive(Clone, Debug, PartialEq)] -pub struct MonteCarloStats { +pub struct PredictionStats { pub mean: f32, pub variance: f32, pub standard_deviation: f32, + pub count: usize, } impl PortBuffer { @@ -19,13 +20,13 @@ impl PortBuffer { tape: Default::default(), observations: Default::default(), tape_len, - monte_carlo_stats: None, + stats: None, } } pub fn reset(&mut self) { self.observations.truncate(0); self.tape.resize(self.tape_len, 0.0); - self.monte_carlo_stats = None; + self.stats = None; } } diff --git a/src/regressor.rs b/src/regressor.rs index baf34cff..0fa78194 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -19,6 +19,7 @@ use crate::graph; use crate::model_instance; use crate::optimizer; use crate::port_buffer; +use crate::port_buffer::PredictionStats; pub const FFM_CONTRA_BUF_LEN: usize = 16384; @@ -390,37 +391,72 @@ impl Regressor { pb.observations.pop().unwrap() } - pub fn predict( + fn predict_internal( &self, fb: &feature_buffer::FeatureBuffer, pb: &mut port_buffer::PortBuffer, - ) -> f32 { - // TODO: we should find a way of not using unsafe + ) { pb.reset(); // empty the tape let further_blocks = &self.blocks_boxes[..]; block_helpers::forward(further_blocks, fb, pb); + } + pub fn predict( + &self, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) -> f32 { + self.predict_internal(fb, pb); self.select_prediction(pb) } - pub fn predict_with_cache( + pub fn predict_and_stats( + &self, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) -> PredictionStats { + self.predict_internal(fb, pb); + self.select_prediction_with_stats(pb) + } + + fn predict_with_cache_internal( &self, fb: &feature_buffer::FeatureBuffer, pb: &mut port_buffer::PortBuffer, caches: &[BlockCache], - ) -> f32 { + ) { pb.reset(); // empty the tape let further_blocks = &self.blocks_boxes[..]; block_helpers::forward_with_cache(further_blocks, fb, pb, caches); + } + + pub fn predict_with_cache( + &self, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + caches: &[BlockCache], + ) -> f32 { + self.predict_with_cache_internal(fb, pb, caches); self.select_prediction(pb) } + pub fn predict_and_stats_with_cache( + &self, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + caches: &[BlockCache], + ) -> PredictionStats { + self.predict_with_cache_internal(fb, pb, caches); + + self.select_prediction_with_stats(pb) + } + fn select_prediction(&self, pb: &mut port_buffer::PortBuffer) -> f32 { - match &pb.monte_carlo_stats { - Some(monte_carlo_stats) => monte_carlo_stats.mean, + match &pb.stats { + Some(stats) => stats.mean, None => { assert_eq!(pb.observations.len(), 1); @@ -429,6 +465,22 @@ impl Regressor { } } + fn select_prediction_with_stats(&self, pb: &mut port_buffer::PortBuffer) -> PredictionStats { + match &pb.stats { + Some(stats) => stats.clone(), + None => { + assert_eq!(pb.observations.len(), 1); + + PredictionStats { + mean: pb.observations.pop().unwrap(), + variance: 0.0, + standard_deviation: 0.0, + count: 1, + } + } + } + } + pub fn setup_cache( &mut self, fb: &feature_buffer::FeatureBuffer, From 62029cb42a41db3468393536cee255e67a080fed Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Sun, 20 Aug 2023 23:17:56 +0200 Subject: [PATCH 08/17] FFI naming made consistent --- src/lib.rs | 29 ++++++++++++++--------------- src/regressor.rs | 10 +++++----- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8957de86..66f7635c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,7 +81,10 @@ impl Predictor { .predict(&self.feature_buffer_translator.feature_buffer, &mut self.pb) } - unsafe fn predict_and_stats(&mut self, input_buffer: &str) -> Result { + unsafe fn predict_and_report_stats( + &mut self, + input_buffer: &str, + ) -> Result { let mut buffered_input = Cursor::new(input_buffer); let reading_result = self.vw_parser.next_vowpal(&mut buffered_input); let buffer = match reading_result { @@ -101,7 +104,7 @@ impl Predictor { self.feature_buffer_translator.translate(buffer, 0); Ok(self .regressor - .predict_and_stats(&self.feature_buffer_translator.feature_buffer, &mut self.pb)) + .predict_and_report_stats(&self.feature_buffer_translator.feature_buffer, &mut self.pb)) } unsafe fn predict_with_cache(&mut self, input_buffer: &str) -> f32 { @@ -133,7 +136,7 @@ impl Predictor { ) } - unsafe fn predict_and_stats_with_cache( + unsafe fn predict_with_cache_and_report_stats( &mut self, input_buffer: &str, ) -> Result { @@ -158,7 +161,7 @@ impl Predictor { }; self.feature_buffer_translator.translate(buffer, 0); - Ok(self.regressor.predict_and_stats_with_cache( + Ok(self.regressor.predict_with_cache_and_report_stats( &self.feature_buffer_translator.feature_buffer, &mut self.pb, self.cache.blocks.as_slice(), @@ -257,7 +260,7 @@ pub unsafe extern "C" fn fw_predict(ptr: *mut FfiPredictor, input_buffer: *const } #[no_mangle] -pub unsafe extern "C" fn fw_predict_and_stats( +pub unsafe extern "C" fn fw_predict_and_report_stats( ptr: *mut FfiPredictor, input_buffer: *const c_char, stats: *mut f32, @@ -265,9 +268,9 @@ pub unsafe extern "C" fn fw_predict_and_stats( ) -> f32 { let str_buffer = c_char_to_str(input_buffer); let predictor: &mut Predictor = from_ptr(ptr); - match predictor.predict_and_stats(str_buffer) { + match predictor.predict_and_report_stats(str_buffer) { Ok(prediction_stats) => { - fill_prediction_stats(&prediction_stats, stats, stats_size); + report_stats(&prediction_stats, stats, stats_size); prediction_stats.mean } Err(e) => e, @@ -285,7 +288,7 @@ pub unsafe extern "C" fn fw_predict_with_cache( } #[no_mangle] -pub unsafe extern "C" fn fw_predict_with_cache_and_stats( +pub unsafe extern "C" fn fw_predict_with_cache_and_report_stats( ptr: *mut FfiPredictor, input_buffer: *const c_char, stats: *mut f32, @@ -293,20 +296,16 @@ pub unsafe extern "C" fn fw_predict_with_cache_and_stats( ) -> f32 { let str_buffer = c_char_to_str(input_buffer); let predictor: &mut Predictor = from_ptr(ptr); - match predictor.predict_and_stats_with_cache(str_buffer) { + match predictor.predict_with_cache_and_report_stats(str_buffer) { Ok(prediction_stats) => { - fill_prediction_stats(&prediction_stats, stats, stats_size); + report_stats(&prediction_stats, stats, stats_size); prediction_stats.mean } Err(e) => e, } } -unsafe fn fill_prediction_stats( - prediction_stats: &PredictionStats, - stats: *mut f32, - stats_size: usize, -) { +unsafe fn report_stats(prediction_stats: &PredictionStats, stats: *mut f32, stats_size: usize) { if stats_size < 4 { log::error!("Stats buffer is too small, expecting 4, was {}", stats_size); return; diff --git a/src/regressor.rs b/src/regressor.rs index 0fa78194..2fa1ffd7 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -411,13 +411,13 @@ impl Regressor { self.select_prediction(pb) } - pub fn predict_and_stats( + pub fn predict_and_report_stats( &self, fb: &feature_buffer::FeatureBuffer, pb: &mut port_buffer::PortBuffer, ) -> PredictionStats { self.predict_internal(fb, pb); - self.select_prediction_with_stats(pb) + self.select_prediction_and_stats(pb) } fn predict_with_cache_internal( @@ -443,7 +443,7 @@ impl Regressor { self.select_prediction(pb) } - pub fn predict_and_stats_with_cache( + pub fn predict_with_cache_and_report_stats( &self, fb: &feature_buffer::FeatureBuffer, pb: &mut port_buffer::PortBuffer, @@ -451,7 +451,7 @@ impl Regressor { ) -> PredictionStats { self.predict_with_cache_internal(fb, pb, caches); - self.select_prediction_with_stats(pb) + self.select_prediction_and_stats(pb) } fn select_prediction(&self, pb: &mut port_buffer::PortBuffer) -> f32 { @@ -465,7 +465,7 @@ impl Regressor { } } - fn select_prediction_with_stats(&self, pb: &mut port_buffer::PortBuffer) -> PredictionStats { + fn select_prediction_and_stats(&self, pb: &mut port_buffer::PortBuffer) -> PredictionStats { match &pb.stats { Some(stats) => stats.clone(), None => { From 35ab593e4d0cea131d555d6d2788c1128adea5fb Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Mon, 21 Aug 2023 15:28:16 +0200 Subject: [PATCH 09/17] Monte Carlo test corrected after incorrect merge --- src/block_monte_carlo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 6e035611..3dd1b8b0 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -264,7 +264,7 @@ mod tests { slearn2(&mut bg, &fb, &mut pb, true); assert_eq!(pb.observations, [2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0]); - spredict2(&mut bg, &fb, &mut pb, false); + spredict2(&mut bg, &fb, &mut pb); let expected_observations = [ 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 0.0, 2.0, 4.0, 4.0, 5.0, ]; From a447cf84d3565d49d5bd8c902cbeeb94423fbae3 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Mon, 21 Aug 2023 21:47:54 +0200 Subject: [PATCH 10/17] Limit monte carlo blocks to more then 0 iterations --- src/block_monte_carlo.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 3dd1b8b0..62870ded 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -22,6 +22,7 @@ pub fn new_monte_carlo_block( ) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); assert_ne!(num_inputs, 0); + assert_ne!(num_iterations, 0); let skip_index_generator = Uniform::from(0..num_inputs); let number_of_inputs_to_skip = (dropout_rate * num_inputs as f32) as usize; From 1b3859ff931d6891ed1b3031a80bb0786c4556c2 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Tue, 29 Aug 2023 20:10:37 +0200 Subject: [PATCH 11/17] First iteration of Monte Carlo block executes run without masking --- src/block_monte_carlo.rs | 62 +++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 62870ded..0da444be 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -90,9 +90,10 @@ impl BlockTrait for BlockMonteCarlo { pb: &mut PortBuffer, ) { unsafe { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward(further_blocks, fb, pb); + if self.number_of_inputs_to_skip == 0 { - self.copy_input_tape_to_output_tape(pb); - block_helpers::forward(further_blocks, fb, pb); return; } @@ -104,7 +105,7 @@ impl BlockTrait for BlockMonteCarlo { let seed = create_seed_from_input_tape(&input_tape); let mut rng = Xoshiro256Plus::seed_from_u64(seed); - for _ in 0..self.num_iterations { + for _ in 1..self.num_iterations { let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, self.input_offset, @@ -133,9 +134,10 @@ impl BlockTrait for BlockMonteCarlo { caches: &[BlockCache], ) { unsafe { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward_with_cache(further_blocks, fb, pb, caches); + if self.number_of_inputs_to_skip == 0 { - self.copy_input_tape_to_output_tape(pb); - block_helpers::forward_with_cache(further_blocks, fb, pb, caches); return; } @@ -147,7 +149,7 @@ impl BlockTrait for BlockMonteCarlo { let seed = create_seed_from_input_tape(&input_tape); let mut rng = Xoshiro256Plus::seed_from_u64(seed); - for _ in 0..self.num_iterations { + for _ in 1..self.num_iterations { let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, self.input_offset, @@ -241,7 +243,45 @@ mod tests { } #[test] - fn test_monte_carlo_block() { + fn test_monte_carlo_block_with_one_run() { + let mut mi = ModelInstance::new_empty().unwrap(); + let mut bg = BlockGraph::new(); + let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let triangle_block = + new_monte_carlo_block(&mut bg, observe_block_backward, 1, 0.3).unwrap(); + let observe_block_forward = + block_misc::new_observe_block(&mut bg, triangle_block, Observe::Forward, None).unwrap(); + block_misc::new_sink_block( + &mut bg, + observe_block_forward, + block_misc::SinkType::Untouched, + ) + .unwrap(); + bg.finalize(); + bg.allocate_and_init_weights(&mi); + + let mut pb = bg.new_port_buffer(); + let fb = fb_vec(); + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, [2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0]); + + spredict2(&mut bg, &fb, &mut pb); + let expected_observations = [2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0]; + assert_eq!(pb.observations.len(), expected_observations.len()); + assert_eq!(pb.observations, expected_observations); + + assert_ne!(pb.stats, None); + + let stats = pb.stats.unwrap(); + assert_epsilon!(stats.mean, 15.0); + assert_epsilon!(stats.variance, 511.0); + assert_epsilon!(stats.standard_deviation, 22.605309); + } + + #[test] + fn test_monte_carlo_block_with_multiple_runs() { let mut mi = ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); @@ -267,7 +307,7 @@ mod tests { spredict2(&mut bg, &fb, &mut pb); let expected_observations = [ - 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 0.0, 2.0, 4.0, 4.0, 5.0, + 2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0, ]; assert_eq!(pb.observations.len(), expected_observations.len()); assert_eq!(pb.observations, expected_observations); @@ -275,8 +315,8 @@ mod tests { assert_ne!(pb.stats, None); let stats = pb.stats.unwrap(); - assert_epsilon!(stats.mean, 11.333333); - assert_epsilon!(stats.variance, 302.88885); - assert_epsilon!(stats.standard_deviation, 17.403702); + assert_epsilon!(stats.mean, 13.0); + assert_epsilon!(stats.variance, 392.33334); + assert_epsilon!(stats.standard_deviation, 19.807405); } } From 69b757363f05a08886b792c5a80f84c3fd48ebac Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Mon, 11 Sep 2023 15:04:02 +0200 Subject: [PATCH 12/17] Handle 0 iteration of Monte Carlo in a more consise way --- src/block_monte_carlo.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 0da444be..2eea2b4c 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -90,10 +90,9 @@ impl BlockTrait for BlockMonteCarlo { pb: &mut PortBuffer, ) { unsafe { - self.copy_input_tape_to_output_tape(pb); - block_helpers::forward(further_blocks, fb, pb); - if self.number_of_inputs_to_skip == 0 { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward(further_blocks, fb, pb); return; } @@ -105,7 +104,7 @@ impl BlockTrait for BlockMonteCarlo { let seed = create_seed_from_input_tape(&input_tape); let mut rng = Xoshiro256Plus::seed_from_u64(seed); - for _ in 1..self.num_iterations { + for i in 0..self.num_iterations { let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, self.input_offset, @@ -115,9 +114,11 @@ impl BlockTrait for BlockMonteCarlo { ); output_tape.copy_from_slice(&input_tape); - for _ in 0..self.number_of_inputs_to_skip { - let skip_index = self.skip_index_generator.sample(&mut rng); - *output_tape.get_unchecked_mut(skip_index) = 0.0; + if i != 0 { + for _ in 0..self.number_of_inputs_to_skip { + let skip_index = self.skip_index_generator.sample(&mut rng); + *output_tape.get_unchecked_mut(skip_index) = 0.0; + } } block_helpers::forward(further_blocks, fb, pb); } @@ -134,10 +135,9 @@ impl BlockTrait for BlockMonteCarlo { caches: &[BlockCache], ) { unsafe { - self.copy_input_tape_to_output_tape(pb); - block_helpers::forward_with_cache(further_blocks, fb, pb, caches); - if self.number_of_inputs_to_skip == 0 { + self.copy_input_tape_to_output_tape(pb); + block_helpers::forward_with_cache(further_blocks, fb, pb, caches); return; } @@ -149,7 +149,7 @@ impl BlockTrait for BlockMonteCarlo { let seed = create_seed_from_input_tape(&input_tape); let mut rng = Xoshiro256Plus::seed_from_u64(seed); - for _ in 1..self.num_iterations { + for i in 0..self.num_iterations { let (_, output_tape) = block_helpers::get_input_output_borrows( &mut pb.tape, self.input_offset, @@ -159,9 +159,11 @@ impl BlockTrait for BlockMonteCarlo { ); output_tape.copy_from_slice(&input_tape); - for _ in 0..self.number_of_inputs_to_skip { - let skip_index = self.skip_index_generator.sample(&mut rng); - *output_tape.get_unchecked_mut(skip_index) = 0.0; + if i != 0 { + for _ in 0..self.number_of_inputs_to_skip { + let skip_index = self.skip_index_generator.sample(&mut rng); + *output_tape.get_unchecked_mut(skip_index) = 0.0; + } } block_helpers::forward_with_cache(further_blocks, fb, pb, caches); } From 8c131a2933d462a733ae1dbe6cb94e5ce5a7e785 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Mon, 11 Sep 2023 15:04:36 +0200 Subject: [PATCH 13/17] ffm_mc_iteration_count and ffm_mc_dropout_rate exposed via cli --- src/cmdline.rs | 8 ++++++++ src/model_instance.rs | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/cmdline.rs b/src/cmdline.rs index 2279c91b..5631b34a 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -194,6 +194,14 @@ pub fn create_expected_args<'a>() -> App<'a, 'a> { .value_name("k") .help("Lenght of a vector to use for FFM") .takes_value(true)) + .arg(Arg::with_name("ffm_mc_iteration_count") + .long("ffm_mc_iteration_count") + .help("Number of Monte Carlo iterations to run") + .takes_value(true)) + .arg(Arg::with_name("ffm_mc_dropout_rate") + .long("ffm_mc_dropout_rate") + .help("Dropout rate for Monte Carlo after FFM") + .takes_value(true)) .arg(Arg::with_name("ffm_bit_precision") .long("ffm_bit_precision") .value_name("N") diff --git a/src/model_instance.rs b/src/model_instance.rs index be901c6a..25f562d5 100644 --- a/src/model_instance.rs +++ b/src/model_instance.rs @@ -382,6 +382,12 @@ impl ModelInstance { } } + if let Some(val) = cl.value_of("ffm_mc_iteration_count") { + mi.ffm_mc_iteration_count = val.parse()?; + } + + mi.ffm_mc_dropout_rate = parse_float("ffm_mc_dropout_rate", mi.ffm_mc_dropout_rate, cl); + if let Some(val) = cl.value_of("ffm_initialization_type") { mi.ffm_initialization_type = val.parse()?; } @@ -543,6 +549,22 @@ impl ModelInstance { } } + if cmd_arguments.is_present("ffm_mc_iteration_count") { + if let Some(val) = cmd_arguments.value_of("ffm_mc_iteration_count") { + let hvalue = val.parse::<>()?; + mi.ffm_mc_iteration_count = hvalue; + replacement_hyperparam_ids.push(("ffm_mc_iteration_count".to_string(), hvalue.to_string())); + } + } + + if cmd_arguments.is_present("ffm_mc_dropout_rate") { + if let Some(val) = cmd_arguments.value_of("ffm_mc_dropout_rate") { + let hvalue = val.parse::()?; + mi.ffm_mc_dropout_rate = hvalue; + replacement_hyperparam_ids.push(("ffm_mc_dropout_rate".to_string(), hvalue.to_string())); + } + } + for (hyper_name, hyper_value) in replacement_hyperparam_ids.into_iter() { log::warn!( "Warning! Updated hyperparameter {} to value {}", From def777c13031829bac78e8cd247ae7ec4fb51ea0 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Wed, 11 Oct 2023 09:55:28 +0200 Subject: [PATCH 14/17] Moving monte carlo block before triangle --- src/block_monte_carlo.rs | 13 +++++++------ src/feature_transform_implementations.rs | 2 +- src/regressor.rs | 7 ++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 2eea2b4c..49aa91c8 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -24,10 +24,10 @@ pub fn new_monte_carlo_block( assert_ne!(num_inputs, 0); assert_ne!(num_iterations, 0); - let skip_index_generator = Uniform::from(0..num_inputs); let number_of_inputs_to_skip = (dropout_rate * num_inputs as f32) as usize; + let skip_index_generator = Uniform::from(0..num_inputs); - let mut block = Box::new(BlockMonteCarlo { + let block = Box::new(BlockMonteCarlo { num_iterations, number_of_inputs_to_skip, skip_index_generator, @@ -41,7 +41,7 @@ pub fn new_monte_carlo_block( } fn create_seed_from_input_tape(input_tape: &[f32]) -> u64 { - (input_tape.iter().sum::() * 1000.0) as u64 + (input_tape.iter().sum::() * 100_000.0).round() as u64 } pub struct BlockMonteCarlo { @@ -225,6 +225,7 @@ impl BlockMonteCarlo { } } +#[cfg(test)] mod tests { use crate::assert_epsilon; use crate::block_helpers::{slearn2, spredict2}; @@ -246,7 +247,7 @@ mod tests { #[test] fn test_monte_carlo_block_with_one_run() { - let mut mi = ModelInstance::new_empty().unwrap(); + let mi = ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); let observe_block_backward = @@ -284,7 +285,7 @@ mod tests { #[test] fn test_monte_carlo_block_with_multiple_runs() { - let mut mi = ModelInstance::new_empty().unwrap(); + let mi = ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); let observe_block_backward = @@ -309,7 +310,7 @@ mod tests { spredict2(&mut bg, &fb, &mut pb); let expected_observations = [ - 2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0, + 2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0 ]; assert_eq!(pb.observations.len(), expected_observations.len()); assert_eq!(pb.observations, expected_observations); diff --git a/src/feature_transform_implementations.rs b/src/feature_transform_implementations.rs index 36c7c479..27db661d 100644 --- a/src/feature_transform_implementations.rs +++ b/src/feature_transform_implementations.rs @@ -622,7 +622,7 @@ mod tests { namespace_descriptor: ns_desc(0), }; - let to_namespace_empty = ExecutorToNamespace { + let _to_namespace_empty = ExecutorToNamespace { namespace_descriptor: ns_desc(1), namespace_seeds: default_seeds(1), // These are precomputed namespace seeds tmp_data: Vec::new(), diff --git a/src/regressor.rs b/src/regressor.rs index dc204cb0..8a1aee44 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -186,19 +186,20 @@ impl Regressor { if mi.ffm_k > 0 { let block_ffm = block_ffm::new_ffm_block(&mut bg, mi).unwrap(); - let triangle_ffm = block_misc::new_triangle_block(&mut bg, block_ffm).unwrap(); if mi.ffm_mc_iteration_count == 0 || mi.ffm_mc_dropout_rate <= 0.0 { + let triangle_ffm = block_misc::new_triangle_block(&mut bg, block_ffm).unwrap(); output = block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); } else { let monte_carlo_ffm = block_monte_carlo::new_monte_carlo_block( &mut bg, - triangle_ffm, + block_ffm, mi.ffm_mc_iteration_count as usize, mi.ffm_mc_dropout_rate, ) .unwrap(); + let triangle_ffm = block_misc::new_triangle_block(&mut bg, monte_carlo_ffm).unwrap(); output = - block_misc::new_join_block(&mut bg, vec![output, monte_carlo_ffm]).unwrap(); + block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); } } From bd8911f9511974e1cb3c98e2003bfc7283fb8c0d Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Wed, 11 Oct 2023 11:08:29 +0200 Subject: [PATCH 15/17] Small correction --- src/block_monte_carlo.rs | 69 +++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 49aa91c8..86de41d2 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -1,7 +1,7 @@ use std::any::Any; use std::error::Error; -use rand::distributions::{Distribution, Uniform}; +use rand::distributions::{Distribution, Open01, Uniform}; use rand_xoshiro::rand_core::SeedableRng; use rand_xoshiro::Xoshiro256Plus; @@ -26,11 +26,14 @@ pub fn new_monte_carlo_block( let number_of_inputs_to_skip = (dropout_rate * num_inputs as f32) as usize; let skip_index_generator = Uniform::from(0..num_inputs); + let increment_generator = Open01 {}; let block = Box::new(BlockMonteCarlo { num_iterations, number_of_inputs_to_skip, + dropout_rate, skip_index_generator, + increment_generator, output_offset: usize::MAX, input_offset: usize::MAX, num_inputs, @@ -47,7 +50,9 @@ fn create_seed_from_input_tape(input_tape: &[f32]) -> u64 { pub struct BlockMonteCarlo { num_iterations: usize, number_of_inputs_to_skip: usize, + dropout_rate: f32, skip_index_generator: Uniform, + increment_generator: Open01, pub num_inputs: usize, pub input_offset: usize, @@ -115,7 +120,12 @@ impl BlockTrait for BlockMonteCarlo { output_tape.copy_from_slice(&input_tape); if i != 0 { - for _ in 0..self.number_of_inputs_to_skip { + let mut number_of_inputs_to_skip = self.number_of_inputs_to_skip; + let sample = self.increment_generator.sample(&mut rng); + if self.dropout_rate >= sample { + number_of_inputs_to_skip += 1; + } + for _ in 0..number_of_inputs_to_skip { let skip_index = self.skip_index_generator.sample(&mut rng); *output_tape.get_unchecked_mut(skip_index) = 0.0; } @@ -160,7 +170,12 @@ impl BlockTrait for BlockMonteCarlo { output_tape.copy_from_slice(&input_tape); if i != 0 { - for _ in 0..self.number_of_inputs_to_skip { + let mut number_of_inputs_to_skip = self.number_of_inputs_to_skip; + let sample = self.increment_generator.sample(&mut rng); + if self.dropout_rate >= sample { + number_of_inputs_to_skip += 1; + } + for _ in 0..number_of_inputs_to_skip { let skip_index = self.skip_index_generator.sample(&mut rng); *output_tape.get_unchecked_mut(skip_index) = 0.0; } @@ -204,6 +219,7 @@ impl BlockMonteCarlo { } fn fill_stats(&self, pb: &mut PortBuffer) { + println!("{:?}", pb.observations); let mean: f32 = pb.observations.iter().sum::() / self.num_iterations as f32; let variance = pb .observations @@ -310,16 +326,53 @@ mod tests { spredict2(&mut bg, &fb, &mut pb); let expected_observations = [ - 2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 0.0, 5.0, 0.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0 + 2.0, 4.0, 4.0, 5.0, 2.0, 0.0, 4.0, 5.0, 2.0, 0.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0 + ]; + assert_eq!(pb.observations, expected_observations); + + assert_ne!(pb.stats, None); + + let stats = pb.stats.unwrap(); + assert_epsilon!(stats.mean, 12.333333); + assert_epsilon!(stats.variance, 354.55554); + assert_epsilon!(stats.standard_deviation, 18.829645); + } + + #[test] + fn test_monte_carlo_block_with_multiple_runs_and_high_dropout_rate() { + let mi = ModelInstance::new_empty().unwrap(); + let mut bg = BlockGraph::new(); + let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let triangle_block = + new_monte_carlo_block(&mut bg, observe_block_backward, 3, 0.7).unwrap(); + let observe_block_forward = + block_misc::new_observe_block(&mut bg, triangle_block, Observe::Forward, None).unwrap(); + block_misc::new_sink_block( + &mut bg, + observe_block_forward, + block_misc::SinkType::Untouched, + ) + .unwrap(); + bg.finalize(); + bg.allocate_and_init_weights(&mi); + + let mut pb = bg.new_port_buffer(); + let fb = fb_vec(); + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, [2.0, 4.0, 4.0, 5.0, 2.0, 4.0, 4.0, 5.0]); + spredict2(&mut bg, &fb, &mut pb); + let expected_observations = [ + 2.0, 4.0, 4.0, 5.0, 2.0, 0.0, 0.0, 5.0, 2.0, 0.0, 0.0, 0.0, 2.0, 4.0, 4.0, 5.0 ]; - assert_eq!(pb.observations.len(), expected_observations.len()); assert_eq!(pb.observations, expected_observations); assert_ne!(pb.stats, None); let stats = pb.stats.unwrap(); - assert_epsilon!(stats.mean, 13.0); - assert_epsilon!(stats.variance, 392.33334); - assert_epsilon!(stats.standard_deviation, 19.807405); + assert_epsilon!(stats.mean, 8.0); + assert_epsilon!(stats.variance, 159.33333); + assert_epsilon!(stats.standard_deviation, 12.62273); } } From 63033c62b528ba9802855d3d63eda173c6168f5f Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Wed, 11 Oct 2023 11:09:25 +0200 Subject: [PATCH 16/17] Remove println --- src/block_monte_carlo.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 86de41d2..343423d5 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -219,7 +219,6 @@ impl BlockMonteCarlo { } fn fill_stats(&self, pb: &mut PortBuffer) { - println!("{:?}", pb.observations); let mean: f32 = pb.observations.iter().sum::() / self.num_iterations as f32; let variance = pb .observations From fd5394ffe1e502a30be33112191cb0a45b7580a4 Mon Sep 17 00:00:00 2001 From: ggaspersic Date: Thu, 9 Nov 2023 13:06:11 +0100 Subject: [PATCH 17/17] Processing review comments --- src/block_monte_carlo.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/block_monte_carlo.rs b/src/block_monte_carlo.rs index 343423d5..2ddb8e78 100644 --- a/src/block_monte_carlo.rs +++ b/src/block_monte_carlo.rs @@ -43,8 +43,11 @@ pub fn new_monte_carlo_block( Ok(block_outputs.pop().unwrap()) } +// Used to generate a larger number when creating a seed from the input tape +const SEED_MULTIPLIER: f32 = 100_000.0; + fn create_seed_from_input_tape(input_tape: &[f32]) -> u64 { - (input_tape.iter().sum::() * 100_000.0).round() as u64 + (input_tape.iter().sum::() * SEED_MULTIPLIER).round() as u64 } pub struct BlockMonteCarlo { @@ -119,7 +122,7 @@ impl BlockTrait for BlockMonteCarlo { ); output_tape.copy_from_slice(&input_tape); - if i != 0 { + if i > 0 { let mut number_of_inputs_to_skip = self.number_of_inputs_to_skip; let sample = self.increment_generator.sample(&mut rng); if self.dropout_rate >= sample { @@ -169,7 +172,7 @@ impl BlockTrait for BlockMonteCarlo { ); output_tape.copy_from_slice(&input_tape); - if i != 0 { + if i > 0 { let mut number_of_inputs_to_skip = self.number_of_inputs_to_skip; let sample = self.increment_generator.sample(&mut rng); if self.dropout_rate >= sample {