From 577e4391b81b5a30cfd834c8876467109175e0a0 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 16:45:04 +0000 Subject: [PATCH 001/108] The world was created in one day. --- Cargo.toml | 22 + src/lib.rs | 841 +++++++++++++++++++++++++++++++ src/nodes.rs | 1262 +++++++++++++++++++++++++++++++++++++++++++++++ src/numerics.rs | 300 +++++++++++ 4 files changed, 2425 insertions(+) create mode 100644 Cargo.toml create mode 100644 src/lib.rs create mode 100644 src/nodes.rs create mode 100644 src/numerics.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..d3549629 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "wyrm" +version = "0.1.0" +authors = ["maciejkula "] +license = "MIT" + +[dependencies] +ndarray = { version = "0.10.13", features = ["serde-1"] } +rand = "0.3.18" +smallvec = { version = "0.5.0", features = ["serde"] } +itertools = "0.7.3" +rayon = "0.9.0" +serde = { version = "1.0.24", features = ["rc"] } +serde_derive = "1.0.24" +stdsimd = "0.0.3" + +[profile.bench] +lto = true +debug = true + +[profile.release] +lto = true \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 00000000..e9a6e5a5 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,841 @@ +#![feature(test)] +//! A backward-mode, define-by-run, low-overhead autodifferentiation library. +//! +//! # Features +//! +//! Performs backpropagation through arbitrary, define-by-run computation graphs, +//! emphasizing low overhead estimation of sparse, small models on the CPU. +//! +//! Highlights: +//! +//! 1. Low overhead. +//! 2. Built-in support for sparse gradients. +//! 3. Define-by-run. +//! 4. Trivial Hogwild-style parallelisation, scaling linearly with the number of CPU cores available. +//! +//! Requires the nightly compiler due to use of SIMD instrinsics. +//! +//! # Quickstart +//! +//! The following defines a univariate linear regression model, then +//! backpropagates through it. +//! +//! ```rust +//! # extern crate rand; +//! # extern crate wyrm; +//! # use wyrm::*; +//! # fn random_matrix(rows: usize, cols: usize) -> Arr { +//! # Arr::zeros((rows, cols)).map(|_| rand::random::()) +//! # } +//! # fn main() { +//! let slope = ParameterNode::new(random_matrix(1, 1)); +//! let intercept = ParameterNode::new(random_matrix(1, 1)); +//! +//! let x = InputNode::new(random_matrix(1, 1)); +//! let y = InputNode::new(random_matrix(1, 1)); +//! +//! let y_hat = slope.clone() * x.clone() + intercept.clone(); +//! let mut loss = (y.clone() - y_hat).square(); +//! # } +//! ``` +//! +//! To optimize the parameters, create an optimizer object and +//! go through several epochs of learning: +//! +//! ```rust +//! # extern crate rand; +//! # extern crate wyrm; +//! # use wyrm::*; +//! # fn random_matrix(rows: usize, cols: usize) -> Arr { +//! # Arr::zeros((rows, cols)).map(|_| rand::random::()) +//! # } +//! # fn main() { +//! # let slope = ParameterNode::new(random_matrix(1, 1)); +//! # let intercept = ParameterNode::new(random_matrix(1, 1)); +//! # let x = InputNode::new(random_matrix(1, 1)); +//! # let y = InputNode::new(random_matrix(1, 1)); +//! # let y_hat = slope.clone() * x.clone() + intercept.clone(); +//! # let mut loss = (y.clone() - y_hat).square(); +//! # let num_epochs = 10; +//! let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); +//! +//! for _ in 0..num_epochs { +//! let x_value: f32 = rand::random(); +//! let y_value = 3.0 * x_value + 5.0; +//! +//! // You can re-use the computation graph +//! // by giving the input nodes new values. +//! x.set_value(x_value); +//! y.set_value(y_value); +//! +//! loss.forward(); +//! loss.backward(1.0); +//! +//! optimizer.step(); +//! optimizer.zero_gradients(); +//! } +//! # } +//! ``` +//! +//! You can use `rayon` to fit your model in parallel, by first creating a set of shared +//! parameters, then building a per-thread copy of the model: +//! +//! ```rust +//! # extern crate rand; +//! # extern crate wyrm; +//! # extern crate rayon; +//! # use std::sync::Arc; +//! # use rayon::prelude::*; +//! # use wyrm::*; +//! # fn random_matrix(rows: usize, cols: usize) -> Arr { +//! # Arr::zeros((rows, cols)).map(|_| rand::random::()) +//! # } +//! # fn main() { +//! let slope_param = Arc::new(HogwildParameter::new(random_matrix(1, 1))); +//! let intercept_param = Arc::new(HogwildParameter::new(random_matrix(1, 1))); +//! let num_epochs = 10; +//! +//! (0..rayon::current_num_threads()) +//! .into_par_iter() +//! .for_each(|_| { +//! let slope = ParameterNode::shared(slope_param.clone()); +//! let intercept = ParameterNode::shared(intercept_param.clone()); +//! let x = InputNode::new(random_matrix(1, 1)); +//! let y = InputNode::new(random_matrix(1, 1)); +//! let y_hat = slope.clone() * x.clone() + intercept.clone(); +//! let mut loss = (y.clone() - y_hat).square(); +//! +//! let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); +//! +//! for _ in 0..num_epochs { +//! let x_value: f32 = rand::random(); +//! let y_value = 3.0 * x_value + 5.0; +//! +//! x.set_value(x_value); +//! y.set_value(y_value); +//! +//! loss.forward(); +//! loss.backward(1.0); +//! +//! optimizer.step(); +//! optimizer.zero_gradients(); +//! } +//! }); +//! # } +//! ``` + +// TODO: pass through of parent values in .value(), +// optimizations in forward +// check for needs gradient +// topological sort to avoid duplicate work +#[macro_use] +extern crate serde_derive; + +extern crate serde; + +extern crate ndarray; +extern crate rand; +extern crate rayon; +extern crate smallvec; +extern crate stdsimd; +extern crate test; + +#[macro_use] +extern crate itertools; + +use ndarray::Axis; + +/// Alias for a `f32` `ndarray` matrix. +pub type Arr = ndarray::Array2; + +use std::cell::RefCell; +use std::ops::{Add, Deref, Div, Mul, Neg, Sub}; +use std::rc::Rc; +use std::clone::Clone; + +mod nodes; +mod numerics; + +use nodes::*; + +pub use numerics::simd_dot; +pub use nodes::{Bor, HogwildParameter, IndexInputNode, InputNode, Node, ParameterNode}; + +fn clamp(x: f32, min: f32, max: f32) -> f32 { + if x > max { + max + } else if x < min { + min + } else { + x + } +} + +/// Trait describing nodes that can accept new values once +/// the graph has been defined. +pub trait DataInput { + fn set_value(&self, T); +} + +/// Handle to a node in the computation graph. The underlying nodes +/// are reference counted, so the handles can be freely cloned to +/// use the nodes multiple times in the same graph. +#[derive(Debug)] +pub struct Variable +where + T: Node, +{ + node: Rc, + grad: Option>, +} + +impl Clone for Variable { + fn clone(&self) -> Self { + Variable { + node: Rc::clone(&self.node), + grad: None, + } + } +} + +impl Variable +where + T: Node, +{ + fn new(node: Rc) -> Self { + Variable { + node: node, + grad: None, + } + } + /// Get the value of the node. + pub fn value(&self) -> Bor { + self.node.value() + } + /// Run the forward pass through the subgraph terminating at this node, + /// recursing through the ancestor nodes. + pub fn forward(&self) { + self.node.forward() + } +} + +impl Variable +where + T: Node, +{ + /// Run the backward pass through the subgraph terminating at this node. + /// The weight parameter scales the gradients. + pub fn backward(&mut self, weight: f32) { + self.grad + .get_or_insert(RefCell::new(self.node.value().map(|_| weight))); + if let Some(ref grad) = self.grad { + { + grad.borrow_mut().map_inplace(|x| *x = weight); + } + self.node.backward(&grad.borrow()); + } + } + + /// Square this variable. + pub fn square(&self) -> Variable> { + Variable::new(Rc::new(SquareNode::new(Rc::clone(&self.node)))) + } + + /// Transpose this variable. + pub fn t(&self) -> Variable> { + Variable::new(Rc::new(TransposeNode::new(Rc::clone(&self.node)))) + } + + /// Exponentiate this variable. + pub fn exp(&self) -> Variable> { + Variable::new(Rc::new(ExpNode::new(Rc::clone(&self.node)))) + } + + /// Compute the softmax of this variable. + pub fn softmax(&self) -> Variable> { + Variable::new(Rc::new(SoftmaxNode::new(Rc::clone(&self.node)))) + } + + /// Compute the sigmoid of this variable. + pub fn sigmoid(&self) -> Variable> { + Variable::new(Rc::new(SigmoidNode::new(Rc::clone(&self.node)))) + } + + /// Compute the row-wise vector dot product of LHS and RHS. + pub fn vector_dot(&self, other: &Variable) -> Variable> + where + S: Node, + { + Variable::new(Rc::new(VectorDotNode::new( + Rc::clone(&self.node), + Rc::clone(&other.node), + ))) + } + + /// Compute the matrix multiplication of LHS and RHS. + pub fn dot(&self, other: &Variable) -> Variable> + where + S: Node, + { + Variable::new(Rc::new(DotNode::new( + Rc::clone(&self.node), + Rc::clone(&other.node), + ))) + } +} + +impl Variable { + /// Zero the accumulated gradients of this parameter node. + pub fn zero_gradients(&self) { + self.node.zero_gradient() + } + /// Return the (dense) gradient value of this node. + pub fn dense_gradient(&self) -> Option { + match self.node.gradient.borrow().dense_gradient { + Some(ref arr) => Some(arr.clone()), + None => None, + } + } + + /// Row-wise indexing of this parameter node. Primiarily used + /// to implement embedding layers. + pub fn index(&self, index: &Variable) -> Variable> { + Variable::new(Rc::new(IndexNode::new( + Rc::clone(&self.node), + Rc::clone(&index.node), + ))) + } +} + +impl<'value> DataInput<&'value Arr> for Variable { + fn set_value(&self, value: &Arr) { + let param_value = + unsafe { &mut *(&self.node.value.deref().value as *const Arr as *mut Arr) }; + param_value.assign(value) + } +} + +impl<'value> DataInput<&'value Arr> for Variable { + fn set_value(&self, value: &Arr) { + self.node.value.borrow_mut().assign(value); + } +} + +impl DataInput for Variable { + fn set_value(&self, value: f32) { + self.node.value.borrow_mut()[(0, 0)] = value; + } +} + +impl<'value> DataInput<&'value [usize]> for Variable { + fn set_value(&self, value: &[usize]) { + let mut node_value = self.node.value.borrow_mut(); + node_value.clear(); + node_value.extend_from_slice(value); + } +} + +impl DataInput for Variable { + fn set_value(&self, value: usize) { + let mut node_value = self.node.value.borrow_mut(); + node_value.clear(); + node_value.push(value); + } +} + +impl Add> for Variable +where + RHS: Node, + LHS: Node, +{ + type Output = Variable>; + fn add(self, other: Variable) -> Self::Output { + Variable::new(Rc::new(AddNode::new(self.node, other.node))) + } +} + +impl Sub> for Variable +where + RHS: Node, + LHS: Node, +{ + type Output = Variable>; + fn sub(self, other: Variable) -> Self::Output { + Variable::new(Rc::new(SubNode::new(self.node, other.node))) + } +} + +impl Mul> for Variable +where + RHS: Node, + LHS: Node, +{ + type Output = Variable>; + fn mul(self, other: Variable) -> Self::Output { + Variable::new(Rc::new(MulNode::new(self.node, other.node))) + } +} + +impl Div> for Variable +where + RHS: Node, + LHS: Node, +{ + type Output = Variable>; + fn div(self, other: Variable) -> Self::Output { + Variable::new(Rc::new(DivNode::new(self.node, other.node))) + } +} + +impl Neg for Variable +where + T: Node, +{ + type Output = Variable>; + fn neg(self) -> Self::Output { + Variable::new(Rc::new(NegNode::new(self.node))) + } +} + +/// Standard stochtic gradient descent optimizer with a fixed learning rate. +pub struct SGD { + learning_rate: f32, + parameters: Vec>, +} + +impl SGD { + /// Create a new optimizer instance with a given set of parameters. + pub fn new(learning_rate: f32, parameters: Vec>) -> Self { + SGD { + learning_rate: learning_rate, + parameters: parameters, + } + } + /// Zero the gradients on each of the parameters being optimized. + pub fn zero_gradients(&self) { + self.parameters.iter().for_each(|x| x.zero_gradients()) + } + + /// Perform a single SGD step. + pub fn step(&mut self) { + let learning_rate = self.learning_rate; + for parameter in &self.parameters { + let mut sink = parameter.node.gradient.borrow_mut(); + let mut param_value = unsafe { + &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) + }; + + if sink.has_dense { + param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); + } + + if sink.has_sparse { + for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let grad_row = grad.subview(Axis(0), grad_idx); + let mut param_row = param_value.subview_mut(Axis(0), param_idx); + + numerics::map_add_assign_slice( + param_row.into_slice().unwrap(), + grad_row.into_slice().unwrap(), + |x| -learning_rate * clamp(x, -10.0, 10.0), + ); + } + } + } + } + } +} + +/// Compute finite difference gradient estimates of the output variable +/// with respect to the input. Use to verify correctness of gradient +/// computations. +pub fn finite_difference( + input: &mut Variable, + output: &mut Variable, +) -> (Arr, Arr) +where + T: Node, +{ + let delta_x = 1e-4; + + let initial_input = { input.value().clone() }; + let mut central_difference = &initial_input * 0.0; + + for (idx, diff) in central_difference.indexed_iter_mut() { + let positive_difference = { + let mut changed_input = initial_input.clone(); + changed_input[idx] += 0.5 * delta_x; + input.set_value(&changed_input); + output.forward(); + output.value().clone() + }; + + let negative_difference = { + let mut changed_input = initial_input.clone(); + changed_input[idx] -= 0.5 * delta_x; + input.set_value(&changed_input); + output.forward(); + output.value().clone() + }; + + let central_difference = positive_difference - negative_difference; + + *diff = central_difference.scalar_sum() / delta_x; + } + + let gradient = { + input.zero_gradients(); + input.set_value(&initial_input); + output.forward(); + output.backward(1.0); + + input.dense_gradient().unwrap() + }; + + (central_difference, gradient) +} + +#[allow(dead_code)] +fn assert_close(x: &Arr, y: &Arr, tol: f32) { + assert!( + x.all_close(y, tol), + "{:#?} not within {} of {:#?}", + x, + tol, + y + ); +} + +#[cfg(test)] +mod tests { + + use std::sync::Arc; + use ndarray::arr2; + use rayon::prelude::*; + + use super::*; + + const TOLERANCE: f32 = 0.1; + + fn random_matrix(rows: usize, cols: usize) -> Arr { + Arr::zeros((rows, cols)).map(|_| rand::random::()) + } + + #[test] + fn add_finite_difference() { + let mut x = ParameterNode::new(random_matrix(1, 1)); + let y = ParameterNode::new(random_matrix(1, 1)); + let mut z = x.clone() + y.clone(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn mul_finite_difference() { + let mut x = ParameterNode::new(random_matrix(1, 1)); + let y = ParameterNode::new(random_matrix(1, 1)); + let mut z = x.clone() * y.clone(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn div_finite_difference() { + let mut x = ParameterNode::new(random_matrix(1, 1)); + let y = ParameterNode::new(random_matrix(1, 1)); + let mut z = (x.clone() + x.clone()) / y.clone(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn vector_dot_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let y = ParameterNode::new(random_matrix(10, 5)); + let mut z = x.vector_dot(&y); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn dot_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let y = ParameterNode::new(random_matrix(5, 10)); + let mut z = (x.clone() + x.clone()).dot(&y); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn square_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = x.square(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn tranpose_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = (x.clone() + x.clone()).t(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn exp_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = (x.clone() + x.clone()).exp(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn dot_square_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let y = ParameterNode::new(random_matrix(10, 5)); + let mut z = x.vector_dot(&y).square(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn sigmoid_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = (x.clone() + x.clone()).sigmoid(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn neg_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = -(x.clone() + x.clone()); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn softmax_finite_difference() { + let mut x = ParameterNode::new(random_matrix(1, 10)); + let mut z = (x.clone() + x.clone()).softmax(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn univariate_regression() { + let slope = ParameterNode::new(random_matrix(1, 1)); + let intercept = ParameterNode::new(random_matrix(1, 1)); + + let num_epochs = 100; + + let x = InputNode::new(random_matrix(1, 1)); + let y = InputNode::new(random_matrix(1, 1)); + + let y_hat = slope.clone() * x.clone() + intercept.clone(); + let diff = y.clone() - y_hat.clone(); + let mut loss = diff.square(); + + let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); + + for _ in 0..num_epochs { + let _x = arr2(&[[rand::random::()]]); + let _y = 3.0 * &_x + 5.0; + + x.set_value(&_x); + y.set_value(&_y); + + loss.forward(); + loss.backward(1.0); + + optimizer.step(); + optimizer.zero_gradients(); + } + + println!( + "Predicted: {} Loss: {} Slope {} Intercept {}", + y_hat.value(), + loss.value(), + slope.value(), + intercept.value() + ); + + assert!(loss.value().scalar_sum() < 1.0e-2); + } + + #[test] + fn multivariate_regression() { + let slope = ParameterNode::new(random_matrix(1, 3)); + let intercept = ParameterNode::new(random_matrix(1, 1)); + + let num_epochs = 200; + + let coefficients = arr2(&[[1.0], [2.0], [3.0]]); + + let x = InputNode::new(random_matrix(1, 3)); + let y = InputNode::new(random_matrix(1, 1)); + + let y_hat = x.vector_dot(&slope) + intercept.clone(); + let diff = y.clone() - y_hat.clone(); + let mut loss = diff.square(); + + let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); + + for _ in 0..num_epochs { + let _x = arr2(&[ + [ + rand::random::(), + rand::random::(), + rand::random::(), + ], + ]); + let _y = &_x.dot(&coefficients) + 5.0; + + x.set_value(&_x); + y.set_value(&_y); + + loss.forward(); + loss.backward(1.0); + + optimizer.step(); + optimizer.zero_gradients(); + } + + println!( + "Predicted: {} Loss: {} Slope {} Intercept {}", + y_hat.value(), + loss.value(), + slope.value(), + intercept.value() + ); + + assert!(loss.value().scalar_sum() < 1.0e-1); + } + + #[test] + fn embedding_factorization() { + let (rows, cols) = (10, 4); + + let true_u = random_matrix(rows, 10); + let true_v = random_matrix(cols, 10); + let x = true_u.dot(&true_v.t()); + + let y = random_matrix(1, 1); + let u_input = vec![0]; + let v_input = vec![0]; + + let output = InputNode::new(y); + + let u_embedding = ParameterNode::new(random_matrix(rows, 10)); + let v_embedding = ParameterNode::new(random_matrix(cols, 10)); + + let u_index = IndexInputNode::new(&u_input); + let v_index = IndexInputNode::new(&v_input); + + let u_vec = u_embedding.index(&u_index); + let v_vec = v_embedding.index(&v_index); + + let y_hat = u_vec.vector_dot(&v_vec); + let mut loss = (output.clone() - y_hat.clone()).square(); + + let num_epochs = 100; + let mut optimizer = SGD::new(0.1, vec![u_embedding.clone(), v_embedding.clone()]); + + let mut loss_val = 0.0; + + for _ in 0..num_epochs { + loss_val = 0.0; + + for row_idx in 0..rows { + for col_idx in 0..cols { + u_index.set_value(row_idx); + v_index.set_value(col_idx); + + output.set_value(x[(row_idx, col_idx)]); + + loss.forward(); + loss.backward(1.0); + + loss_val += loss.value().scalar_sum(); + + optimizer.step(); + optimizer.zero_gradients(); + } + } + + println!("Loss {}", loss_val) + } + + assert!(loss_val < 1e-3); + } + + #[test] + fn hogwild_embedding_factorization() { + let (rows, cols) = (10, 4); + + let true_u = random_matrix(rows, 10); + let true_v = random_matrix(cols, 10); + let x = true_u.dot(&true_v.t()); + + let u_input = vec![0]; + let v_input = vec![0]; + + let u_parameters = Arc::new(HogwildParameter::new(random_matrix(rows, 10))); + let v_parameters = Arc::new(HogwildParameter::new(random_matrix(cols, 10))); + + let losses: Vec = (0..rayon::current_num_threads()) + .into_par_iter() + .map(|_| { + let u_embedding = ParameterNode::shared(u_parameters.clone()); + let v_embedding = ParameterNode::shared(v_parameters.clone()); + + let u_index = IndexInputNode::new(&u_input); + let v_index = IndexInputNode::new(&v_input); + let output = InputNode::new(random_matrix(1, 1)); + + let u_vec = u_embedding.index(&u_index); + let v_vec = v_embedding.index(&v_index); + + let y_hat = u_vec.vector_dot(&v_vec); + let mut loss = (output.clone() - y_hat.clone()).square(); + + let num_epochs = 100; + + let mut optimizer = SGD::new(0.1, vec![u_embedding.clone(), v_embedding.clone()]); + + let mut loss_val = 0.0; + + for _ in 0..num_epochs { + loss_val = 0.0; + + for row_idx in 0..rows { + for col_idx in 0..cols { + u_index.set_value(row_idx); + v_index.set_value(col_idx); + + output.set_value(x[(row_idx, col_idx)]); + + loss.forward(); + loss.backward(1.0); + + loss_val += loss.value().scalar_sum(); + + optimizer.step(); + optimizer.zero_gradients(); + } + } + } + + println!("Loss val {}", loss_val); + + loss_val + }) + .collect(); + + let sum_loss: f32 = losses.iter().sum(); + + assert!(sum_loss / (losses.len() as f32) < 1e-3); + } +} diff --git a/src/nodes.rs b/src/nodes.rs new file mode 100644 index 00000000..4a2eb27c --- /dev/null +++ b/src/nodes.rs @@ -0,0 +1,1262 @@ +use std; +use std::fmt; +use std::ops::{AddAssign, Deref, DerefMut, DivAssign, MulAssign, SubAssign}; +use std::cell::{Ref, RefCell}; +use std::sync::Arc; +use std::rc::Rc; + +use ndarray; +use ndarray::Axis; +use ndarray::linalg::general_mat_mul; + +use smallvec::SmallVec; + +use numerics; + +use super::{Arr, Variable}; + +/// Generalisation over borrowed RefCell values +/// and simple references. +#[derive(Debug)] +pub enum Bor<'value, T: 'value> { + RefGuard(Ref<'value, T>), + Reference(&'value T), +} + +impl<'value, T: 'value> Deref for Bor<'value, T> { + type Target = T; + fn deref(&self) -> &T { + match *self { + Bor::RefGuard(ref val) => val.deref(), + Bor::Reference(ref val) => val.deref(), + } + } +} + +impl<'value, T: 'value + fmt::Display> fmt::Display for Bor<'value, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.deref()) + } +} + +/// Trait representing a computation node. Structs implementing +/// this trait can be used as elements of the computation graph. +pub trait Node: fmt::Debug + Sized { + /// Type of the node's value. + type Value; + /// Type of the input gradient the node receives + /// during backpropagation. + type InputGradient; + /// Type of the gradient the node passes down + /// to its ancestors during backpropagation. + type OutputGradient; + /// Perform the forward step. Should recursively call + /// the forward methods of its ancestors. + fn forward(&self); + /// Perform the backward step. Should recursively call + /// the backward methods of its ancestors. + fn backward(&self, &Ref); + /// Return the value of the node. + fn value(&self) -> Bor; + /// If the node needs to used in the backward step. + fn needs_gradient(&self) -> bool; +} + +#[derive(Debug)] +pub struct AddNode { + value: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, +} + +impl AddNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc) -> Self { + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + let value = lhs.value().deref() + rhs.value().deref(); + + AddNode { + value: RefCell::new(value), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + } + } +} + +impl Node for AddNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.lhs.forward(); + self.rhs.forward(); + + let lhs_value = self.lhs.value(); + let rhs_value = self.rhs.value(); + + debug_assert_eq!( + lhs_value.shape(), + self.value().shape(), + "LHS operand changed shape." + ); + debug_assert_eq!( + rhs_value.shape(), + self.value().shape(), + "RHS operand changed shape." + ); + + let mut self_value = self.value.borrow_mut(); + + numerics::map_assign_binary( + self_value.deref_mut(), + lhs_value.deref(), + rhs_value.deref(), + |x, y| x + y, + ); + } + fn backward(&self, gradient: &Ref) { + self.lhs.backward(gradient); + self.rhs.backward(gradient); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +/// Input node for the graph. +#[derive(Debug)] +pub struct InputNode { + pub value: RefCell, +} + +impl InputNode { + /// Create a new input node with a given value. This fixes the shape + /// of the node in the graph. + pub fn new(value: Arr) -> Variable { + Variable::new(Rc::new(InputNode { + value: RefCell::new(value), + })) + } +} + +impl Node for InputNode { + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = (); + fn forward(&self) {} + fn backward(&self, _: &Ref) {} + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + false + } +} + +#[derive(Debug)] +pub struct GradientAccumulator { + pub dense_shape: (usize, usize), + pub dense_gradient: Option, + pub sparse_gradient: SmallVec<[(SmallVec<[usize; 4]>, Arr); 4]>, + pub has_dense: bool, + pub has_sparse: bool, +} + +impl GradientAccumulator { + fn new(dense_shape: (usize, usize)) -> Self { + GradientAccumulator { + dense_shape: dense_shape, + dense_gradient: None, + sparse_gradient: SmallVec::new(), + has_dense: false, + has_sparse: false, + } + } + pub fn dense_gradient(&mut self) -> &mut Arr { + self.dense_gradient + .get_or_insert(Arr::zeros(self.dense_shape)) + } + fn zero_gradient(&mut self) { + if self.has_dense { + self.dense_gradient().fill(0.0); + } + + if self.has_sparse { + for &mut (ref mut index_vec, ref mut grad) in self.sparse_gradient.iter_mut() { + index_vec.clear(); + grad.fill(0.0) + } + } + + self.has_dense = false; + self.has_sparse = false; + } +} + +pub trait GradientSink { + fn accumulate_gradient(&mut self, gradient: T); +} + +impl<'a, 'b> GradientSink<&'a Ref<'b, Arr>> for GradientAccumulator { + fn accumulate_gradient(&mut self, gradient: &Ref) { + self.dense_gradient().add_assign(gradient.deref()); + self.has_dense = true; + } +} + +impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { + fn accumulate_gradient(&mut self, gradient: (&'a [usize], &'a Arr)) { + let (index, value) = gradient; + self.has_sparse = true; + let gradients = &mut self.sparse_gradient; + + // Check if we can reuse one of the gradient accumulators + for &mut (ref mut index_vec, ref mut grad) in gradients.iter_mut() { + if index_vec.is_empty() { + index_vec.extend_from_slice(&index[..]); + grad.assign(value); + return; + } + } + + // Otherwise create one + gradients.push((SmallVec::from(&index[..]), value.clone())); + } +} + +unsafe impl Sync for HogwildParameter {} + +/// Struct used to hold parameters that need to be shared among +/// multiple `ParameterNode`s for asynchronous, parallel optimization. +#[derive(Debug, Serialize, Deserialize)] +pub struct HogwildParameter { + pub value: Arr, +} + +impl HogwildParameter { + /// Create a new parameter object. + pub fn new(value: Arr) -> Self { + HogwildParameter { value: value } + } +} + +/// Parameter node, holds the optimizable parameters of the model. +#[derive(Debug)] +pub struct ParameterNode { + pub value: Arc, + pub gradient: RefCell, +} + +impl ParameterNode { + /// Create a parameter node that shares its parameter values + /// with other parameter nodes via the `HogwildParameter` object. + pub fn shared(value: Arc) -> Variable { + let shape = (value.value.rows(), value.value.cols()); + + Variable::new(Rc::new(ParameterNode { + value: value, + gradient: RefCell::new(GradientAccumulator::new(shape)), + })) + } + /// Create a new parameter node. The parameters held by this node + /// cannot be shared and optimized in parallel. + pub fn new(value: Arr) -> Variable { + let shape = (value.rows(), value.cols()); + + Variable::new(Rc::new(ParameterNode { + value: Arc::new(HogwildParameter::new(value)), + gradient: RefCell::new(GradientAccumulator::new(shape)), + })) + } + /// Zero the accumulated gradients of this node. + pub fn zero_gradient(&self) { + self.gradient.borrow_mut().zero_gradient(); + } +} + +impl Node for ParameterNode { + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = (); + fn forward(&self) {} + fn backward(&self, gradient: &Ref) { + self.gradient.borrow_mut().accumulate_gradient(gradient); + } + fn value(&self) -> Bor { + Bor::Reference(&self.value.value) + } + fn needs_gradient(&self) -> bool { + true + } +} + +#[derive(Debug)] +pub struct SubNode +where + LHS: Node, + RHS: Node, +{ + value: RefCell, + rhs_gradient: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, +} + +impl SubNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc) -> Self { + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + let value = lhs.value().deref() + rhs.value().deref(); + + let rhs_gradient = rhs.value().deref() * 0.0; + + SubNode { + value: RefCell::new(value), + rhs_gradient: RefCell::new(rhs_gradient), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + } + } +} + +impl Node for SubNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.lhs.forward(); + self.rhs.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.lhs.value().deref()); + dest.sub_assign(self.rhs.value().deref()); + } + + fn backward(&self, gradient: &Ref) { + { + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + rhs_gradient.assign(gradient); + rhs_gradient.mul_assign(-1.0); + } + + self.lhs.backward(gradient); + self.rhs.backward(&self.rhs_gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct MulNode { + value: RefCell, + lhs_gradient: RefCell, + rhs_gradient: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, +} + +impl MulNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc) -> Self { + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + let value = lhs.value().deref() * rhs.value().deref(); + + let lhs_gradient = &value * 0.0; + let rhs_gradient = &value * 0.0; + + MulNode { + value: RefCell::new(value), + lhs_gradient: RefCell::new(lhs_gradient), + rhs_gradient: RefCell::new(rhs_gradient), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + } + } +} + +impl Node for MulNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.lhs.forward(); + self.rhs.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.lhs.value().deref()); + dest.mul_assign(self.rhs.value().deref()); + } + fn backward(&self, gradient: &Ref) { + { + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + lhs_gradient.assign(self.rhs.value().deref()); + lhs_gradient.mul_assign(gradient.deref()); + + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + rhs_gradient.assign(self.lhs.value().deref()); + rhs_gradient.mul_assign(gradient.deref()); + } + + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct DivNode { + value: RefCell, + lhs_gradient: RefCell, + rhs_gradient: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, +} + +impl DivNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc) -> Self { + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + let value = lhs.value().deref() / rhs.value().deref(); + + let lhs_gradient = &value * 0.0; + let rhs_gradient = &value * 0.0; + + DivNode { + value: RefCell::new(value), + lhs_gradient: RefCell::new(lhs_gradient), + rhs_gradient: RefCell::new(rhs_gradient), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + } + } +} + +impl Node for DivNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.lhs.forward(); + self.rhs.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.lhs.value().deref()); + dest.div_assign(self.rhs.value().deref()); + } + fn backward(&self, gradient: &Ref) { + { + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let rhs_value = self.rhs.value(); + + izip!(lhs_gradient.iter_mut(), rhs_value.iter(), gradient.iter()) + .for_each(|(dest, rhs_val, grad_val)| *dest = grad_val / rhs_val); + + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + izip!( + rhs_gradient.iter_mut(), + self.lhs.value().iter(), + rhs_value.iter(), + gradient.iter() + ).for_each(|(dest, lhs_val, rhs_val, grad_val)| { + *dest = -lhs_val / rhs_val.powi(2) * grad_val + }); + } + + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct DotNode { + value: RefCell, + lhs_gradient: RefCell, + rhs_gradient: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, +} + +impl DotNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc) -> Self { + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + let value = lhs.value().dot(rhs.value().deref()); + + let lhs_gradient = lhs.value().deref() * 0.0; + let rhs_gradient = rhs.value().deref() * 0.0; + + DotNode { + value: RefCell::new(value), + lhs_gradient: RefCell::new(lhs_gradient), + rhs_gradient: RefCell::new(rhs_gradient), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + } + } +} + +impl Node for DotNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + + fn forward(&self) { + self.lhs.forward(); + self.rhs.forward(); + + general_mat_mul( + 1.0, + self.lhs.value().deref(), + self.rhs.value().deref(), + 0.0, + self.value.borrow_mut().deref_mut(), + ); + } + + fn backward(&self, gradient: &Ref) { + { + let rhs_value = self.rhs.value(); + let lhs_value = self.lhs.value(); + + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + general_mat_mul(1.0, gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); + general_mat_mul(1.0, &lhs_value.t(), gradient, 0.0, &mut rhs_gradient); + } + + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct VectorDotNode { + value: RefCell, + lhs_gradient: RefCell, + rhs_gradient: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, +} + +impl VectorDotNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc) -> Self { + let (value, lhs_gradient, rhs_gradient, needs_gradient) = { + let lhs_value = lhs.value(); + let rhs_value = rhs.value(); + + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + + assert_eq!( + lhs_value.shape(), + rhs_value.shape(), + "LHS and RHS must be the same shape for vector dot product." + ); + + let mut value = Arr::zeros((lhs_value.shape()[0], 1)); + + for (result, lhs, rhs) in izip!( + value.as_slice_mut().unwrap(), + lhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + rhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()) + ) { + *result = numerics::simd_dot(lhs, rhs); + } + + let lhs_gradient = lhs_value.deref() * 0.0; + let rhs_gradient = rhs_value.deref() * 0.0; + + (value, lhs_gradient, rhs_gradient, needs_gradient) + }; + + VectorDotNode { + value: RefCell::new(value), + lhs_gradient: RefCell::new(lhs_gradient), + rhs_gradient: RefCell::new(rhs_gradient), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + } + } +} + +impl Node for VectorDotNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + + fn forward(&self) { + self.lhs.forward(); + self.rhs.forward(); + + let lhs_value = self.lhs.value(); + let rhs_value = self.rhs.value(); + + for (result, lhs, rhs) in izip!( + self.value.borrow_mut().as_slice_mut().unwrap(), + lhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + rhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()) + ) { + *result = numerics::simd_dot(lhs, rhs); + } + } + + fn backward(&self, gradient: &Ref) { + { + let mut lhs_grad = self.lhs_gradient.borrow_mut(); + let mut rhs_grad = self.rhs_gradient.borrow_mut(); + + let lhs_value = self.lhs.value(); + let rhs_value = self.rhs.value(); + + for (backward_row, rhs_row, &gradient) in izip!( + lhs_grad + .genrows_mut() + .into_iter() + .map(|x| x.into_slice().unwrap()), + rhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + gradient.as_slice().unwrap() + ) { + numerics::simd_scaled_assign(backward_row, rhs_row, gradient) + } + for (backward_row, lhs_row, &gradient) in izip!( + rhs_grad + .genrows_mut() + .into_iter() + .map(|x| x.into_slice().unwrap()), + lhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + gradient.as_slice().unwrap() + ) { + numerics::simd_scaled_assign(backward_row, lhs_row, gradient) + } + } + + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct SquareNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl SquareNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = operand.value().map(|x| x.powi(2)); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + SquareNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for SquareNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.operand.value().deref()); + dest.map_inplace(|x| *x = x.powi(2)); + } + + fn backward(&self, gradient: &Ref) { + for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest = operand_val * 2.0 * grad_val; + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct SigmoidNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl SigmoidNode +where + T: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = operand.value().deref().map(|x| 1.0 / (1.0 + (-x).exp())); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + SigmoidNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for SigmoidNode +where + T: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + + numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { + 1.0 / (1.0 + (-x).exp()) + }); + } + + fn backward(&self, gradient: &Ref) { + { + let mut operand_gradient = self.operand_gradient.borrow_mut(); + + numerics::map_assign_binary( + &mut operand_gradient, + self.value.borrow().deref(), + gradient, + |sigmoid, grad| grad * sigmoid * (1.0 - sigmoid), + ); + } + + self.operand.backward(&self.operand_gradient.borrow()) + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct NegNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl NegNode +where + T: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = -operand.value().deref(); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + NegNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for NegNode +where + T: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + + fn forward(&self) { + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.operand.value().deref()); + dest.map_inplace(|x| *x = -*x); + } + + fn backward(&self, gradient: &Ref) { + for (dest, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + gradient.iter() + ) { + *dest = -grad_val; + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct ExpNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl ExpNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = operand.value().deref().map(|x| x.exp()); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + ExpNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for ExpNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + let mut dest = self.value.borrow_mut(); + + dest.assign(self.operand.value().deref()); + dest.map_inplace(|x| *x = x.exp()); + } + fn backward(&self, gradient: &Ref) { + for (dest, self_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.value.borrow().iter(), + gradient.iter() + ) { + *dest = self_val * grad_val; + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct TransposeNode { + value: RefCell, + gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl TransposeNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let needs_gradient = operand.needs_gradient(); + let value = RefCell::new(&operand.value().t() * 1.0); + let gradient = RefCell::new(operand.value().deref() * 0.0); + + TransposeNode { + value: value, + gradient: gradient, + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for TransposeNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + self.value.borrow_mut().assign(&self.operand.value().t()); + } + fn backward(&self, gradient: &Ref) { + { + self.gradient.borrow_mut().assign(&gradient.t()); + } + + self.operand.backward(&self.gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +#[derive(Debug)] +pub struct SoftmaxNode { + value: RefCell, + jacobian: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl SoftmaxNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = { + let max = operand + .value() + .deref() + .as_slice() + .unwrap() + .iter() + .fold(std::f32::MIN, |x, y| x.max(*y)); + let numerator = operand.value().map(|x| (x - max).exp()); + let denominator = numerator.scalar_sum(); + + numerator / denominator + }; + + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + let dim = value.shape()[1]; + + SoftmaxNode { + value: RefCell::new(value), + jacobian: RefCell::new(ndarray::Array2::zeros((dim, dim))), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for SoftmaxNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + let mut dest = self.value.borrow_mut(); + dest.assign(self.operand.value().deref()); + + let max = self.operand + .value() + .deref() + .iter() + .fold(std::f32::MIN, |x, y| x.max(*y)); + dest.map_inplace(|x| *x = (*x - max).exp()); + let denominator = dest.scalar_sum(); + dest.map_inplace(|x| *x /= denominator); + } + fn backward(&self, gradient: &Ref) { + let value = self.value.borrow(); + let mut jacobian = self.jacobian.borrow_mut(); + + for (row_idx, (mut row, row_val)) in jacobian + .genrows_mut() + .into_iter() + .zip(value.iter()) + .enumerate() + { + for (col_idx, (grad, col_val)) in row.iter_mut().zip(value.iter()).enumerate() { + if row_idx == col_idx { + *grad = row_val * (1.0 - col_val); + } else { + *grad = -row_val * col_val; + } + } + } + + { + general_mat_mul( + 1.0, + gradient, + jacobian.deref_mut(), + 0.0, + self.operand_gradient.borrow_mut().deref_mut(), + ); + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + +/// An input node for integer indices into `ParameterNode`s, used +/// for implementing indexable embedding layers. +#[derive(Debug)] +pub struct IndexInputNode { + pub value: RefCell>, +} + +impl IndexInputNode { + /// Create a new index input node. + pub fn new(value: &[usize]) -> Variable { + Variable::new(Rc::new(IndexInputNode { + value: RefCell::new(SmallVec::from(value)), + })) + } +} + +impl Node for IndexInputNode { + type Value = SmallVec<[usize; 4]>; + type InputGradient = Arr; + type OutputGradient = (); + fn forward(&self) {} + fn backward(&self, _: &Ref) {} + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + false + } +} + +#[derive(Debug)] +pub struct IndexNode { + value: RefCell, + index_value: RefCell>, + operand_gradient: RefCell, + index: Rc, + operand: Rc, + needs_gradient: bool, +} + +impl IndexNode +where + OP: Node, +{ + pub fn new(operand: Rc, index: Rc) -> Self { + let value = operand.value().select(Axis(0), &index.value()[..]); + let grad = &value * 0.0; + let idx_value = index.value().clone(); + let needs_gradient = operand.needs_gradient(); + + IndexNode { + value: RefCell::new(value), + index_value: RefCell::new(idx_value), + operand_gradient: RefCell::new(grad), + index: index, + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for IndexNode { + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + let operand_value = self.operand.value(); + + let mut idx_value = self.index_value.borrow_mut(); + idx_value.clear(); + idx_value.extend_from_slice(&self.index.value()[..]); + + let mut arr_value = self.value.borrow_mut(); + + debug_assert_eq!( + arr_value.shape()[0], + idx_value.len(), + "Result of indexing operation must maintain consistent shape between iterations." + ); + + for (&idx, mut row) in idx_value.iter().zip(arr_value.genrows_mut()) { + let new_val = operand_value.subview(Axis(0), idx); + + numerics::slice_assign(row.into_slice().unwrap(), new_val.as_slice().unwrap()) + } + } + + fn backward(&self, gradient: &Ref) { + self.operand + .gradient + .borrow_mut() + .accumulate_gradient((&self.index_value.borrow()[..], gradient.deref())); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} diff --git a/src/numerics.rs b/src/numerics.rs new file mode 100644 index 00000000..da613ab9 --- /dev/null +++ b/src/numerics.rs @@ -0,0 +1,300 @@ +use std::cmp; +use stdsimd; + +use super::Arr; + +/// SIMD-enabled vector-vector dot product. +pub fn simd_dot(xs: &[f32], ys: &[f32]) -> f32 { + let mut simd_result = stdsimd::simd::f32x8::splat(0.0); + let mut scalar_result = 0.0; + let stride = 8; + + let split_idx = cmp::min(xs.len(), ys.len()) / stride * stride; + let (simd_xs, scalar_xs) = xs.split_at(split_idx); + let (simd_ys, scalar_ys) = ys.split_at(split_idx); + + for (x, y) in simd_xs.chunks(stride).zip(simd_ys.chunks(stride)) { + unsafe { + simd_result = simd_result + + stdsimd::simd::f32x8::load_unchecked(x, 0) + * stdsimd::simd::f32x8::load_unchecked(y, 0); + } + } + + for (x_scalar, y_scalar) in scalar_xs.iter().zip(scalar_ys.iter()) { + scalar_result += x_scalar * y_scalar; + } + + scalar_result + (0..8).map(|idx| simd_result.extract(idx)).sum::() +} + +pub fn simd_scaled_assign(xs: &mut [f32], ys: &[f32], alpha: f32) { + let stride = 8; + let simd_alpha = stdsimd::simd::f32x8::splat(alpha); + + let split_idx = xs.len() / stride * stride; + let (simd_xs, scalar_xs) = xs.split_at_mut(split_idx); + let (simd_ys, scalar_ys) = ys.split_at(split_idx); + + for (x, y) in simd_xs.chunks_mut(stride).zip(simd_ys.chunks(stride)) { + unsafe { + let elem = stdsimd::simd::f32x8::load_unchecked(y, 0) * simd_alpha; + elem.store_unchecked(x, 0); + } + } + + for (x_scalar, y_scalar) in scalar_xs.iter_mut().zip(scalar_ys.iter()) { + *x_scalar = y_scalar * alpha; + } +} + +pub fn slice_assign(xs: &mut [f32], ys: &[f32]) { + for (x, &y) in xs.iter_mut().zip(ys.iter()) { + *x = y; + } +} + +pub fn map_assign(xs: &mut Arr, ys: &Arr, func: F) +where + F: Fn(f32) -> f32, +{ + let xs = xs.as_slice_mut().expect("Unable to convert LHS to slice."); + let ys = ys.as_slice().expect("Unable to convert RHS to slice."); + + for (x, &y) in xs.iter_mut().zip(ys.iter()) { + *x = func(y); + } +} + +pub fn map_add_assign_slice(xs: &mut [f32], ys: &[f32], func: F) +where + F: Fn(f32) -> f32, +{ + for (x, &y) in xs.iter_mut().zip(ys.iter()) { + *x += func(y); + } +} + +pub fn map_assign_binary(xs: &mut Arr, ys: &Arr, zs: &Arr, func: F) +where + F: Fn(f32, f32) -> f32, +{ + let xs = xs.as_slice_mut() + .expect("Unable to convert operand to slice."); + let ys = ys.as_slice().expect("Unable to convert operand to slice."); + let zs = zs.as_slice().expect("Unable to convert operand to slice."); + + for (x, &y, &z) in izip!(xs.iter_mut(), ys.iter(), zs.iter()) { + *x = func(y, z); + } +} + +#[allow(dead_code)] +pub fn map_inplace_assign(xs: &mut Arr, ys: &Arr, func: F) +where + F: Fn(&mut f32, f32), +{ + let xs = xs.as_slice_mut() + .expect("Unable to convert operand to slice."); + let ys = ys.as_slice().expect("Unable to convert operand to slice."); + + for (x, &y) in izip!(xs.iter_mut(), ys.iter()) { + func(x, y); + } +} + +#[allow(dead_code)] +pub fn map_inplace_assign_binary(xs: &mut Arr, ys: &Arr, zs: &Arr, func: F) +where + F: Fn(&mut f32, f32, f32), +{ + let xs = xs.as_slice_mut() + .expect("Unable to convert operand to slice."); + let ys = ys.as_slice().expect("Unable to convert operand to slice."); + let zs = zs.as_slice().expect("Unable to convert operand to slice."); + + for (x, &y, &z) in izip!(xs.iter_mut(), ys.iter(), zs.iter()) { + func(x, y, z); + } +} + +#[cfg(test)] +mod tests { + + use std; + + use super::*; + + use rand; + use rand::Rng; + use test::Bencher; + + fn random_matrix(rows: usize, cols: usize) -> Arr { + Arr::zeros((rows, cols)).map(|_| rand::random::()) + } + + fn array_scaled_assign(xs: &mut Arr, ys: &Arr, alpha: f32) { + for (x, y) in xs.iter_mut().zip(ys.iter()) { + *x = y * alpha; + } + } + + fn scaled_assign(xs: &mut Arr, ys: &Arr, alpha: f32) { + // assert_eq!(xs.shape(), ys.shape(), "Operands do not have the same shape."); + + let xs = xs.as_slice_mut().expect("Unable to convert LHS to slice."); + let ys = ys.as_slice().expect("Unable to convert RHS to slice."); + + simd_scaled_assign(xs, ys, alpha); + } + + fn dot(lhs: &[f32], rhs: &[f32]) -> f32 { + lhs.iter().zip(rhs.iter()).map(|(x, y)| x * y).sum() + } + + fn array_assign(xs: &mut Arr, ys: &Arr) { + xs.assign(ys); + } + + fn assign(xs: &mut Arr, ys: &Arr) { + assert_eq!( + xs.shape(), + ys.shape(), + "Operands do not have the same shape." + ); + + let xs = xs.as_slice_mut().expect("Unable to convert LHS to slice."); + let ys = ys.as_slice().expect("Unable to convert RHS to slice."); + + for (x, &y) in xs.iter_mut().zip(ys.iter()) { + *x = y; + } + } + + fn unrolled_dot(xs: &[f32], ys: &[f32]) -> f32 { + let len = std::cmp::min(xs.len(), ys.len()); + let mut xs = &xs[..len]; + let mut ys = &ys[..len]; + + let mut s = 0.; + let (mut p0, mut p1, mut p2, mut p3, mut p4, mut p5, mut p6, mut p7) = + (0., 0., 0., 0., 0., 0., 0., 0.); + + while xs.len() >= 8 { + p0 += xs[0] * ys[0]; + p1 += xs[1] * ys[1]; + p2 += xs[2] * ys[2]; + p3 += xs[3] * ys[3]; + p4 += xs[4] * ys[4]; + p5 += xs[5] * ys[5]; + p6 += xs[6] * ys[6]; + p7 += xs[7] * ys[7]; + + xs = &xs[8..]; + ys = &ys[8..]; + } + s += p0 + p4; + s += p1 + p5; + s += p2 + p6; + s += p3 + p7; + + for i in 0..xs.len() { + s += xs[i] * ys[i]; + } + + s + } + + #[test] + fn test_dot() { + for len in 0..32 { + let xs = (0..len) + .map(|_| rand::thread_rng().gen()) + .collect::>(); + let ys = (0..len) + .map(|_| rand::thread_rng().gen()) + .collect::>(); + + let _dot = dot(&xs[..], &ys[..]); + let _unrolled_dot = unrolled_dot(&xs[..], &ys[..]); + let _simd_dot = simd_dot(&xs[..], &ys[..]); + + let epsilon = 1e-5; + + assert!((_dot - _unrolled_dot).abs() < epsilon); + assert!((_dot - _simd_dot).abs() < epsilon, "{} {}", _dot, _simd_dot); + } + } + + #[test] + fn test_scaled_assign() { + for len in 0..32 { + let mut xs_1 = random_matrix(len, 1); + let mut xs_2 = xs_1.clone(); + let ys = random_matrix(len, 1); + + let alpha = 3.5; + + array_scaled_assign(&mut xs_1, &ys, alpha); + scaled_assign(&mut xs_2, &ys, alpha); + + assert_eq!(xs_1, xs_2); + } + } + + #[bench] + fn bench_dot(b: &mut Bencher) { + let xs = vec![0.0; 256]; + let ys = vec![0.0; 256]; + + b.iter(|| dot(&xs[..], &ys[..])); + } + + #[bench] + fn bench_unrolled_dot(b: &mut Bencher) { + let xs = vec![0.0; 256]; + let ys = vec![0.0; 256]; + + b.iter(|| unrolled_dot(&xs[..], &ys[..])); + } + + #[bench] + fn bench_simd_dot(b: &mut Bencher) { + let xs = vec![0.0; 256]; + let ys = vec![0.0; 256]; + + b.iter(|| simd_dot(&xs[..], &ys[..])); + } + + #[bench] + fn bench_array_scaled_assign(b: &mut Bencher) { + let mut xs = random_matrix(256, 1); + let ys = random_matrix(256, 1); + + b.iter(|| array_scaled_assign(&mut xs, &ys, 3.5)); + } + + #[bench] + fn bench_slice_scaled_assign(b: &mut Bencher) { + let mut xs = random_matrix(256, 1); + let ys = random_matrix(256, 1); + + b.iter(|| scaled_assign(&mut xs, &ys, 3.5)); + } + + #[bench] + fn bench_array_assign(b: &mut Bencher) { + let mut xs = random_matrix(256, 1); + let ys = random_matrix(256, 1); + + b.iter(|| array_assign(&mut xs, &ys)); + } + + #[bench] + fn bench_slice_assign(b: &mut Bencher) { + let mut xs = random_matrix(256, 1); + let ys = random_matrix(256, 1); + + b.iter(|| assign(&mut xs, &ys)); + } +} From 4b8d2ed715470d183ae9efe5bfa1323d3b89f9f9 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 16:51:23 +0000 Subject: [PATCH 002/108] Update manifest. --- Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d3549629..008c17cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,11 @@ [package] name = "wyrm" version = "0.1.0" -authors = ["maciejkula "] +authors = ["Maciej Kula"] license = "MIT" +description = "A low-overhead, define-by-run autodifferentiation library." +repository = "https://github.com/maciejkula/wyrm" +readme = "readme.md" [dependencies] ndarray = { version = "0.10.13", features = ["serde-1"] } From af83d511570b473496dc018dcd10fa4a0808deaf Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 18:33:32 +0000 Subject: [PATCH 003/108] Update reamde. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e9a6e5a5..3faf8087 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ #![feature(test)] -//! A backward-mode, define-by-run, low-overhead autodifferentiation library. +//! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! //! # Features //! From caf6fd9a8b36b229038cd42c16a6a5b2566eb9c7 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 21:31:26 +0000 Subject: [PATCH 004/108] Spelling. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 3faf8087..7e626b4c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ //! 3. Define-by-run. //! 4. Trivial Hogwild-style parallelisation, scaling linearly with the number of CPU cores available. //! -//! Requires the nightly compiler due to use of SIMD instrinsics. +//! Requires the nightly compiler due to use of SIMD intrinsics. //! //! # Quickstart //! From ea6ca061a03a7a928943574f4aee5bb6dd50b0d2 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 21:34:51 +0000 Subject: [PATCH 005/108] More spelling. --- src/nodes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nodes.rs b/src/nodes.rs index 4a2eb27c..58690101 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -58,7 +58,7 @@ pub trait Node: fmt::Debug + Sized { fn backward(&self, &Ref); /// Return the value of the node. fn value(&self) -> Bor; - /// If the node needs to used in the backward step. + /// If the node needs to be used in the backward step. fn needs_gradient(&self) -> bool; } From ff17b0e312451ba06a8a8a5f197ba6f5ded22422 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 21:42:57 +0000 Subject: [PATCH 006/108] Hide ParameterNode fields. --- Cargo.toml | 2 +- src/nodes.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 008c17cd..c2283eaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.1.0" +version = "0.2.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." diff --git a/src/nodes.rs b/src/nodes.rs index 58690101..5505eafd 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -255,8 +255,8 @@ impl HogwildParameter { /// Parameter node, holds the optimizable parameters of the model. #[derive(Debug)] pub struct ParameterNode { - pub value: Arc, - pub gradient: RefCell, + pub(crate) value: Arc, + pub(crate) gradient: RefCell, } impl ParameterNode { From a1b72b8b07fdba6c4a6fe522144876ebad51f619 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 26 Dec 2017 22:05:51 +0000 Subject: [PATCH 007/108] Spelling. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 7e626b4c..b93fbf39 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -397,7 +397,7 @@ where } } -/// Standard stochtic gradient descent optimizer with a fixed learning rate. +/// Standard stochastic gradient descent optimizer with a fixed learning rate. pub struct SGD { learning_rate: f32, parameters: Vec>, From 9b917866eb394ddc2f12ae879f7a590c8aad37e6 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 30 Dec 2017 18:49:44 +0000 Subject: [PATCH 008/108] Add scalar sum and ln nodes. --- src/lib.rs | 28 +++++++++++- src/nodes.rs | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index b93fbf39..d007b163 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -241,6 +241,16 @@ where Variable::new(Rc::new(SquareNode::new(Rc::clone(&self.node)))) } + /// Sum this variable. + pub fn scalar_sum(&self) -> Variable> { + Variable::new(Rc::new(SumNode::new(Rc::clone(&self.node)))) + } + + /// Take the natural logarithm of this variable. + pub fn ln(&self) -> Variable> { + Variable::new(Rc::new(LogNode::new(Rc::clone(&self.node)))) + } + /// Transpose this variable. pub fn t(&self) -> Variable> { Variable::new(Rc::new(TransposeNode::new(Rc::clone(&self.node)))) @@ -576,7 +586,23 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] - fn tranpose_finite_difference() { + fn ln_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = (x.clone() + x.clone()).ln(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn sum_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = (x.clone() + x.clone()).scalar_sum(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] + fn transpose_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut z = (x.clone() + x.clone()).t(); diff --git a/src/nodes.rs b/src/nodes.rs index 5505eafd..0cd29e67 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -813,6 +813,69 @@ where } } +#[derive(Debug)] +pub struct LogNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl LogNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = operand.value().map(|x| x.ln()); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + LogNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for LogNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.operand.value().deref()); + dest.map_inplace(|x| *x = x.ln()); + } + + fn backward(&self, gradient: &Ref) { + for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest = grad_val / operand_val; + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + #[derive(Debug)] pub struct SigmoidNode { value: RefCell, @@ -1158,6 +1221,67 @@ where } } +#[derive(Debug)] +pub struct SumNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, +} + +impl SumNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = { + let mut value = Arr::zeros((1, 1)); + value.fill(operand.value().scalar_sum()); + value + }; + + let gradient = operand.value().deref() * 0.0; + let needs_gradient = operand.needs_gradient(); + + SumNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + } + } +} + +impl Node for SumNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + dest[(0, 0)] = self.operand.value().scalar_sum(); + } + fn backward(&self, gradient: &Ref) { + debug_assert!(gradient.len() == 1, "Input gradient must be a scalar."); + + { + self.operand_gradient.borrow_mut().fill(gradient[(0, 0)]); + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } +} + /// An input node for integer indices into `ParameterNode`s, used /// for implementing indexable embedding layers. #[derive(Debug)] From 5dbbe693a8afb9be6b470b3414eed11b43ac81f3 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 31 Dec 2017 20:40:58 +0000 Subject: [PATCH 009/108] Add node reuse benchmark. --- src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index d007b163..6f013637 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -864,4 +864,20 @@ mod tests { assert!(sum_loss / (losses.len() as f32) < 1e-3); } + + use test::Bencher; + + #[bench] + fn bench_node_reuse(b: &mut Bencher) { + let dim = 128; + + let x = ParameterNode::new(random_matrix(1, dim)); + let y = ParameterNode::new(random_matrix(dim, 10)); + let v = x.dot(&y); + let z = v.clone() + v.clone() + v.clone() + v.clone(); + + b.iter(|| { + z.forward(); + }); + } } From 85f3a2979c0fe3a0f03a3cb0336a303643c9c4f0 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 1 Jan 2018 11:12:37 +0000 Subject: [PATCH 010/108] Easier tests, numerically. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6f013637..c1cd3d4c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -587,7 +587,7 @@ mod tests { } #[test] fn ln_finite_difference() { - let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut x = ParameterNode::new(random_matrix(2, 2)); let mut z = (x.clone() + x.clone()).ln(); let (finite_difference, gradient) = finite_difference(&mut x, &mut z); From 1e39796d90865a6b71d066342633c86e75b11b25 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 30 Dec 2017 12:55:34 +0000 Subject: [PATCH 011/108] Start not repeating work. --- src/nodes.rs | 433 ++++++++++++++++++++++++++++++++++++++---------- src/numerics.rs | 21 +++ 2 files changed, 370 insertions(+), 84 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index 0cd29e67..f51c6397 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1,7 +1,7 @@ use std; use std::fmt; use std::ops::{AddAssign, Deref, DerefMut, DivAssign, MulAssign, SubAssign}; -use std::cell::{Ref, RefCell}; +use std::cell::{Cell, Ref, RefCell}; use std::sync::Arc; use std::rc::Rc; @@ -15,6 +15,58 @@ use numerics; use super::{Arr, Variable}; +#[derive(Debug, PartialEq)] +pub enum ForwardAction { + Evaluate, + Cached, +} + +#[derive(Debug, PartialEq)] +pub enum BackwardAction { + Set, + Increment, +} + +#[derive(Debug, Default)] +struct PassCounter { + forward_count: Cell, + backward_count: Cell, +} + +impl PassCounter { + fn clear(&self) { + self.forward_count.set(0); + self.backward_count.set(0); + } + fn recurse_backward(&self) -> bool { + let backward_count = self.backward_count.get(); + let forward_count = self.forward_count.get(); + + backward_count == forward_count + } + fn forward(&self) -> ForwardAction { + let count = self.forward_count.get(); + self.forward_count.set(count + 1); + + match count { + 0 => ForwardAction::Evaluate, + _ => ForwardAction::Cached, + } + } + fn backward(&self) -> BackwardAction { + let backward_count = self.backward_count.get(); + + let action = match backward_count { + 0 => BackwardAction::Set, + _ => BackwardAction::Increment, + }; + + self.backward_count.set(backward_count + 1); + + action + } +} + /// Generalisation over borrowed RefCell values /// and simple references. #[derive(Debug)] @@ -60,6 +112,7 @@ pub trait Node: fmt::Debug + Sized { fn value(&self) -> Bor; /// If the node needs to be used in the backward step. fn needs_gradient(&self) -> bool; + fn zero_gradient(&self) {} } #[derive(Debug)] @@ -68,6 +121,7 @@ pub struct AddNode { lhs: Rc, rhs: Rc, needs_gradient: bool, + counter: PassCounter, } impl AddNode @@ -84,6 +138,7 @@ where lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -97,6 +152,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.lhs.forward(); self.rhs.forward(); @@ -133,6 +192,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + } } /// Input node for the graph. @@ -163,6 +227,8 @@ impl Node for InputNode { fn needs_gradient(&self) -> bool { false } + fn zero_gradient(&self) { + } } #[derive(Debug)] @@ -300,6 +366,9 @@ impl Node for ParameterNode { fn needs_gradient(&self) -> bool { true } + fn zero_gradient(&self) { + self.gradient.borrow_mut().zero_gradient(); + } } #[derive(Debug)] @@ -313,6 +382,7 @@ where lhs: Rc, rhs: Rc, needs_gradient: bool, + counter: PassCounter, } impl SubNode @@ -332,6 +402,7 @@ where lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -345,6 +416,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.lhs.forward(); self.rhs.forward(); @@ -355,15 +430,24 @@ where } fn backward(&self, gradient: &Ref) { - { - let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + match self.counter.backward() { + BackwardAction::Set => { + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + rhs_gradient.assign(gradient); + rhs_gradient.mul_assign(-1.0); + }, + BackwardAction::Increment => { + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - rhs_gradient.assign(gradient); - rhs_gradient.mul_assign(-1.0); + rhs_gradient.scaled_add(-1.0, gradient); + } } - self.lhs.backward(gradient); - self.rhs.backward(&self.rhs_gradient.borrow()); + if self.counter.recurse_backward() { + self.lhs.backward(gradient); + self.rhs.backward(&self.rhs_gradient.borrow()); + } } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) @@ -371,6 +455,12 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + + } } #[derive(Debug)] @@ -381,6 +471,7 @@ pub struct MulNode { lhs: Rc, rhs: Rc, needs_gradient: bool, + counter: PassCounter, } impl MulNode @@ -402,6 +493,7 @@ where lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -415,6 +507,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.lhs.forward(); self.rhs.forward(); @@ -424,19 +520,40 @@ where dest.mul_assign(self.rhs.value().deref()); } fn backward(&self, gradient: &Ref) { - { - let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - lhs_gradient.assign(self.rhs.value().deref()); - lhs_gradient.mul_assign(gradient.deref()); - - let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - - rhs_gradient.assign(self.lhs.value().deref()); - rhs_gradient.mul_assign(gradient.deref()); + match self.counter.backward() { + BackwardAction::Set => { + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + lhs_gradient.assign(self.rhs.value().deref()); + lhs_gradient.mul_assign(gradient.deref()); + + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + rhs_gradient.assign(self.lhs.value().deref()); + rhs_gradient.mul_assign(gradient.deref()); + }, + BackwardAction::Increment => { + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + numerics::map_inplace_assign_binary(lhs_gradient.deref_mut(), + self.rhs.value().deref(), + gradient.deref(), + |out, rhs, grad| { + *out += rhs * grad + }); + numerics::map_inplace_assign_binary(rhs_gradient.deref_mut(), + self.lhs.value().deref(), + gradient.deref(), + |out, rhs, grad| { + *out += rhs * grad + }); + } } - self.lhs.backward(&self.lhs_gradient.borrow()); - self.rhs.backward(&self.rhs_gradient.borrow()); + if self.counter.recurse_backward() { + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) @@ -444,6 +561,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + } } #[derive(Debug)] @@ -454,6 +576,7 @@ pub struct DivNode { lhs: Rc, rhs: Rc, needs_gradient: bool, + counter: PassCounter, } impl DivNode @@ -475,6 +598,7 @@ where lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -488,6 +612,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.lhs.forward(); self.rhs.forward(); @@ -497,27 +625,49 @@ where dest.div_assign(self.rhs.value().deref()); } fn backward(&self, gradient: &Ref) { - { - let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - let rhs_value = self.rhs.value(); - - izip!(lhs_gradient.iter_mut(), rhs_value.iter(), gradient.iter()) - .for_each(|(dest, rhs_val, grad_val)| *dest = grad_val / rhs_val); - - let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - - izip!( - rhs_gradient.iter_mut(), - self.lhs.value().iter(), - rhs_value.iter(), - gradient.iter() - ).for_each(|(dest, lhs_val, rhs_val, grad_val)| { - *dest = -lhs_val / rhs_val.powi(2) * grad_val - }); + match self.counter.backward() { + BackwardAction::Set => { + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let rhs_value = self.rhs.value(); + + izip!(lhs_gradient.iter_mut(), rhs_value.iter(), gradient.iter()) + .for_each(|(dest, rhs_val, grad_val)| *dest = grad_val / rhs_val); + + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + izip!( + rhs_gradient.iter_mut(), + self.lhs.value().iter(), + rhs_value.iter(), + gradient.iter() + ).for_each(|(dest, lhs_val, rhs_val, grad_val)| { + *dest = -lhs_val / rhs_val.powi(2) * grad_val + }); + }, + BackwardAction::Increment => { + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let rhs_value = self.rhs.value(); + + izip!(lhs_gradient.iter_mut(), rhs_value.iter(), gradient.iter()) + .for_each(|(dest, rhs_val, grad_val)| *dest += grad_val / rhs_val); + + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + izip!( + rhs_gradient.iter_mut(), + self.lhs.value().iter(), + rhs_value.iter(), + gradient.iter() + ).for_each(|(dest, lhs_val, rhs_val, grad_val)| { + *dest += -lhs_val / rhs_val.powi(2) * grad_val + }); + } } - self.lhs.backward(&self.lhs_gradient.borrow()); - self.rhs.backward(&self.rhs_gradient.borrow()); + if self.counter.recurse_backward() { + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } } fn value(&self) -> Bor { @@ -527,6 +677,12 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + } } #[derive(Debug)] @@ -537,6 +693,7 @@ pub struct DotNode { lhs: Rc, rhs: Rc, needs_gradient: bool, + counter: PassCounter, } impl DotNode @@ -558,6 +715,7 @@ where lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -572,6 +730,10 @@ where type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.lhs.forward(); self.rhs.forward(); @@ -585,17 +747,22 @@ where } fn backward(&self, gradient: &Ref) { - { - let rhs_value = self.rhs.value(); - let lhs_value = self.lhs.value(); - let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + // TODO: handle the non-overwrite case + match self.counter.backward() { + _ => { + let rhs_value = self.rhs.value(); + let lhs_value = self.lhs.value(); + + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - general_mat_mul(1.0, gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); - general_mat_mul(1.0, &lhs_value.t(), gradient, 0.0, &mut rhs_gradient); + general_mat_mul(1.0, gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); + general_mat_mul(1.0, &lhs_value.t(), gradient, 0.0, &mut rhs_gradient); + } } + // Always recurse because we haven't actually handled accumulation yet self.lhs.backward(&self.lhs_gradient.borrow()); self.rhs.backward(&self.rhs_gradient.borrow()); } @@ -607,6 +774,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + } } #[derive(Debug)] @@ -617,6 +789,7 @@ pub struct VectorDotNode { lhs: Rc, rhs: Rc, needs_gradient: bool, + counter: PassCounter, } impl VectorDotNode @@ -666,6 +839,7 @@ where lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -680,6 +854,10 @@ where type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.lhs.forward(); self.rhs.forward(); @@ -702,43 +880,75 @@ where } fn backward(&self, gradient: &Ref) { - { - let mut lhs_grad = self.lhs_gradient.borrow_mut(); - let mut rhs_grad = self.rhs_gradient.borrow_mut(); - - let lhs_value = self.lhs.value(); - let rhs_value = self.rhs.value(); + let mut lhs_grad = self.lhs_gradient.borrow_mut(); + let mut rhs_grad = self.rhs_gradient.borrow_mut(); - for (backward_row, rhs_row, &gradient) in izip!( - lhs_grad - .genrows_mut() - .into_iter() - .map(|x| x.into_slice().unwrap()), - rhs_value - .genrows() - .into_iter() - .map(|x| x.into_slice().unwrap()), - gradient.as_slice().unwrap() - ) { - numerics::simd_scaled_assign(backward_row, rhs_row, gradient) - } - for (backward_row, lhs_row, &gradient) in izip!( - rhs_grad - .genrows_mut() - .into_iter() - .map(|x| x.into_slice().unwrap()), - lhs_value - .genrows() - .into_iter() - .map(|x| x.into_slice().unwrap()), - gradient.as_slice().unwrap() - ) { - numerics::simd_scaled_assign(backward_row, lhs_row, gradient) + let lhs_value = self.lhs.value(); + let rhs_value = self.rhs.value(); + + match self.counter.backward() { + BackwardAction::Set => { + for (backward_row, rhs_row, &gradient) in izip!( + lhs_grad + .genrows_mut() + .into_iter() + .map(|x| x.into_slice().unwrap()), + rhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + gradient.as_slice().unwrap() + ) { + numerics::simd_scaled_assign(backward_row, rhs_row, gradient) + } + for (backward_row, lhs_row, &gradient) in izip!( + rhs_grad + .genrows_mut() + .into_iter() + .map(|x| x.into_slice().unwrap()), + lhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + gradient.as_slice().unwrap() + ) { + numerics::simd_scaled_assign(backward_row, lhs_row, gradient) + } + }, + BackwardAction::Increment => { + for (backward_row, rhs_row, &gradient) in izip!( + lhs_grad + .genrows_mut() + .into_iter() + .map(|x| x.into_slice().unwrap()), + rhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + gradient.as_slice().unwrap() + ) { + numerics::simd_scaled_add(backward_row, rhs_row, gradient) + } + for (backward_row, lhs_row, &gradient) in izip!( + rhs_grad + .genrows_mut() + .into_iter() + .map(|x| x.into_slice().unwrap()), + lhs_value + .genrows() + .into_iter() + .map(|x| x.into_slice().unwrap()), + gradient.as_slice().unwrap() + ) { + numerics::simd_scaled_add(backward_row, lhs_row, gradient) + } } } - self.lhs.backward(&self.lhs_gradient.borrow()); - self.rhs.backward(&self.rhs_gradient.borrow()); + if self.counter.recurse_backward() { + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } } fn value(&self) -> Bor { @@ -748,6 +958,12 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + } } #[derive(Debug)] @@ -756,6 +972,7 @@ pub struct SquareNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl SquareNode @@ -772,6 +989,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -784,6 +1002,9 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } self.operand.forward(); let mut dest = self.value.borrow_mut(); @@ -793,15 +1014,30 @@ where } fn backward(&self, gradient: &Ref) { - for (dest, operand_val, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.operand.value().iter(), - gradient.iter() - ) { - *dest = operand_val * 2.0 * grad_val; + match self.counter.backward() { + BackwardAction::Set => { + for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest = operand_val * 2.0 * grad_val; + } + }, + BackwardAction::Increment => { + for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest += operand_val * 2.0 * grad_val; + } + } } - self.operand.backward(&self.operand_gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } } fn value(&self) -> Bor { @@ -811,6 +1047,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1384,3 +1625,27 @@ impl Node for IndexNode { self.needs_gradient } } + +#[cfg(test)] +mod tests { + use rand; + + use super::*; + + fn random_matrix(rows: usize, cols: usize) -> Arr { + Arr::zeros((rows, cols)).map(|_| rand::random::()) + } + + #[test] + fn test_sub_counter() { + let x = ParameterNode::new(random_matrix(1, 1)); + let y = x.clone() - x.clone(); + + let mut z = y.clone() + y.clone() + y.clone(); + + z.forward(); + assert_eq!(y.node.counter.forward_count.get(), 3); + z.backward(1.0); + assert_eq!(y.node.counter.backward_count.get(), 3); + } +} diff --git a/src/numerics.rs b/src/numerics.rs index da613ab9..aa53c0f5 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -48,6 +48,27 @@ pub fn simd_scaled_assign(xs: &mut [f32], ys: &[f32], alpha: f32) { } } +pub fn simd_scaled_add(xs: &mut [f32], ys: &[f32], alpha: f32) { + let stride = 8; + let simd_alpha = stdsimd::simd::f32x8::splat(alpha); + + let split_idx = xs.len() / stride * stride; + let (simd_xs, scalar_xs) = xs.split_at_mut(split_idx); + let (simd_ys, scalar_ys) = ys.split_at(split_idx); + + for (x, y) in simd_xs.chunks_mut(stride).zip(simd_ys.chunks(stride)) { + unsafe { + let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) + + stdsimd::simd::f32x8::load_unchecked(y, 0) * simd_alpha; + elem.store_unchecked(x, 0); + } + } + + for (x_scalar, y_scalar) in scalar_xs.iter_mut().zip(scalar_ys.iter()) { + *x_scalar += y_scalar * alpha; + } +} + pub fn slice_assign(xs: &mut [f32], ys: &[f32]) { for (x, &y) in xs.iter_mut().zip(ys.iter()) { *x = y; From 149f3a0fae81cfce92df5d34783eaf7f9cde318a Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 31 Dec 2017 20:26:09 +0000 Subject: [PATCH 012/108] Finish adding pass marking. --- src/lib.rs | 30 +++--- src/nodes.rs | 251 ++++++++++++++++++++++++++++++++++-------------- src/numerics.rs | 4 +- 3 files changed, 194 insertions(+), 91 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c1cd3d4c..b0b770c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,7 @@ //! loss.backward(1.0); //! //! optimizer.step(); -//! optimizer.zero_gradients(); +//! loss.zero_gradient(); //! } //! # } //! ``` @@ -118,7 +118,7 @@ //! loss.backward(1.0); //! //! optimizer.step(); -//! optimizer.zero_gradients(); +//! loss.zero_gradient(); //! } //! }); //! # } @@ -217,6 +217,10 @@ where pub fn forward(&self) { self.node.forward() } + /// Zero the gradients. Must be called after a backward step or whenever inputs change. + pub fn zero_gradient(&self) { + self.node.zero_gradient(); + } } impl Variable @@ -295,10 +299,6 @@ where } impl Variable { - /// Zero the accumulated gradients of this parameter node. - pub fn zero_gradients(&self) { - self.node.zero_gradient() - } /// Return the (dense) gradient value of this node. pub fn dense_gradient(&self) -> Option { match self.node.gradient.borrow().dense_gradient { @@ -421,10 +421,6 @@ impl SGD { parameters: parameters, } } - /// Zero the gradients on each of the parameters being optimized. - pub fn zero_gradients(&self) { - self.parameters.iter().for_each(|x| x.zero_gradients()) - } /// Perform a single SGD step. pub fn step(&mut self) { @@ -474,6 +470,7 @@ where for (idx, diff) in central_difference.indexed_iter_mut() { let positive_difference = { + output.zero_gradient(); let mut changed_input = initial_input.clone(); changed_input[idx] += 0.5 * delta_x; input.set_value(&changed_input); @@ -482,6 +479,7 @@ where }; let negative_difference = { + output.zero_gradient(); let mut changed_input = initial_input.clone(); changed_input[idx] -= 0.5 * delta_x; input.set_value(&changed_input); @@ -495,7 +493,7 @@ where } let gradient = { - input.zero_gradients(); + output.zero_gradient(); input.set_value(&initial_input); output.forward(); output.backward(1.0); @@ -503,6 +501,8 @@ where input.dense_gradient().unwrap() }; + output.zero_gradient(); + (central_difference, gradient) } @@ -677,7 +677,7 @@ mod tests { loss.backward(1.0); optimizer.step(); - optimizer.zero_gradients(); + loss.zero_gradient(); } println!( @@ -726,7 +726,7 @@ mod tests { loss.backward(1.0); optimizer.step(); - optimizer.zero_gradients(); + loss.zero_gradient(); } println!( @@ -787,7 +787,7 @@ mod tests { loss_val += loss.value().scalar_sum(); optimizer.step(); - optimizer.zero_gradients(); + loss.zero_gradient(); } } @@ -849,7 +849,7 @@ mod tests { loss_val += loss.value().scalar_sum(); optimizer.step(); - optimizer.zero_gradients(); + loss.zero_gradient(); } } } diff --git a/src/nodes.rs b/src/nodes.rs index f51c6397..d17a39ec 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -112,7 +112,7 @@ pub trait Node: fmt::Debug + Sized { fn value(&self) -> Bor; /// If the node needs to be used in the backward step. fn needs_gradient(&self) -> bool; - fn zero_gradient(&self) {} + fn zero_gradient(&self); } #[derive(Debug)] @@ -155,7 +155,7 @@ where if self.counter.forward() == ForwardAction::Cached { return; } - + self.lhs.forward(); self.rhs.forward(); @@ -227,8 +227,7 @@ impl Node for InputNode { fn needs_gradient(&self) -> bool { false } - fn zero_gradient(&self) { - } + fn zero_gradient(&self) {} } #[derive(Debug)] @@ -419,7 +418,7 @@ where if self.counter.forward() == ForwardAction::Cached { return; } - + self.lhs.forward(); self.rhs.forward(); @@ -436,7 +435,7 @@ where rhs_gradient.assign(gradient); rhs_gradient.mul_assign(-1.0); - }, + } BackwardAction::Increment => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -459,7 +458,6 @@ where self.counter.clear(); self.lhs.zero_gradient(); self.rhs.zero_gradient(); - } } @@ -530,23 +528,23 @@ where rhs_gradient.assign(self.lhs.value().deref()); rhs_gradient.mul_assign(gradient.deref()); - }, + } BackwardAction::Increment => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::map_inplace_assign_binary(lhs_gradient.deref_mut(), - self.rhs.value().deref(), - gradient.deref(), - |out, rhs, grad| { - *out += rhs * grad - }); - numerics::map_inplace_assign_binary(rhs_gradient.deref_mut(), - self.lhs.value().deref(), - gradient.deref(), - |out, rhs, grad| { - *out += rhs * grad - }); + numerics::map_inplace_assign_binary( + lhs_gradient.deref_mut(), + self.rhs.value().deref(), + gradient.deref(), + |out, rhs, grad| *out += rhs * grad, + ); + numerics::map_inplace_assign_binary( + rhs_gradient.deref_mut(), + self.lhs.value().deref(), + gradient.deref(), + |out, rhs, grad| *out += rhs * grad, + ); } } @@ -615,7 +613,7 @@ where if self.counter.forward() == ForwardAction::Cached { return; } - + self.lhs.forward(); self.rhs.forward(); @@ -643,7 +641,7 @@ where ).for_each(|(dest, lhs_val, rhs_val, grad_val)| { *dest = -lhs_val / rhs_val.powi(2) * grad_val }); - }, + } BackwardAction::Increment => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); @@ -733,7 +731,7 @@ where if self.counter.forward() == ForwardAction::Cached { return; } - + self.lhs.forward(); self.rhs.forward(); @@ -747,7 +745,6 @@ where } fn backward(&self, gradient: &Ref) { - // TODO: handle the non-overwrite case match self.counter.backward() { _ => { @@ -880,14 +877,14 @@ where } fn backward(&self, gradient: &Ref) { - let mut lhs_grad = self.lhs_gradient.borrow_mut(); - let mut rhs_grad = self.rhs_gradient.borrow_mut(); - let lhs_value = self.lhs.value(); let rhs_value = self.rhs.value(); - + match self.counter.backward() { BackwardAction::Set => { + let mut lhs_grad = self.lhs_gradient.borrow_mut(); + let mut rhs_grad = self.rhs_gradient.borrow_mut(); + for (backward_row, rhs_row, &gradient) in izip!( lhs_grad .genrows_mut() @@ -914,8 +911,11 @@ where ) { numerics::simd_scaled_assign(backward_row, lhs_row, gradient) } - }, + } BackwardAction::Increment => { + let mut lhs_grad = self.lhs_gradient.borrow_mut(); + let mut rhs_grad = self.rhs_gradient.borrow_mut(); + for (backward_row, rhs_row, &gradient) in izip!( lhs_grad .genrows_mut() @@ -1015,24 +1015,20 @@ where fn backward(&self, gradient: &Ref) { match self.counter.backward() { - BackwardAction::Set => { - for (dest, operand_val, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.operand.value().iter(), - gradient.iter() - ) { - *dest = operand_val * 2.0 * grad_val; - } + BackwardAction::Set => for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest = operand_val * 2.0 * grad_val; + }, + BackwardAction::Increment => for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest += operand_val * 2.0 * grad_val; }, - BackwardAction::Increment => { - for (dest, operand_val, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.operand.value().iter(), - gradient.iter() - ) { - *dest += operand_val * 2.0 * grad_val; - } - } } if self.counter.recurse_backward() { @@ -1076,6 +1072,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1088,6 +1085,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); let mut dest = self.value.borrow_mut(); @@ -1115,6 +1116,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1123,6 +1129,7 @@ pub struct SigmoidNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl SigmoidNode @@ -1139,6 +1146,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1161,18 +1169,32 @@ where } fn backward(&self, gradient: &Ref) { - { - let mut operand_gradient = self.operand_gradient.borrow_mut(); - - numerics::map_assign_binary( - &mut operand_gradient, - self.value.borrow().deref(), - gradient, - |sigmoid, grad| grad * sigmoid * (1.0 - sigmoid), - ); + match self.counter.backward() { + BackwardAction::Set => { + let mut operand_gradient = self.operand_gradient.borrow_mut(); + + numerics::map_assign_binary( + &mut operand_gradient, + self.value.borrow().deref(), + gradient, + |sigmoid, grad| grad * sigmoid * (1.0 - sigmoid), + ); + } + BackwardAction::Increment => { + let mut operand_gradient = self.operand_gradient.borrow_mut(); + + numerics::map_inplace_assign_binary( + &mut operand_gradient, + self.value.borrow().deref(), + gradient, + |dest, sigmoid, grad| *dest += grad * sigmoid * (1.0 - sigmoid), + ); + } } - self.operand.backward(&self.operand_gradient.borrow()) + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()) + } } fn value(&self) -> Bor { @@ -1182,6 +1204,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1190,6 +1217,7 @@ pub struct NegNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl NegNode @@ -1206,6 +1234,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1219,6 +1248,10 @@ where type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); let mut dest = self.value.borrow_mut(); @@ -1228,14 +1261,24 @@ where } fn backward(&self, gradient: &Ref) { - for (dest, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - gradient.iter() - ) { - *dest = -grad_val; + match self.counter.backward() { + BackwardAction::Set => for (dest, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + gradient.iter() + ) { + *dest = -grad_val; + }, + BackwardAction::Increment => for (dest, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + gradient.iter() + ) { + *dest += -grad_val; + }, } - self.operand.backward(&self.operand_gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } } fn value(&self) -> Bor { @@ -1245,6 +1288,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1253,6 +1300,7 @@ pub struct ExpNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl ExpNode @@ -1281,6 +1329,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); let mut dest = self.value.borrow_mut(); @@ -1288,15 +1340,25 @@ where dest.map_inplace(|x| *x = x.exp()); } fn backward(&self, gradient: &Ref) { - for (dest, self_val, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.value.borrow().iter(), - gradient.iter() - ) { - *dest = self_val * grad_val; + match self.counter.backward() { + BackwardAction::Set => for (dest, self_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.value.borrow().iter(), + gradient.iter() + ) { + *dest = self_val * grad_val; + }, + BackwardAction::Increment => for (dest, self_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.value.borrow().iter(), + gradient.iter() + ) { + *dest += self_val * grad_val; + }, + } + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); } - - self.operand.backward(&self.operand_gradient.borrow()); } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) @@ -1304,6 +1366,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1312,6 +1379,7 @@ pub struct TransposeNode { gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl TransposeNode @@ -1328,6 +1396,7 @@ where gradient: gradient, operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1340,15 +1409,26 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); self.value.borrow_mut().assign(&self.operand.value().t()); } fn backward(&self, gradient: &Ref) { - { - self.gradient.borrow_mut().assign(&gradient.t()); + match self.counter.backward() { + BackwardAction::Set => { + self.gradient.borrow_mut().assign(&gradient.t()); + } + BackwardAction::Increment => { + self.gradient.borrow_mut().add_assign(&gradient.t()); + } } - self.operand.backward(&self.gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.gradient.borrow()); + } } fn value(&self) -> Bor { @@ -1367,6 +1447,7 @@ pub struct SoftmaxNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl SoftmaxNode @@ -1398,6 +1479,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1410,6 +1492,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); let mut dest = self.value.borrow_mut(); dest.assign(self.operand.value().deref()); @@ -1424,6 +1510,7 @@ where dest.map_inplace(|x| *x /= denominator); } fn backward(&self, gradient: &Ref) { + // TODO: accumulate gradients let value = self.value.borrow(); let mut jacobian = self.jacobian.borrow_mut(); @@ -1521,6 +1608,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } /// An input node for integer indices into `ParameterNode`s, used @@ -1551,6 +1643,7 @@ impl Node for IndexInputNode { fn needs_gradient(&self) -> bool { false } + fn zero_gradient(&self) {} } #[derive(Debug)] @@ -1561,6 +1654,7 @@ pub struct IndexNode { index: Rc, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl IndexNode @@ -1580,6 +1674,7 @@ where index: index, operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1589,6 +1684,10 @@ impl Node for IndexNode { type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + let operand_value = self.operand.value(); let mut idx_value = self.index_value.borrow_mut(); @@ -1624,12 +1723,16 @@ impl Node for IndexNode { fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[cfg(test)] mod tests { use rand; - + use super::*; fn random_matrix(rows: usize, cols: usize) -> Arr { diff --git a/src/numerics.rs b/src/numerics.rs index aa53c0f5..841d3c41 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -58,8 +58,8 @@ pub fn simd_scaled_add(xs: &mut [f32], ys: &[f32], alpha: f32) { for (x, y) in simd_xs.chunks_mut(stride).zip(simd_ys.chunks(stride)) { unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) + - stdsimd::simd::f32x8::load_unchecked(y, 0) * simd_alpha; + let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) + + stdsimd::simd::f32x8::load_unchecked(y, 0) * simd_alpha; elem.store_unchecked(x, 0); } } From 9919338c5b06a78a6c8c186b9d3f68588318abc9 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 31 Dec 2017 20:43:51 +0000 Subject: [PATCH 013/108] Add node reuse benchmark. --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index b0b770c4..b44219cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -878,6 +878,7 @@ mod tests { b.iter(|| { z.forward(); + z.zero_gradient(); }); } } From 5e208984e00cb45a6daf743b106d2ba43fe6cfee Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 1 Jan 2018 10:56:06 +0000 Subject: [PATCH 014/108] Rebase on master and fix new nodes. --- src/lib.rs | 4 +++- src/nodes.rs | 61 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b44219cb..ddfb472a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -498,7 +498,9 @@ where output.forward(); output.backward(1.0); - input.dense_gradient().unwrap() + input + .dense_gradient() + .expect("Expecting a gradient but gradient not present.") }; output.zero_gradient(); diff --git a/src/nodes.rs b/src/nodes.rs index d17a39ec..60a809f8 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1056,6 +1056,7 @@ pub struct LogNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl LogNode @@ -1098,15 +1099,26 @@ where } fn backward(&self, gradient: &Ref) { - for (dest, operand_val, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.operand.value().iter(), - gradient.iter() - ) { - *dest = grad_val / operand_val; + match self.counter.backward() { + BackwardAction::Set => for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest = grad_val / operand_val; + }, + BackwardAction::Increment => for (dest, operand_val, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.operand.value().iter(), + gradient.iter() + ) { + *dest += grad_val / operand_val; + }, } - self.operand.backward(&self.operand_gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } } fn value(&self) -> Bor { @@ -1159,6 +1171,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); let mut dest = self.value.borrow_mut(); @@ -1317,6 +1333,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1438,6 +1455,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1547,6 +1569,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } } #[derive(Debug)] @@ -1555,6 +1581,7 @@ pub struct SumNode { operand_gradient: RefCell, operand: Rc, needs_gradient: bool, + counter: PassCounter, } impl SumNode @@ -1576,6 +1603,7 @@ where operand_gradient: RefCell::new(gradient), operand: operand, needs_gradient: needs_gradient, + counter: PassCounter::default(), } } } @@ -1588,6 +1616,10 @@ where type InputGradient = Arr; type OutputGradient = Arr; fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.operand.forward(); let mut dest = self.value.borrow_mut(); @@ -1596,11 +1628,20 @@ where fn backward(&self, gradient: &Ref) { debug_assert!(gradient.len() == 1, "Input gradient must be a scalar."); - { - self.operand_gradient.borrow_mut().fill(gradient[(0, 0)]); + match self.counter.backward() { + BackwardAction::Set => { + self.operand_gradient.borrow_mut().fill(gradient[(0, 0)]); + } + BackwardAction::Increment => { + self.operand_gradient + .borrow_mut() + .add_assign(gradient[(0, 0)]); + } } - self.operand.backward(&self.operand_gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) From 00b66fb6b48188b89e5667d352e34d7b961451f8 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 2 Jan 2018 17:52:26 +0000 Subject: [PATCH 015/108] Add instructions on using BLAS. --- Cargo.toml | 2 +- src/lib.rs | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c2283eaf..6b72f216 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/maciejkula/wyrm" readme = "readme.md" [dependencies] -ndarray = { version = "0.10.13", features = ["serde-1"] } +ndarray = { version = "0.11.0", features = ["serde-1"] } rand = "0.3.18" smallvec = { version = "0.5.0", features = ["serde"] } itertools = "0.7.3" diff --git a/src/lib.rs b/src/lib.rs index ddfb472a..717e6959 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,6 +123,16 @@ //! }); //! # } //! ``` +//! +//! ## BLAS support +//! You should enable BLAS support to get (much) better performance out of matrix-multiplication-heavy +//! workloads. To do so, add the following to your `Cargo.toml`: +//! +//! ``` +//! ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } +//! blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } +//! openblas-src = { version = "0.5.6", default-features = false, features = ["cblas"] } +//! ``` // TODO: pass through of parent values in .value(), // optimizations in forward @@ -427,9 +437,8 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = unsafe { - &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) - }; + let mut param_value = + unsafe { &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); @@ -883,4 +892,29 @@ mod tests { z.zero_gradient(); }); } + + #[bench] + fn bench_matrix_multiply(b: &mut Bencher) { + let dim = 64; + let num_epochs = 20; + + let x_data = Arc::new(HogwildParameter::new(random_matrix(1, dim))); + let y_data = Arc::new(HogwildParameter::new(random_matrix(dim, 10))); + + b.iter(|| { + (0..rayon::current_num_threads()) + .into_par_iter() + .for_each(|_| { + let x = ParameterNode::shared(x_data.clone()); + let y = ParameterNode::shared(y_data.clone()); + + let v = x.dot(&y); + + for _ in 0..num_epochs { + v.forward(); + v.zero_gradient(); + } + }); + }); + } } From 99c3a11e861e7ea1ef50647ecb08c44cc8693714 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 2 Jan 2018 21:34:06 +0000 Subject: [PATCH 016/108] Ignore code section in docs. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 717e6959..b5065c85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -128,7 +128,7 @@ //! You should enable BLAS support to get (much) better performance out of matrix-multiplication-heavy //! workloads. To do so, add the following to your `Cargo.toml`: //! -//! ``` +//! ```text //! ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } //! blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } //! openblas-src = { version = "0.5.6", default-features = false, features = ["cblas"] } From b3d363615b017436da65eff2c8b2261e49ca6931 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 2 Jan 2018 22:05:33 +0000 Subject: [PATCH 017/108] Bump version to 0.3.0. --- Cargo.toml | 6 +++++- src/lib.rs | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6b72f216..69a39854 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,16 @@ [package] name = "wyrm" -version = "0.2.0" +version = "0.3.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." +documentation = "https://docs.rs/wyrm/" repository = "https://github.com/maciejkula/wyrm" readme = "readme.md" +[badges] +travis-ci = { repository = "https://github.com/maciejkula/wyrm", branch = "master" } + [dependencies] ndarray = { version = "0.11.0", features = ["serde-1"] } rand = "0.3.18" diff --git a/src/lib.rs b/src/lib.rs index b5065c85..36b502dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -437,8 +437,9 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = - unsafe { &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) }; + let mut param_value = unsafe { + &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) + }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); From 78cb887e57de09ed2d3f39a0df61f77d025f0ee5 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 2 Jan 2018 22:11:57 +0000 Subject: [PATCH 018/108] Fix travis badge in Cargo.toml. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 69a39854..91ee7df2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/maciejkula/wyrm" readme = "readme.md" [badges] -travis-ci = { repository = "https://github.com/maciejkula/wyrm", branch = "master" } +travis-ci = { repository = "maciejkula/wyrm", branch = "master" } [dependencies] ndarray = { version = "0.11.0", features = ["serde-1"] } From 5a75d4c6579d8c0565b638c2a4cbbb4a8ed6d512 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 3 Jan 2018 10:39:57 +0000 Subject: [PATCH 019/108] Push 0.3.1. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 91ee7df2..8d1451ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.3.0" +version = "0.3.1" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From e64983b97c3c3a4719f6eff2154d77d6eafcff58 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 6 Jan 2018 13:35:49 +0000 Subject: [PATCH 020/108] Extract parameters of a graph from its root. --- src/lib.rs | 127 +++++++++++++++++++++++++++++++++++++++------------ src/nodes.rs | 32 +++++++++---- 2 files changed, 120 insertions(+), 39 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 36b502dc..06a46649 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -187,6 +187,10 @@ pub trait DataInput { fn set_value(&self, T); } +fn merge_parameters(xs: &[Rc], ys: &[Rc]) -> Vec> { + xs.iter().chain(ys.iter()).cloned().collect() +} + /// Handle to a node in the computation graph. The underlying nodes /// are reference counted, so the handles can be freely cloned to /// use the nodes multiple times in the same graph. @@ -197,6 +201,7 @@ where { node: Rc, grad: Option>, + parameters: Vec>, } impl Clone for Variable { @@ -204,6 +209,7 @@ impl Clone for Variable { Variable { node: Rc::clone(&self.node), grad: None, + parameters: self.parameters.clone(), } } } @@ -212,10 +218,11 @@ impl Variable where T: Node, { - fn new(node: Rc) -> Self { + fn new(node: Rc, parameters: Vec>) -> Self { Variable { node: node, grad: None, + parameters: parameters, } } /// Get the value of the node. @@ -231,6 +238,18 @@ where pub fn zero_gradient(&self) { self.node.zero_gradient(); } + + /// Return the parameters of the graph. + pub fn parameters(&self) -> Vec> { + let mut unique_params = self.parameters.clone(); + unique_params.sort_unstable_by_key(|x| x.deref() as *const ParameterNode); + unique_params.dedup_by_key(|x| (*x).deref() as *const ParameterNode); + + unique_params + .iter() + .map(|x| Variable::new(x.clone(), Vec::new())) + .collect() + } } impl Variable @@ -252,37 +271,58 @@ where /// Square this variable. pub fn square(&self) -> Variable> { - Variable::new(Rc::new(SquareNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(SquareNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Sum this variable. pub fn scalar_sum(&self) -> Variable> { - Variable::new(Rc::new(SumNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(SumNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Take the natural logarithm of this variable. pub fn ln(&self) -> Variable> { - Variable::new(Rc::new(LogNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(LogNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Transpose this variable. pub fn t(&self) -> Variable> { - Variable::new(Rc::new(TransposeNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(TransposeNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Exponentiate this variable. pub fn exp(&self) -> Variable> { - Variable::new(Rc::new(ExpNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(ExpNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Compute the softmax of this variable. pub fn softmax(&self) -> Variable> { - Variable::new(Rc::new(SoftmaxNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(SoftmaxNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Compute the sigmoid of this variable. pub fn sigmoid(&self) -> Variable> { - Variable::new(Rc::new(SigmoidNode::new(Rc::clone(&self.node)))) + Variable::new( + Rc::new(SigmoidNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) } /// Compute the row-wise vector dot product of LHS and RHS. @@ -290,10 +330,13 @@ where where S: Node, { - Variable::new(Rc::new(VectorDotNode::new( - Rc::clone(&self.node), - Rc::clone(&other.node), - ))) + Variable::new( + Rc::new(VectorDotNode::new( + Rc::clone(&self.node), + Rc::clone(&other.node), + )), + merge_parameters(&self.parameters, &other.parameters), + ) } /// Compute the matrix multiplication of LHS and RHS. @@ -301,10 +344,10 @@ where where S: Node, { - Variable::new(Rc::new(DotNode::new( - Rc::clone(&self.node), - Rc::clone(&other.node), - ))) + Variable::new( + Rc::new(DotNode::new(Rc::clone(&self.node), Rc::clone(&other.node))), + merge_parameters(&self.parameters, &other.parameters), + ) } } @@ -320,10 +363,13 @@ impl Variable { /// Row-wise indexing of this parameter node. Primiarily used /// to implement embedding layers. pub fn index(&self, index: &Variable) -> Variable> { - Variable::new(Rc::new(IndexNode::new( - Rc::clone(&self.node), - Rc::clone(&index.node), - ))) + Variable::new( + Rc::new(IndexNode::new( + Rc::clone(&self.node), + Rc::clone(&index.node), + )), + merge_parameters(&self.parameters, &index.parameters), + ) } } @@ -370,7 +416,10 @@ where { type Output = Variable>; fn add(self, other: Variable) -> Self::Output { - Variable::new(Rc::new(AddNode::new(self.node, other.node))) + Variable::new( + Rc::new(AddNode::new(self.node, other.node)), + merge_parameters(&self.parameters, &other.parameters), + ) } } @@ -381,7 +430,10 @@ where { type Output = Variable>; fn sub(self, other: Variable) -> Self::Output { - Variable::new(Rc::new(SubNode::new(self.node, other.node))) + Variable::new( + Rc::new(SubNode::new(self.node, other.node)), + merge_parameters(&self.parameters, &other.parameters), + ) } } @@ -392,7 +444,10 @@ where { type Output = Variable>; fn mul(self, other: Variable) -> Self::Output { - Variable::new(Rc::new(MulNode::new(self.node, other.node))) + Variable::new( + Rc::new(MulNode::new(self.node, other.node)), + merge_parameters(&self.parameters, &other.parameters), + ) } } @@ -403,7 +458,10 @@ where { type Output = Variable>; fn div(self, other: Variable) -> Self::Output { - Variable::new(Rc::new(DivNode::new(self.node, other.node))) + Variable::new( + Rc::new(DivNode::new(self.node, other.node)), + merge_parameters(&self.parameters, &other.parameters), + ) } } @@ -413,7 +471,7 @@ where { type Output = Variable>; fn neg(self) -> Self::Output { - Variable::new(Rc::new(NegNode::new(self.node))) + Variable::new(Rc::new(NegNode::new(self.node)), self.parameters.clone()) } } @@ -544,6 +602,17 @@ mod tests { Arr::zeros((rows, cols)).map(|_| rand::random::()) } + #[test] + fn parameter_deduplication() { + let x = ParameterNode::new(random_matrix(1, 1)); + let y = ParameterNode::new(random_matrix(1, 1)); + + let z = x + y; + let z = z.clone() + z.clone(); + + assert_eq!(z.parameters().len(), 2); + } + #[test] fn add_finite_difference() { let mut x = ParameterNode::new(random_matrix(1, 1)); @@ -676,7 +745,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); + let mut optimizer = SGD::new(0.1, loss.parameters()); for _ in 0..num_epochs { let _x = arr2(&[[rand::random::()]]); @@ -719,7 +788,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); + let mut optimizer = SGD::new(0.1, loss.parameters()); for _ in 0..num_epochs { let _x = arr2(&[ @@ -779,7 +848,7 @@ mod tests { let mut loss = (output.clone() - y_hat.clone()).square(); let num_epochs = 100; - let mut optimizer = SGD::new(0.1, vec![u_embedding.clone(), v_embedding.clone()]); + let mut optimizer = SGD::new(0.1, loss.parameters()); let mut loss_val = 0.0; @@ -841,7 +910,7 @@ mod tests { let num_epochs = 100; - let mut optimizer = SGD::new(0.1, vec![u_embedding.clone(), v_embedding.clone()]); + let mut optimizer = SGD::new(0.1, loss.parameters()); let mut loss_val = 0.0; diff --git a/src/nodes.rs b/src/nodes.rs index 60a809f8..90e7807f 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -209,9 +209,12 @@ impl InputNode { /// Create a new input node with a given value. This fixes the shape /// of the node in the graph. pub fn new(value: Arr) -> Variable { - Variable::new(Rc::new(InputNode { - value: RefCell::new(value), - })) + Variable::new( + Rc::new(InputNode { + value: RefCell::new(value), + }), + Vec::new(), + ) } } @@ -330,20 +333,26 @@ impl ParameterNode { pub fn shared(value: Arc) -> Variable { let shape = (value.value.rows(), value.value.cols()); - Variable::new(Rc::new(ParameterNode { + let node = Rc::new(ParameterNode { value: value, gradient: RefCell::new(GradientAccumulator::new(shape)), - })) + }); + let params = vec![node.clone()]; + + Variable::new(node, params) } /// Create a new parameter node. The parameters held by this node /// cannot be shared and optimized in parallel. pub fn new(value: Arr) -> Variable { let shape = (value.rows(), value.cols()); - Variable::new(Rc::new(ParameterNode { + let node = Rc::new(ParameterNode { value: Arc::new(HogwildParameter::new(value)), gradient: RefCell::new(GradientAccumulator::new(shape)), - })) + }); + let params = vec![node.clone()]; + + Variable::new(node, params) } /// Zero the accumulated gradients of this node. pub fn zero_gradient(&self) { @@ -1666,9 +1675,12 @@ pub struct IndexInputNode { impl IndexInputNode { /// Create a new index input node. pub fn new(value: &[usize]) -> Variable { - Variable::new(Rc::new(IndexInputNode { - value: RefCell::new(SmallVec::from(value)), - })) + Variable::new( + Rc::new(IndexInputNode { + value: RefCell::new(SmallVec::from(value)), + }), + Vec::new(), + ) } } From 4cf2410ed77a5a6169cd8b3955b489a4e8b9c589 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 6 Jan 2018 14:51:33 +0000 Subject: [PATCH 021/108] Add stack/concat node. --- src/lib.rs | 52 ++++++++++++++ src/nodes.rs | 197 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 06a46649..3c3ab24b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -349,6 +349,24 @@ where merge_parameters(&self.parameters, &other.parameters), ) } + + pub fn stack( + &self, + other: &Variable, + axis: ndarray::Axis, + ) -> Variable> + where + S: Node, + { + Variable::new( + Rc::new(ConcatenateNode::new( + Rc::clone(&self.node), + Rc::clone(&other.node), + axis, + )), + merge_parameters(&self.parameters, &other.parameters), + ) + } } impl Variable { @@ -732,6 +750,40 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn rowwise_stack_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut y = ParameterNode::new(random_matrix(10, 5)); + let v = x.clone() + y.clone(); + + let mut z = v.stack(&v, ndarray::Axis(0)).sigmoid(); + + assert_eq!(z.value().rows(), 20); + assert_eq!(z.value().cols(), 5); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] + fn columnwise_stack_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut y = ParameterNode::new(random_matrix(10, 5)); + let v = x.clone() + y.clone(); + + let mut z = v.stack(&v, ndarray::Axis(1)).sigmoid(); + + assert_eq!(z.value().rows(), 10); + assert_eq!(z.value().cols(), 10); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] fn univariate_regression() { let slope = ParameterNode::new(random_matrix(1, 1)); let intercept = ParameterNode::new(random_matrix(1, 1)); diff --git a/src/nodes.rs b/src/nodes.rs index 90e7807f..b407a6b9 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -199,6 +199,203 @@ where } } +fn row_wise_stack(dest: &mut Arr, lhs: &Arr, rhs: &Arr) { + for (mut dest_row, source_row) in dest.genrows_mut() + .into_iter() + .zip(lhs.genrows().into_iter().chain(rhs.genrows())) + { + numerics::slice_assign( + dest_row.as_slice_mut().unwrap(), + source_row.as_slice().unwrap(), + ); + } +} + +fn column_wise_stack(dest: &mut Arr, lhs: &Arr, rhs: &Arr) { + for (mut dest_row, lhs_row, rhs_row) in izip!( + dest.genrows_mut().into_iter(), + lhs.genrows().into_iter(), + rhs.genrows().into_iter() + ) { + let dest_row = dest_row.as_slice_mut().unwrap(); + let lhs_row = lhs_row.as_slice().unwrap(); + let rhs_row = rhs_row.as_slice().unwrap(); + + let (left, right) = dest_row.split_at_mut(lhs_row.len()); + numerics::slice_assign(left, lhs_row); + numerics::slice_assign(right, rhs_row); + } +} + +fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: BackwardAction) { + for (grad_row, mut lhs_row, mut rhs_row) in izip!( + gradient.genrows().into_iter(), + lhs.genrows_mut().into_iter(), + rhs.genrows_mut().into_iter() + ) { + let grad_row = grad_row.as_slice().unwrap(); + let lhs_row = lhs_row.as_slice_mut().unwrap(); + let rhs_row = rhs_row.as_slice_mut().unwrap(); + + let (left, right) = grad_row.split_at(lhs_row.len()); + + match op { + BackwardAction::Increment => { + for (x, y) in lhs_row.iter_mut().zip(left.iter()) { + *x += y; + } + for (x, y) in rhs_row.iter_mut().zip(right.iter()) { + *x += y; + } + } + BackwardAction::Set => { + for (x, &y) in lhs_row.iter_mut().zip(left.iter()) { + *x = y; + } + for (x, &y) in rhs_row.iter_mut().zip(right.iter()) { + *x = y; + } + } + } + } +} + +fn row_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: BackwardAction) { + for (grad_row, mut dest_row) in gradient + .genrows() + .into_iter() + .zip(lhs.genrows_mut().into_iter().chain(rhs.genrows_mut())) + { + let grad_row = grad_row.as_slice().unwrap(); + let dest_row = dest_row.as_slice_mut().unwrap(); + + match op { + BackwardAction::Increment => for (x, y) in dest_row.iter_mut().zip(grad_row.iter()) { + *x += y; + }, + BackwardAction::Set => for (x, &y) in dest_row.iter_mut().zip(grad_row.iter()) { + *x = y; + }, + } + } +} + +#[derive(Debug)] +pub struct ConcatenateNode { + axis: ndarray::Axis, + value: RefCell, + lhs_gradient: RefCell, + rhs_gradient: RefCell, + lhs: Rc, + rhs: Rc, + needs_gradient: bool, + counter: PassCounter, +} + +impl ConcatenateNode +where + LHS: Node, + RHS: Node, +{ + pub fn new(lhs: Rc, rhs: Rc, axis: ndarray::Axis) -> Self { + let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); + + let value = ndarray::stack( + axis, + &[lhs.value().deref().view(), rhs.value().deref().view()], + ).expect("Unable to concatenate arrays."); + + let lhs_gradient = lhs.value().deref() * 0.0; + let rhs_gradient = rhs.value().deref() * 0.0; + + ConcatenateNode { + axis: axis, + value: RefCell::new(value), + lhs_gradient: RefCell::new(lhs_gradient), + rhs_gradient: RefCell::new(rhs_gradient), + lhs: lhs, + rhs: rhs, + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } +} + +impl Node for ConcatenateNode +where + LHS: Node, + RHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + + self.lhs.forward(); + self.rhs.forward(); + + let lhs_value = self.lhs.value(); + let rhs_value = self.rhs.value(); + + let mut self_value = self.value.borrow_mut(); + + match self.axis { + // Vertically + ndarray::Axis(0) => { + row_wise_stack(self_value.deref_mut(), lhs_value.deref(), rhs_value.deref()) + } + // Horizontally + ndarray::Axis(1) => { + column_wise_stack(self_value.deref_mut(), lhs_value.deref(), rhs_value.deref()) + } + // Not allowed + _ => panic!("Stacking tensors not allowed."), + } + } + fn backward(&self, gradient: &Ref) { + { + let mut lhs_grad = self.lhs_gradient.borrow_mut(); + let mut rhs_grad = self.rhs_gradient.borrow_mut(); + + match self.axis { + ndarray::Axis(0) => row_wise_stack_gradient( + gradient, + lhs_grad.deref_mut(), + rhs_grad.deref_mut(), + self.counter.backward(), + ), + ndarray::Axis(1) => column_wise_stack_gradient( + gradient, + lhs_grad.deref_mut(), + rhs_grad.deref_mut(), + self.counter.backward(), + ), + _ => panic!("Stacking tensors not allowed."), + } + } + + if self.counter.recurse_backward() { + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + fn zero_gradient(&self) { + self.counter.clear(); + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + } +} + /// Input node for the graph. #[derive(Debug)] pub struct InputNode { From e710cb526e8d992486e27e4f36512372b70fbde9 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 6 Jan 2018 18:03:47 +0000 Subject: [PATCH 022/108] Add tanh node. --- src/lib.rs | 20 +++++++++++- src/nodes.rs | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3c3ab24b..6cfae2d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -247,7 +247,7 @@ where unique_params .iter() - .map(|x| Variable::new(x.clone(), Vec::new())) + .map(|x| Variable::new(Rc::clone(x), Vec::new())) .collect() } } @@ -293,6 +293,14 @@ where ) } + /// Take the tanh of this variable. + pub fn tanh(&self) -> Variable> { + Variable::new( + Rc::new(TanhNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) + } + /// Transpose this variable. pub fn t(&self) -> Variable> { Variable::new( @@ -350,6 +358,8 @@ where ) } + /// Stack/concatenate LHS and RHS, either row-wise (`ndarray::Axis(0)`) or + /// column-wise (`ndarray::Axis(1)`). pub fn stack( &self, other: &Variable, @@ -693,6 +703,14 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn tanh_finite_difference() { + let mut x = ParameterNode::new(random_matrix(2, 2)); + let mut z = (x.clone() + x.clone()).tanh(); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] fn sum_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut z = (x.clone() + x.clone()).scalar_sum(); diff --git a/src/nodes.rs b/src/nodes.rs index b407a6b9..af217e1b 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -67,7 +67,7 @@ impl PassCounter { } } -/// Generalisation over borrowed RefCell values +/// Generalisation over borrowed `RefCell` values /// and simple references. #[derive(Debug)] pub enum Bor<'value, T: 'value> { @@ -534,7 +534,7 @@ impl ParameterNode { value: value, gradient: RefCell::new(GradientAccumulator::new(shape)), }); - let params = vec![node.clone()]; + let params = vec![Rc::clone(&node)]; Variable::new(node, params) } @@ -547,7 +547,7 @@ impl ParameterNode { value: Arc::new(HogwildParameter::new(value)), gradient: RefCell::new(GradientAccumulator::new(shape)), }); - let params = vec![node.clone()]; + let params = vec![Rc::clone(&node)]; Variable::new(node, params) } @@ -1341,6 +1341,91 @@ where } } +#[derive(Debug)] +pub struct TanhNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, + counter: PassCounter, +} + +impl TanhNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = operand.value().map(|x| x.tanh()); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + TanhNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } +} + +impl Node for TanhNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + + dest.assign(self.operand.value().deref()); + dest.map_inplace(|x| *x = x.tanh()); + } + + fn backward(&self, gradient: &Ref) { + match self.counter.backward() { + BackwardAction::Set => for (dest, value, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.value().iter(), + gradient.iter() + ) { + *dest = grad_val * (1.0 - value.powi(2)); + }, + BackwardAction::Increment => for (dest, value, grad_val) in izip!( + self.operand_gradient.borrow_mut().iter_mut(), + self.value().iter(), + gradient.iter() + ) { + *dest += grad_val * (1.0 - value.powi(2)); + }, + } + + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + + fn zero_gradient(&self) { + self.counter.clear(); + self.operand.zero_gradient(); + } +} + #[derive(Debug)] pub struct SigmoidNode { value: RefCell, From d533bc3386464990d1bff3b6fcd054d914bc2814 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 6 Jan 2018 19:10:55 +0000 Subject: [PATCH 023/108] Start adding LSTM cells. --- src/layers/mod.rs | 1 + src/layers/recurrent.rs | 169 ++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + 3 files changed, 172 insertions(+) create mode 100644 src/layers/mod.rs create mode 100644 src/layers/recurrent.rs diff --git a/src/layers/mod.rs b/src/layers/mod.rs new file mode 100644 index 00000000..9998b574 --- /dev/null +++ b/src/layers/mod.rs @@ -0,0 +1 @@ +pub mod recurrent; diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs new file mode 100644 index 00000000..606ae1ed --- /dev/null +++ b/src/layers/recurrent.rs @@ -0,0 +1,169 @@ +use std::sync::Arc; + +use rand; +use ndarray; + +use nodes; +use nodes::{HogwildParameter, InputNode, Node, ParameterNode}; + +use {Arr, Variable}; + +fn random_matrix(rows: usize, cols: usize) -> Arr { + Arr::zeros((rows, cols)).map(|_| rand::random::()) +} + +pub struct LSTMParameters { + input_dim: usize, + hidden_dim: usize, + + forget_weights: Arc, + forget_biases: Arc, + + update_gate_weights: Arc, + update_gate_biases: Arc, + + update_value_weights: Arc, + update_value_biases: Arc, + + output_gate_weights: Arc, + output_gate_biases: Arc, +} + +impl LSTMParameters { + pub fn new(input_dim: usize, hidden_dim: usize) -> Self { + Self { + input_dim: input_dim, + hidden_dim: hidden_dim, + + forget_weights: Arc::new(HogwildParameter::new(random_matrix( + input_dim + hidden_dim, + hidden_dim, + ))), + forget_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + + update_gate_weights: Arc::new(HogwildParameter::new(random_matrix( + input_dim + hidden_dim, + hidden_dim, + ))), + update_gate_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + + update_value_weights: Arc::new(HogwildParameter::new(random_matrix( + input_dim + hidden_dim, + hidden_dim, + ))), + update_value_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + + output_gate_weights: Arc::new(HogwildParameter::new(random_matrix( + input_dim + hidden_dim, + hidden_dim, + ))), + output_gate_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + } + } + + pub fn build(&self) -> LSTMCell { + LSTMCell { + _input_dim: self.input_dim, + _hidden_dim: self.hidden_dim, + + forget_weights: ParameterNode::shared(self.forget_weights.clone()), + forget_biases: ParameterNode::shared(self.forget_biases.clone()), + + update_gate_weights: ParameterNode::shared(self.update_gate_weights.clone()), + update_gate_biases: ParameterNode::shared(self.update_gate_biases.clone()), + + update_value_weights: ParameterNode::shared(self.update_value_weights.clone()), + update_value_biases: ParameterNode::shared(self.update_value_biases.clone()), + + output_gate_weights: ParameterNode::shared(self.output_gate_weights.clone()), + output_gate_biases: ParameterNode::shared(self.output_gate_biases.clone()), + } + } +} + +pub struct LSTMCell { + _input_dim: usize, + _hidden_dim: usize, + + forget_weights: Variable, + forget_biases: Variable, + + update_gate_weights: Variable, + update_gate_biases: Variable, + + update_value_weights: Variable, + update_value_biases: Variable, + + output_gate_weights: Variable, + output_gate_biases: Variable, +} + +impl LSTMCell { + pub fn forward( + &self, + cell: Variable, + hidden: Variable, + input: Variable, + ) -> ( + Variable>, + Variable>, + ) + where + C: Node, + H: Node, + I: Node, + { + let stacked_input = hidden.stack(&input, ndarray::Axis(1)); + + // Forget part of the cell state + let forget_gate = + (stacked_input.dot(&self.forget_weights) + self.forget_biases.clone()).sigmoid(); + let cell = forget_gate * cell; + + // Update the cell state with new input + let update_gate = (stacked_input.dot(&self.update_gate_weights) + + self.update_gate_biases.clone()) + .sigmoid(); + let update_value = (stacked_input.dot(&self.update_value_weights) + + self.update_value_biases.clone()) + .tanh(); + let update = update_gate * update_value; + let cell = cell + update; + + // Emit a hidden state + let output_value = cell.tanh(); + let output_gate = (stacked_input.dot(&self.output_gate_weights) + + self.output_gate_biases.clone()) + .sigmoid(); + let hidden = output_gate * output_value; + + (cell, hidden) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_basic_lstm() { + let input_dim = 10; + let hidden_dim = 5; + + let lstm_params = LSTMParameters::new(input_dim, hidden_dim); + let lstm = lstm_params.build(); + + let state = InputNode::new(Arr::zeros((1, hidden_dim))); + let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); + let input = InputNode::new(random_matrix(1, input_dim)); + + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + + hidden.zero_gradient(); + hidden.forward(); + } + +} diff --git a/src/lib.rs b/src/lib.rs index 6cfae2d8..86e4dda7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(conservative_impl_trait)] #![feature(test)] //! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! @@ -165,6 +166,7 @@ use std::clone::Clone; mod nodes; mod numerics; +pub mod layers; use nodes::*; From c7599ccb2fa75aa05f6971c4d1da63659689d5cd Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 6 Jan 2018 20:39:20 +0000 Subject: [PATCH 024/108] Initial LSTM cell tests. --- src/layers/recurrent.rs | 101 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index 606ae1ed..431a1c6a 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -1,6 +1,8 @@ use std::sync::Arc; +use std::ops::Deref; use rand; +use rand::distributions::{IndependentSample, Normal}; use ndarray; use nodes; @@ -12,6 +14,11 @@ fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) } +fn xavier_normal(rows: usize, cols: usize) -> Arr { + let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); + Arr::zeros((rows, cols)).map(|_| normal.ind_sample(&mut rand::thread_rng()) as f32) +} + pub struct LSTMParameters { input_dim: usize, hidden_dim: usize, @@ -35,29 +42,29 @@ impl LSTMParameters { input_dim: input_dim, hidden_dim: hidden_dim, - forget_weights: Arc::new(HogwildParameter::new(random_matrix( + forget_weights: Arc::new(HogwildParameter::new(xavier_normal( input_dim + hidden_dim, hidden_dim, ))), - forget_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + forget_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), - update_gate_weights: Arc::new(HogwildParameter::new(random_matrix( + update_gate_weights: Arc::new(HogwildParameter::new(xavier_normal( input_dim + hidden_dim, hidden_dim, ))), - update_gate_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + update_gate_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), - update_value_weights: Arc::new(HogwildParameter::new(random_matrix( + update_value_weights: Arc::new(HogwildParameter::new(xavier_normal( input_dim + hidden_dim, hidden_dim, ))), - update_value_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + update_value_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), - output_gate_weights: Arc::new(HogwildParameter::new(random_matrix( + output_gate_weights: Arc::new(HogwildParameter::new(xavier_normal( input_dim + hidden_dim, hidden_dim, ))), - output_gate_biases: Arc::new(HogwildParameter::new(random_matrix(1, hidden_dim))), + output_gate_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), } } @@ -145,6 +152,8 @@ impl LSTMCell { mod tests { use super::*; + use DataInput; + use SGD; #[test] fn test_basic_lstm() { @@ -166,4 +175,80 @@ mod tests { hidden.forward(); } + fn predicted_label(softmax_output: &Arr) -> usize { + softmax_output + .iter() + .enumerate() + .max_by(|&(_, x), &(_, y)| x.partial_cmp(y).unwrap()) + .unwrap() + .0 + } + + #[test] + fn test_sequential_numbers() { + let max_number = 100; + let num_epochs = 1000; + + let input_dim = 16; + let hidden_dim = 32; + + let lstm_params = LSTMParameters::new(input_dim, hidden_dim); + let lstm = lstm_params.build(); + + let final_layer = ParameterNode::new(random_matrix(hidden_dim, max_number)); + let embeddings = ParameterNode::new(random_matrix(max_number, input_dim)); + let index = nodes::IndexInputNode::new(&vec![0]); + let mut labels = Arr::zeros((1, max_number)); + let y = nodes::InputNode::new(labels.clone()); + + let state = InputNode::new(Arr::zeros((1, hidden_dim))); + let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); + let input = embeddings.index(&index); + + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + //let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + + let prediction = hidden.dot(&final_layer).softmax(); + let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); + + let mut optimizer = SGD::new(0.5, loss.parameters()); + + for _ in 0..num_epochs { + let mut loss_val = 0.0; + let mut correct = 0; + let mut total = 0; + + for number in 0..max_number { + index.set_value(number); + + labels *= 0.0; + labels[(0, number)] = 1.0; + + y.set_value(&labels); + + loss.forward(); + loss.backward(1.0); + + loss_val += loss.value().scalar_sum(); + + optimizer.step(); + loss.zero_gradient(); + + if number == predicted_label(prediction.value().deref()) { + correct += 1; + } + + total += 1; + } + + println!( + "Loss {}, accuracy {}", + loss_val, + correct as f32 / total as f32 + ); + } + } + } From 414330cd26ee21496f69e45cd2e68b13126cebb4 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 6 Jan 2018 23:54:51 +0000 Subject: [PATCH 025/108] Add pi prediction test. --- src/layers/pi.txt | 1 + src/layers/recurrent.rs | 90 ++++++++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 src/layers/pi.txt diff --git a/src/layers/pi.txt b/src/layers/pi.txt new file mode 100644 index 00000000..af23acee --- /dev/null +++ b/src/layers/pi.txt @@ -0,0 +1 @@ +14159265358979323846264338327950288419716939937510582097494459230781640628620899862803482534211706798214808651328230664709384460955058223172535940812848111745028410270193852110555964462294895493038196442881097566593344612847564823378678316527120190914564856692346034861045432664821339360726024914127372458700660631558817488152092096282925409171536436789259036001133053054882046652138414695194151160943305727036575959195309218611738193261179310511854807446237996274956735188575272489122793818301194912983367336244065664308602139494639522473719070217986094370277053921717629317675238467481846766940513200056812714526356082778577134275778960917363717872146844090122495343014654958537105079227968925892354201995611212902196086403441815981362977477130996051870721134999999837297804995105973173281609631859502445945534690830264252230825334468503526193118817101000313783875288658753320838142061717766914730359825349042875546873115956286388235378759375195778185778053217122680661300192787661119590921642019893809525720106548586327886593615338182796823030195203530185296899577362259941389124972177528347913151557485724245415069595082953311686172785588907509838175463746493931925506040092770167113900984882401285836160356370766010471018194295559619894676783744944825537977472684710404753464620804668425906949129331367702898915210475216205696602405803815019351125338243003558764024749647326391419927260426992279678235478163600934172164121992458631503028618297455570674983850549458858692699569092721079750930295532116534498720275596023648066549911988183479775356636980742654252786255181841757467289097777279380008164706001614524919217321721477235014144197356854816136115735255213347574184946843852332390739414333454776241686251898356948556209921922218427255025425688767179049460165346680498862723279178608578438382796797668145410095388378636095068006422512520511739298489608412848862694560424196528502221066118630674427862203919494504712371378696095636437191728746776465757396241389086583264599581339047802759009946576407895126946839835259570982582262052248940772671947826848260147699090264013639443745530506820349625245174939965143142980919065925093722169646151570985838741059788595977297549893016175392846813826868386894277415599185592524595395943104997252468084598727364469584865383673622262609912460805124388439045124413654976278079771569143599770012961608944169486855584840635342207222582848864815845602850601684273945226746767889525213852254995466672782398645659611635488623057745649803559363456817432411251507606947945109659609402522887971089314566913686722874894056010150330861792868092087476091782493858900971490967598526136554978189312978482168299894872265880485756401427047755513237964145152374623436454285844479526586782105114135473573952311342716610213596953623144295248493718711014576540359027993440374200731057853906219838744780847848968332144571386875194350643021845319104848100537061468067491927819119793995206141966342875444064374512371819217999839101591956181467514269123974894090718649423196156794520809514655022523160388193014209376213785595663893778708303906979207734672218256259966150142150306803844773454920260541466592520149744285073251866600213243408819071048633173464965145390579626856100550810665879699816357473638405257145910289706414011097120628043903975951567715770042033786993600723055876317635942187312514712053292819182618612586732157919841484882916447060957527069572209175671167229109816909152801735067127485832228718352093539657251210835791513698820914442100675103346711031412671113699086585163983150197016515116851714376576183515565088490998985998238734552833163550764791853589322618548963213293308985706420467525907091548141654985946163718027098199430992448895757128289059232332609729971208443357326548938239119325974636673058360414281388303203824903758985243744170291327656180937734440307074692112019130203303801976211011004492932151608424448596376698389522868478312355265821314495768572624334418930396864262434107732269780280731891544110104468232527162010526522721116603966655730925471105578537634668206531098965269186205647693125705863566201855810072936065987648611791045334885034611365768675324944166803962657978771855608455296541266540853061434443185867697514566140680070023787765913440171274947042056223053899456131407112700040785473326993908145466464588079727082668306343285878569830523580893306575740679545716377525420211495576158140025012622859413021647155097925923099079654737612551765675135751782966645477917450112996148903046399471329621073404375189573596145890193897131117904297828564750320319869151402870808599048010941214722131794764777262241425485454033215718530614228813758504306332175182979866223717215916077166925474873898665494945011465406284336639379003976926567214638530673609657120918076383271664162748888007869256029022847210403172118608204190004229661711963779213375751149595015660496318629472654736425230817703675159067350235072835405670403867435136222247715891504953098444893330963408780769325993978054193414473774418426312986080998886874132604721569516239658645730216315981931951673538129741677294786724229246543668009806769282382806899640048243540370141631496589794092432378969070697794223625082216889573837986230015937764716512289357860158816175578297352334460428151262720373431465319777741603199066554187639792933441952154134189948544473456738316249934191318148092777710386387734317720754565453220777092120190516609628049092636019759882816133231666365286193266863360627356763035447762803504507772355471058595487027908143562401451718062464362679456127531813407833033625423278394497538243720583531147711992606381334677687969597030983391307710987040859133746414428227726346594704745878477872019277152807317679077071572134447306057007334924369311383504931631284042512192565179806941135280131470130478164378851852909285452011658393419656213491434159562586586557055269049652098580338507224264829397285847831630577775606888764462482468579260395352773480304802900587607582510474709164396136267604492562742042083208566119062545433721315359584506877246029016187667952406163425225771954291629919306455377991403734043287526288896399587947572917464263574552540790914513571113694109119393251910760208252026187985318877058429725916778131496990090192116971737278476847268608490033770242429165130050051683233643503895170298939223345172201381280696501178440874519601212285993716231301711444846409038906449544400619869075485160263275052983491874078668088183385102283345085048608250393021332197155184306354550076682829493041377655279397517546139539846833936383047461199665385815384205685338621867252334028308711232827892125077126294632295639898989358211674562701021835646220134967151881909730381198004973407239610368540664319395097901906996395524530054505806855019567302292191393391856803449039820595510022635353619204199474553859381023439554495977837790237421617271117236434354394782218185286240851400666044332588856986705431547069657474585503323233421073015459405165537906866273337995851156257843229882737231989875714159578111963583300594087306812160287649628674460477464915995054973742562690104903778198683593814657412680492564879855614537234786733039046883834363465537949864192705638729317487233208376011230299113679386270894387993620162951541337142489283072201269014754668476535761647737946752004907571555278196536213239264061601363581559074220202031872776052772190055614842555187925303435139844253223415762336106425063904975008656271095359194658975141310348227693062474353632569160781547818115284366795706110861533150445212747392454494542368288606134084148637767009612071512491404302725386076482363414334623518975766452164137679690314950191085759844239198629164219399490723623464684411739403265918404437805133389452574239950829659122850855582157250310712570126683024029295252201187267675622041542051618416348475651699981161410100299607838690929160302884002691041407928862150784245167090870006992821206604183718065355672525325675328612910424877618258297651579598470356222629348600341587229805349896502262917487882027342092222453398562647669149055628425039127577102840279980663658254889264880254566101729670266407655904290994568150652653053718294127033693137851786090407086671149655834343476933857817113864558736781230145876871266034891390956200993936103102916161528813843790990423174733639480457593149314052976347574811935670911013775172100803155902485309066920376719220332290943346768514221447737939375170344366199104033751117354719185504644902636551281622882446257591633303910722538374218214088350865739177150968288747826569959957449066175834413752239709683408005355984917541738188399944697486762655165827658483588453142775687900290951702835297163445621296404352311760066510124120065975585127617858382920419748442360800719304576189323492292796501987518721272675079812554709589045563579212210333466974992356302549478024901141952123828153091140790738602515227429958180724716259166854513331239480494707911915326734302824418604142636395480004480026704962482017928964766975831832713142517029692348896276684403232609275249603579964692565049368183609003238092934595889706953653494060340216654437558900456328822505452556405644824651518754711962184439658253375438856909411303150952617937800297412076651479394259029896959469955657612186561967337862362561252163208628692221032748892186543648022967807057656151446320469279068212073883778142335628236089632080682224680122482611771858963814091839036736722208883215137556003727983940041529700287830766709444745601345564172543709069793961225714298946715435784687886144458123145935719849225284716050492212424701412147805734551050080190869960330276347870810817545011930714122339086639383395294257869050764310063835198343893415961318543475464955697810382930971646514384070070736041123735998434522516105070270562352660127648483084076118301305279320542746286540360367453286510570658748822569815793678976697422057505968344086973502014102067235850200724522563265134105592401902742162484391403599895353945909440704691209140938700126456001623742880210927645793106579229552498872758461012648369998922569596881592056001016552563756785667227966198857827948488558343975187445455129656344348039664205579829368043522027709842942325330225763418070394769941597915945300697521482933665556615678736400536665641654732170439035213295435291694145990416087532018683793702348886894791510716378529023452924407736594956305100742108714261349745956151384987137570471017879573104229690666702144986374645952808243694457897723300487647652413390759204340196340391147320233807150952220106825634274716460243354400515212669324934196739770415956837535551667302739007497297363549645332888698440611964961627734495182736955882207573551766515898551909866653935494810688732068599075407923424023009259007017319603622547564789406475483466477604114632339056513433068449539790709030234604614709616968868850140834704054607429586991382966824681857103188790652870366508324319744047718556789348230894310682870272280973624809399627060747264553992539944280811373694338872940630792615959954626246297070625948455690347119729964090894180595343932512362355081349490043642785271383159125689892951964272875739469142725343669415323610045373048819855170659412173524625895487301676002988659257866285612496655235338294287854253404830833070165372285635591525347844598183134112900199920598135220511733658564078264849427644113763938669248031183644536985891754426473998822846218449008777697763127957226726555625962825427653183001340709223343657791601280931794017185985999338492354956400570995585611349802524990669842330173503580440811685526531170995708994273287092584878944364600504108922669178352587078595129834417295351953788553457374260859029081765155780390594640873506123226112009373108048548526357228257682034160504846627750450031262008007998049254853469414697751649327095049346393824322271885159740547021482897111777923761225788734771881968254629812686858170507402725502633290449762778944236216741191862694396506715157795867564823993917604260176338704549901761436412046921823707648878341968968611815581587360629386038101712158552726683008238340465647588040513808016336388742163714064354955618689641122821407533026551004241048967835285882902436709048871181909094945331442182876618103100735477054981596807720094746961343609286148494178501718077930681085469000944589952794243981392135055864221964834915126390128038320010977386806628779239718014613432445726400973742570073592100315415089367930081699805365202760072774967458400283624053460372634165542590276018348403068113818551059797056640075094260878857357960373245141467867036880988060971642584975951380693094494015154222219432913021739125383559150310033303251117491569691745027149433151558854039221640972291011290355218157628232831823425483261119128009282525619020526301639114772473314857391077758744253876117465786711694147764214411112635835538713610110232679877564102468240322648346417663698066378576813492045302240819727856471983963087815432211669122464159117767322532643356861461865452226812688726844596844241610785401676814208088502800541436131462308210259417375623899420757136275167457318918945628352570441335437585753426986994725470316566139919996826282472706413362221789239031760854289437339356188916512504244040089527198378738648058472689546243882343751788520143956005710481194988423906061369573423155907967034614914344788636041031823507365027785908975782727313050488939890099239135033732508559826558670892426124294736701939077271307068691709264625484232407485503660801360466895118400936686095463250021458529309500009071510582362672932645373821049387249966993394246855164832611341461106802674466373343753407642940266829738652209357016263846485285149036293201991996882851718395366913452224447080459239660281715655156566611135982311225062890585491450971575539002439315351909021071194573002438801766150352708626025378817975194780610137150044899172100222013350131060163915415895780371177927752259787428919179155224171895853616805947412341933984202187456492564434623925319531351033114763949119950728584306583619353693296992898379149419394060857248639688369032655643642166442576079147108699843157337496488352927693282207629472823815374099615455987982598910937171262182830258481123890119682214294576675807186538065064870261338928229949725745303328389638184394477077940228435988341003583854238973542439564755568409522484455413923941000162076936368467764130178196593799715574685419463348937484391297423914336593604100352343777065888677811394986164787471407932638587386247328896456435987746676384794665040741118256583788784548581489629612739984134427260860618724554523606431537101127468097787044640947582803487697589483282412392929605829486191966709189580898332012103184303401284951162035342801441276172858302435598300320420245120728725355811958401491809692533950757784000674655260314461670508276827722235341911026341631571474061238504258459884199076112872580591139356896014316682831763235673254170734208173322304629879928049085140947903688786878949305469557030726190095020764334933591060245450864536289354568629585313153371838682656178622736371697577418302398600659148161640494496501173213138957470620884748023653710311508984279927544268532779743113951435741722197597993596852522857452637962896126915723579866205734083757668738842664059909935050008133754324546359675048442352848747014435454195762584735642161981340734685411176688311865448937769795665172796623267148103386439137518659467300244345005449953997423723287124948347060440634716063258306498297955101095418362350303094530973358344628394763047756450150085075789495489313939448992161255255977014368589435858775263796255970816776438001254365023714127834679261019955852247172201777237004178084194239487254068015560359983905489857235467456423905858502167190313952629445543913166313453089390620467843877850542393905247313620129476918749751910114723152893267725339181466073000890277689631148109022097245207591672970078505807171863810549679731001678708506942070922329080703832634534520380278609905569001341371823683709919495164896007550493412678764367463849020639640197666855923356546391383631857456981471962108410809618846054560390384553437291414465134749407848844237721751543342603066988317683310011331086904219390310801437843341513709243530136776310849135161564226984750743032971674696406665315270353254671126675224605511995818319637637076179919192035795820075956053023462677579439363074630569010801149427141009391369138107258137813578940055995001835425118417213605572752210352680373572652792241737360575112788721819084490061780138897107708229310027976659358387589093956881485602632243937265624727760378908144588378550197028437793624078250527048758164703245812908783952324532378960298416692254896497156069811921865849267704039564812781021799132174163058105545988013004845629976511212415363745150056350701278159267142413421033015661653560247338078430286552572227530499988370153487930080626018096238151613669033411113865385109193673938352293458883225508870645075394739520439680790670868064450969865488016828743437861264538158342807530618454859037982179945996811544197425363443996029025100158882721647450068207041937615845471231834600726293395505482395571372568402322682130124767945226448209102356477527230820810635188991526928891084555711266039650343978962782500161101532351605196559042118449499077899920073294769058685778787209829013529566139788848605097860859570177312981553149516814671769597609942100361835591387778176984587581044662839988060061622984861693533738657877359833616133841338536842119789389001852956919678045544828584837011709672125353387586215823101331038776682721157269495181795897546939926421979155233857662316762754757035469941489290413018638611943919628388705436777432242768091323654494853667680000010652624854730558615989991401707698385483188750142938908995068545307651168033373222651756622075269517914422528081651716677667279303548515420402381746089232839170327542575086765511785939500279338959205766827896776445318404041855401043513483895312013263783692835808271937831265496174599705674507183320650345566440344904536275600112501843356073612227659492783937064784264567633881880756561216896050416113903906396016202215368494109260538768871483798955999911209916464644119185682770045742434340216722764455893301277815868695250694993646101756850601671453543158148010545886056455013320375864548584032402987170934809105562116715468484778039447569798042631809917564228098739987669732376957370158080682290459921236616890259627304306793165311494017647376938735140933618332161428021497633991898354848756252987524238730775595559554651963944018218409984124898262367377146722606163364329640633572810707887581640438148501884114318859882769449011932129682715888413386943468285900666408063140777577257056307294004929403024204984165654797367054855804458657202276378404668233798528271057843197535417950113472736257740802134768260450228515797957976474670228409995616015691089038458245026792659420555039587922981852648007068376504183656209455543461351341525700659748819163413595567196496540321872716026485930490397874895890661272507948282769389535217536218507962977851461884327192232238101587444505286652380225328438913752738458923844225354726530981715784478342158223270206902872323300538621634798850946954720047952311201504329322662827276321779088400878614802214753765781058197022263097174950721272484794781695729614236585957820908307332335603484653187302930266596450137183754288975579714499246540386817992138934692447419850973346267933210726868707680626399193619650440995421676278409146698569257150743157407938053239252394775574415918458215625181921552337096074833292349210345146264374498055961033079941453477845746999921285999993996122816152193148887693880222810830019860165494165426169685867883726095877456761825072759929508931805218729246108676399589161458550583972742098090978172932393010676638682404011130402470073508578287246271349463685318154696904669686939254725194139929146524238577625500474852954768147954670070503479995888676950161249722820403039954632788306959762493615101024365553522306906129493885990157346610237122354789112925476961760050479749280607212680392269110277722610254414922157650450812067717357120271802429681062037765788371669091094180744878140490755178203856539099104775941413215432844062503018027571696508209642734841469572639788425600845312140659358090412711359200419759851362547961606322887361813673732445060792441176399759746193835845749159880976674470930065463424234606342374746660804317012600520559284936959414340814685298150539471789004518357551541252235905906872648786357525419112888773717663748602766063496035367947026923229718683277173932361920077745221262475186983349515101986426988784717193966497690708252174233656627259284406204302141137199227852699846988477023238238400556555178890876613601304770984386116870523105531491625172837327286760072481729876375698163354150746088386636406934704372066886512756882661497307886570156850169186474885416791545965072342877306998537139043002665307839877638503238182155355973235306860430106757608389086270498418885951380910304235957824951439885901131858358406674723702971497850841458530857813391562707603563907639473114554958322669457024941398316343323789759556808568362972538679132750555425244919435891284050452269538121791319145135009938463117740179715122837854601160359554028644059024964669307077690554810288502080858008781157738171917417760173307385547580060560143377432990127286772530431825197579167929699650414607066457125888346979796429316229655201687973000356463045793088403274807718115553309098870255052076804630346086581653948769519600440848206596737947316808641564565053004988161649057883115434548505266006982309315777650037807046612647060214575057932709620478256152471459189652236083966456241051955105223572397395128818164059785914279148165426328920042816091369377737222999833270820829699557377273756676155271139225880552018988762011416800546873655806334716037342917039079863965229613128017826797172898229360702880690877686605932527463784053976918480820410219447197138692560841624511239806201131845412447820501107987607171556831540788654390412108730324020106853419472304766667217498698685470767812051247367924791931508564447753798537997322344561227858432968466475133365736923872014647236794278700425032555899268843495928761240075587569464137056251400117971331662071537154360068764773186755871487839890810742953094106059694431584775397009439883949144323536685392099468796450665339857388878661476294434140104988899316005120767810358861166020296119363968213496075011164983278563531614516845769568710900299976984126326650234771672865737857908574664607722834154031144152941880478254387617707904300015669867767957609099669360755949651527363498118964130433116627747123388174060373174397054067031096767657486953587896700319258662594105105335843846560233917967492678447637084749783336555790073841914731988627135259546251816043422537299628632674968240580602964211463864368642247248872834341704415734824818333016405669596688667695634914163284264149745333499994800026699875888159350735781519588990053951208535103572613736403436753471410483601754648830040784641674521673719048310967671134434948192626811107399482506073949507350316901973185211955263563258433909982249862406703107683184466072912487475403161796994113973877658998685541703188477886759290260700432126661791922352093822787888098863359911608192353555704646349113208591897961327913197564909760001399623444553501434642686046449586247690943470482932941404111465409239883444351591332010773944111840741076849810663472410482393582740194493566516108846312567852977697346843030614624180358529331597345830384554103370109167677637427621021370135485445092630719011473184857492331816720721372793556795284439254815609137281284063330393735624200160456645574145881660521666087387480472433912129558777639069690370788285277538940524607584962315743691711317613478388271941686066257210368513215664780014767523103935786068961112599602818393095487090590738613519145918195102973278755710497290114871718971800469616977700179139196137914171627070189584692143436967629274591099400600849835684252019155937037010110497473394938778859894174330317853487076032219829705797511914405109942358830345463534923498268836240433272674155403016195056806541809394099820206099941402168909007082133072308966211977553066591881411915778362729274615618571037217247100952142369648308641025928874579993223749551912219519034244523075351338068568073544649951272031744871954039761073080602699062580760202927314552520780799141842906388443734996814582733720726639176702011830046481900024130835088465841521489912761065137415394356572113903285749187690944137020905170314877734616528798482353382972601361109845148418238081205409961252745808810994869722161285248974255555160763716750548961730168096138038119143611439921063800508321409876045993093248510251682944672606661381517457125597549535802399831469822036133808284993567055755247129027453977621404931820146580080215665360677655087838043041343105918046068008345911366408348874080057412725867047922583191274157390809143831384564241509408491339180968402511639919368532255573389669537490266209232613188558915808324555719484538756287861288590041060060737465014026278240273469625282171749415823317492396835301361786536737606421667781377399510065895288774276626368418306801908046098498094697636673356622829151323527888061577682781595886691802389403330764419124034120223163685778603572769415417788264352381319050280870185750470463129333537572853866058889045831114507739429352019943219711716422350056440429798920815943071670198574692738486538334361457946341759225738985880016980147574205429958012429581054565108310462972829375841611625325625165724980784920998979906200359365099347215829651741357984910471116607915874369865412223483418877229294463351786538567319625598520260729476740726167671455736498121056777168934849176607717052771876011999081441130586455779105256843048114402619384023224709392498029335507318458903553971330884461741079591625117148648744686112476054286734367090466784686702740918810142497111496578177242793470702166882956108777944050484375284433751088282647719785400065097040330218625561473321177711744133502816088403517814525419643203095760186946490886815452856213469883554445602495566684366029221951248309106053772019802183101032704178386654471812603971906884623708575180800353270471856594994761242481109992886791589690495639476246084240659309486215076903149870206735338483495508363660178487710608098042692471324100094640143736032656451845667924566695510015022983307984960799498824970617236744936122622296179081431141466094123415935930958540791390872083227335495720807571651718765994498569379562387555161757543809178052802946420044721539628074636021132942559160025707356281263873310600589106524570802447493754318414940148211999627645310680066311838237616396631809314446712986155275982014514102756006892975024630401735148919457636078935285550531733141645705049964438909363084387448478396168405184527328840323452024705685164657164771393237755172947951261323982296023945485797545865174587877133181387529598094121742273003522965080891777050682592488223221549380483714547816472139768209633205083056479204820859204754998573203888763916019952409189389455767687497308569559580106595265030362661597506622250840674288982659075106375635699682115109496697445805472886936310203678232501823237084597901115484720876182124778132663304120762165873129708112307581598212486398072124078688781145016558251361789030708608701989758898074566439551574153631931919810705753366337380382721527988493503974800158905194208797113080512339332219034662499171691509485414018710603546037946433790058909577211808044657439628061867178610171567409676620802957665770512912099079443046328929473061595104309022214393718495606340561893425130572682914657832933405246350289291754708725648426003496296116541382300773133272983050016025672401418515204189070115428857992081219844931569990591820118197335001261877280368124819958770702075324063612593134385955425477819611429351635612234966615226147353996740515849986035529533292457523888101362023476246690558164389678630976273655047243486430712184943734853006063876445662721866617012381277156213797461498613287441177145524447089971445228856629424402301847912054784985745216346964489738920624019435183100882834802492490854030778638751659113028739587870981007727182718745290139728366148421428717055317965430765045343246005363614726181809699769334862640774351999286863238350887566835950972655748154319401955768504372480010204137498318722596773871549583997184449072791419658459300839426370208756353982169620553248032122674989114026785285996734052420310917978999057188219493913207534317079800237365909853755202389116434671855829068537118979526262344924833924963424497146568465912489185566295893299090352392333336474352037077010108438800329075983421701855422838616172104176030116459187805393674474720599850235828918336929223373239994804371084196594731626548257480994825099918330069765693671596893644933488647442135008407006608835972350395323401795825570360169369909886711321097988970705172807558551912699306730992507040702455685077867906947661262980822516331363995211709845280926303759224267425755998928927837047444521893632034894155210445972618838003006776179313813991620580627016510244588692476492468919246121253102757313908404700071435613623169923716948481325542009145304103713545329662063921054798243921251725401323149027405858920632175894943454890684639931375709103463327141531622328055229729795380188016285907357295541627886764982741861642187898857410716490691918511628152854867941736389066538857642291583425006736124538491606741373401735727799563410433268835695078149313780073623541800706191802673285511919426760912210359874692411728374931261633950012395992405084543756985079570462226646190001035004901830341535458428337643781119885563187777925372011667185395418359844383052037628194407615941068207169703022851522505731260930468984234331527321313612165828080752126315477306044237747535059522871744026663891488171730864361113890694202790881431194487994171540421034121908470940802540239329429454938786402305129271190975135360009219711054120966831115163287054230284700731206580326264171161659576132723515666625366727189985341998952368848309993027574199164638414270779887088742292770538912271724863220288984251252872178260305009945108247835729056919885554678860794628053712270424665431921452817607414824038278358297193010178883456741678113989547504483393146896307633966572267270433932167454218245570625247972199786685427989779923395790575818906225254735822052364248507834071101449804787266919901864388229323053823185597328697809222535295910173414073348847610055640182423921926950620831838145469839236646136398910121021770959767049083050818547041946643713122996923588953849301363565761861060622287055994233716310212784574464639897381885667462608794820186474876727272220626764653380998019668836809941590757768526398651462533363124505364026105696055131838131742611844201890888531963569869627950367384243130113317533053298020166888174813429886815855778103432317530647849832106297184251843855344276201282345707169885305183261796411785796088881503296022907056144762209150947390359466469162353968092013945781758910889319921122600739281491694816152738427362642980982340632002440244958944561291670495082358124873917996486411334803247577752197089327722623494860150466526814398770516153170266969297049283162855042128981467061953319702695072143782304768752802873541261663917082459251700107141808548006369232594620190022780874098597719218051585321473926532515590354102092846659252999143537918253145452905984158176370589279069098969111643811878094353715213322614436253144901274547726957393934815469163116249288735747188240715039950094467319543161938554852076657388251396391635767231510055560372633948672082078086537349424401157996675073607111593513319591971209489647175530245313647709420946356969822266737752099451684506436238242118535348879893956731878066061078854400055082765703055874485418057788917192078814233511386629296671796434687600770479995378833878703487180218424373421122739402557176908196030920182401884270570460926225641783752652633583242406612533115294234579655695025068100183109004112453790153329661569705223792103257069370510908307894799990049993953221536227484766036136776979785673865846709366795885837887956259464648913766521995882869338018360119323685785585581955560421562508836502033220245137621582046181067051953306530606065010548871672453779428313388716313955969058320834168984760656071183471362181232462272588419902861420872849568796393254642853430753011052857138296437099903569488852851904029560473461311382638788975517885604249987483163828040468486189381895905420398898726506976202019955484126500053944282039301274816381585303964399254702016727593285743666616441109625663373054092195196751483287348089574777752783442210910731113518280460363471981856555729571447476825528578633493428584231187494400032296906977583159038580393535213588600796003420975473922967333106493956018122378128545843176055617338611267347807458506760630482294096530411183066710818930311088717281675195796753471885372293096161432040063813224658411111577583585811350185690478153689381377184728147519983505047812977185990847076219746058874232569958288925350419379582606162118423687685114183160683158679946016520577405294230536017803133572632670547903384012573059123396018801378254219270947673371919872873852480574212489211834708766296672072723256505651293331260595057777275424712416483128329820723617505746738701282095755443059683955556868611883971355220844528526400812520276655576774959696266126045652456840861392382657685833846984997787267065551918544686984694784957346226062942196245570853712727765230989554501930377321666491825781546772920052126671434632096378918523232150189761260343736840671941930377468809992968775824410478781232662531818459604538535438391144967753128642609252115376732588667226040425234910870269580996475958057946639734190640100363619040420331135793365424263035614570090112448008900208014780566037101541223288914657223931450760716706435568274377439657890679726874384730763464516775621030986040927170909512808630902973850445271828927496892121066700816485833955377359191369501531620189088874842107987068991148046692706509407620465027725286507289053285485614331608126930056937854178610969692025388650345771831766868859236814884752764984688219497397297077371871884004143231276365048145311228509900207424092558592529261030210673681543470152523487863516439762358604191941296976904052648323470099111542426012734380220893310966863678986949779940012601642276092608234930411806438291383473546797253992623387915829984864592717340592256207491053085315371829116816372193951887009577881815868504645076993439409874335144316263303172477474868979182092394808331439708406730840795893581089665647758599055637695252326536144247802308268118310377358870892406130313364773710116282146146616794040905186152603600925219472188909181073358719641421444786548995285823439470500798303885388608310357193060027711945580219119428999227223534587075662469261776631788551443502182870266856106650035310502163182060176092179846849368631612937279518730789726373537171502563787335797718081848784588665043358243770041477104149349274384575871071597315594394264125702709651251081155482479394035976811881172824721582501094960966253933953809221955919181885526780621499231727631632183398969380756168559117529984501320671293924041445938623988093812404521914848316462101473891825101090967738690664041589736104764365000680771056567184862814963711188321924456639458144914861655004956769826903089111856879869294705135248160917432430153836847072928989828460222373014526556798986277679680914697983782687643115988321090437156112997665215396354644208691975673700057387649784376862876817924974694384274652563163230055513041742273416464551278127845777724575203865437542828256714128858345444351325620544642410110379554641905811686230596447695870540721419852121067343324107567675758184569906930460475227701670056845439692340417110898889934163505851578873534308155208117720718803791040469830695786854739376564336319797868036718730796939242363214484503547763156702553900654231179201534649779290662415083288583952905426376876689688050333172278001858850697362324038947004718976193473443084374437599250341788079722358591342458131440498477017323616947197657153531977549971627856631190469126091825912498903676541769799036237552865263757337635269693443544004730671988689019681474287677908669796885225016369498567302175231325292653758964151714795595387842784998664563028788319620998304945198743963690706827626574858104391122326187940599415540632701319898957037611053236062986748037791537675115830432084987209202809297526498125691634250005229088726469252846661046653921714820801305022980526378364269597337070539227891535105688839381132497570713310295044303467159894487868471164383280506925077662745001220035262037094660234146489983902525888301486781621967751945831677187627572005054397944124599007711520515461993050983869825428464072555409274031325716326407929341833421470904125425335232480219322770753555467958716383587501815933871742360615511710131235256334858203651461418700492057043720182617331947157008675785393360786227395581857975872587441025420771054753612940474601000940954449596628814869159038990718659805636171376922272907641977551777201042764969496110562205925024202177042696221549587264539892276976603105249808557594716310758701332088614632664125911486338812202844406941694882615295776253250198703598706743804698219420563812558334364219492322759372212890564209430823525440841108645453694049692714940033197828613181861888111184082578659287574263844500599442295685864604810330153889114994869354360302218109434667640000223625505736312946262960961987605642599639461386923308371962659547392346241345977957485246478379807956931986508159776753505539189911513352522987361127791827485420086895396583594219633315028695611920122988898870060799927954111882690230789131076036176347794894320321027733594169086500719328040171638406449878717537567811853213284082165711075495282949749362146082155832056872321855740651610962748743750980922302116099826330339154694946444910045152809250897450748967603240907689836529406579201983152654106581368237919840906457124689484702093577611931399802468134052003947819498662026240089021501661638135383815150377350229660746279529103840686855690701575166241929872444827194293310048548244545807188976330032325258215812803274679620028147624318286221710543528983482082734516801861317195933247110746622285087106661177034653528395776259977446721857158161264111432717943478859908928084866949141390977167369002777585026866465405659503948678411107901161040085727445629384254941675946054871172359464291058509099502149587931121961359083158826206823321561530868337308381732793281969838750870834838804638847844188400318471269745437093732983624028751979208023218787448828728437273780178270080587824107493575148899789117397461293203510814327032514090304874622629423443275712600866425083331876886507564292716055252895449215376517514921963671810494353178583834538652556566406572513635750643532365089367904317025978781771903148679638408288102094614900797151377170990619549696400708676671023300486726314755105372317571143223174114116806228642063889062101923552235467116621374996932693217370431059872250394565749246169782609702533594750209138366737728944386964000281103440260847128990007468077648440887113413525033678773167977093727786821661178653442317322646378476978751443320953400016506921305464768909850502030150448808342618452087305309731894929164253229336124315143065782640702838984098416029503092418971209716016492656134134334222988279099217860426798124572853458013382609958771781131021673402565627440072968340661984806766158050216918337236803990279316064204368120799003162644491461902194582296909921227885539487835383056468648816555622943156731282743908264506116289428035016613366978240517701552196265227254558507386405852998303791803504328767038092521679075712040612375963276856748450791511473134400018325703449209097124358094479004624943134550289006806487042935340374360326258205357901183956490893543451013429696175452495739606214902887289327925206965353863964432253883275224996059869747598823299162635459733244451637553343774929289905811757863555556269374269109471170021654117182197505198317871371060510637955585889055688528879890847509157646390746936198815078146852621332524738376511929901561091897779220087057933964638274906806987691681974923656242260871541761004306089043779766785196618914041449252704808819714988015420577870065215940092897776013307568479669929554336561398477380603943688958876460549838714789684828053847017308711177611596635050399793438693391197898871091565417091330826076474063057114110988393880954814378284745288383680794188843426662220704387228874139478010177213922819119923654055163958934742639538248296090369002883593277458550608013179884071624465639979482757836501955142215513392819782269842786383916797150912624105487257009240700454884856929504481107380879965474815689139353809434745569721289198271770207666136024895814681191336141212587838955773571949863172108443989014239484966592517313881716026632619310653665350414730708044149391693632623737677770958503132559900957627319573086480424677012123270205337426670531424482081681303063973787366424836725398374876909806021827857862165127385635132901489035098832706172589325753639939790557291751600976154590447716922658063151110280384360173747421524760851520990161585823125715907334217365762671423904782795872815050956330928026684589376496497702329736413190609827406335310897924642421345837409011693919642504591288134034988106354008875968200544083643865166178805576089568967275315380819420773325979172784376256611843198910250074918290864751497940031607038455494653859460274524474668123146879434416109933389089926384118474252570445725174593257389895651857165759614812660203107976282541655905060424791140169579003383565748692528007430256234194982864679144763227740055294609039401775363356554719310001754300475047191448998410400158679461792416100164547165513370740739502604427695385538343975505488710997852054011751697475813449260794336895437832211724506873442319898788441285420647428097356258070669831069799352606933921356858813912148073547284632277849080870024677763036055512323866562951788537196730346347012229395816067925091532174890308408865160611190114984434123501246469280288059961342835118847154497712784733617662850621697787177438243625657117794500644777183702219991066950216567576440449979407650379999548450027106659878136038023141268369057831904607927652972776940436130230517870805465115424693952651271010529270703066730244471259739399505146284047674313637399782591845411764133279064606365841529270190302760173394748669603486949765417524293060407270050590395031485229213925755948450788679779252539317651564161971684435243697944473559642606333910551268260615957262170366985064732812667245219890605498802807828814297963366967441248059821921463395657457221022986775997467381260693670691340815594120161159601902377535255563006062479832612498812881929373434768626892192397778339107331065882568137771723283153290825250927330478507249771394483338925520811756084529665905539409655685417060011798572938139982583192936791003918440992865756059935989100029698644609747147184701015312837626311467742091455740418159088000649432378558393085308283054760767995243573916312218860575496738322431956506554608528812019023636447127037486344217272578795034284863129449163184753475314350413920961087960577309872013524840750576371992536504709085825139368634638633680428917671076021111598288755399401200760139470336617937153963061398636554922137415979051190835882900976566473007338793146789131814651093167615758213514248604422924453041131606527009743300884990346754055186406773426035834096086055337473627609356588531097609942383473822220872924644976845605795625167655740884103217313456277358560523582363895320385340248422733716391239732159954408284216666360232965456947035771848734420342277066538373875061692127680157661810954200977083636043611105924091178895403380214265239489296864398089261146354145715351943428507213534530183158756282757338982688985235577992957276452293915674775666760510878876484534936360682780505646228135988858792599409464460417052044700463151379754317371877560398159626475014109066588661621800382669899619655805872086397211769952194667898570117983324406018115756580742841829106151939176300591943144346051540477105700543390001824531177337189558576036071828605063564799790041397618089553636696031621931132502238517916720551806592635180362512145759262383693482226658955769946604919381124866090997981285718234940066155521961122072030922776462009993152442735894887105766238946938894464950939603304543408421024624010487233287500817491798755438793873814398942380117627008371960530943839400637561164585609431295175977139353960743227924892212670458081833137641658182695621058728924477400359470092686626596514220506300785920024882918608397437323538490839643261470005324235406470420894992102504047267810590836440074663800208701266642094571817029467522785400745085523777208905816839184465928294170182882330149715542352359117748186285929676050482038643431087795628929254056389466219482687110428281638939757117577869154301650586029652174595819888786804081103284327398671986213062055598552660364050462821523061545944744899088390819997387474529698107762014871340001225355222466954093152131153379157980269795557105085074738747507580687653764457825244326380461430428892359348529610582693821034980004052484070844035611678171705128133788057056434506161193304244407982603779511985486945591520519600930412710072778493015550388953603382619293437970818743209499141595933963681106275572952780042548630600545238391510689989135788200194117865356821491185282078521301255185184937115034221595422445119002073935396274002081104655302079328672547405436527175958935007163360763216147258154076420530200453401835723382926619153083540951202263291650544261236191970516138393573266937601569144299449437448568097756963031295887191611292946818849363386473927476012269641588489009657170861605981472044674286642087653347998582220906198021732116142304194777549907387385679411898246609130916917722742072333676350326783405863019301932429963972044451792881228544782119535308989101253429755247276357302262813820918074397486714535907786335301608215599113141442050914472935350222308171936635093468658586563148555758624478186201087118897606529698992693281787055764351433820601410773292610634315253371822433852635202177354407152818981376987551575745469397271504884697936195004777209705617939138289898453274262272886471088832701737232588182446584362495805925603381052156062061557132991560848920643403033952622634514542836786982880742514225674518061841495646861116354049718976821542277224794740335715274368194098920501136534001238467142965518673441537416150425632567134302476551252192180357801692403266995417460875924092070046693403965101781348578356944407604702325407555577647284507518268904182939661133101601311190773986324627782190236506603740416067249624901374332172464540974129955705291424382080760983648234659738866913499197840131080155813439791948528304367390124820824448141280954437738983200598649091595053228579145768849625786658859991798675205545580990045564611787552493701245532171701942828846174027366499784755082942280202329012216301023097721515694464279098021908266898688342630716092079140851976952355534886577434252775311972474308730436195113961190800302558783876442060850447306312992778889427291897271698905759252446796601897074829609491906487646937027507738664323919190422542902353189233772931667360869962280325571853089192844038050710300647768478632431910002239297852553723755662136447400967605394398382357646069924652600890906241059042154539279044115295803453345002562441010063595300395988644661695956263518780606885137234627079973272331346939714562855426154676506324656766202792452085813477176085216913409465203076733918411475041401689241213198268815686645614853802875393311602322925556189410429953356400957864953409351152664540244187759493169305604486864208627572011723195264050230997745676478384889734643172159806267876718380052476968840849891850861490034324034767426862459523958903585821350064509981782446360873177543788596776729195261112138591947254514003011805034378752776644027626189410175768726804281766238606804778852428874302591452470739505465251353394595987896197789110418902929438185672050709646062635417329446495766126519534957018600154126239622864138977967333290705673769621564981845068422636903678495559700260798679962610190393312637685569687670292953711625280055431007864087289392257145124811357786276649024251619902774710903359333093049483805978566288447874414698414990671237647895822632949046798120899848571635710878311918486302545016209298058292083348136384054217200561219893536693713367333924644161252231969434712064173754912163570085736943973059797097197266666422674311177621764030686813103518991122713397240368870009968629225464650063852886203938005047782769128356033725482557939129852515068299691077542576474883253414121328006267170940090982235296579579978030182824284902214707481111240186076134151503875698309186527806588966823625239378452726345304204188025084423631903833183845505223679923577529291069250432614469501098610888999146585518818735825281643025209392852580779697376208456374821144339881627100317031513344023095263519295886806908213558536801610002137408511544849126858412686958991741491338205784928006982551957402018181056412972508360703568510553317878408290000415525118657794539633175385320921497205266078312602819611648580986845875251299974040927976831766399146553861089375879522149717317281315179329044311218158710235187407572221001237687219447472093493123241070650806185623725267325407333248757544829675734500193219021991199607979893733836732425761039389853492787774739805080800155447640610535222023254094435677187945654304067358964910176107759483645408234861302547184764851895758366743997915085128580206078205544629917232020282229148869593997299742974711553718589242384938558585954074381048826246487880533042714630119415898963287926783273224561038521970111304665871005000832851773117764897352309266612345888731028835156264460236719966445547276083101187883891511493409393447500730258558147561908813987523578123313422798665035227253671712307568610450045489703600795698276263923441071465848957802414081584052295369374997106655948944592462866199635563506526234053394391421112718106910522900246574236041300936918892558657846684612156795542566054160050712766417660568742742003295771606434486062012398216982717231978268166282499387149954491373020518436690767235774000539326626227603236597517189259018011042903842741855078948874388327030632832799630072006980122443651163940869222207453202446241211558043545420642151215850568961573564143130688834431852808539759277344336553841883403035178229462537020157821573732655231857635540989540332363823192198921711774494694036782961859208034038675758341115188241774391450773663840718804893582568685420116450313576333555094403192367203486510105610498727264721319865434354504091318595131451812764373104389725070049819870521762724940652146199592321423144397765467083517147493679861865527917158240806510637995001842959387991583501715807598837849622573985121298103263793762183224565942366853767991131401080431397323354490908249104991433258432988210339846981417157560108297065830652113470768036806953229719905999044512090872757762253510409023928887794246304832803191327104954785991801969678353214644411892606315266181674431935508170818754770508026540252941092182648582138575266881555841131985600221351588872103656960875150631875330029421186822218937755460272272912905042922597877106678738400006167721546384412923711935218284998243509208918016855727981564218581911974909857305703326676464607287574305653726027689823732597450844796495456480307715981539558277791393736017174229960273531027687194494449179397851446315973144353518504914139415573293820485421235081739125497498193087143966151329420459193801062314217741991840601803479498876910515579055548069538785400664533759818628464199052204528033062636956264909108276271159038569950512465299960628554438383303276385998007929228466595035512112452840875162290602620118577753137479493620554964010730013488531507354873539056029089335264007132747326219603117734339436733857591245081493357369116645412817881714540230547506671365182582848980995121391939956332413365567770980030819102720409971486874181346670060940510214626902804491596465453301077546954130887141653125448130611924078211886900560277818242350226961893443525476335735364856193632544177566139817039306328721669057222597452091929172621998444096461582694563802395028371216864465617852355651641277128269186886155727162014749340522769465957121983149433816221140069363074304441732847861017777438379770372317952554341072234455125555899986461838767649039724611679590181000350989286412041951635511087632042676129798265294258829511412758412627327907988075597518515768412647422094797218433093529726652100156625145529947451276315509176367302594621329301904028379542463232585503010967069227202270748634190054383026506812141421350571541750575086399076739463351462090828889349383764393992569006040673114220933121959362029829723511632593867722414779116295727807523950562515816031333593823115005186268905306583681299881086632632719806112715488587980934879129137074982305759290918629391950147211975860672700925477180257503377307993971345395326461952699965963856549175904583335857991020127132045839032008538788816336376851820837278851311752277696097879621423721625452145912818317982160441113116714069148271709810154577819392023115638719508050246797257924976057726259133285597263712112019057207714091486450740949267180358151575715140503976109638467555692989703835473141002238025834687673501297754132795320609711545064842121859364909979177668747744818828706323155158650328981642282882327468661065927321979071623846421534898524762167890502609980452664839295423572873439776804957740914495383915755654854590589764951985138010079580107837599457752991967005476022525520344539887125387801719607181640781248478472579124078245443616823452395706895142722697504318736332630111030534233358216093331912188066082683414289104151732472160533558499932245487307788229052523242348615315209769384610425828497149634753418375620030149157032796853018686315724884015266398356895636346574353217834931998255421173084677452970858395076164582296303244243282377374505170285606980678895217681981567107816334052667595394249262807569683261074953233905362230908070814559198373553777487420290390181429373115293346444681512129450975965343062842153194457271186149000176505581770953024688752632501197052094761594167687277844720001927891372518416228577837922844390843011811214963664246590336341945406571835447719124466212593926566203068885200555991212353637182269225317814587925937504414489339816086579008761650246351970458288954817937566810464746141051424988702521399368705093723054477341126413548928068410591077166778212383328102621855877513127211793444482014404257450830639447383637939062830089733062413806145894142276947479316657176231824721683506780764875734204915576282175839729751344789906965895325489403356156131674032764724692125057591162515296545685446334981143176702572956618447754874693784642337372389819206620485118943788682248072793520225017965453437572741639107919729529508129429222053477173041844779156739917384183117103625243957161527146690058147000026330104526435478659032907332054683388720787354447626479252976901709120078741837367350877133769776834963442524199499513883150748775374338494582597655609965559543180409201784971846854973706962120885243770138537576814166327224126344239821529416453780004925072627651507890850712659970367087266927643083772296859851691223050374627443108529343052730788652839773352460174635277032059381791253969156210636376258829375713738407544064689647831007045806134467312715911946084359358259877828352665311510650416232953290477721740835593497237585521380483050900096466760883015406128243087406455944318534137552201663058121110334531207450868243394321590435944303124312274713858420303901060709403152355561727679941600203939750998976293353258555756248089966918298642226775023601932579747267425782111197347094023574572222712125268523842958742735015636600931880454933389897415714905441825597380808715652814301026704602843168192303925352977957658624143927015497408792731310516361191375770089295648233236482982630246079758757677453771601024908046243018565241617566556001608591215345562676021926899828553778725831451440826545834844094784631787773747946535801699607794055687011923286080411309046293508718271259346687127666948738998245985277864995691654640294589350649643358098247659651651420909867552038083092032304873427034682887516040715466538346196112230137594515792526967436425319273900360386082364507626988274976187235754767628899507521148048525279508450339585708381304769378813211236742813194879502280663201700224603319896719706491637411758548518784840120548446725888514015627250198217190669608126277854859648183696214107217142149863619187747545096503089570994709343378569816744658282679119406119560378453978558392407612763441057667510243075598145527861678159496570625597550743065210853015979080733437360794328667578905334836695554868039134337201564988342208933999716414797469386969054800891930671380571715058573071488156499207140867582596028760564597824237702424698053280566327870419267684671162668794634869504645074202193739452592626686135529406247813612062026364981999994984051438682852589563422643287076632993048917234007254717641886853513723326678779217383475414800228033929973579361524127558295692768372312347989894462743304545667900620324205163962825884430854383072014956721064605332385372031432421126074244858450945804940818209276391400085404220235562602185643489941454399504109805918179488826280520664410863190016885681551692294862030107388971810077092905904807490924271410189335428184299959881696609938369616443815288772140852680887574882932587358099056707558170179491619061140019085537448827262009366856044755965574764856740081773817033073803054769736097865438593821872205839023444435088674998665060406458743460053318274362961778625180818931443632512051070946908135864405192295129324500788333987884293393424351263433652043858129128343452973086529097833006712617981303167943855357262969987403595704584522308563900989131794759487521263970783759448611394519602867512105616389760088800927461158608002078033415914517970730368351969777660763737853330120241201120469886092093390853657732223924124490515327809509558664594776344822699860748132973026309750288121035177231244650953496536930900186377640940943498373132513218620802148099226855029484546618147155574447096695301776904342720318927706047177845279391604722815343798035396798614243709566832214914654380145938292773933960327540480095522318166673803571839327570771420467238386246178039762923771312095807893638414479298025880655221292620936239306373134966401866195108115834711733120258058667276399927635790780638188130691563662741254312595899361196476261014055635033995231403231138196562363271989618372548453337020625634642239527669435683767613687119629218187545760816170530315907288287007123136663087227549186613957737305460659974378109876498024140112421427736680827513909593134041558262667895108467761186659576601659981780894149857549762843878561002637965431783136340251358141611519020964991335487331311150227006819301359295959716401971960536250335584799809634887180391116128135959685654788683258564378961731597620024196215528962979048198221994622694871374624447290934564700285376949588595916067892824910544125159963007813683674902093749157328962700286568293444313423473512392982591667395034259958689706972673325827359031212887466604514614878503461428277659916080903986525757172630818334944418201935333850712923457743755793440621787113300631060033240539916936826037461766385657588775802012293663532702671006812618251729146082025418928859352444910701382062115538277935652969145765020486432828655579347072096348073726921411868954673227677513356901901537236690368653891612916888878764075254934942497334271811788927599315967193547589880979245252623636590363200708544407845447973482918020820449266706344204375553250505275228337788870408040335319234076856301093477721256390886404131010738178533383160381352808281190408325644018420537467929926220376987180180611226244909092426419858208617511771137890516091403815750033664241560952163281971223350231674226005679412814062172196418427057843289598028823350598282081966662490358577899403331522748177769528436816300885317696947836905806710648280835980466988410981351586549069333195223943632879239905348109878302745001720654336990661177845543646877236318444647680691428280045510746866453928053994091087549391660957316197150331669683099294663491427987808422572206971488755806374803088629951184731871247772919100702275888934869394562895158029653721504096031077612898312635899648934102470360366450586872875890514068412381242473863854279082827338279733268855049358743031602747490631295723497426112215174171531336186224109138695006888358989623492763173164783400774608866555987333821138299287769114954921841920877716060684728746736818861675072210172611038306717878566948129487850489430630861699487987031605158841082823512741535385133658953329486294944950618685147791058046960390693726626703865129052011378108586161888869479576074135855345851517680519733344334952301203957707396237713160302428872005373209982530089776189731298178819446717311606472314762484575519287327828251271824468078242152164695678192940982389262849437602488522790036202193866964822156280936053731780408637272684266964219299468192149087017075333610947913818040632873875938482695355830773957614479972700034728801827852813895032179863452161110666088393140532269449054555278678944175792024400214507801920998044613825478058580484424164047750315360549065914300781583724301231375115622840158386442708907182848167575271238467824595343344496220100960710513706084618011875431207254913349942476171156333214089346091565615506003173842187015702261031019166038870646614388977363187809407115275281746895764015810470169652475577408916445686777171585005832699434016772021567677240681283665652641229824394651331973591997094032759385026695574702318132032437164205861410336065245369391600506449530601612678226489424373971667176612310489750318857321655549883421218028469125290861014855278152776256237504563757694977343368460156077270355096290493924870884062810679436224187047470083688426710225583024035998416459511224852726336326451140173952480861946358407837535568856223171155209472230654370926067973510005655493812245754837285457117973936157561676416928958052572975223385586113883221711073622658162188424431788574887981090266537934266642169909140565364322493013348679881548866286650523469972355747384248305904236771432787923164224038777643301926001922847783138376325361210253369358126240868666997382759773656822279072158324788886423693463961643633087301398142114303060087306661648036789840913359262934023043249749268878316436026810113095707161419128306865773235326396536773903176613613159655535849993986005651559219367599777179330197446881483711032065036931928945214026509154651843099365534933371834252984336799159394174662239003895276738133306177476295749438687169784537672194935065908757119177208754771071899379608947745126547575018711948707387367858902006173733210756933022163206284320656711920969505857611739616323262177089454262146098584102378132158177276022227381334954104810030732751077999489919779638835307344434575329759142637684054422647842160631227696469671564739990437159033239065607266441164386054048388471619121090087010191307260710441141432419767968285478855247794764818029597360494397004795960402927462992035720997619501403483153809477146010563334469988208221205872815107291829712119178764248803546723169165418522567292344291871281632325969654135485895771332083399112887759172261152733790103413620856145779923987783250835507301998184590259583559892605532996737704917224549353296833000022301815172265757875240588322490858212800897479093261007625787704286560069961762121768454789964407050662417102133274867962374302291553582007801411653480656474882306150033920689837947662550365498228053296628621179306284301704924023019857199789488368971830438051821744191476604297524372516834354112170386313794114220952958857980601529387527537990309388716835720957607152219002793792927863036372687658226812419933848081660216037221547101430073775377926990695871212892880190520316012858618254944133538207848834653116326504076424283908701210151942319616522684220037112304643006734420647477180213530701240988603533991526679238711017062218658835737812109351797756044256346949997872511254408545222748109148743072598696020402759411789425812818821599523596589791811440776533543217575952555361581280011638467203193465072968079907939637149617743121194020212975731251652537680173591015573381537720019524445436200718484756634154074423286210609976132434875488474345396659813387174660930205350702719529839432714253711557666000257844230310734295515339450604862227649666876240793243531929926392537310768921353525723210808898193391686682789482811704726245019484097009757609209837240900747179733407881418251958425980962417476101382526439551352593118850456362641883003385396524359974169313228947198783084276004013680747039040972384739458348961865397905941185993103561684368692194853820557803957738813606795499000851232594425297244866667668346414021899159445653094234406506678519484177667794704720419588220432953803263105374948831221803912796784461001397267538921951191178365876625280836900532490045974109470687729123282143046353372835199536482743258331191444590178096077828835837301118575436599589827245319253105881150263075425714939430244539318701799236081666113054262539958338979429716020703387678150330102801200959972522222808014235710947603519255444349299867678178910455590630159538097618759203589373419789623589311259839025983102671933041892151096891562250696591198283234555030590817307351955037216658702880539921385760370353771051780212801295668419841403628727256232144287543022109094727210734741349755141907370433182766261772759968888260272252471336833534528166927795913288613817663498577289369009657495622871030243625907724122190943008717556926257580657099120166596224360802428700245473620363948412559548817272724736534677836472019183039987176270375157246499222894679323226936191776416146187956139566995677830682903165896994307673335082349907906241002025061340573443006957454746821756904416515406365846804636926212742110753990421887161276177870142588648257752238891845995233762923779155857445494773612955259522265786364621183775984737003479714082069941455807190802135907322692331008317595106590191212947954086036407573587502058902087045796700070552625058114206639074592152733094068236494415908910092202966805233252661989113118420162916310768940847235643668081821686572196882683584027855007828040434537101836510969517823357430305048526537380735310741859177056103973950626403554422751561011072617793706347238049906669221619711942591204450846417463835899382399465173955090008594799901360266742614942900664671150671754221770387745076735637421547829059110126191575558702389570014051178226469899449179083017954758767601680941001358376135785913569244556477644641786671153919513576961048649224900834467154863830544779143300976804868783481846727337584368927243104474068076852786255851650920882638132336231487333367147645204508766276149503899495048095604609896043291233583488599902945264002849942808786240398118148847673012167541611066299955536681931232874257020637383520200868636913117334697317412191536332467453256308713473027921749562270146873258678917345583799643513588009593508775563562488104938529990076751355135277924124292774885658885665132473025147102105753525165118148509027504768455182520963318990685276144351382136621523688905787866994322888160283774820355060160298940091197138501798716836337441392759736440170070147637066557035043381211135764150184518214136198234951596010647527125759351853043328755377830575095674254426847122196187091785607839361445113833356491032564057338986671781239722375193164306170138595394743678433926709867124522111896908402363274114966012434830989299417380305884171666130730400675883804321115553794406054977217059428215148861656727712409033877277456290971101348851843741186956554497457368452180669829110450580042998879538990278043835962824094218605562877884288021275538848037286400194416142574999042720095952046541705981049899675045119364711727722204361026140797508096869751766002371877483480161203102346805671126447661237476278521902412025699435347162266608936752198331118135111465038548950251206557726361454736044268594980743969323312971273771573470997139522911826534851555871373366291202427143025037632695013509116129529937858646813072264860082708813335381937036825988678933212383270532976258573827900978264605455985551318366888446282651337984916678394097613537662517982582496634587719501243840403591408492097337546424744881761840700235695801774101776969250778148933866725578985645898510568919609243988415692806969833522402256345704973122452693541938370048431833571965166267215755241934019330990183193091965829209696562476676836596470195957547393455143374137087615173236772042273856742791706982045499530959188724349395240944416789988463198455048523936629720797774528143994182567894577957125524268260899408633173715388962628896294021121088844273765686245276121303710173007851357154045330415079594477761435974378037424366469732471384104921243141389035790924160364063140381498314819052517209371039640268089948325722979545640427017577229041732347960736187878899133183058430693948259613187138164234672187308451338772190869751049428437693250249816566738162606159417682525099937416728839517440669325496534031014522253161890092353764863784828813442098700480962271712264074895719390029185733074601043607291909457679946149292904279816877294264877299528584346477753869069501489841339245403941446802636254021186143170312511175776428299146445334089209769616990983726523617687456058947049681701369749095230720826828878907301900182534258053434217059287139317379931424108526473909482845964180936141384758311361305761084623668372376959134926158245162215521348792441450417568480641206365201703863301295327776990231186480200675569056822950163549319923059142463962170253297475731140942201801993680350264956369558664259067626856873721103391567938398957655651931778830002416135395624377778408017488193730950206999008908993280883974303677365955248913001566332940779071396154645340887915103006513219344866732482759079468078798194250195826223203951312520141099605312606965554042486705499867869230217469890095478507256729787947698888310934874644264007181831603316555115342761556224054744733780492462149521332585276988473362691826491743389878247892784689188280546699823036899397834137475870258057163494135684339293960681920617733317917382085624364336353598634944968907810640196740744365836670715869245211829978938040771375012908586465789057714268335827689785547176871844277261205092664861020515356428406323684818072879407171279668200607275595559040402331787494473464547606281895415121391629184442976510669479693540168660100551960776873353965116149309375709685545593815137895690392510149532656281470119983269922000663928753747131352364215892651262040728877165783584052196460541054354436421665622445650429990102565869272791427529311720827939377513261060528812353734510683729398935808712438693859343891757133763007203197608166044646839377258069092372975234867029169104263692620901996052041210240776481903160140858635584276095370865581642739953493465463145040401995285372520049578052546562511541092524379913262627136090994029022620628367521323050651839340574501120993414649184333236465693717259144893241590062420206128857329261335968087265000456282845575745965921205303413101118275013069615098355156320043107846019065654938065425252291619918199596027523277022498557388248998827074659363557685825605180689642853768507720122203479209939361792682065901421656159253067379445689490708532635681968318617722682499114726157320358076462981162440133167378927886892290325933498617970219949819257396176730758344170985592221701718257127775344915082052784309046194608352174020058386728497094110232669539214454610662150064106747402070091899119513764669044812672536915371622907913854039375600778351533741677479421003840023089518509945487790393461222208650601605003517762648316111533255877050735412792499098593734737870811942530551214369797499149518605359204038302357163527276308746932196221900642608861836761033460022554774778136410126919065696864950126883762969072339612762872230411418136100602640440300359969889199458273976241146137448040596970625767647237660655416185746905272292382282751867991569833907476711461030227766060200612468764777288190967916133540198814027579921741676787992316039635694928515136336472195406111717673873725557285229400543617851765023075446938693078734991103521825329297260445532107978877114498988709115112372506042387537348412570860640690520584521227545338480082053024504565176695185769132000428167580549248117805198326460324457928297301291053183856368212062155312886685649565126138922613670640939533345705269869596923503530942245438652786776730275404027022463844835532399147513634410440500923303612714960813554905315390210022995957565837053812619656831442860579566966221547216956208700137277685369608407048333251327931122325071486302069512453950037357233468070946564830892098015348787056334910923660575540508641115214414814346304372732710450277686619531078583233348578402971609252153260925589326556006721243594642550659967717703884453961816328796144608177892721718369088801267782074301064225246348074543004764928855534090621851536543554741254761527697726677697727770583158014121856880117050283652755432148034880044429799980621579045641619572127845089284898064264974270905791290692178072987694779751124473059914060506299468942809310342164166299356148281309988707452927160484336308184041264696379258430941854422163590845761460785585624738149314270782662151855416038702068769804617474008083243436653823545551094494984310934947599446726736653525176627067721941831919771963780157021699336750837600571634546436717767233875886434056448715669643210412825956453498413884128904206820470076155969168430389993483667935425492103281133631847225923055543830582069416756299920133731754891220372303490726810685344540359935618235763128377676406310131253352121419946118693508331765878520471123643312267651299641713252175135532618676819423387903654689080018271352835848884441117612341011799187092365071848578562210211040097769944531217950224795780695065329659403839873699072407976790408267940076187295478359634927939045769736616434053597922192858705749574816966940623342726197335181366260637359825755524965098072601236682836059283418558480269584137725589708837899429105498003311138846034019391661221866960584915714857335682861495000190975911252188003964197621635593757437180114805594422987304181968080856472657135476128316292004498803154021055305970766663627493283089168809323592900817874119857383171926167288349184024297212904349655269427264025596414635259143484006758676903503823205729341329815935330444464968294413673234421583807616948312193331198190610961429522015361702985751055943264614685054526849757648078080092213358113781977492717685450755383287688744745915937311624706010912446098294248412875202244625944776387494919978404468292573609685345498432665368628444893657041118177938064416165312236002149187687694673984075171763075168498563592014868929431059402024579696229245666448819675762943495353263821716133957577907663707645695702597388004384158058943361371065518599876007549241872117148892952217377211460811543449826654798725800566747240511220073834592715757277152185899469481179406444663994323700442911407472181802248258377360173466853007449855647154200361235933973129144585915228874087195087086322188372882628228846318437172619033057771476515641438223067918473860391476831081413582757558536435977216500282778037134228696887873497950960311088991961433866640684506974207877002805093672033872326296378560386532164323488155575570184690890746478791224363755566686780676105449550172607911429308312857612544819444494732448190937953690082063846316782250648095318104065702543276043857035059228189198780658654121842992172737209551032422510797180778330426090867942734289557355592527238055114404380012390416877164451802264916816419274011064516224311017000566911217331894234005479596846698042980173625704067332821299621536848814041021944634246462207455756439604529853130714090846084996537678037932018991408658146621753193376659701143306086250098295669176388460567629729314649114937046244693519840395344491351411936679333019366176636525551491749823079870722808608596261126605042892969665356525166888855721122768027727437089173896397722575648905334010388559311256799915165890250164869614272070059160561661597024519890518329692789355503039346812197615821839804839605625230914626384473862960398489243861872985077759287927220685548072104978176532862101874767668972488411395603494803767270363169210073508340738652616845074824964485974281349364803724261167042668708319250409976153190768557703274217850100064419841242073964001396036015838105659284136845741191027364202741637234882145241013477165296031284086584197879511165115298278146203791398550063999603265912485253084936903131301007999771913622308660110999291428712493885416120380204113401888872196934779044975274542880728035093058287544207551348166609278793535665212556201399882496284787262144323628536765025914504683776352825876521391564809721419296755493843755826002531685363567313792624758780494459441834291727569883762262618463654527434976624111384513054814498363117897844897320767195087841586188796929558197332506999514026015116755297505754378102422389579257865621284327312022007167305740692868693639301867659582513264991459502609170693475194089753574640168308117988464524736189560564794263580705625632811892696630264795359510971276591362331808669215357886078127599105371714022045061860753748663063505914839164676567232057145168861707909846959322367249467375830996070425892204815507991327520885837811176852142693347869218952406226579210436203488529262679840139532164587911515790504605797108389833718640380244175113472264725470107947939969535546696197267632552299146549334996632341859514503609803440922122067125676987234279407088570704742931733291885238967219713539244924261786411886377909628144869178694681775917171506691114800207594320120619696377951032270890295660855622254526026104607361313688690092817210681986185537809820184711541636303262656992834241550236009780464171085255376127289053350455061356841437758544296779770146602943876872251153638011917581540281208182556064854107879335989210644272448986189616294134180012951306836386092941000831366733721530083526962357371753307386533382048421903081864491840937239440334052449095545580164064607615810103017674884750176619086929460987692016912021816882910408707095609514704169211470274133900522533408348128703530310239196999785974139085936054335996970756044601342424536824960987725813110247327985620721265724990034682938868723048955622532044636026398542252584164643242716114198178024825955635449072192265838636626637508359443148776351561457107455280161596770484427141944351832756984075526779264112617652506159652354571879566731709133193587616282559207830801852068901515047133403861003100559148178521103847545429333891884441205179439699701941126951195265649195941899754183932346474242907027188752235343936736336632003072327470374071239825620246626519740901997624520561985576257600087081730832883443818310700545144935458854226785785519153722923795554943334101744201696000906964156127322977702212179518683763590822551288164700219923488640439591530184640047143211863606225270115411222838027785389110984902013427410141215597699654388771974853764311582298385331230717511329619045590079380642766958190148426279912217929479873489018684716765038273285520590829845298062592503521284519259279865935061329619467962523739725655841578537445675589980324054921869628884903325608514553443916602262577755129162007727968526293879375304541810807292858919897153817973434961872329276147478501926114504132748732429705834084711123337462746172746265824153242710593225062553023147387592517247873228814914559156050363345754242337791603749525024930223514819613811625639114156103268449580725082734317659440540982697652693445798634797097431244982719331138638731596363612186234972614095560799206283169994200720548115253533939460768500199098865538614334957816500899616490796781429011483876456821749140756237676184537751440314754112067601607264605568592577993220703373333989163695043466906948284366299800374145276277165476238255461708831898108688068478537055364804693509588180253605297407935386765111950793732820831462689600710751755206144337841145499501364324463281933463890509365457145069008644834401804283633905135781572739733345372842633721740657757710798305175557210367959769018899584941301959995730179012401939086813565855396619413717944876320798688003716073032205474235722668968018821234243918859841689722776521940324932273147936692340048489760590379580946960417542796137825537812239476461478329269765451622902817011004378460387565441517394339600489153188175766505009516974024156447712936566142539493688842305174001299205568542898538979426699567770270891465137368922061044154816621568042198384767308717875902792091759006952734566820265133731115180001814341209626016586298210766635233617740078377834237091526440630540718078433580610729611055500204151316963730468492133568372654003075098290893646120478911147530370498939528334578240828173864413227100029683119402033234564208264732762338302946393789983758365545599193408662350909679611340048670271231765266637107787251118603540375544874186935197336566217723592293967764632515620234875701137957120962377234313702120310049651521119760131764194082034373485128526029133349151250831198028501778557107253731491392157091051309650598859999315608636554774035518981667335358800482146650997414337611827777233519107412175728415925808725913150746060256349037772633739144613770380213183474473011130326702969173350477016321066162278300272692833655840117914194478087482533607144032962522857750098085996090409363126356213281620714534061042241120830100085872642521122624801426475194261843258533867538740547434910727100497542811594660171361225904401589916002298278017960351940800465135347526987776095278399843680869089891978396935321799801391354425527179102253970108106321430485113782914985113819691430434975001899806816444121232733283071928243624067331965546926778511931527751134464689055042481133614349846048490512583456832664415284897139723760403282126602535166939140820499473204860216277597917712347510975024030789357599377150950217516935558270725339118923340702238320775858021371747783787783910152341320984894234596136923404979982793041444631627072147961174569757196812392919137409829258055619552074342432959828989805292333664154192563673806894942014712413405250722040617943552525552250087487900865683145428351677505422948032747830440564385815919526667582829297052261276287110401348017872248017896840524079243605827424674430767216452703134513541676496689012747868010102951338626986497482121186290403376915685762406992963724930972016287072001898354236903641492702369619385473724803298550451120891928798298744678641291594175316756025334353106267452545071141814832398806072971402347255207134907983989823552687239509093656678789923837125789762487559904432288953883773173489411227570714109597900479193010467407504114353817824646307959895556389918847737813413470702467473621120489862269918885174562517325193413520381158633501239130544419100736284475675141610504109735058527620444891909789019843154852805339857778443139338839943104444656692445508859463140817512203313906815965925105468580131338381521764182104334297888261196304431113887962587460902261309008499754303957712432306169062629194039214397402708947776637024881554993224588259790206312574369109463932528062416424768684954553249380176393716156368478598237159023854212658406153672286071317026747401311452610637653833903159219434698176053583803106128878520515469336392410884676320095670897183674905781630851581381619668822220475704375906143380407258538620835651769984267745231958241826836982701602374149383634966293515768540613973427464708996856181701605511048809715548591186171896680259735417054239851355600187203350790609464212711439931960465274240508822253597734815191354385712532585404939460108657937980586201433660788252197178090258173708709164604527279771535099103407364250203863867182205228796944583876529479510486607173902293274554267856697768659399234168341222746630150621553205026553414609952493560508549217565491348309589065361756938176374736441833789742297007035452066631709296075919896277324230902523974438610142630986877339138825186843165010279649114977375828889134503411488659486702154921010843280807834280894172980089832975369406449699031253998639195816014689952208806622854084148642747862819755466292788146216071713818801808405720847158689068369193933818642784545379567192723979723646516675920110579956639625985355127635587681402134098290162968734298507924718460568748283313812591619624761569028759010727331032991406238646083333786382579263023915900035576090324772813388873391780969666014696150317542267511259933155296742133363002229649064809345820081810618021002276645804002782133367585730190113717546727630590443531313190360924890972464279284555499134900051802957070829190525567818899138996251386623193800536113462242946102489540724048571232566288889317221164329478161905548680549434410340906807160880282279596869501336438142682521704728708630101373011552368614169083756757476372397631857570381094433905645644685241830281481079983769185121272019350440418046047216269394457883770901059746932197205581140787759897720720096893822493032368305158626572811146379969831375179376232151112523497343052406221052442343537329056551634066695061658928782187077567941760807129737813351871179316500331555238224877306534441794534153952024244497034101208740721881093882681675120422994049481794494727328947701115741394412284555218284249222406587526891722727806071167540469730080370396187877966948825556146743843925701158295466613586786718976612973112672000729715536130275035561678177654422874421147298816148027052438068176535732755786025058470840132088379328160087690813004924914736825170353822196190390149995234953871059973511434782923394991879366086923013755963685323738067035911442432685615121094042595826393016780171286692392832310576588517140202111969570647998140315056330451415644146231637638099044028162569175764891425697141635984393174332702378123369380430128926263753826677950341693343236075002481757418087503884750949394548962097404854426356371649959499209808842947903636662975260032438563529458447289445471662092974954966168774141208821304770228161164560440072363515811497297392189667373826472047226422212420165601502849713063327958143025160136948255670147809357908896571349261581613469018069650895563101212184918058479227206918716963163300448580201028606578585912699746376617414639341595695395542033146280265189511679380745733157598460861737026878676029436777805002446733913324316698803540732323882818475010516413311895370364884226902704780527424906034920829547550540034571601840725745369381455311753542107265578356154998744474804273234578800618731493415660463529797794550753593047956872093167245365472083816858556060438019770307642460834898761013457093948770029461757920619525492557571090385251714885252656710453498134198033906415298763436954202560802776144219143189213939088345431317696851018401038444723489488695209819435319065065553546173358140455448378847525262539496658699920584176527801253410338964698186424300341467913806190280596078548880107897055169462152287730901044674624979799926271209516847795684825833414022664772108433624375937416105367340419547389641978954253350363018614009515347669614762556518738232924685473569358028960115367917873035531593783630822486151777705415775765617593585120166929431111388635821596676188303261041646517148469793854226216871614001223782137797741312689772667129920259220174087700769562834739322010881593562862819285635718933849588506038531581797606794798408783609759601497334205727046035217906056476032855692762734951822032361441125841824262477120120357763888959743182328278713146080535335744942976217967890345681698895535185044783256163807094769516990862471000197488092050095219436323787197648703392238115403634754886268459561597551937654101150140670012269274743938885899438597302454148010612359080362745852884935632515853843832424932526660875889083187007091002373771065769850564339288543376583425967506537150053335144899082938877373520514593330496265314151413861244379358850709446880454869753581702129084907873478068143663233228194158273456713564431715379678180581958524648400840329099819437817181773023170039897330504953873561162610239994332597801268934326055847102787649010709234438846340117355568659035852449193701810416262085042992586974358170981338940459344719374938776242324098528327622666049423851297094532455862521036008292866497241749191419889661295580767709795947953060131191590117739431042090490794244488685130868444937059090260061206494257447103535476578592427081304106185462198818300906345881870387558562749115873754210646679513464875867715438380185213482819158124625993351601989355951679689328522058247994210345127158771633452229954188396804488355297533612868372259353900792016669413390911687588039888288692160023732573615882071635162713328105181876021048521806755266486739089009071951380586267351243122156916379022773287054108420378415256832887180469879525130732663402785190594173389203585403956770356113293544825856282876106106982297214209619935093313121711878910787668720445488760894101747986471378824621539559333332755620094395804345379197822805903959599274369137937786649409640487778417483364326840262829324062600819080818043909145563519368560630450891422896452199877988493474777291327972660276584016678901364905087411421268619698620441269652829810870454798615595453380212011556469799767857389201862435993267776894540605082188382279098336271671244900267611784982643770330020818445900097172352043319947082420987715144497510170556430295428218196700092025156158441742059336581481349026931115170938722600264586305613256057925609273322655793462808056834439213736884056504343073965740610177793701414246154930707413608054421002956000956635889778992676305177187819437067614982175641865901161608654086353915130392013168057690341725964536923508064174465623515239290504094799531840748621512105618338545661766526063937136588025216662235761322019417013726649660732520107719479312652827633024138051649071745659648537483546691945235803153019691604809946068149040378198297323609300871357607986214254220964190043679054790499300783724215819545354183711293686584305538427176280352791288211293083515756565999447417884383815651484342298587042455924346932952328218035083337262837918302165918361815542171574484657784201343299825945668845582661719790121808494803324487872581837748055222681510113717453684178702802744524429054745182346749195641885512444213377835214238659799259882032870851093383868299065719946149062902574276860388505110326385445404191849588665385450405713236296810691468148478696591668618427567984600418687622980555629630459532279230516167215919686758495236352989357885077460815373214546429847923105116763577494946229525694976603594739624309953433104049942096778838270027144784940690370732491064441516960532565605867787574174721108274357743151940607579835636291433263978122189462874477981198072256467146640548501310096567863148800903037493388753641831651349825466946733161181233648543976493250261795493572043054021829748712511074040116114058999110930624923128131163405492625713567218186289327861388337180285350565035919527414008695109261675414767926680321092374670872136062783329223864136195941213392780361182763241060047409711110481400036233427145144833346416754663546997314947566434236594934968458845515241507563766050866328274247941360628760412906449138285194564026431532258586240431418386695906332450630003922131926476259626915109044576953014440546180378575030366862124622786397527466678701210033929848733750144756003221006223580293437749550320370127384681630610265703008722754629667968808905871276763610662257223522297392064430935243272281008599730951325286306011054979156447918450046180467624089289256809129305929606423570210615246462050232489665939873249339673769520239917608984745718435319366465291258480644801965201628387951894993367592414856261369959453072872545324632915291101287637706055706095313775277518679232921349552451330898679691651290738413021675732386375758200803635757280027544903279530799007994425411087256931880146679355958346764328688769666100973957499678365933978463469599489506104903836474095046952260638580467580730699122904740898791668721171475276447116044019527181695082897335371485309289370463844208932997711258568408466083399340456890267875160087754612679880154658565220612109534907967073655397025761994313766399606060611064069593308281718764260435734253617569437848484952501082664883951597004905983808121052211110919433239511360514464598342107990580820937164645231277040231600721385437234612672609978703856570919985075956346132484601884098501942876879022687345565005191215465440638292538512763176639220509383452043007730170299403626154340013227639109129883278639204123004455516840548898090807791746360924393349126411642400938807463566072623366958427645836982687348158819610585718357674620096505260659292635482914990457683072108932458570737016607173981944850288426039636607460311847862258310565808708703055675958613417007454029656876347741764310517510367328692455585820823720386017817394051751304379948688223200443780431031709210342616749980000730160948145863744887785222730763304953839443453827706087607635420984450083062476302535727810327834617669705442871553153400164970766571959850417481990872014908756860377835919947193433527729472855379257876848323011018593658007172911869676176550537750302930338307064489128114120255061508964110076238245744886551825810581403453201247547232690875475070785776597325428444593530449920700145387489482265564422236963655441942254413382122254774975354946248276805333369832841561386923634433585538684711114304982483989918031654586382893537991305352228334301379533729540162576232280811384994918761441413229337671065634925288145282395062090223578766846501166600973827536604054469416534222390521083145858470355293522199282727605748212660652913855303455497445514703449394868634294596584310241907859236802245607639367841662705185551787029040735573046206396924533077957822459497104201880430001838814290081730394505073427870131244668600927785818110409115117293748736278878749074652855654347488868310641100510230208751077689187815256227352515503795324448577872776170019648537035551676552091193393437628662846198440262952521836785223674751088097815070989784130862458815226609635514018744958369269177990471207264949057372642860052114035812310760066995185361248627467563758962252991164960668765082617341784847893372950567390078786179253514406210453662506404637288156982323175005962610809219552111508593029556549675388626129723399146283584760486276270273097392020014322487075823373549152460856082103288829741839064788699232736913600488374366152235170584377055452108155133612621429118156153017588825735948925071088792621286413924433093837973338678061317952373152667738208580247014335270092438032669517421195076708843263464427491275589077468635821621660427413151702124585860562336314931646469139465624974717419583542186077487110573384584336899396459137406033821593522435947516262391886853078228217639832373061802042465604775279431047961897242995330297924974816840528937910449470045908649918727273454135081019838818646736093925719305119686456018557824502182310658894379865224320506773799661969554724405859224179530068204517953700434724517628935667705084902131077366257516973355274623029430312035962609534235743972496592110106578178261087453188748031874308235736991951563409571627009924449297491054898515196586647401482251063353679497371425102293418825851173719944991150975837461301055050641977215319293548753711916302620303285886585284801935092258757755974252765840117213423236480840271433563675420463751825525249443296570438613878659019657388028684018940876728167141370336617326501205786539157807030887142615190750014925761129276751930967284539711602136063030905422439663206743235827978893323244057791992784846333397777376559018705748068286783479656241461028995084873996929707504327530299728722973279344429886464127253481606037797072982991730292963086958019963124133049393504933254123550710544611825911411164545347103298810478440677801380771314654000993863064812666143308582068113958383191695455582594268957698414288937434670841079463189325391069639557807060212459748982935646135607889834724199794785643620420946134123876131988653523583129968622689486084084566556068769545012744866314050547353517468730098063227804689122468214608067276277084024022661554850240089528916571176174390203375848778429112896232470591918746910420058483261406773337510271956539946971625172483122306339193287079838007484857265161234349332733566644733585564302352808839243482787608861649432893991663992104883078477770480457284914563033532650700295889062659154985094079727675671297950100982294762289618915914415200322838787734851309790810191292672271037788980539641563623641691549857684083984688616843754070651210390625061281076637990479088796747780697384731704752534421563903872012388063236880370179493089549007763315230635483742568166533616066419800301882871237674818983302468363714883092592833759022789425880600872860388591688497306939480205112217663591382515242786700944069423551202015683777788518246700256517085092496237477268136942843500629388144299879053010562173754591826799732177350293689280652100253962688074980926434580116557158867004435039765053234782873273688408635400027406767838219635222265392909398073673913640828987220177767471681181958561337215831190546829360832369761134502817578302029348459829250008956826302712632958662921476531422333517930933879513570953463771836840924444220963193312956203055755173400679737406141621079236334238056468500920371671526425563718538895714164197723874226105966673969971731681694154350952831935564177056686222152179911513556397071433128936575538446483262012064243380169558626985610224606460693307938478588143674070005997697036490192733288261353293631124036506986521606389872502672380874033967443978302582968942568967418643361349794752455262914265228424192430833881035800537870239995421721136865502753413622116931406946695131869281025747959856051450050217159133177516099578655519818861932112821107094422872404424811534060558959583558152320121846058205635926993034788511320686266275887714460359966561084307256965005630644891875994665967728471715395736121081808415472731426617489331341746326623542220726001460127012069346395205644455432916629866607830890681187900908152950636267820756143888157813511346953663038784120923469428687308393204323338727754968052103028215443247233888452153437272501285897476914608083144041258681815400491877722878698018534545370065266556491709154295227567092222174741120627206566229898060328916720687436549482461086973672255474048128892424718543236057534116728507575520571311566979545848873987422281358879858407831350605482905514827852948911219053831956242287194847594078593980479010941940706717644390327307121358873850499936388382055016834027774960702768448802819122206368886368110435695293006521955282615269912716372773884189932871305634646882273982887631986457098363089177864870866761854856800476725526754147428510281458074031529921978145577568436811101853174981670164266478840902626828244482580275320945499151045185177165463118049045679857132575281179136562781581112888165622858760308759749638494352756766121689592614850307853620452745077529506310124803418045840594329260798544356200937080918215239203717906781219922804960697382387433126267303067959439609549571895772179155973005886936468455766760924509060882022122357192545367151918348725874239194108904441159599327600445065562064611646556654875942473692523369559930303550958176261762318495619064948396730020377638743693439998294302091470736189479326927624451865602395590537051289781634554233201149759948962784243274837880327014186769526211809750064051497558896502930048676052080104915378854139094245316917199876289412772211294645682948602814931815602496778879498137772162293594378110044480607976724292762495107841534464291508427645200020427694706980417758322090970202916573472515829046309103590378429775726517208772447409522671663060054697163879431711968734846887381866567512792985750163634113146275304990191356468238043299706957701507893377286580357127909137674208056554 diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index 431a1c6a..abeabfeb 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -148,6 +148,23 @@ impl LSTMCell { } } +// fn lstm_cell_accumulate(lstm: &LSTMCell, cell: Variable, hidden: Variable, inputs: &[Variable]) -> +// Variable> +// where +// C: Node, +// H: Node, +// I: Node, +// { +// let (head, tail) = inputs.split_at(1); +// let (cell, hidden) = lstm.forward(cell.clone(), hidden.clone(), head.first().unwrap().clone()); + +// if tail.len() == 0 { +// hidden +// } else { +// lstm_cell_accumulate(lstm, cell, hidden, tail) +// } +// } + #[cfg(test)] mod tests { @@ -155,6 +172,16 @@ mod tests { use DataInput; use SGD; + fn pi_digits(num: usize) -> Vec { + let pi_str = include_str!("pi.txt"); + pi_str + .chars() + .filter_map(|x| x.to_digit(10)) + .map(|x| x as usize) + .take(num) + .collect() + } + #[test] fn test_basic_lstm() { let input_dim = 10; @@ -185,46 +212,68 @@ mod tests { } #[test] - fn test_sequential_numbers() { - let max_number = 100; - let num_epochs = 1000; + fn test_pi_digits() { + let num_epochs = 50; + let sequence_length = 4; + let num_digits = 10; let input_dim = 16; let hidden_dim = 32; let lstm_params = LSTMParameters::new(input_dim, hidden_dim); let lstm = lstm_params.build(); - let final_layer = ParameterNode::new(random_matrix(hidden_dim, max_number)); - let embeddings = ParameterNode::new(random_matrix(max_number, input_dim)); - let index = nodes::IndexInputNode::new(&vec![0]); - let mut labels = Arr::zeros((1, max_number)); + let final_layer = ParameterNode::new(random_matrix(hidden_dim, num_digits)); + let embeddings = ParameterNode::new(random_matrix(num_digits, input_dim)); + + let mut labels = Arr::zeros((1, num_digits)); let y = nodes::InputNode::new(labels.clone()); let state = InputNode::new(Arr::zeros((1, hidden_dim))); let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); - let input = embeddings.index(&index); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - //let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let inputs: Vec<_> = (0..sequence_length) + .map(|_| nodes::IndexInputNode::new(&vec![0])) + .collect(); + let embeddings: Vec<_> = inputs + .iter() + .map(|input| embeddings.index(&input)) + .collect(); + + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[0].clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[1].clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[2].clone()); + let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[3].clone()); let prediction = hidden.dot(&final_layer).softmax(); let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); - let mut optimizer = SGD::new(0.5, loss.parameters()); + let mut optimizer = SGD::new(0.05, loss.parameters()); + + let digits = pi_digits(100); + + let mut correct = 0; + let mut total = 0; for _ in 0..num_epochs { let mut loss_val = 0.0; - let mut correct = 0; - let mut total = 0; - for number in 0..max_number { - index.set_value(number); + correct = 0; + total = 0; + + for i in 0..(digits.len() - sequence_length - 1) { + let digit_chunk = &digits[i..(i + sequence_length + 1)]; + if digit_chunk.len() < sequence_length + 1 { + break; + } + + for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { + input.set_value(digit); + } + let target_digit = *digit_chunk.last().unwrap(); labels *= 0.0; - labels[(0, number)] = 1.0; + labels[(0, target_digit)] = 1.0; y.set_value(&labels); @@ -236,7 +285,7 @@ mod tests { optimizer.step(); loss.zero_gradient(); - if number == predicted_label(prediction.value().deref()) { + if target_digit == predicted_label(prediction.value().deref()) { correct += 1; } @@ -249,6 +298,7 @@ mod tests { correct as f32 / total as f32 ); } - } + assert!((correct as f32 / total as f32) > 0.75); + } } From d496c445a7654a4979f280f540cefd7126f01ba0 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 7 Jan 2018 17:21:22 +0000 Subject: [PATCH 026/108] Try to fuse the LSTM nodes. --- src/layers/dense.rs | 84 ++++++++++++++++ src/layers/mod.rs | 11 +++ src/layers/recurrent.rs | 213 +++++++++++++++++++++++++++++++++++++--- src/lib.rs | 4 + src/nodes.rs | 10 +- 5 files changed, 306 insertions(+), 16 deletions(-) create mode 100644 src/layers/dense.rs diff --git a/src/layers/dense.rs b/src/layers/dense.rs new file mode 100644 index 00000000..363c081c --- /dev/null +++ b/src/layers/dense.rs @@ -0,0 +1,84 @@ +use std::cell::{Cell, Ref, RefCell}; +use std::ops::{AddAssign, Deref, DerefMut, DivAssign, MulAssign, SubAssign}; + +use std::sync::Arc; +use std::rc::Rc; + +use rand; +use rand::distributions::{IndependentSample, Normal}; +use ndarray; + +use nodes; +use nodes::{AddNode, BackwardAction, Bor, DotNode, ForwardAction, HogwildParameter, Node, + ParameterNode, PassCounter}; + +use {Arr, Variable}; + +#[derive(Debug)] +pub struct DenseNode +where + OP: Node, +{ + weights: Variable, + biases: Variable, + // activation: Rc< + output: Variable, ParameterNode>>, + needs_gradient: bool, + counter: PassCounter, +} + +impl DenseNode +where + OP: Node, +{ + pub fn new( + weights: Variable, + biases: Variable, + operand: Variable, + ) -> Self { + let result = operand.dot(&weights) + biases.clone(); + + let needs_gradient = + operand.needs_gradient() || weights.needs_gradient() || biases.needs_gradient(); + + Self { + weights: weights, + biases: biases, + output: result, + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } +} + +impl Node for DenseNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + self.output.forward(); + } + + fn backward(&self, gradient: &Ref) { + self.output.node.backward(gradient) + } + + fn value(&self) -> Bor { + self.output.value() + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + + fn zero_gradient(&self) { + self.counter.clear(); + self.output.zero_gradient(); + } +} diff --git a/src/layers/mod.rs b/src/layers/mod.rs index 9998b574..2140be70 100644 --- a/src/layers/mod.rs +++ b/src/layers/mod.rs @@ -1 +1,12 @@ pub mod recurrent; +pub mod dense; + +use rand; +use rand::distributions::{IndependentSample, Normal}; + +use Arr; + +pub fn xavier_normal(rows: usize, cols: usize) -> Arr { + let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); + Arr::zeros((rows, cols)).map(|_| normal.ind_sample(&mut rand::thread_rng()) as f32) +} diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index abeabfeb..1d57258b 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -1,22 +1,20 @@ use std::sync::Arc; -use std::ops::Deref; +use std::rc::Rc; +use std::cell::{Ref}; +use std::ops::{Deref}; use rand; -use rand::distributions::{IndependentSample, Normal}; use ndarray; use nodes; -use nodes::{HogwildParameter, InputNode, Node, ParameterNode}; +use nodes::{HogwildParameter, Node, ParameterNode}; -use {Arr, Variable}; +use layers::xavier_normal; -fn random_matrix(rows: usize, cols: usize) -> Arr { - Arr::zeros((rows, cols)).map(|_| rand::random::()) -} +use {Arr, Bor, Variable}; -fn xavier_normal(rows: usize, cols: usize) -> Arr { - let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); - Arr::zeros((rows, cols)).map(|_| normal.ind_sample(&mut rand::thread_rng()) as f32) +pub fn random_matrix(rows: usize, cols: usize) -> Arr { + Arr::zeros((rows, cols)).map(|_| rand::random::()) } pub struct LSTMParameters { @@ -146,6 +144,196 @@ impl LSTMCell { (cell, hidden) } + + pub fn build( + &self, + cell: Variable, + hidden: Variable, + input: Variable, + ) -> (Variable>, Variable>) + where + C: Node, + H: Node, + I: Node, + { + let stacked_input = hidden.stack(&input, ndarray::Axis(1)); + + // Forget part of the cell state + let forget_gate = + (stacked_input.dot(&self.forget_weights) + self.forget_biases.clone()).sigmoid(); + let cell = forget_gate * cell; + + // Update the cell state with new input + let update_gate = (stacked_input.dot(&self.update_gate_weights) + + self.update_gate_biases.clone()) + .sigmoid(); + let update_value = (stacked_input.dot(&self.update_value_weights) + + self.update_value_biases.clone()) + .tanh(); + let update = update_gate * update_value; + let cell = cell + update; + + // Emit a hidden state + let output_value = cell.tanh(); + let output_gate = (stacked_input.dot(&self.output_gate_weights) + + self.output_gate_biases.clone()) + .sigmoid(); + let hidden = output_gate * output_value; + + let state_node = Rc::new(LSTMCellState { + cell_state: cell.clone(), + }); + let hidden_node = Rc::new(LSTMCellHidden { + hidden_state: hidden.clone(), + }); + + (Variable::new(Rc::clone(&state_node), + cell.parameters.clone()), + Variable::new(Rc::clone(&hidden_node), + hidden.parameters.clone())) + } +} + +#[derive(Debug)] +pub struct LSTMCellState +where + C: Node, + H: Node, + I: Node, +{ + cell_state: Variable< + nodes::AddNode< + nodes::MulNode< + nodes::SigmoidNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + C, + >, + nodes::MulNode< + nodes::SigmoidNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + nodes::TanhNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + >, + >, + >, +} + +#[derive(Debug)] +pub struct LSTMCellHidden +where + C: Node, + H: Node, + I: Node, +{ + hidden_state: Variable< + nodes::MulNode< + nodes::SigmoidNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + nodes::TanhNode< + nodes::AddNode< + nodes::MulNode< + nodes::SigmoidNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + C, + >, + nodes::MulNode< + nodes::SigmoidNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + nodes::TanhNode< + nodes::AddNode< + nodes::DotNode, nodes::ParameterNode>, + nodes::ParameterNode, + >, + >, + >, + >, + >, + >, + >, +} + +impl Node for LSTMCellState + where + C: Node, + H: Node, + I: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.cell_state.forward(); + } + + fn backward(&self, gradient: &Ref) { + self.cell_state.node.backward(gradient) + } + + fn value(&self) -> Bor { + self.cell_state.value() + } + + fn needs_gradient(&self) -> bool { + self.cell_state.needs_gradient() + } + + fn zero_gradient(&self) { + self.cell_state.zero_gradient(); + } +} + +impl Node for LSTMCellHidden + where + C: Node, + H: Node, + I: Node, +{ + type Value = Arr; + type InputGradient = Arr; + type OutputGradient = Arr; + fn forward(&self) { + self.hidden_state.forward(); + } + + fn backward(&self, gradient: &Ref) { + self.hidden_state.node.backward(gradient) + } + + fn value(&self) -> Bor { + self.hidden_state.value() + } + + fn needs_gradient(&self) -> bool { + self.hidden_state.needs_gradient() + } + + fn zero_gradient(&self) { + self.hidden_state.zero_gradient(); + } } // fn lstm_cell_accumulate(lstm: &LSTMCell, cell: Variable, hidden: Variable, inputs: &[Variable]) -> @@ -168,9 +356,12 @@ impl LSTMCell { #[cfg(test)] mod tests { + use std::ops::Deref; + use super::*; use DataInput; use SGD; + use nodes::InputNode; fn pi_digits(num: usize) -> Vec { let pi_str = include_str!("pi.txt"); @@ -196,7 +387,7 @@ mod tests { let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let (_, hidden) = lstm.build(state.clone(), hidden.clone(), input.clone()); hidden.zero_gradient(); hidden.forward(); diff --git a/src/lib.rs b/src/lib.rs index 86e4dda7..f9f1ee19 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -241,6 +241,10 @@ where self.node.zero_gradient(); } + pub fn needs_gradient(&self) -> bool { + self.node.needs_gradient() + } + /// Return the parameters of the graph. pub fn parameters(&self) -> Vec> { let mut unique_params = self.parameters.clone(); diff --git a/src/nodes.rs b/src/nodes.rs index af217e1b..e946410e 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -28,23 +28,23 @@ pub enum BackwardAction { } #[derive(Debug, Default)] -struct PassCounter { +pub struct PassCounter { forward_count: Cell, backward_count: Cell, } impl PassCounter { - fn clear(&self) { + pub fn clear(&self) { self.forward_count.set(0); self.backward_count.set(0); } - fn recurse_backward(&self) -> bool { + pub fn recurse_backward(&self) -> bool { let backward_count = self.backward_count.get(); let forward_count = self.forward_count.get(); backward_count == forward_count } - fn forward(&self) -> ForwardAction { + pub fn forward(&self) -> ForwardAction { let count = self.forward_count.get(); self.forward_count.set(count + 1); @@ -53,7 +53,7 @@ impl PassCounter { _ => ForwardAction::Cached, } } - fn backward(&self) -> BackwardAction { + pub fn backward(&self) -> BackwardAction { let backward_count = self.backward_count.get(); let action = match backward_count { From 13ece9f2d33c637ccb4396815befe538b06278ca Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 7 Jan 2018 19:51:38 +0000 Subject: [PATCH 027/108] Box nodes to manage type complexity. --- src/layers/dense.rs | 1 - src/layers/recurrent.rs | 210 +++------------------------------------- src/lib.rs | 9 ++ src/nodes.rs | 45 ++++----- 4 files changed, 42 insertions(+), 223 deletions(-) diff --git a/src/layers/dense.rs b/src/layers/dense.rs index 363c081c..d3939980 100644 --- a/src/layers/dense.rs +++ b/src/layers/dense.rs @@ -57,7 +57,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index 1d57258b..64608f9d 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::rc::Rc; -use std::cell::{Ref}; -use std::ops::{Deref}; +use std::cell::Ref; +use std::ops::Deref; use rand; use ndarray; @@ -110,8 +110,8 @@ impl LSTMCell { hidden: Variable, input: Variable, ) -> ( - Variable>, - Variable>, + Variable>>, + Variable>>, ) where C: Node, @@ -142,197 +142,7 @@ impl LSTMCell { .sigmoid(); let hidden = output_gate * output_value; - (cell, hidden) - } - - pub fn build( - &self, - cell: Variable, - hidden: Variable, - input: Variable, - ) -> (Variable>, Variable>) - where - C: Node, - H: Node, - I: Node, - { - let stacked_input = hidden.stack(&input, ndarray::Axis(1)); - - // Forget part of the cell state - let forget_gate = - (stacked_input.dot(&self.forget_weights) + self.forget_biases.clone()).sigmoid(); - let cell = forget_gate * cell; - - // Update the cell state with new input - let update_gate = (stacked_input.dot(&self.update_gate_weights) - + self.update_gate_biases.clone()) - .sigmoid(); - let update_value = (stacked_input.dot(&self.update_value_weights) - + self.update_value_biases.clone()) - .tanh(); - let update = update_gate * update_value; - let cell = cell + update; - - // Emit a hidden state - let output_value = cell.tanh(); - let output_gate = (stacked_input.dot(&self.output_gate_weights) - + self.output_gate_biases.clone()) - .sigmoid(); - let hidden = output_gate * output_value; - - let state_node = Rc::new(LSTMCellState { - cell_state: cell.clone(), - }); - let hidden_node = Rc::new(LSTMCellHidden { - hidden_state: hidden.clone(), - }); - - (Variable::new(Rc::clone(&state_node), - cell.parameters.clone()), - Variable::new(Rc::clone(&hidden_node), - hidden.parameters.clone())) - } -} - -#[derive(Debug)] -pub struct LSTMCellState -where - C: Node, - H: Node, - I: Node, -{ - cell_state: Variable< - nodes::AddNode< - nodes::MulNode< - nodes::SigmoidNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - C, - >, - nodes::MulNode< - nodes::SigmoidNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - nodes::TanhNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - >, - >, - >, -} - -#[derive(Debug)] -pub struct LSTMCellHidden -where - C: Node, - H: Node, - I: Node, -{ - hidden_state: Variable< - nodes::MulNode< - nodes::SigmoidNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - nodes::TanhNode< - nodes::AddNode< - nodes::MulNode< - nodes::SigmoidNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - C, - >, - nodes::MulNode< - nodes::SigmoidNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - nodes::TanhNode< - nodes::AddNode< - nodes::DotNode, nodes::ParameterNode>, - nodes::ParameterNode, - >, - >, - >, - >, - >, - >, - >, -} - -impl Node for LSTMCellState - where - C: Node, - H: Node, - I: Node, -{ - type Value = Arr; - type InputGradient = Arr; - type OutputGradient = Arr; - fn forward(&self) { - self.cell_state.forward(); - } - - fn backward(&self, gradient: &Ref) { - self.cell_state.node.backward(gradient) - } - - fn value(&self) -> Bor { - self.cell_state.value() - } - - fn needs_gradient(&self) -> bool { - self.cell_state.needs_gradient() - } - - fn zero_gradient(&self) { - self.cell_state.zero_gradient(); - } -} - -impl Node for LSTMCellHidden - where - C: Node, - H: Node, - I: Node, -{ - type Value = Arr; - type InputGradient = Arr; - type OutputGradient = Arr; - fn forward(&self) { - self.hidden_state.forward(); - } - - fn backward(&self, gradient: &Ref) { - self.hidden_state.node.backward(gradient) - } - - fn value(&self) -> Bor { - self.hidden_state.value() - } - - fn needs_gradient(&self) -> bool { - self.hidden_state.needs_gradient() - } - - fn zero_gradient(&self) { - self.hidden_state.zero_gradient(); + (cell.boxed(), hidden.boxed()) } } @@ -387,7 +197,7 @@ mod tests { let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - let (_, hidden) = lstm.build(state.clone(), hidden.clone(), input.clone()); + let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); hidden.zero_gradient(); hidden.forward(); @@ -406,7 +216,7 @@ mod tests { fn test_pi_digits() { let num_epochs = 50; - let sequence_length = 4; + let sequence_length = 8; let num_digits = 10; let input_dim = 16; let hidden_dim = 32; @@ -434,7 +244,11 @@ mod tests { let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[0].clone()); let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[1].clone()); let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[2].clone()); - let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[3].clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[3].clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[4].clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[5].clone()); + let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[6].clone()); + let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[7].clone()); let prediction = hidden.dot(&final_layer).softmax(); let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); diff --git a/src/lib.rs b/src/lib.rs index f9f1ee19..39c88e08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -262,6 +262,15 @@ impl Variable where T: Node, { + /// Box the variable, erasing its specific type. Use to manage the complexity + /// of variable types in deep computation graphs. + pub fn boxed(&self) -> Variable>> { + Variable::new( + Rc::new(self.node.clone() as Rc>), + self.parameters.clone(), + ) + } + /// Run the backward pass through the subgraph terminating at this node. /// The weight parameter scales the gradients. pub fn backward(&mut self, weight: f32) { diff --git a/src/nodes.rs b/src/nodes.rs index e946410e..4509f09a 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -93,15 +93,12 @@ impl<'value, T: 'value + fmt::Display> fmt::Display for Bor<'value, T> { /// Trait representing a computation node. Structs implementing /// this trait can be used as elements of the computation graph. -pub trait Node: fmt::Debug + Sized { +pub trait Node: fmt::Debug + 'static { /// Type of the node's value. type Value; /// Type of the input gradient the node receives /// during backpropagation. type InputGradient; - /// Type of the gradient the node passes down - /// to its ancestors during backpropagation. - type OutputGradient; /// Perform the forward step. Should recursively call /// the forward methods of its ancestors. fn forward(&self); @@ -115,6 +112,26 @@ pub trait Node: fmt::Debug + Sized { fn zero_gradient(&self); } +impl Node for Rc> { + type Value = Arr; + type InputGradient = Arr; + fn forward(&self) { + self.deref().forward() + } + fn backward(&self, gradient: &Ref) { + self.deref().backward(gradient) + } + fn value(&self) -> Bor { + self.deref().value() + } + fn needs_gradient(&self) -> bool { + self.deref().needs_gradient() + } + fn zero_gradient(&self) { + self.deref().zero_gradient() + } +} + #[derive(Debug)] pub struct AddNode { value: RefCell, @@ -150,7 +167,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -328,7 +344,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -418,7 +433,6 @@ impl InputNode { impl Node for InputNode { type Value = Arr; type InputGradient = Arr; - type OutputGradient = (); fn forward(&self) {} fn backward(&self, _: &Ref) {} fn value(&self) -> Bor { @@ -560,7 +574,6 @@ impl ParameterNode { impl Node for ParameterNode { type Value = Arr; type InputGradient = Arr; - type OutputGradient = (); fn forward(&self) {} fn backward(&self, gradient: &Ref) { self.gradient.borrow_mut().accumulate_gradient(gradient); @@ -619,7 +632,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -709,7 +721,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -814,7 +825,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -931,7 +941,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { @@ -1054,7 +1063,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { @@ -1206,7 +1214,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1290,7 +1297,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1375,7 +1381,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1460,7 +1465,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1552,7 +1556,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { @@ -1635,7 +1638,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1715,7 +1717,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1803,7 +1804,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1905,7 +1905,6 @@ where { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; @@ -1969,7 +1968,6 @@ impl IndexInputNode { impl Node for IndexInputNode { type Value = SmallVec<[usize; 4]>; type InputGradient = Arr; - type OutputGradient = (); fn forward(&self) {} fn backward(&self, _: &Ref) {} fn value(&self) -> Bor { @@ -2017,7 +2015,6 @@ where impl Node for IndexNode { type Value = Arr; type InputGradient = Arr; - type OutputGradient = Arr; fn forward(&self) { if self.counter.forward() == ForwardAction::Cached { return; From ca7e780892f2e044ed24838735391f30783e8c0f Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 7 Jan 2018 23:09:01 +0000 Subject: [PATCH 028/108] Update tests. --- src/layers/dense.rs | 83 ----------------------------------------- src/layers/mod.rs | 1 - src/layers/recurrent.rs | 24 ++++++------ src/lib.rs | 7 +++- 4 files changed, 18 insertions(+), 97 deletions(-) delete mode 100644 src/layers/dense.rs diff --git a/src/layers/dense.rs b/src/layers/dense.rs deleted file mode 100644 index d3939980..00000000 --- a/src/layers/dense.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::cell::{Cell, Ref, RefCell}; -use std::ops::{AddAssign, Deref, DerefMut, DivAssign, MulAssign, SubAssign}; - -use std::sync::Arc; -use std::rc::Rc; - -use rand; -use rand::distributions::{IndependentSample, Normal}; -use ndarray; - -use nodes; -use nodes::{AddNode, BackwardAction, Bor, DotNode, ForwardAction, HogwildParameter, Node, - ParameterNode, PassCounter}; - -use {Arr, Variable}; - -#[derive(Debug)] -pub struct DenseNode -where - OP: Node, -{ - weights: Variable, - biases: Variable, - // activation: Rc< - output: Variable, ParameterNode>>, - needs_gradient: bool, - counter: PassCounter, -} - -impl DenseNode -where - OP: Node, -{ - pub fn new( - weights: Variable, - biases: Variable, - operand: Variable, - ) -> Self { - let result = operand.dot(&weights) + biases.clone(); - - let needs_gradient = - operand.needs_gradient() || weights.needs_gradient() || biases.needs_gradient(); - - Self { - weights: weights, - biases: biases, - output: result, - needs_gradient: needs_gradient, - counter: PassCounter::default(), - } - } -} - -impl Node for DenseNode -where - OP: Node, -{ - type Value = Arr; - type InputGradient = Arr; - fn forward(&self) { - if self.counter.forward() == ForwardAction::Cached { - return; - } - self.output.forward(); - } - - fn backward(&self, gradient: &Ref) { - self.output.node.backward(gradient) - } - - fn value(&self) -> Bor { - self.output.value() - } - - fn needs_gradient(&self) -> bool { - self.needs_gradient - } - - fn zero_gradient(&self) { - self.counter.clear(); - self.output.zero_gradient(); - } -} diff --git a/src/layers/mod.rs b/src/layers/mod.rs index 2140be70..707bb8f4 100644 --- a/src/layers/mod.rs +++ b/src/layers/mod.rs @@ -1,5 +1,4 @@ pub mod recurrent; -pub mod dense; use rand; use rand::distributions::{IndependentSample, Normal}; diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index 64608f9d..0c073b83 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -1,7 +1,5 @@ use std::sync::Arc; use std::rc::Rc; -use std::cell::Ref; -use std::ops::Deref; use rand; use ndarray; @@ -11,7 +9,7 @@ use nodes::{HogwildParameter, Node, ParameterNode}; use layers::xavier_normal; -use {Arr, Bor, Variable}; +use {Arr, Variable}; pub fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) @@ -149,6 +147,7 @@ impl LSTMCell { // fn lstm_cell_accumulate(lstm: &LSTMCell, cell: Variable, hidden: Variable, inputs: &[Variable]) -> // Variable> // where +// // C: Node, // H: Node, // I: Node, @@ -195,9 +194,14 @@ mod tests { let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); let input = InputNode::new(random_matrix(1, input_dim)); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); - let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + let (mut state, mut hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + + // Test a deep RNN + for _ in 0..10 { + let step = lstm.forward(state.clone(), hidden.clone(), input.clone()); + state = step.0; + hidden = step.1; + } hidden.zero_gradient(); hidden.forward(); @@ -216,7 +220,7 @@ mod tests { fn test_pi_digits() { let num_epochs = 50; - let sequence_length = 8; + let sequence_length = 4; let num_digits = 10; let input_dim = 16; let hidden_dim = 32; @@ -244,11 +248,7 @@ mod tests { let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[0].clone()); let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[1].clone()); let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[2].clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[3].clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[4].clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[5].clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[6].clone()); - let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[7].clone()); + let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[3].clone()); let prediction = hidden.dot(&final_layer).softmax(); let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); diff --git a/src/lib.rs b/src/lib.rs index 39c88e08..47399602 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -190,7 +190,12 @@ pub trait DataInput { } fn merge_parameters(xs: &[Rc], ys: &[Rc]) -> Vec> { - xs.iter().chain(ys.iter()).cloned().collect() + let mut unique_params: Vec<_> = xs.iter().chain(ys.iter()).cloned().collect(); + + unique_params.sort_unstable_by_key(|x| x.deref() as *const ParameterNode); + unique_params.dedup_by_key(|x| (*x).deref() as *const ParameterNode); + + unique_params } /// Handle to a node in the computation graph. The underlying nodes From e586ffd08cc154fdd5f70094370123a97f3f874c Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 8 Jan 2018 22:15:57 +0000 Subject: [PATCH 029/108] Solve problems with zeroing gradients in deep graphs. --- src/layers/recurrent.rs | 48 +++++++-------- src/nodes.rs | 127 ++++++++++++++++++++++++++-------------- 2 files changed, 106 insertions(+), 69 deletions(-) diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index 0c073b83..fb2f538c 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -1,3 +1,5 @@ +//! Module holding building blocks for recurrent neural networks. + use std::sync::Arc; use std::rc::Rc; @@ -15,6 +17,10 @@ pub fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) } +/// Holds shared parameters for an LSTM cell. +/// +/// Construct this first, then use the `build` method to instantiate +/// LSTM cell nodes. pub struct LSTMParameters { input_dim: usize, hidden_dim: usize, @@ -33,6 +39,7 @@ pub struct LSTMParameters { } impl LSTMParameters { + /// Create a new LSTM parameters object. pub fn new(input_dim: usize, hidden_dim: usize) -> Self { Self { input_dim: input_dim, @@ -64,6 +71,7 @@ impl LSTMParameters { } } + /// Build an LSTM cell. pub fn build(&self) -> LSTMCell { LSTMCell { _input_dim: self.input_dim, @@ -84,6 +92,7 @@ impl LSTMParameters { } } +/// An LSTM cell. pub struct LSTMCell { _input_dim: usize, _hidden_dim: usize, @@ -102,6 +111,10 @@ pub struct LSTMCell { } impl LSTMCell { + /// Run a single LSTM iteration over inputs. + /// + /// If this is the first cell, initialize the cell state and the hidden state; + /// otherwise pass the cell and hidden states from previous iterations. pub fn forward( &self, cell: Variable, @@ -144,24 +157,6 @@ impl LSTMCell { } } -// fn lstm_cell_accumulate(lstm: &LSTMCell, cell: Variable, hidden: Variable, inputs: &[Variable]) -> -// Variable> -// where -// -// C: Node, -// H: Node, -// I: Node, -// { -// let (head, tail) = inputs.split_at(1); -// let (cell, hidden) = lstm.forward(cell.clone(), hidden.clone(), head.first().unwrap().clone()); - -// if tail.len() == 0 { -// hidden -// } else { -// lstm_cell_accumulate(lstm, cell, hidden, tail) -// } -// } - #[cfg(test)] mod tests { @@ -197,14 +192,14 @@ mod tests { let (mut state, mut hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); // Test a deep RNN - for _ in 0..10 { + for _ in 0..200 { let step = lstm.forward(state.clone(), hidden.clone(), input.clone()); state = step.0; hidden = step.1; } - hidden.zero_gradient(); hidden.forward(); + hidden.zero_gradient(); } fn predicted_label(softmax_output: &Arr) -> usize { @@ -245,10 +240,15 @@ mod tests { .map(|input| embeddings.index(&input)) .collect(); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[0].clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[1].clone()); - let (state, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[2].clone()); - let (_, hidden) = lstm.forward(state.clone(), hidden.clone(), embeddings[3].clone()); + let (mut state, mut hidden) = + lstm.forward(state.clone(), hidden.clone(), embeddings[0].clone()); + + for i in 1..sequence_length { + let out = lstm.forward(state.clone(), hidden.clone(), embeddings[i].clone()); + + state = out.0; + hidden = out.1; + } let prediction = hidden.dot(&final_layer).softmax(); let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); diff --git a/src/nodes.rs b/src/nodes.rs index 4509f09a..8678066e 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -38,6 +38,9 @@ impl PassCounter { self.forward_count.set(0); self.backward_count.set(0); } + pub fn is_zero(&self) -> bool { + self.forward_count.get() == 0 + } pub fn recurse_backward(&self) -> bool { let backward_count = self.backward_count.get(); let forward_count = self.forward_count.get(); @@ -209,9 +212,11 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -405,9 +410,11 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -565,10 +572,10 @@ impl ParameterNode { Variable::new(node, params) } - /// Zero the accumulated gradients of this node. - pub fn zero_gradient(&self) { - self.gradient.borrow_mut().zero_gradient(); - } + // /// Zero the accumulated gradients of this node. + // pub fn zero_gradient(&self) { + // //self.gradient.borrow_mut().zero_gradient(); + // } } impl Node for ParameterNode { @@ -673,9 +680,11 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -777,9 +786,11 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -893,9 +904,11 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -987,9 +1000,11 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -1174,9 +1189,11 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.rhs.zero_gradient(); + self.counter.clear(); + } } } @@ -1258,8 +1275,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1342,8 +1361,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1426,8 +1447,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1517,8 +1540,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1599,8 +1624,10 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1678,8 +1705,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1749,8 +1778,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1861,8 +1892,10 @@ where self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -1941,8 +1974,10 @@ where } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } @@ -2056,8 +2091,10 @@ impl Node for IndexNode { self.needs_gradient } fn zero_gradient(&self) { - self.counter.clear(); - self.operand.zero_gradient(); + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } } } From f2c4cf85d69867432f7f94cfe1d183b7b765b51b Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 10 Jan 2018 20:08:51 +0000 Subject: [PATCH 030/108] Add docs for LSTM layers. --- src/layers/mod.rs | 3 ++ src/layers/recurrent.rs | 96 +++++++++++++++++++++++++++++++---------- src/lib.rs | 5 +-- 3 files changed, 79 insertions(+), 25 deletions(-) diff --git a/src/layers/mod.rs b/src/layers/mod.rs index 707bb8f4..a10a4204 100644 --- a/src/layers/mod.rs +++ b/src/layers/mod.rs @@ -1,3 +1,5 @@ +//! Neural network layers. + pub mod recurrent; use rand; @@ -5,6 +7,7 @@ use rand::distributions::{IndependentSample, Normal}; use Arr; +/// Return a Xavier-normal initialised random array. pub fn xavier_normal(rows: usize, cols: usize) -> Arr { let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); Arr::zeros((rows, cols)).map(|_| normal.ind_sample(&mut rand::thread_rng()) as f32) diff --git a/src/layers/recurrent.rs b/src/layers/recurrent.rs index fb2f538c..7bf91eab 100644 --- a/src/layers/recurrent.rs +++ b/src/layers/recurrent.rs @@ -1,9 +1,58 @@ //! Module holding building blocks for recurrent neural networks. - +//! +//! You can create an LSTM layer by repeatedly applying an LSTM cell +//! to your inputs: +//! +//! ```rust +//! # extern crate rand; +//! # extern crate wyrm; +//! # extern crate ndarray; +//! # use std::sync::Arc; +//! # use std::rc::Rc; +//! # +//! # use wyrm::{HogwildParameter, InputNode, Node, ParameterNode}; +//! # +//! # use wyrm::layers::xavier_normal; +//! # use wyrm::layers::recurrent::*; +//! # +//! # use wyrm::{Arr, Variable}; +//! # fn main() { +//! let input_dim = 10; +//! let hidden_dim = 5; +//! +//! // Initialize the parameters. +//! let lstm_params = LSTMParameters::new(input_dim, hidden_dim); +//! let lstm = lstm_params.build(); +//! +//! // Initialize the cell state and hidden state. +//! let state = InputNode::new(Arr::zeros((1, hidden_dim))); +//! let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); +//! +//! // Construct the input node. +//! let input = InputNode::new(xavier_normal(1, input_dim)); +//! +//! // The forward method outputs a tuple of (cell_state, hidden_state). +//! let mut state = lstm.forward((state, hidden), input.clone()); +//! +//! // Construct an LSTM with 200 steps of recursion. Usually +//! // we'd use different inputs for every step, but here we re-use +//! // the same input for simplicity. +//! for _ in 0..200 { +//! state = lstm.forward(state.clone(), input.clone()); +//! } +//! +//! // Unpack the cell and hidden state. +//! let (_, mut hidden) = state; +//! +//! // Run as usual. +//! hidden.forward(); +//! hidden.backward(1.0); +//! hidden.zero_gradient(); +//! # } +//! ``` use std::sync::Arc; use std::rc::Rc; -use rand; use ndarray; use nodes; @@ -13,10 +62,6 @@ use layers::xavier_normal; use {Arr, Variable}; -pub fn random_matrix(rows: usize, cols: usize) -> Arr { - Arr::zeros((rows, cols)).map(|_| rand::random::()) -} - /// Holds shared parameters for an LSTM cell. /// /// Construct this first, then use the `build` method to instantiate @@ -117,8 +162,7 @@ impl LSTMCell { /// otherwise pass the cell and hidden states from previous iterations. pub fn forward( &self, - cell: Variable, - hidden: Variable, + state: (Variable, Variable), input: Variable, ) -> ( Variable>>, @@ -129,6 +173,8 @@ impl LSTMCell { H: Node, I: Node, { + let (cell, hidden) = state; + let stacked_input = hidden.stack(&input, ndarray::Axis(1)); // Forget part of the cell state @@ -182,23 +228,31 @@ mod tests { let input_dim = 10; let hidden_dim = 5; + // Initialize the parameters. let lstm_params = LSTMParameters::new(input_dim, hidden_dim); let lstm = lstm_params.build(); + // Initialize the cell state and hidden state. let state = InputNode::new(Arr::zeros((1, hidden_dim))); let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); - let input = InputNode::new(random_matrix(1, input_dim)); - let (mut state, mut hidden) = lstm.forward(state.clone(), hidden.clone(), input.clone()); + // Construct the input node. + let input = InputNode::new(xavier_normal(1, input_dim)); - // Test a deep RNN + // The forward method outputs a tuple of (cell_state, hidden_state). + let mut state = lstm.forward((state, hidden), input.clone()); + + // Construct a deep RNN. for _ in 0..200 { - let step = lstm.forward(state.clone(), hidden.clone(), input.clone()); - state = step.0; - hidden = step.1; + state = lstm.forward(state.clone(), input.clone()); } + // Unpack the cell and hidden state. + let (_, mut hidden) = state; + + // Run as usual. hidden.forward(); + hidden.backward(1.0); hidden.zero_gradient(); } @@ -223,8 +277,8 @@ mod tests { let lstm_params = LSTMParameters::new(input_dim, hidden_dim); let lstm = lstm_params.build(); - let final_layer = ParameterNode::new(random_matrix(hidden_dim, num_digits)); - let embeddings = ParameterNode::new(random_matrix(num_digits, input_dim)); + let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); + let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); let mut labels = Arr::zeros((1, num_digits)); let y = nodes::InputNode::new(labels.clone()); @@ -240,16 +294,14 @@ mod tests { .map(|input| embeddings.index(&input)) .collect(); - let (mut state, mut hidden) = - lstm.forward(state.clone(), hidden.clone(), embeddings[0].clone()); + let mut state = lstm.forward((state, hidden), embeddings[0].clone()); for i in 1..sequence_length { - let out = lstm.forward(state.clone(), hidden.clone(), embeddings[i].clone()); - - state = out.0; - hidden = out.1; + state = lstm.forward(state.clone(), embeddings[i].clone()); } + let (_, hidden) = state; + let prediction = hidden.dot(&final_layer).softmax(); let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); diff --git a/src/lib.rs b/src/lib.rs index 47399602..346aecb7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -543,9 +543,8 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = unsafe { - &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) - }; + let mut param_value = + unsafe { &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); From 2272edfa52663b3bf5027dc57ca1b99c284b6789 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 10 Jan 2018 20:09:04 +0000 Subject: [PATCH 031/108] Head off potential UB in HogwildParameters. --- src/lib.rs | 6 ++---- src/nodes.rs | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 346aecb7..d994c93b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -423,8 +423,7 @@ impl Variable { impl<'value> DataInput<&'value Arr> for Variable { fn set_value(&self, value: &Arr) { - let param_value = - unsafe { &mut *(&self.node.value.deref().value as *const Arr as *mut Arr) }; + let param_value = unsafe { &mut *(self.node.value.deref().value.as_ptr()) }; param_value.assign(value) } } @@ -543,8 +542,7 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = - unsafe { &mut *(¶meter.node.value.deref().value as *const Arr as *mut Arr) }; + let mut param_value = unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); diff --git a/src/nodes.rs b/src/nodes.rs index 8678066e..f868a512 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -528,13 +528,15 @@ unsafe impl Sync for HogwildParameter {} /// multiple `ParameterNode`s for asynchronous, parallel optimization. #[derive(Debug, Serialize, Deserialize)] pub struct HogwildParameter { - pub value: Arr, + pub value: RefCell, } impl HogwildParameter { /// Create a new parameter object. pub fn new(value: Arr) -> Self { - HogwildParameter { value: value } + HogwildParameter { + value: RefCell::new(value), + } } } @@ -549,7 +551,14 @@ impl ParameterNode { /// Create a parameter node that shares its parameter values /// with other parameter nodes via the `HogwildParameter` object. pub fn shared(value: Arc) -> Variable { - let shape = (value.value.rows(), value.value.cols()); + let shape = unsafe { + // This method can be called in multiple threads, so borrowing + // (even immutably) will read to borrow failures. + ( + (*value.value.as_ptr()).rows(), + (*value.value.as_ptr()).cols(), + ) + }; let node = Rc::new(ParameterNode { value: value, @@ -586,7 +595,7 @@ impl Node for ParameterNode { self.gradient.borrow_mut().accumulate_gradient(gradient); } fn value(&self) -> Bor { - Bor::Reference(&self.value.value) + Bor::Reference(unsafe { &*(self.value.value.as_ptr() as *const Arr) }) } fn needs_gradient(&self) -> bool { true From 43c63121f3b1d0380e195ef97d4d8e8df1239754 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 10 Jan 2018 21:44:15 +0000 Subject: [PATCH 032/108] Bump to 0.4.0. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8d1451ce..42f73850 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.3.1" +version = "0.4.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 6a3078f29f30cbdd565cd106f73d389934e541a6 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 13 Jan 2018 09:48:43 +0000 Subject: [PATCH 033/108] Speed up softmax. --- src/lib.rs | 1 - src/nodes.rs | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d994c93b..e54a9e00 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(conservative_impl_trait)] #![feature(test)] //! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! diff --git a/src/nodes.rs b/src/nodes.rs index f868a512..9f15dd22 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1873,7 +1873,10 @@ where .zip(value.iter()) .enumerate() { - for (col_idx, (grad, col_val)) in row.iter_mut().zip(value.iter()).enumerate() { + for (col_idx, (grad, col_val)) in row.as_slice_mut() + .unwrap() + .iter_mut() + .zip(value.as_slice().unwrap()).enumerate() { if row_idx == col_idx { *grad = row_val * (1.0 - col_val); } else { From d3f980fa4950dc4e0428405c2f4466c9ae34b27d Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 13 Jan 2018 09:52:11 +0000 Subject: [PATCH 034/108] Move layers to nn --- src/lib.rs | 2 +- src/{layers => nn}/mod.rs | 0 src/{layers => nn}/pi.txt | 0 src/{layers => nn}/recurrent.rs | 6 +++--- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/{layers => nn}/mod.rs (100%) rename src/{layers => nn}/pi.txt (100%) rename src/{layers => nn}/recurrent.rs (99%) diff --git a/src/lib.rs b/src/lib.rs index e54a9e00..16a7dffe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -165,7 +165,7 @@ use std::clone::Clone; mod nodes; mod numerics; -pub mod layers; +pub mod nn; use nodes::*; diff --git a/src/layers/mod.rs b/src/nn/mod.rs similarity index 100% rename from src/layers/mod.rs rename to src/nn/mod.rs diff --git a/src/layers/pi.txt b/src/nn/pi.txt similarity index 100% rename from src/layers/pi.txt rename to src/nn/pi.txt diff --git a/src/layers/recurrent.rs b/src/nn/recurrent.rs similarity index 99% rename from src/layers/recurrent.rs rename to src/nn/recurrent.rs index 7bf91eab..3788a49e 100644 --- a/src/layers/recurrent.rs +++ b/src/nn/recurrent.rs @@ -12,8 +12,8 @@ //! # //! # use wyrm::{HogwildParameter, InputNode, Node, ParameterNode}; //! # -//! # use wyrm::layers::xavier_normal; -//! # use wyrm::layers::recurrent::*; +//! # use wyrm::nn::xavier_normal; +//! # use wyrm::nn::recurrent::*; //! # //! # use wyrm::{Arr, Variable}; //! # fn main() { @@ -58,7 +58,7 @@ use ndarray; use nodes; use nodes::{HogwildParameter, Node, ParameterNode}; -use layers::xavier_normal; +use nn::xavier_normal; use {Arr, Variable}; From c4799a92bfe9fdff0850762e7e4a2f9838802c16 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 13 Jan 2018 11:00:48 +0000 Subject: [PATCH 035/108] Add log-softmax node. --- src/lib.rs | 22 ++++++++- src/nn/mod.rs | 3 +- src/nodes.rs | 117 +++++++++++++++++++++++++++++++++++++++++++++++- src/numerics.rs | 27 ++++++++++- 4 files changed, 165 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 16a7dffe..c6615d35 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -344,6 +344,14 @@ where ) } + /// Compute the log-softmax of this variable. + pub fn log_softmax(&self) -> Variable> { + Variable::new( + Rc::new(LogSoftmaxNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) + } + /// Compute the sigmoid of this variable. pub fn sigmoid(&self) -> Variable> { Variable::new( @@ -541,7 +549,8 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; + let mut param_value = + unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); @@ -784,6 +793,17 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn log_softmax_finite_difference() { + let mut x = ParameterNode::new(random_matrix(1, 10)); + let mut z = (x.clone() + x.clone()).log_softmax(); + let v = (x.clone() + x.clone()).softmax().ln(); + + assert_close(v.value().deref(), z.value().deref(), TOLERANCE); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] fn rowwise_stack_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut y = ParameterNode::new(random_matrix(10, 5)); diff --git a/src/nn/mod.rs b/src/nn/mod.rs index a10a4204..d25bdb9e 100644 --- a/src/nn/mod.rs +++ b/src/nn/mod.rs @@ -1,5 +1,6 @@ -//! Neural network layers. +//! Neural network components. +pub mod losses; pub mod recurrent; use rand; diff --git a/src/nodes.rs b/src/nodes.rs index 9f15dd22..96012cc5 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1876,7 +1876,9 @@ where for (col_idx, (grad, col_val)) in row.as_slice_mut() .unwrap() .iter_mut() - .zip(value.as_slice().unwrap()).enumerate() { + .zip(value.as_slice().unwrap()) + .enumerate() + { if row_idx == col_idx { *grad = row_val * (1.0 - col_val); } else { @@ -1911,6 +1913,119 @@ where } } +#[derive(Debug)] +pub struct LogSoftmaxNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, + counter: PassCounter, +} + +impl LogSoftmaxNode +where + OP: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = { + let operand_value = operand.value(); + let operand_slice = operand_value.deref().as_slice().unwrap(); + let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); + + let denominator = max + + operand_slice + .iter() + .map(|&x| (x - max).exp()) + .sum::() + .ln(); + + operand_value.deref() - denominator + }; + + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + let dim = value.shape()[1]; + + LogSoftmaxNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } +} + +impl Node for LogSoftmaxNode +where + OP: Node, +{ + type Value = Arr; + type InputGradient = Arr; + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + + self.operand.forward(); + let mut dest = self.value.borrow_mut(); + dest.assign(self.operand.value().deref()); + + let operand_value = self.operand.value(); + let operand_slice = operand_value.deref().as_slice().unwrap(); + let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); + + let denominator = max + + operand_slice + .iter() + .map(|&x| (x - max).exp()) + .sum::() + .ln(); + + dest.as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x -= denominator); + } + fn backward(&self, gradient: &Ref) { + // TODO: accumulate gradients + { + let value = self.value.borrow(); + let value_slice = value.as_slice().expect("Can't get value slice."); + + let gradient_slice = gradient + .as_slice() + .expect("Can't get input gradient slice."); + let mut downstream_gradient = self.operand_gradient.borrow_mut(); + let downstream_gradient_slice = downstream_gradient + .as_slice_mut() + .expect("Can't get output gradient slice"); + + let gradient_sum = numerics::simd_sum(gradient_slice); + + for (out_grad, in_grad, val) in + izip!(downstream_gradient_slice, gradient_slice, value_slice) + { + *out_grad = in_grad - val.exp() * gradient_sum; + } + } + + self.operand.backward(&self.operand_gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + fn zero_gradient(&self) { + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } + } +} + #[derive(Debug)] pub struct SumNode { value: RefCell, diff --git a/src/numerics.rs b/src/numerics.rs index 841d3c41..4b524118 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -25,7 +25,32 @@ pub fn simd_dot(xs: &[f32], ys: &[f32]) -> f32 { scalar_result += x_scalar * y_scalar; } - scalar_result + (0..8).map(|idx| simd_result.extract(idx)).sum::() + scalar_result + + (0..stride as u32) + .map(|idx| simd_result.extract(idx)) + .sum::() +} + +pub fn simd_sum(xs: &[f32]) -> f32 { + let mut simd_result = stdsimd::simd::f32x8::splat(0.0); + let mut scalar_result = 0.0; + let stride = 8; + + let split_idx = (xs.len() / stride) * stride; + let (simd_xs, scalar_xs) = xs.split_at(split_idx); + + for x in simd_xs.chunks(stride) { + unsafe { simd_result = simd_result + stdsimd::simd::f32x8::load_unchecked(x, 0) } + } + + for x_scalar in scalar_xs.iter() { + scalar_result += x_scalar; + } + + scalar_result + + (0..stride as u32) + .map(|idx| simd_result.extract(idx)) + .sum::() } pub fn simd_scaled_assign(xs: &mut [f32], ys: &[f32], alpha: f32) { From dd966622320c7b6982e4fd31f15c616d989aade9 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 13 Jan 2018 12:59:53 +0000 Subject: [PATCH 036/108] Add sparse cross entropy loss. --- src/lib.rs | 22 +++++++ src/nn/losses.rs | 145 +++++++++++++++++++++++++++++++++++++++++++++++ src/nodes.rs | 1 - 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 src/nn/losses.rs diff --git a/src/lib.rs b/src/lib.rs index c6615d35..827ddc7f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -428,6 +428,18 @@ impl Variable { } } +impl Variable> where + T: Node, { + /// Return the log-softmax predictions from a sparse categorical + /// cross-entropy node. + /// + /// Calling `.value()` on the node returns the value of the loss; + /// this function allows getting the predictins with low overhead. + pub fn predictions(&self) -> Bor { + self.node.predictions() + } +} + impl<'value> DataInput<&'value Arr> for Variable { fn set_value(&self, value: &Arr) { let param_value = unsafe { &mut *(self.node.value.deref().value.as_ptr()) }; @@ -804,6 +816,16 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn sparse_categorical_cross_entropy_finite_difference() { + let mut x = ParameterNode::new(random_matrix(1, 10)); + let z = x.clone() + x.clone(); + let idx = IndexInputNode::new(&vec![0][..]); + let mut loss = nn::losses::sparse_categorical_crossentropy(&z, &idx); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut loss); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] fn rowwise_stack_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut y = ParameterNode::new(random_matrix(10, 5)); diff --git a/src/nn/losses.rs b/src/nn/losses.rs new file mode 100644 index 00000000..e425439a --- /dev/null +++ b/src/nn/losses.rs @@ -0,0 +1,145 @@ +//! Loss functions. + +use std::rc::Rc; +use std::cell::{Ref, RefCell}; +use std::ops::Deref; + +use {Arr, Node, Variable}; +use nodes::{Bor, ForwardAction, IndexInputNode, LogSoftmaxNode, PassCounter}; + +/// Sparse categorical cross entropy loss. +/// +/// Note that this performs a log-softmax operation +/// internally, so there is no need to perform a softmax +/// manually. +pub fn sparse_categorical_crossentropy( + x: &Variable, + y: &Variable, +) -> Variable> +where + T: Node, +{ + let node = SparseCategoricalCrossentropyNode::new(Rc::clone(&x.node), Rc::clone(&y.node)); + + Variable::new(Rc::new(node), x.parameters.clone()) +} + +#[derive(Debug)] +pub struct SparseCategoricalCrossentropyNode { + operand: Rc, + log_softmax: LogSoftmaxNode, + y: Rc, + loss_value: RefCell, + gradient: RefCell, + needs_gradient: bool, + counter: PassCounter, +} + +impl SparseCategoricalCrossentropyNode +where + LHS: Node, +{ + pub fn new(operand: Rc, y: Rc) -> Self { + assert!( + operand.value().rows() == 1, + "Minibatches not supported: rows must be 1." + ); + + let log_softmax = LogSoftmaxNode::new(Rc::clone(&operand)); + let scalar_loss = { + let log_softmax_value = log_softmax.value(); + + let mut scalar_loss = 0.0; + + for &idx in y.value().iter() { + scalar_loss += -log_softmax_value[(0, idx)]; + } + + scalar_loss + }; + + let mut loss_value = Arr::zeros((1, 1)); + loss_value.fill(scalar_loss); + + let gradient = operand.value().deref() * 0.0; + let needs_gradient = operand.needs_gradient(); + + SparseCategoricalCrossentropyNode { + operand: operand, + log_softmax: log_softmax, + y: y, + loss_value: RefCell::new(loss_value), + gradient: RefCell::new(gradient), + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } + + pub fn predictions(&self) -> Bor { + self.log_softmax.value() + } +} + +impl Node for SparseCategoricalCrossentropyNode +where + LHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + + self.log_softmax.forward(); + self.y.forward(); + + let softmax_value = self.log_softmax.value(); + debug_assert!( + softmax_value.rows() == 1, + "Minibatches not supported: rows must be 1." + ); + let softmax_slice = softmax_value.deref().as_slice().unwrap(); + + let mut loss_value = 0.0; + + for &idx in self.y.value().iter() { + loss_value += -softmax_slice[idx]; + } + + self.loss_value.borrow_mut().fill(loss_value); + } + fn backward(&self, _: &Ref) { + // TODO: actually use the input gradient + { + let mut gradient = self.gradient.borrow_mut(); + let gradient_slice = gradient.as_slice_mut().unwrap(); + + let value = self.log_softmax.value(); + let value_slice = value.as_slice().unwrap(); + + for (grad, val) in izip!(gradient_slice.iter_mut(), value_slice.iter()) { + *grad = val.exp(); + } + + for &idx in self.y.value().iter() { + gradient_slice[idx] -= 1.0; + } + } + + self.operand.backward(&self.gradient.borrow()); + } + fn value(&self) -> Bor { + Bor::RefGuard(self.loss_value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + fn zero_gradient(&self) { + if !self.counter.is_zero() { + self.log_softmax.zero_gradient(); + self.counter.clear(); + } + } +} diff --git a/src/nodes.rs b/src/nodes.rs index 96012cc5..5bbc02b2 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1944,7 +1944,6 @@ where let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); - let dim = value.shape()[1]; LogSoftmaxNode { value: RefCell::new(value), From 485d687e258a82e9eac38d63ec7e937f65e52334 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 13 Jan 2018 14:31:00 +0000 Subject: [PATCH 037/108] Bump to 0.5.0. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 42f73850..b6f54d6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.4.0" +version = "0.5.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 79a0cfc22cba0b51d7dcff113e0658d83cb0cc25 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 13 Jan 2018 22:05:23 +0000 Subject: [PATCH 038/108] Improve LSTM layer ergonomics. --- src/lib.rs | 6 +- src/nn/{recurrent.rs => lstm.rs} | 156 +++++++++++++++++++------------ src/nn/mod.rs | 2 +- 3 files changed, 100 insertions(+), 64 deletions(-) rename src/nn/{recurrent.rs => lstm.rs} (74%) diff --git a/src/lib.rs b/src/lib.rs index 827ddc7f..a031a677 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -428,8 +428,10 @@ impl Variable { } } -impl Variable> where - T: Node, { +impl Variable> +where + T: Node, +{ /// Return the log-softmax predictions from a sparse categorical /// cross-entropy node. /// diff --git a/src/nn/recurrent.rs b/src/nn/lstm.rs similarity index 74% rename from src/nn/recurrent.rs rename to src/nn/lstm.rs index 3788a49e..ffde967e 100644 --- a/src/nn/recurrent.rs +++ b/src/nn/lstm.rs @@ -1,7 +1,7 @@ -//! Module holding building blocks for recurrent neural networks. +//! Module for LSTM layers. //! -//! You can create an LSTM layer by repeatedly applying an LSTM cell -//! to your inputs: +//! You can create an LSTM layer by first initializing its parameters, +//! then applying it to your inputs: //! //! ```rust //! # extern crate rand; @@ -13,7 +13,7 @@ //! # use wyrm::{HogwildParameter, InputNode, Node, ParameterNode}; //! # //! # use wyrm::nn::xavier_normal; -//! # use wyrm::nn::recurrent::*; +//! # use wyrm::nn::lstm; //! # //! # use wyrm::{Arr, Variable}; //! # fn main() { @@ -21,33 +21,25 @@ //! let hidden_dim = 5; //! //! // Initialize the parameters. -//! let lstm_params = LSTMParameters::new(input_dim, hidden_dim); -//! let lstm = lstm_params.build(); +//! let parameters = lstm::Parameters::new(input_dim, hidden_dim); +//! let lstm = parameters.build(); +//! +//! // Construct the input nodes. +//! let input: Vec<_> = (0..200) +//! .map(|_| InputNode::new(xavier_normal(1, input_dim))).collect(); //! -//! // Initialize the cell state and hidden state. -//! let state = InputNode::new(Arr::zeros((1, hidden_dim))); -//! let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); +//! // Construct an LSTM with 200 steps of recursion. +//! let mut hidden = lstm.forward(&input); //! -//! // Construct the input node. -//! let input = InputNode::new(xavier_normal(1, input_dim)); -//! -//! // The forward method outputs a tuple of (cell_state, hidden_state). -//! let mut state = lstm.forward((state, hidden), input.clone()); -//! -//! // Construct an LSTM with 200 steps of recursion. Usually -//! // we'd use different inputs for every step, but here we re-use -//! // the same input for simplicity. -//! for _ in 0..200 { -//! state = lstm.forward(state.clone(), input.clone()); -//! } -//! -//! // Unpack the cell and hidden state. -//! let (_, mut hidden) = state; +//! let mut last_hidden = hidden.last_mut().unwrap(); //! //! // Run as usual. -//! hidden.forward(); -//! hidden.backward(1.0); -//! hidden.zero_gradient(); +//! last_hidden.forward(); +//! last_hidden.backward(1.0); +//! last_hidden.zero_gradient(); +//! +//! // Reset the hidden state between sequences +//! lstm.reset_state(); //! # } //! ``` use std::sync::Arc; @@ -60,13 +52,13 @@ use nodes::{HogwildParameter, Node, ParameterNode}; use nn::xavier_normal; -use {Arr, Variable}; +use {Arr, DataInput, Variable}; /// Holds shared parameters for an LSTM cell. /// /// Construct this first, then use the `build` method to instantiate /// LSTM cell nodes. -pub struct LSTMParameters { +pub struct Parameters { input_dim: usize, hidden_dim: usize, @@ -83,7 +75,7 @@ pub struct LSTMParameters { output_gate_biases: Arc, } -impl LSTMParameters { +impl Parameters { /// Create a new LSTM parameters object. pub fn new(input_dim: usize, hidden_dim: usize) -> Self { Self { @@ -116,11 +108,16 @@ impl LSTMParameters { } } + /// Build an LSTM layer. + pub fn build(&self) -> Layer { + Layer::new(self.build_cell()) + } + /// Build an LSTM cell. - pub fn build(&self) -> LSTMCell { - LSTMCell { - _input_dim: self.input_dim, - _hidden_dim: self.hidden_dim, + pub fn build_cell(&self) -> Cell { + Cell { + input_dim: self.input_dim, + hidden_dim: self.hidden_dim, forget_weights: ParameterNode::shared(self.forget_weights.clone()), forget_biases: ParameterNode::shared(self.forget_biases.clone()), @@ -138,9 +135,10 @@ impl LSTMParameters { } /// An LSTM cell. -pub struct LSTMCell { - _input_dim: usize, - _hidden_dim: usize, +#[derive(Debug)] +pub struct Cell { + input_dim: usize, + hidden_dim: usize, forget_weights: Variable, forget_biases: Variable, @@ -155,7 +153,7 @@ pub struct LSTMCell { output_gate_biases: Variable, } -impl LSTMCell { +impl Cell { /// Run a single LSTM iteration over inputs. /// /// If this is the first cell, initialize the cell state and the hidden state; @@ -203,6 +201,55 @@ impl LSTMCell { } } +/// An LSTM layer. +#[derive(Debug)] +pub struct Layer { + cell: Cell, + state: Variable, + hidden: Variable, +} + +impl Layer { + fn new(cell: Cell) -> Self { + let hidden_dim = cell.hidden_dim; + + Layer { + cell: cell, + state: nodes::InputNode::new(Arr::zeros((1, hidden_dim))), + hidden: nodes::InputNode::new(Arr::zeros((1, hidden_dim))), + } + } + /// Construct an LSTM layer over given inputs, returning the emitted + /// hidden states. + /// + /// The state of the layer is initialized with zero vectors. Use + /// `Cell` for custom initialization. + pub fn forward( + &self, + inputs: &[Variable], + ) -> Vec>>> + where + T: Node, + { + let mut state = (self.state.clone().boxed(), self.hidden.clone().boxed()); + + let outputs: Vec<_> = inputs + .iter() + .map(|input| { + state = self.cell.forward(state.clone(), input.clone()); + state.1.clone() + }) + .collect(); + + outputs + } + /// Reset the internal state of the layer. + pub fn reset_state(&self) { + self.state.set_value(0.0); + self.hidden.set_value(0.0); + } +} + #[cfg(test)] mod tests { @@ -212,6 +259,7 @@ mod tests { use DataInput; use SGD; use nodes::InputNode; + use nn::losses::sparse_categorical_crossentropy; fn pi_digits(num: usize) -> Vec { let pi_str = include_str!("pi.txt"); @@ -229,8 +277,8 @@ mod tests { let hidden_dim = 5; // Initialize the parameters. - let lstm_params = LSTMParameters::new(input_dim, hidden_dim); - let lstm = lstm_params.build(); + let lstm_params = Parameters::new(input_dim, hidden_dim); + let lstm = lstm_params.build_cell(); // Initialize the cell state and hidden state. let state = InputNode::new(Arr::zeros((1, hidden_dim))); @@ -274,17 +322,12 @@ mod tests { let input_dim = 16; let hidden_dim = 32; - let lstm_params = LSTMParameters::new(input_dim, hidden_dim); + let lstm_params = Parameters::new(input_dim, hidden_dim); let lstm = lstm_params.build(); let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); - - let mut labels = Arr::zeros((1, num_digits)); - let y = nodes::InputNode::new(labels.clone()); - - let state = InputNode::new(Arr::zeros((1, hidden_dim))); - let hidden = InputNode::new(Arr::zeros((1, hidden_dim))); + let y = nodes::IndexInputNode::new(&vec![0]); let inputs: Vec<_> = (0..sequence_length) .map(|_| nodes::IndexInputNode::new(&vec![0])) @@ -294,17 +337,11 @@ mod tests { .map(|input| embeddings.index(&input)) .collect(); - let mut state = lstm.forward((state, hidden), embeddings[0].clone()); - - for i in 1..sequence_length { - state = lstm.forward(state.clone(), embeddings[i].clone()); - } - - let (_, hidden) = state; - - let prediction = hidden.dot(&final_layer).softmax(); - let mut loss = (-(y.clone() * prediction.ln())).scalar_sum(); + let hidden_states = lstm.forward(&embeddings); + let hidden = hidden_states.last().unwrap(); + let prediction = hidden.dot(&final_layer); + let mut loss = sparse_categorical_crossentropy(&prediction, &y); let mut optimizer = SGD::new(0.05, loss.parameters()); let digits = pi_digits(100); @@ -329,10 +366,7 @@ mod tests { } let target_digit = *digit_chunk.last().unwrap(); - labels *= 0.0; - labels[(0, target_digit)] = 1.0; - - y.set_value(&labels); + y.set_value(target_digit); loss.forward(); loss.backward(1.0); diff --git a/src/nn/mod.rs b/src/nn/mod.rs index d25bdb9e..c05b9a7c 100644 --- a/src/nn/mod.rs +++ b/src/nn/mod.rs @@ -1,7 +1,7 @@ //! Neural network components. pub mod losses; -pub mod recurrent; +pub mod lstm; use rand; use rand::distributions::{IndependentSample, Normal}; From 2b0914f6a33fc2f93a94c5020c8cf2bf52929f12 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 14 Jan 2018 17:40:19 +0000 Subject: [PATCH 039/108] Add gemv/gemm dispatch optimizations. Currently, BLAS gemv is not triggered if M is not row-major, making vector-matrix operations around 3x slower than they should be if gemv were called correctly. This is an issue in ndarray (opened an issue upstream). --- Cargo.toml | 5 ++ src/nn/lstm.rs | 2 +- src/nodes.rs | 15 ++-- src/numerics.rs | 224 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b6f54d6c..470fc340 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,11 @@ serde = { version = "1.0.24", features = ["rc"] } serde_derive = "1.0.24" stdsimd = "0.0.3" +[dev-dependencies] +ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } +blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } +openblas-src = { version = "0.5.6", default-features = false, features = ["static"] } + [profile.bench] lto = true debug = true diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index ffde967e..a8d3f6d0 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -23,7 +23,7 @@ //! // Initialize the parameters. //! let parameters = lstm::Parameters::new(input_dim, hidden_dim); //! let lstm = parameters.build(); -//! +//! //! // Construct the input nodes. //! let input: Vec<_> = (0..200) //! .map(|_| InputNode::new(xavier_normal(1, input_dim))).collect(); diff --git a/src/nodes.rs b/src/nodes.rs index 5bbc02b2..8ca5fcc1 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -7,7 +7,6 @@ use std::rc::Rc; use ndarray; use ndarray::Axis; -use ndarray::linalg::general_mat_mul; use smallvec::SmallVec; @@ -972,7 +971,7 @@ where self.lhs.forward(); self.rhs.forward(); - general_mat_mul( + numerics::mat_mul( 1.0, self.lhs.value().deref(), self.rhs.value().deref(), @@ -991,8 +990,14 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - general_mat_mul(1.0, gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); - general_mat_mul(1.0, &lhs_value.t(), gradient, 0.0, &mut rhs_gradient); + numerics::mat_mul(1.0, gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); + numerics::mat_mul( + 1.0, + &lhs_value.t(), + gradient.deref(), + 0.0, + &mut rhs_gradient, + ); } } @@ -1888,7 +1893,7 @@ where } { - general_mat_mul( + numerics::mat_mul( 1.0, gradient, jacobian.deref_mut(), diff --git a/src/numerics.rs b/src/numerics.rs index 4b524118..a11e803e 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -1,8 +1,134 @@ use std::cmp; use stdsimd; +use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; +use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; + use super::Arr; +fn vec_mat_mul( + alpha: f32, + lhs: &ArrayBase, + rhs: &ArrayBase, + beta: f32, + out: &mut ArrayBase, +) where + S1: Data, + S2: Data, + S3: DataMut, +{ + if lhs.len() != rhs.rows() || rhs.cols() != out.len() { + panic!( + "Shapes are incompatible for a vec-mat mul: LHS: {} vs RHS ({} x {}) vs out {}", + lhs.len(), + rhs.rows(), + rhs.cols(), + out.len() + ) + } + + let out_slice = out.as_slice_mut().expect("Vec-mat result not contiguous."); + let lhs_slice = lhs.as_slice().expect("LHS not contiguous"); + + for (row_idx, (&x, row)) in lhs_slice.iter().zip(rhs.genrows()).enumerate() { + let row_slice = row.as_slice().expect("RHS row not C-contiguous."); + + if row_idx == 0 { + saxpy(x * alpha, row_slice, beta, out_slice); + } else { + saxpy(x * alpha, row_slice, 1.0, out_slice); + } + } +} + +fn saxpy(alpha: f32, xs: &[f32], beta: f32, outs: &mut [f32]) { + let stride = 8; + let simd_alpha = stdsimd::simd::f32x8::splat(alpha); + let simd_beta = stdsimd::simd::f32x8::splat(beta); + + let split_idx = xs.len() / stride * stride; + let (simd_xs, scalar_xs) = xs.split_at(split_idx); + let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); + + for (x, out) in izip!(simd_xs.chunks(stride), + simd_outs.chunks_mut(stride)) { + unsafe { + let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) + * simd_alpha + + stdsimd::simd::f32x8::load_unchecked(out, 0) + * simd_beta; + elem.store_unchecked(out, 0); + } + } + + for (&x, out) in izip!(scalar_xs.iter(), + scalar_outs.iter_mut()) { + *out = x * alpha + *out * beta; + } +} + +enum MatrixLayout { + RowMajor, + ColumnMajor, +} + +fn layout>(matrix: &ArrayBase) -> MatrixLayout { + match matrix.strides()[0] { + 1 => MatrixLayout::ColumnMajor, + _ => MatrixLayout::RowMajor, + } +} + +pub fn mat_mul( + alpha: f32, + lhs: &ArrayBase, + rhs: &ArrayBase, + beta: f32, + out: &mut ArrayBase, +) where + S1: Data, + S2: Data, + S3: DataMut, +{ + match (lhs.rows(), rhs.cols()) { + (_, 1) => { + general_mat_vec_mul( + alpha, + lhs, + &rhs.subview(Axis(1), 0), + beta, + &mut out.subview_mut(Axis(1), 0), + ); + } + (1, _) => { + // general_mat_vec_mul( + // alpha, + // &rhs, + // &lhs.subview(Axis(0), 0), + // beta, + // &mut out.subview_mut(Axis(0), 0), + // ); + match layout(rhs) { + MatrixLayout::RowMajor => vec_mat_mul( + alpha, + &lhs.subview(Axis(0), 0), + rhs, + beta, + &mut out.subview_mut(Axis(0), 0)), + MatrixLayout::ColumnMajor => general_mat_vec_mul( + alpha, + &rhs.t(), + &lhs.subview(Axis(0), 0), + beta, + &mut out.subview_mut(Axis(0), 0)) + } + } + _ => { + general_mat_mul(alpha, lhs, rhs, beta, out); + } + } +} + /// SIMD-enabled vector-vector dot product. pub fn simd_dot(xs: &[f32], ys: &[f32]) -> f32 { let mut simd_result = stdsimd::simd::f32x8::splat(0.0); @@ -288,6 +414,59 @@ mod tests { } } + #[allow(dead_code)] + fn assert_close(x: &Arr, y: &Arr, tol: f32) { + assert!( + x.all_close(y, tol), + "{:#?} not within {} of {:#?}", + x, + tol, + y + ); + } + + #[test] + fn test_dot_node_specializations_mm() { + let x = random_matrix(64, 64); + let y = random_matrix(64, 64); + + let mut result = random_matrix(64, 64); + let mut expected = random_matrix(64, 64); + + mat_mul(1.0, &x, &y, 0.0, &mut result); + general_mat_mul(1.0, &x, &y, 0.0, &mut expected); + + assert_close(&result, &expected, 0.001); + } + + #[test] + fn test_dot_node_specializations_mv() { + let x = random_matrix(64, 64); + let y = random_matrix(64, 1); + + let mut result = random_matrix(64, 1); + let mut expected = random_matrix(64, 1); + + mat_mul(1.0, &x, &y, 0.0, &mut result); + general_mat_mul(1.0, &x, &y, 0.0, &mut expected); + + assert_close(&result, &expected, 0.001); + } + + #[test] + fn test_dot_node_specializations_vm() { + let x = random_matrix(1, 64); + let y = random_matrix(64, 64); + + let mut result = random_matrix(1, 64); + let mut expected = random_matrix(1, 64); + + mat_mul(1.0, &x, &y, 0.0, &mut result); + general_mat_mul(1.0, &x, &y, 0.0, &mut expected); + + assert_close(&result, &expected, 0.001); + } + #[bench] fn bench_dot(b: &mut Bencher) { let xs = vec![0.0; 256]; @@ -343,4 +522,49 @@ mod tests { b.iter(|| assign(&mut xs, &ys)); } + + #[bench] + fn dot_node_specializations_mm(b: &mut Bencher) { + let x = random_matrix(64, 64); + let y = random_matrix(64, 64); + let mut z = random_matrix(64, 64); + + b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); + } + + #[bench] + fn dot_node_general_vm(b: &mut Bencher) { + let x = random_matrix(1, 64); + let y = random_matrix(64, 64); + let mut z = random_matrix(1, 64); + + b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); + } + + #[bench] + fn dot_node_specializations_vm(b: &mut Bencher) { + let x = random_matrix(1, 64); + let y = random_matrix(64, 64); + let mut z = random_matrix(1, 64); + + b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); + } + + #[bench] + fn dot_node_specializations_mv(b: &mut Bencher) { + let x = random_matrix(64, 64); + let y = random_matrix(64, 1); + let mut z = random_matrix(64, 1); + + b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); + } + + #[bench] + fn dot_node_general_mv(b: &mut Bencher) { + let x = random_matrix(64, 64); + let y = random_matrix(64, 1); + let mut z = random_matrix(64, 1); + + b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); + } } From 6f9f1c50b0fa51aefe80868101955349769721e1 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 15 Jan 2018 17:50:12 +0000 Subject: [PATCH 040/108] Bump to 0.6.0. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 470fc340..37230a47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.5.0" +version = "0.6.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 3014ac9511147197986a4da2995f4c7ee8d0b627 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 17 Jan 2018 19:11:04 +0000 Subject: [PATCH 041/108] Add fast-math option to enable fast approximations for transcendental functions. --- Cargo.toml | 3 ++ src/lib.rs | 8 +++- src/nn/losses.rs | 5 +- src/nn/lstm.rs | 55 ++++++++++++++++++++++ src/nodes.rs | 49 +++++++++++--------- src/numerics.rs | 117 +++++++++++++++++++++++++++++++++++++++++++---- 6 files changed, 201 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 37230a47..1c651b5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,9 @@ readme = "readme.md" [badges] travis-ci = { repository = "maciejkula/wyrm", branch = "master" } +[features] +fast-math = [] + [dependencies] ndarray = { version = "0.11.0", features = ["serde-1"] } rand = "0.3.18" diff --git a/src/lib.rs b/src/lib.rs index a031a677..186639f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -133,6 +133,11 @@ //! blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } //! openblas-src = { version = "0.5.6", default-features = false, features = ["cblas"] } //! ``` +//! +//! ## Fast numerics +//! +//! Enable the `fast-math` option to use fast approximations to transcendental functions. +//! This should give substantial speed gains in networks that are `exp`, `ln`, or `tanh`-heavy. // TODO: pass through of parent values in .value(), // optimizations in forward @@ -563,8 +568,7 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = - unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; + let mut param_value = unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); diff --git a/src/nn/losses.rs b/src/nn/losses.rs index e425439a..5d14c245 100644 --- a/src/nn/losses.rs +++ b/src/nn/losses.rs @@ -6,6 +6,7 @@ use std::ops::Deref; use {Arr, Node, Variable}; use nodes::{Bor, ForwardAction, IndexInputNode, LogSoftmaxNode, PassCounter}; +use numerics; /// Sparse categorical cross entropy loss. /// @@ -119,8 +120,8 @@ where let value = self.log_softmax.value(); let value_slice = value.as_slice().unwrap(); - for (grad, val) in izip!(gradient_slice.iter_mut(), value_slice.iter()) { - *grad = val.exp(); + for (grad, &val) in izip!(gradient_slice.iter_mut(), value_slice.iter()) { + *grad = numerics::exp(val); } for &idx in self.y.value().iter() { diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index a8d3f6d0..019d8d42 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -254,6 +254,7 @@ impl Layer { mod tests { use std::ops::Deref; + use test::Bencher; use super::*; use DataInput; @@ -392,4 +393,58 @@ mod tests { assert!((correct as f32 / total as f32) > 0.75); } + + #[bench] + fn bench_lstm(b: &mut Bencher) { + let sequence_length = 4; + let num_digits = 10; + let input_dim = 16; + let hidden_dim = 32; + + let lstm_params = Parameters::new(input_dim, hidden_dim); + let lstm = lstm_params.build(); + + let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); + let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); + let y = nodes::IndexInputNode::new(&vec![0]); + + let inputs: Vec<_> = (0..sequence_length) + .map(|_| nodes::IndexInputNode::new(&vec![0])) + .collect(); + let embeddings: Vec<_> = inputs + .iter() + .map(|input| embeddings.index(&input)) + .collect(); + + let hidden_states = lstm.forward(&embeddings); + let hidden = hidden_states.last().unwrap(); + + let prediction = hidden.dot(&final_layer); + let mut loss = sparse_categorical_crossentropy(&prediction, &y); + let mut optimizer = SGD::new(0.05, loss.parameters()); + + let digits = pi_digits(100); + + b.iter(|| { + for i in 0..(digits.len() - sequence_length - 1) { + let digit_chunk = &digits[i..(i + sequence_length + 1)]; + if digit_chunk.len() < sequence_length + 1 { + break; + } + + for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { + input.set_value(digit); + } + + let target_digit = *digit_chunk.last().unwrap(); + y.set_value(target_digit); + + loss.forward(); + loss.backward(1.0); + + optimizer.step(); + loss.zero_gradient(); + } + }); + } } diff --git a/src/nodes.rs b/src/nodes.rs index 8ca5fcc1..b2f0b164 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1310,7 +1310,7 @@ where OP: Node, { pub fn new(operand: Rc) -> Self { - let value = operand.value().map(|x| x.ln()); + let value = operand.value().map(|&x| numerics::ln(x)); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -1340,7 +1340,7 @@ where let mut dest = self.value.borrow_mut(); dest.assign(self.operand.value().deref()); - dest.map_inplace(|x| *x = x.ln()); + dest.map_inplace(|x| *x = numerics::ln(*x)); } fn backward(&self, gradient: &Ref) { @@ -1396,7 +1396,7 @@ where OP: Node, { pub fn new(operand: Rc) -> Self { - let value = operand.value().map(|x| x.tanh()); + let value = operand.value().map(|&x| numerics::tanh(x)); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -1426,7 +1426,7 @@ where let mut dest = self.value.borrow_mut(); dest.assign(self.operand.value().deref()); - dest.map_inplace(|x| *x = x.tanh()); + dest.map_inplace(|x| *x = numerics::tanh(*x)); } fn backward(&self, gradient: &Ref) { @@ -1482,7 +1482,10 @@ where T: Node, { pub fn new(operand: Rc) -> Self { - let value = operand.value().deref().map(|x| 1.0 / (1.0 + (-x).exp())); + let value = operand + .value() + .deref() + .map(|x| 1.0 / (1.0 + numerics::exp(-x))); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -1512,7 +1515,7 @@ where let mut dest = self.value.borrow_mut(); numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { - 1.0 / (1.0 + (-x).exp()) + 1.0 / (1.0 + numerics::exp(-x)) }); } @@ -1659,7 +1662,7 @@ where OP: Node, { pub fn new(operand: Rc) -> Self { - let value = operand.value().deref().map(|x| x.exp()); + let value = operand.value().deref().map(|&x| numerics::exp(x)); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -1688,7 +1691,7 @@ where let mut dest = self.value.borrow_mut(); dest.assign(self.operand.value().deref()); - dest.map_inplace(|x| *x = x.exp()); + dest.map_inplace(|x| *x = numerics::exp(*x)); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { @@ -1822,7 +1825,7 @@ where .unwrap() .iter() .fold(std::f32::MIN, |x, y| x.max(*y)); - let numerator = operand.value().map(|x| (x - max).exp()); + let numerator = operand.value().map(|x| numerics::exp(x - max)); let denominator = numerator.scalar_sum(); numerator / denominator @@ -1863,7 +1866,7 @@ where .deref() .iter() .fold(std::f32::MIN, |x, y| x.max(*y)); - dest.map_inplace(|x| *x = (*x - max).exp()); + dest.map_inplace(|x| *x = numerics::exp(*x - max)); let denominator = dest.scalar_sum(); dest.map_inplace(|x| *x /= denominator); } @@ -1938,11 +1941,12 @@ where let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); let denominator = max - + operand_slice - .iter() - .map(|&x| (x - max).exp()) - .sum::() - .ln(); + + numerics::ln( + operand_slice + .iter() + .map(|&x| numerics::exp(x - max)) + .sum::(), + ); operand_value.deref() - denominator }; @@ -1980,11 +1984,12 @@ where let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); let denominator = max - + operand_slice - .iter() - .map(|&x| (x - max).exp()) - .sum::() - .ln(); + + numerics::ln( + operand_slice + .iter() + .map(|&x| numerics::exp(x - max)) + .sum::(), + ); dest.as_slice_mut() .unwrap() @@ -2007,10 +2012,10 @@ where let gradient_sum = numerics::simd_sum(gradient_slice); - for (out_grad, in_grad, val) in + for (out_grad, in_grad, &val) in izip!(downstream_gradient_slice, gradient_slice, value_slice) { - *out_grad = in_grad - val.exp() * gradient_sum; + *out_grad = in_grad - numerics::exp(val) * gradient_sum; } } diff --git a/src/numerics.rs b/src/numerics.rs index a11e803e..f49200b1 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -1,4 +1,5 @@ use std::cmp; +use std::mem; use stdsimd; use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; @@ -6,6 +7,57 @@ use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; use super::Arr; +#[inline(always)] +fn expf_fast(x: f32) -> f32 { + let u = (12102203.0 * x + 1064866805.0) as i32; + + unsafe { mem::transmute::(u) } +} + +/// Uses approximate e^x when the fast-math feature is enabled. +#[inline(always)] +pub fn exp(x: f32) -> f32 { + if cfg!(feature = "fast-math") { + expf_fast(x) + } else { + x.exp() + } +} + +fn lnf_fast(x: f32) -> f32 { + (unsafe { mem::transmute::(x) } - 1064866805) as f32 * 8.262958405176314e-8 +} + +/// Uses approximate ln(x) when the fast-math feature is enabled. +#[inline(always)] +pub fn ln(x: f32) -> f32 { + if cfg!(feature = "fast-math") { + lnf_fast(x) + } else { + x.ln() + } +} + +fn tanhf_fast(x: f32) -> f32 { + if x < -3.0 { + -1.0 + } else if x > 3.0 { + 1.0 + } else { + x * (27.0 + x.powi(2)) / (27.0 + 9.0 * x.powi(2)) + } +} + +/// Uses approximate ln(x) when the fast-math feature is enabled. +#[inline(always)] +pub fn tanh(x: f32) -> f32 { + if cfg!(feature = "fast-math") { + tanhf_fast(x) + } else { + x.tanh() + } +} + fn vec_mat_mul( alpha: f32, lhs: &ArrayBase, @@ -50,19 +102,15 @@ fn saxpy(alpha: f32, xs: &[f32], beta: f32, outs: &mut [f32]) { let (simd_xs, scalar_xs) = xs.split_at(split_idx); let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); - for (x, out) in izip!(simd_xs.chunks(stride), - simd_outs.chunks_mut(stride)) { + for (x, out) in izip!(simd_xs.chunks(stride), simd_outs.chunks_mut(stride)) { unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) - * simd_alpha - + stdsimd::simd::f32x8::load_unchecked(out, 0) - * simd_beta; + let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) * simd_alpha + + stdsimd::simd::f32x8::load_unchecked(out, 0) * simd_beta; elem.store_unchecked(out, 0); } } - for (&x, out) in izip!(scalar_xs.iter(), - scalar_outs.iter_mut()) { + for (&x, out) in izip!(scalar_xs.iter(), scalar_outs.iter_mut()) { *out = x * alpha + *out * beta; } } @@ -114,13 +162,15 @@ pub fn mat_mul( &lhs.subview(Axis(0), 0), rhs, beta, - &mut out.subview_mut(Axis(0), 0)), + &mut out.subview_mut(Axis(0), 0), + ), MatrixLayout::ColumnMajor => general_mat_vec_mul( alpha, &rhs.t(), &lhs.subview(Axis(0), 0), beta, - &mut out.subview_mut(Axis(0), 0)) + &mut out.subview_mut(Axis(0), 0), + ), } } _ => { @@ -377,6 +427,35 @@ mod tests { s } + #[test] + fn test_expf() { + let values: Vec = vec![-0.5, -0.1, 0.0, 0.1, 0.5]; + for &x in &values { + println!("Input: {}, stdlib: {}, fast: {}", x, x.exp(), expf_fast(x)); + } + } + + #[test] + fn test_lnf() { + let values: Vec = vec![0.1, 0.5, 1.0, 5.0, 10.0]; + for &x in &values { + println!("Input: {}, stdlib: {}, fast: {}", x, x.ln(), lnf_fast(x)); + } + } + + #[test] + fn test_tanh() { + let values: Vec = vec![-0.5, -0.1, 0.0, 0.1, 0.5]; + for &x in &values { + println!( + "Input: {}, stdlib: {}, fast: {}", + x, + x.tanh(), + tanhf_fast(x) + ); + } + } + #[test] fn test_dot() { for len in 0..32 { @@ -467,6 +546,24 @@ mod tests { assert_close(&result, &expected, 0.001); } + #[bench] + fn bench_exp(b: &mut Bencher) { + let x: Vec = vec![1.0; 32]; + + let mut v = 0.0; + + b.iter(|| x.iter().for_each(|&y| v += y.exp())); + } + + #[bench] + fn bench_exp_fast(b: &mut Bencher) { + let x: Vec = vec![1.0; 32]; + + let mut v = 0.0; + + b.iter(|| x.iter().for_each(|&y| v += expf_fast(y))); + } + #[bench] fn bench_dot(b: &mut Bencher) { let xs = vec![0.0; 256]; From 3ff6944f33cc3e7ac4655addf1f0fc699af97fcf Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 17 Jan 2018 22:25:22 +0000 Subject: [PATCH 042/108] Bump version. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1c651b5f..a2798ca7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.6.0" +version = "0.7.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 62a884a6204c7adf6078ba6aebd11252d8312157 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 17 Jan 2018 23:27:53 +0000 Subject: [PATCH 043/108] Use more robust approximate log function. --- src/fast_approx.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 ++- src/numerics.rs | 12 ++++---- 3 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 src/fast_approx.rs diff --git a/src/fast_approx.rs b/src/fast_approx.rs new file mode 100644 index 00000000..7ad4e228 --- /dev/null +++ b/src/fast_approx.rs @@ -0,0 +1,73 @@ +// Direct translations from +// https://github.com/etheory/fastapprox/blob/master/fastapprox/src/fastlog.h + +/*=====================================================================* + * Copyright (C) 2011 Paul Mineiro * + * All rights reserved. * + * * + * Redistribution and use in source and binary forms, with * + * or without modification, are permitted provided that the * + * following conditions are met: * + * * + * * Redistributions of source code must retain the * + * above copyright notice, this list of conditions and * + * the following disclaimer. * + * * + * * Redistributions in binary form must reproduce the * + * above copyright notice, this list of conditions and * + * the following disclaimer in the documentation and/or * + * other materials provided with the distribution. * + * * + * * Neither the name of Paul Mineiro nor the names * + * of other contributors may be used to endorse or promote * + * products derived from this software without specific * + * prior written permission. * + * * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND * + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, * + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER * + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE * + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * + * POSSIBILITY OF SUCH DAMAGE. * + * * + * Contact: Paul Mineiro * + *=====================================================================*/ + +#[repr(C)] +union FloatUint { + f: f32, + i: u32, +} + +#[repr(C)] +union UintFloat { + i: u32, + f: f32, +} + +#[inline(always)] +pub fn fastlog2(x: f32) -> f32 { + unsafe { + let vx = FloatUint { f: x }; + let mx = UintFloat { + i: (vx.i & 0x007FFFFF) | 0x3f000000, + }; + let mut y = vx.i as f32; + y *= 1.1920928955078125e-7; + + y - 124.22551499 - 1.498030302 * mx.f - 1.72587999 / (0.3520887068 + mx.f) + } +} + +#[inline(always)] +pub fn fastlog(x: f32) -> f32 { + 0.69314718 * fastlog2(x) +} diff --git a/src/lib.rs b/src/lib.rs index 186639f1..cc9e3714 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -170,6 +170,7 @@ use std::clone::Clone; mod nodes; mod numerics; +mod fast_approx; pub mod nn; use nodes::*; @@ -568,7 +569,8 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; + let mut param_value = + unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); diff --git a/src/numerics.rs b/src/numerics.rs index f49200b1..3ecb1a02 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -5,6 +5,8 @@ use stdsimd; use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; +use fast_approx::fastlog; + use super::Arr; #[inline(always)] @@ -24,15 +26,11 @@ pub fn exp(x: f32) -> f32 { } } -fn lnf_fast(x: f32) -> f32 { - (unsafe { mem::transmute::(x) } - 1064866805) as f32 * 8.262958405176314e-8 -} - /// Uses approximate ln(x) when the fast-math feature is enabled. #[inline(always)] pub fn ln(x: f32) -> f32 { if cfg!(feature = "fast-math") { - lnf_fast(x) + fastlog(x) } else { x.ln() } @@ -436,10 +434,10 @@ mod tests { } #[test] - fn test_lnf() { + fn test_fastlog() { let values: Vec = vec![0.1, 0.5, 1.0, 5.0, 10.0]; for &x in &values { - println!("Input: {}, stdlib: {}, fast: {}", x, x.ln(), lnf_fast(x)); + println!("Input: {}, stdlib: {}, fast: {}", x, x.ln(), fastlog(x)); } } From 56b44a693c1763b5fe090bd5b39c0ee7f68487a4 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 17 Jan 2018 23:28:13 +0000 Subject: [PATCH 044/108] Bump version to 0.7.1. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a2798ca7..016977cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.7.0" +version = "0.7.1" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 98bdb4bd80821d14e25cf14e47f57b39e6f60ca5 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 18 Jan 2018 21:05:56 +0000 Subject: [PATCH 045/108] Optimize elementwise op nodes. --- src/fast_approx.rs | 20 ++++++ src/lib.rs | 3 +- src/nodes.rs | 95 ++++++++++++++--------------- src/numerics.rs | 148 +++++++++++++++++++++++++++++++++++++++------ 4 files changed, 194 insertions(+), 72 deletions(-) diff --git a/src/fast_approx.rs b/src/fast_approx.rs index 7ad4e228..b2e7926b 100644 --- a/src/fast_approx.rs +++ b/src/fast_approx.rs @@ -41,6 +41,8 @@ * Contact: Paul Mineiro * *=====================================================================*/ +use std::mem; + #[repr(C)] union FloatUint { f: f32, @@ -71,3 +73,21 @@ pub fn fastlog2(x: f32) -> f32 { pub fn fastlog(x: f32) -> f32 { 0.69314718 * fastlog2(x) } + +#[inline(always)] +pub fn expf_fast(x: f32) -> f32 { + let u = (12102203.0 * x + 1064866805.0) as i32; + + unsafe { mem::transmute::(u) } +} + +#[inline(always)] +pub fn tanhf_fast(x: f32) -> f32 { + if x < -3.0 { + -1.0 + } else if x > 3.0 { + 1.0 + } else { + x * (27.0 + x.powi(2)) / (27.0 + 9.0 * x.powi(2)) + } +} diff --git a/src/lib.rs b/src/lib.rs index cc9e3714..216aa73a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -569,8 +569,7 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = - unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; + let mut param_value = unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); diff --git a/src/nodes.rs b/src/nodes.rs index b2f0b164..75d15d8d 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1,6 +1,6 @@ use std; use std::fmt; -use std::ops::{AddAssign, Deref, DerefMut, DivAssign, MulAssign, SubAssign}; +use std::ops::{AddAssign, Deref, DerefMut}; use std::cell::{Cell, Ref, RefCell}; use std::sync::Arc; use std::rc::Rc; @@ -193,12 +193,9 @@ where let mut self_value = self.value.borrow_mut(); - numerics::map_assign_binary( - self_value.deref_mut(), - lhs_value.deref(), - rhs_value.deref(), - |x, y| x + y, - ); + numerics::add(lhs_value.deref(), + rhs_value.deref(), + self_value.deref_mut()); } fn backward(&self, gradient: &Ref) { self.lhs.backward(gradient); @@ -657,8 +654,9 @@ where let mut dest = self.value.borrow_mut(); - dest.assign(self.lhs.value().deref()); - dest.sub_assign(self.rhs.value().deref()); + numerics::sub(self.lhs.value().deref(), + self.rhs.value().deref(), + dest.deref_mut()); } fn backward(&self, gradient: &Ref) { @@ -666,8 +664,9 @@ where BackwardAction::Set => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - rhs_gradient.assign(gradient); - rhs_gradient.mul_assign(-1.0); + numerics::simd_scaled_assign(rhs_gradient.as_slice_mut().unwrap(), + gradient.as_slice().unwrap(), + -1.0); } BackwardAction::Increment => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -748,37 +747,35 @@ where let mut dest = self.value.borrow_mut(); - dest.assign(self.lhs.value().deref()); - dest.mul_assign(self.rhs.value().deref()); + numerics::mul(self.lhs.value().deref(), + self.rhs.value().deref(), + dest.deref_mut()); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { BackwardAction::Set => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - lhs_gradient.assign(self.rhs.value().deref()); - lhs_gradient.mul_assign(gradient.deref()); + + numerics::mul(self.rhs.value().deref(), + gradient.deref(), + lhs_gradient.deref_mut()); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - rhs_gradient.assign(self.lhs.value().deref()); - rhs_gradient.mul_assign(gradient.deref()); + numerics::mul(self.lhs.value().deref(), + gradient.deref(), + rhs_gradient.deref_mut()); } BackwardAction::Increment => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::map_inplace_assign_binary( - lhs_gradient.deref_mut(), - self.rhs.value().deref(), - gradient.deref(), - |out, rhs, grad| *out += rhs * grad, - ); - numerics::map_inplace_assign_binary( - rhs_gradient.deref_mut(), - self.lhs.value().deref(), - gradient.deref(), - |out, rhs, grad| *out += rhs * grad, - ); + numerics::increment_mul(self.rhs.value().deref(), + gradient.deref(), + lhs_gradient.deref_mut()); + numerics::increment_mul(self.lhs.value().deref(), + gradient.deref(), + rhs_gradient.deref_mut()); } } @@ -854,8 +851,9 @@ where let mut dest = self.value.borrow_mut(); - dest.assign(self.lhs.value().deref()); - dest.div_assign(self.rhs.value().deref()); + numerics::div(self.lhs.value().deref(), + self.rhs.value().deref(), + dest.deref_mut()); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { @@ -863,8 +861,9 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); - izip!(lhs_gradient.iter_mut(), rhs_value.iter(), gradient.iter()) - .for_each(|(dest, rhs_val, grad_val)| *dest = grad_val / rhs_val); + numerics::div(gradient.deref(), + rhs_value.deref(), + lhs_gradient.deref_mut()); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -881,8 +880,9 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); - izip!(lhs_gradient.iter_mut(), rhs_value.iter(), gradient.iter()) - .for_each(|(dest, rhs_val, grad_val)| *dest += grad_val / rhs_val); + numerics::increment_div(gradient.deref(), + rhs_value.deref(), + lhs_gradient.deref_mut()); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -1424,9 +1424,9 @@ where self.operand.forward(); let mut dest = self.value.borrow_mut(); - - dest.assign(self.operand.value().deref()); - dest.map_inplace(|x| *x = numerics::tanh(*x)); + numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { + numerics::tanh(x) + }); } fn backward(&self, gradient: &Ref) { @@ -1941,12 +1941,11 @@ where let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); let denominator = max - + numerics::ln( - operand_slice - .iter() - .map(|&x| numerics::exp(x - max)) - .sum::(), - ); + + operand_slice + .iter() + .map(|&x| numerics::exp(x - max)) + .sum::() + .ln(); operand_value.deref() - denominator }; @@ -1983,13 +1982,7 @@ where let operand_slice = operand_value.deref().as_slice().unwrap(); let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); - let denominator = max - + numerics::ln( - operand_slice - .iter() - .map(|&x| numerics::exp(x - max)) - .sum::(), - ); + let denominator = max + numerics::softmax_exp_sum(operand_slice, max).ln(); dest.as_slice_mut() .unwrap() diff --git a/src/numerics.rs b/src/numerics.rs index 3ecb1a02..f266087f 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -1,21 +1,13 @@ use std::cmp; -use std::mem; use stdsimd; use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; -use fast_approx::fastlog; +use fast_approx::{expf_fast, fastlog, tanhf_fast}; use super::Arr; -#[inline(always)] -fn expf_fast(x: f32) -> f32 { - let u = (12102203.0 * x + 1064866805.0) as i32; - - unsafe { mem::transmute::(u) } -} - /// Uses approximate e^x when the fast-math feature is enabled. #[inline(always)] pub fn exp(x: f32) -> f32 { @@ -36,16 +28,6 @@ pub fn ln(x: f32) -> f32 { } } -fn tanhf_fast(x: f32) -> f32 { - if x < -3.0 { - -1.0 - } else if x > 3.0 { - 1.0 - } else { - x * (27.0 + x.powi(2)) / (27.0 + 9.0 * x.powi(2)) - } -} - /// Uses approximate ln(x) when the fast-math feature is enabled. #[inline(always)] pub fn tanh(x: f32) -> f32 { @@ -56,6 +38,37 @@ pub fn tanh(x: f32) -> f32 { } } +pub fn softmax_exp_sum(xs: &[f32], max: f32) -> f32 { + let mut xs = xs; + let mut s = 0.; + + let (mut p0, mut p1, mut p2, mut p3, mut p4, mut p5, mut p6, mut p7) = + (0., 0., 0., 0., 0., 0., 0., 0.); + + while xs.len() >= 8 { + p0 += exp(xs[0] - max); + p1 += exp(xs[1] - max); + p2 += exp(xs[2] - max); + p3 += exp(xs[3] - max); + p4 += exp(xs[4] - max); + p5 += exp(xs[5] - max); + p6 += exp(xs[6] - max); + p7 += exp(xs[7] - max); + + xs = &xs[8..]; + } + s += p0 + p4; + s += p1 + p5; + s += p2 + p6; + s += p3 + p7; + + for i in 0..xs.len() { + s += exp(xs[i] - max) + } + + s +} + fn vec_mat_mul( alpha: f32, lhs: &ArrayBase, @@ -268,6 +281,85 @@ pub fn simd_scaled_add(xs: &mut [f32], ys: &[f32], alpha: f32) { } } + +macro_rules! simd_binary_op { + ( $name:ident, $simd_name:ident, $increment_name:ident, $simd_increment_name:ident, $op:tt ) => { + pub fn $name(xs: &Arr, ys: &Arr, out: &mut Arr) { + $simd_name(xs.as_slice().unwrap(), + ys.as_slice().unwrap(), + out.as_slice_mut().unwrap()); + } + + fn $simd_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { + let stride = 8; + + let split_idx = xs.len() / stride * stride; + let (simd_xs, scalar_xs) = xs.split_at(split_idx); + let (simd_ys, scalar_ys) = ys.split_at(split_idx); + let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); + + for (x, y, out) in izip!( + simd_xs.chunks(stride), + simd_ys.chunks(stride), + simd_outs.chunks_mut(stride) + ) { + unsafe { + let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) + $op stdsimd::simd::f32x8::load_unchecked(y, 0); + elem.store_unchecked(out, 0); + } + } + + for (&x_scalar, &y_scalar, out_scalar) in + izip!(scalar_xs.iter(), scalar_ys.iter(), scalar_outs.iter_mut()) + { + *out_scalar = x_scalar $op y_scalar; + } + } + + #[allow(dead_code)] + pub fn $increment_name(xs: &Arr, ys: &Arr, out: &mut Arr) { + $simd_increment_name(xs.as_slice().unwrap(), + ys.as_slice().unwrap(), + out.as_slice_mut().unwrap()); + } + + #[allow(dead_code)] + fn $simd_increment_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { + let stride = 8; + + let split_idx = xs.len() / stride * stride; + let (simd_xs, scalar_xs) = xs.split_at(split_idx); + let (simd_ys, scalar_ys) = ys.split_at(split_idx); + let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); + + for (x, y, out) in izip!( + simd_xs.chunks(stride), + simd_ys.chunks(stride), + simd_outs.chunks_mut(stride) + ) { + unsafe { + let elem = stdsimd::simd::f32x8::load_unchecked(out, 0) + + (stdsimd::simd::f32x8::load_unchecked(x, 0) + $op stdsimd::simd::f32x8::load_unchecked(y, 0)); + elem.store_unchecked(out, 0); + } + } + + for (&x_scalar, &y_scalar, out_scalar) in + izip!(scalar_xs.iter(), scalar_ys.iter(), scalar_outs.iter_mut()) + { + *out_scalar += x_scalar $op y_scalar; + } + } + } +} + +simd_binary_op!(add, simd_add, increment_add, increment_simd_add, +); +simd_binary_op!(sub, simd_sub, increment_sub, increment_simd_sub, -); +simd_binary_op!(mul, simd_mul, increment_mul, increment_simd_mul, *); +simd_binary_op!(div, simd_div, increment_div, increment_simd_div, /); + pub fn slice_assign(xs: &mut [f32], ys: &[f32]) { for (x, &y) in xs.iter_mut().zip(ys.iter()) { *x = y; @@ -349,6 +441,8 @@ mod tests { use rand::Rng; use test::Bencher; + use numerics; + fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) } @@ -544,6 +638,22 @@ mod tests { assert_close(&result, &expected, 0.001); } + #[bench] + fn bench_sofmax_exp_sum(b: &mut Bencher) { + let x = vec![1.0; 32]; + let max = 1.0; + + b.iter(|| x.iter().map(|&x| numerics::exp(x - max)).sum::().ln()) + } + + #[bench] + fn bench_sofmax_exp_sum_unrolled(b: &mut Bencher) { + let x = vec![1.0; 32]; + let max = 1.0; + + b.iter(|| softmax_exp_sum(&x, max).ln()) + } + #[bench] fn bench_exp(b: &mut Bencher) { let x: Vec = vec![1.0; 32]; From b99e30021060f1517b53645f75d9c55daabedd5c Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 19 Jan 2018 09:31:08 +0000 Subject: [PATCH 046/108] More accurate fastexp function. --- src/fast_approx.rs | 21 ++++++++++++ src/lib.rs | 3 +- src/nodes.rs | 84 ++++++++++++++++++++++++++++------------------ src/numerics.rs | 16 ++++----- 4 files changed, 82 insertions(+), 42 deletions(-) diff --git a/src/fast_approx.rs b/src/fast_approx.rs index b2e7926b..7ab2f962 100644 --- a/src/fast_approx.rs +++ b/src/fast_approx.rs @@ -91,3 +91,24 @@ pub fn tanhf_fast(x: f32) -> f32 { x * (27.0 + x.powi(2)) / (27.0 + 9.0 * x.powi(2)) } } + +#[inline(always)] +pub fn fastpow2(x: f32) -> f32 { + let offset = if x < 0.0 { 1.0 } else { 0.0 }; + let clip = if x < -126.0 { -126.0 } else { x }; + let w = clip as i32; + let z = clip - w as f32 + offset; + + let v = UintFloat { + i: ((1 << 23) as f32 + * (clip + 121.2740575 + 27.7280233 / (4.84252568 - z) - 1.49012907 * z)) + as u32, + }; + + unsafe { v.f } +} + +#[inline(always)] +pub fn fastexp(x: f32) -> f32 { + fastpow2(1.442695040 * x) +} diff --git a/src/lib.rs b/src/lib.rs index 216aa73a..cc9e3714 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -569,7 +569,8 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; + let mut param_value = + unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); diff --git a/src/nodes.rs b/src/nodes.rs index 75d15d8d..0db7c605 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -193,9 +193,7 @@ where let mut self_value = self.value.borrow_mut(); - numerics::add(lhs_value.deref(), - rhs_value.deref(), - self_value.deref_mut()); + numerics::add(lhs_value.deref(), rhs_value.deref(), self_value.deref_mut()); } fn backward(&self, gradient: &Ref) { self.lhs.backward(gradient); @@ -654,9 +652,11 @@ where let mut dest = self.value.borrow_mut(); - numerics::sub(self.lhs.value().deref(), - self.rhs.value().deref(), - dest.deref_mut()); + numerics::sub( + self.lhs.value().deref(), + self.rhs.value().deref(), + dest.deref_mut(), + ); } fn backward(&self, gradient: &Ref) { @@ -664,9 +664,11 @@ where BackwardAction::Set => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::simd_scaled_assign(rhs_gradient.as_slice_mut().unwrap(), - gradient.as_slice().unwrap(), - -1.0); + numerics::simd_scaled_assign( + rhs_gradient.as_slice_mut().unwrap(), + gradient.as_slice().unwrap(), + -1.0, + ); } BackwardAction::Increment => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -747,35 +749,45 @@ where let mut dest = self.value.borrow_mut(); - numerics::mul(self.lhs.value().deref(), - self.rhs.value().deref(), - dest.deref_mut()); + numerics::mul( + self.lhs.value().deref(), + self.rhs.value().deref(), + dest.deref_mut(), + ); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { BackwardAction::Set => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - numerics::mul(self.rhs.value().deref(), - gradient.deref(), - lhs_gradient.deref_mut()); + numerics::mul( + self.rhs.value().deref(), + gradient.deref(), + lhs_gradient.deref_mut(), + ); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::mul(self.lhs.value().deref(), - gradient.deref(), - rhs_gradient.deref_mut()); + numerics::mul( + self.lhs.value().deref(), + gradient.deref(), + rhs_gradient.deref_mut(), + ); } BackwardAction::Increment => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::increment_mul(self.rhs.value().deref(), - gradient.deref(), - lhs_gradient.deref_mut()); - numerics::increment_mul(self.lhs.value().deref(), - gradient.deref(), - rhs_gradient.deref_mut()); + numerics::increment_mul( + self.rhs.value().deref(), + gradient.deref(), + lhs_gradient.deref_mut(), + ); + numerics::increment_mul( + self.lhs.value().deref(), + gradient.deref(), + rhs_gradient.deref_mut(), + ); } } @@ -851,9 +863,11 @@ where let mut dest = self.value.borrow_mut(); - numerics::div(self.lhs.value().deref(), - self.rhs.value().deref(), - dest.deref_mut()); + numerics::div( + self.lhs.value().deref(), + self.rhs.value().deref(), + dest.deref_mut(), + ); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { @@ -861,9 +875,11 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); - numerics::div(gradient.deref(), - rhs_value.deref(), - lhs_gradient.deref_mut()); + numerics::div( + gradient.deref(), + rhs_value.deref(), + lhs_gradient.deref_mut(), + ); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -880,9 +896,11 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); - numerics::increment_div(gradient.deref(), - rhs_value.deref(), - lhs_gradient.deref_mut()); + numerics::increment_div( + gradient.deref(), + rhs_value.deref(), + lhs_gradient.deref_mut(), + ); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); diff --git a/src/numerics.rs b/src/numerics.rs index f266087f..cf5d7362 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -4,7 +4,7 @@ use stdsimd; use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; -use fast_approx::{expf_fast, fastlog, tanhf_fast}; +use fast_approx::{fastexp, fastlog, tanhf_fast}; use super::Arr; @@ -12,7 +12,7 @@ use super::Arr; #[inline(always)] pub fn exp(x: f32) -> f32 { if cfg!(feature = "fast-math") { - expf_fast(x) + fastexp(x) } else { x.exp() } @@ -281,9 +281,9 @@ pub fn simd_scaled_add(xs: &mut [f32], ys: &[f32], alpha: f32) { } } - macro_rules! simd_binary_op { - ( $name:ident, $simd_name:ident, $increment_name:ident, $simd_increment_name:ident, $op:tt ) => { + ( $name:ident, $simd_name:ident, + $increment_name:ident,$simd_increment_name:ident, $op:tt ) => { pub fn $name(xs: &Arr, ys: &Arr, out: &mut Arr) { $simd_name(xs.as_slice().unwrap(), ys.as_slice().unwrap(), @@ -520,10 +520,10 @@ mod tests { } #[test] - fn test_expf() { + fn test_fastexp() { let values: Vec = vec![-0.5, -0.1, 0.0, 0.1, 0.5]; for &x in &values { - println!("Input: {}, stdlib: {}, fast: {}", x, x.exp(), expf_fast(x)); + println!("Input: {}, stdlib: {}, fast: {}", x, x.exp(), fastexp(x)); } } @@ -664,12 +664,12 @@ mod tests { } #[bench] - fn bench_exp_fast(b: &mut Bencher) { + fn bench_fastexp(b: &mut Bencher) { let x: Vec = vec![1.0; 32]; let mut v = 0.0; - b.iter(|| x.iter().for_each(|&y| v += expf_fast(y))); + b.iter(|| x.iter().for_each(|&y| v += fastexp(y))); } #[bench] From c9796ad936e5d210966d831f03bc346a22962ae4 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 19 Jan 2018 22:01:18 +0000 Subject: [PATCH 047/108] Remove unused code. --- src/fast_approx.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/fast_approx.rs b/src/fast_approx.rs index 7ab2f962..abf59239 100644 --- a/src/fast_approx.rs +++ b/src/fast_approx.rs @@ -41,8 +41,6 @@ * Contact: Paul Mineiro * *=====================================================================*/ -use std::mem; - #[repr(C)] union FloatUint { f: f32, @@ -74,13 +72,6 @@ pub fn fastlog(x: f32) -> f32 { 0.69314718 * fastlog2(x) } -#[inline(always)] -pub fn expf_fast(x: f32) -> f32 { - let u = (12102203.0 * x + 1064866805.0) as i32; - - unsafe { mem::transmute::(u) } -} - #[inline(always)] pub fn tanhf_fast(x: f32) -> f32 { if x < -3.0 { From 87246ee9b64266d8382891626618507a6fd04571 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 20 Jan 2018 12:09:42 +0000 Subject: [PATCH 048/108] Accumulate gradients in DotNodes. --- src/lib.rs | 15 ++++++++++++++- src/nodes.rs | 47 +++++++++++++++++++++++++---------------------- src/numerics.rs | 5 +++++ 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cc9e3714..2154c343 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -669,7 +669,7 @@ mod tests { use super::*; - const TOLERANCE: f32 = 0.1; + const TOLERANCE: f32 = 0.2; fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) @@ -732,6 +732,19 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn dot_accumulation_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut y = ParameterNode::new(random_matrix(5, 10)); + let z = x.clone().dot(&y); + let mut v = z.clone() * z.clone(); + + let (difference, gradient) = finite_difference(&mut x, &mut v); + assert_close(&difference, &gradient, TOLERANCE); + + let (difference, gradient) = finite_difference(&mut y, &mut v); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] fn square_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut z = x.square(); diff --git a/src/nodes.rs b/src/nodes.rs index 0db7c605..3b34bdec 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -999,29 +999,32 @@ where } fn backward(&self, gradient: &Ref) { - // TODO: handle the non-overwrite case - match self.counter.backward() { - _ => { - let rhs_value = self.rhs.value(); - let lhs_value = self.lhs.value(); + let beta = match self.counter.backward() { + BackwardAction::Set => 0.0, + BackwardAction::Increment => 1.0, + }; - let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + { + let rhs_value = self.rhs.value(); + let lhs_value = self.lhs.value(); - numerics::mat_mul(1.0, gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); - numerics::mat_mul( - 1.0, - &lhs_value.t(), - gradient.deref(), - 0.0, - &mut rhs_gradient, - ); - } + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + numerics::mat_mul(1.0, gradient, &rhs_value.t(), beta, &mut lhs_gradient); + numerics::mat_mul( + 1.0, + &lhs_value.t(), + gradient.deref(), + beta, + &mut rhs_gradient, + ); } - // Always recurse because we haven't actually handled accumulation yet - self.lhs.backward(&self.lhs_gradient.borrow()); - self.rhs.backward(&self.rhs_gradient.borrow()); + if self.counter.recurse_backward() { + self.lhs.backward(&self.lhs_gradient.borrow()); + self.rhs.backward(&self.rhs_gradient.borrow()); + } } fn value(&self) -> Bor { @@ -1532,9 +1535,9 @@ where let mut dest = self.value.borrow_mut(); - numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { - 1.0 / (1.0 + numerics::exp(-x)) - }); + numerics::map_assign(dest.deref_mut(), + self.operand.value().deref(), + |x| numerics::sigmoid(x)); } fn backward(&self, gradient: &Ref) { diff --git a/src/numerics.rs b/src/numerics.rs index cf5d7362..5544002b 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -38,6 +38,11 @@ pub fn tanh(x: f32) -> f32 { } } +#[inline(always)] +pub fn sigmoid(x: f32) -> f32 { + 1.0 / (1.0 + exp(-x)) +} + pub fn softmax_exp_sum(xs: &[f32], max: f32) -> f32 { let mut xs = xs; let mut s = 0.; From 69533dec185ca005f92f2d69d1fa44edf64cbb10 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 21 Jan 2018 21:44:18 +0000 Subject: [PATCH 049/108] Bump version. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 016977cb..3b7927ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.7.1" +version = "0.7.2" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From c4a44cfd515ab939b009d8a17c58e3be3f0717ac Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 25 Jan 2018 17:48:07 +0000 Subject: [PATCH 050/108] Add an adagrad implementation. --- src/lib.rs | 64 ++++++++++++++++++++++++++++++++++++++++++++++--- src/nodes.rs | 26 +++++++++++++++++--- src/numerics.rs | 11 ++++++++- 3 files changed, 94 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2154c343..9ee477a8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -569,8 +569,7 @@ impl SGD { let learning_rate = self.learning_rate; for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = - unsafe { &mut *(parameter.node.value.deref().value.as_ptr()) }; + let mut param_value = unsafe { parameter.node.value.value_mut() }; if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); @@ -594,6 +593,65 @@ impl SGD { } } +/// Adagrad optimizer, scaled the learning rate by the inverse of previously +/// accumulated gradients. +pub struct Adagrad { + learning_rate: f32, + parameters: Vec>, +} + +impl Adagrad { + /// Create a new optimizer instance with a given set of parameters. + pub fn new(learning_rate: f32, parameters: Vec>) -> Self { + Adagrad { + learning_rate: learning_rate, + parameters: parameters, + } + } + + /// Perform a single SGD step. + pub fn step(&mut self) { + let learning_rate = self.learning_rate; + + for parameter in &self.parameters { + let mut sink = parameter.node.gradient.borrow_mut(); + let mut param_value = unsafe { parameter.node.value.value_mut() }; + let mut squared_gradient = + unsafe { parameter.node.value.squared_gradient_mut() }; + + if sink.has_dense { + for (value, gradient, squared_gradient) in izip!( + param_value.as_slice_mut().unwrap(), + sink.dense_gradient().as_slice().unwrap(), + squared_gradient.as_slice_mut().unwrap() + ) { + *value -= learning_rate * gradient / squared_gradient.sqrt(); + *squared_gradient += gradient.powi(2); + } + } + + if sink.has_sparse { + for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let grad_row = grad.subview(Axis(0), grad_idx); + let mut param_row = param_value.subview_mut(Axis(0), param_idx); + let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); + + for (value, gradient, squared_gradient) in izip!( + param_row.as_slice_mut().unwrap(), + grad_row.into_slice().unwrap(), + squared_row.as_slice_mut().unwrap() + ) { + *value -= learning_rate * gradient / squared_gradient.sqrt(); + *squared_gradient += gradient.powi(2); + } + } + } + } + } + } +} + /// Compute finite difference gradient estimates of the output variable /// with respect to the input. Use to verify correctness of gradient /// computations. @@ -997,7 +1055,7 @@ mod tests { let mut loss = (output.clone() - y_hat.clone()).square(); let num_epochs = 100; - let mut optimizer = SGD::new(0.1, loss.parameters()); + let mut optimizer = Adagrad::new(0.1, loss.parameters()); let mut loss_val = 0.0; diff --git a/src/nodes.rs b/src/nodes.rs index 3b34bdec..c08b2ed1 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -523,15 +523,35 @@ unsafe impl Sync for HogwildParameter {} #[derive(Debug, Serialize, Deserialize)] pub struct HogwildParameter { pub value: RefCell, + pub squared_gradients: RefCell, } impl HogwildParameter { /// Create a new parameter object. pub fn new(value: Arr) -> Self { + let squared_gradients = &value * 0.0 + 1.0; + HogwildParameter { value: RefCell::new(value), + squared_gradients: RefCell::new(squared_gradients), } } + + pub fn value(&self) -> Ref { + self.value.borrow() + } + + pub fn squared_gradients(&self) -> Ref { + self.squared_gradients.borrow() + } + + pub(crate) unsafe fn value_mut(&self) -> &mut Arr { + &mut *(self.value.as_ptr()) + } + + pub(crate) unsafe fn squared_gradient_mut(&self) -> &mut Arr { + &mut *(self.squared_gradients.as_ptr()) + } } /// Parameter node, holds the optimizable parameters of the model. @@ -1535,9 +1555,9 @@ where let mut dest = self.value.borrow_mut(); - numerics::map_assign(dest.deref_mut(), - self.operand.value().deref(), - |x| numerics::sigmoid(x)); + numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { + numerics::sigmoid(x) + }); } fn backward(&self, gradient: &Ref) { diff --git a/src/numerics.rs b/src/numerics.rs index 5544002b..9e252b84 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -4,7 +4,7 @@ use stdsimd; use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; -use fast_approx::{fastexp, fastlog, tanhf_fast}; +use fast_approx::{fastexp, fastlog, tanhf_fast, fastpow2}; use super::Arr; @@ -43,6 +43,15 @@ pub fn sigmoid(x: f32) -> f32 { 1.0 / (1.0 + exp(-x)) } +#[inline(always)] +pub fn pow2(x: f32) -> f32 { + if cfg!(feature = "fast-math") { + fastpow2(x) + } else { + x.powi(2) + } +} + pub fn softmax_exp_sum(xs: &[f32], max: f32) -> f32 { let mut xs = xs; let mut s = 0.; From c83e60bbcb90a9809800692769eb37bd473a3237 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 26 Feb 2018 18:23:46 +0000 Subject: [PATCH 051/108] Don't use fast numerics for pow and sigmoid. --- src/lib.rs | 26 ++++++++++++-------------- src/numerics.rs | 8 ++------ 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9ee477a8..7a32d08e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,7 +57,7 @@ //! # let y_hat = slope.clone() * x.clone() + intercept.clone(); //! # let mut loss = (y.clone() - y_hat).square(); //! # let num_epochs = 10; -//! let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); +//! let mut optimizer = SGD::new(0.1, loss.parameters()); //! //! for _ in 0..num_epochs { //! let x_value: f32 = rand::random(); @@ -105,7 +105,7 @@ //! let y_hat = slope.clone() * x.clone() + intercept.clone(); //! let mut loss = (y.clone() - y_hat).square(); //! -//! let mut optimizer = SGD::new(0.1, vec![slope.clone(), intercept.clone()]); +//! let mut optimizer = SGD::new(0.1, loss.parameters()); //! //! for _ in 0..num_epochs { //! let x_value: f32 = rand::random(); @@ -142,7 +142,6 @@ // TODO: pass through of parent values in .value(), // optimizations in forward // check for needs gradient -// topological sort to avoid duplicate work #[macro_use] extern crate serde_derive; @@ -616,17 +615,16 @@ impl Adagrad { for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); let mut param_value = unsafe { parameter.node.value.value_mut() }; - let mut squared_gradient = - unsafe { parameter.node.value.squared_gradient_mut() }; + let mut squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; if sink.has_dense { - for (value, gradient, squared_gradient) in izip!( + for (value, &gradient, squared_gradient) in izip!( param_value.as_slice_mut().unwrap(), sink.dense_gradient().as_slice().unwrap(), squared_gradient.as_slice_mut().unwrap() ) { *value -= learning_rate * gradient / squared_gradient.sqrt(); - *squared_gradient += gradient.powi(2); + *squared_gradient += numerics::pow2(gradient); } } @@ -637,13 +635,13 @@ impl Adagrad { let mut param_row = param_value.subview_mut(Axis(0), param_idx); let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); - for (value, gradient, squared_gradient) in izip!( + for (value, &gradient, squared_gradient) in izip!( param_row.as_slice_mut().unwrap(), grad_row.into_slice().unwrap(), squared_row.as_slice_mut().unwrap() ) { *value -= learning_rate * gradient / squared_gradient.sqrt(); - *squared_gradient += gradient.powi(2); + *squared_gradient += numerics::pow2(gradient); } } } @@ -943,7 +941,7 @@ mod tests { let slope = ParameterNode::new(random_matrix(1, 1)); let intercept = ParameterNode::new(random_matrix(1, 1)); - let num_epochs = 100; + let num_epochs = 200; let x = InputNode::new(random_matrix(1, 1)); let y = InputNode::new(random_matrix(1, 1)); @@ -952,11 +950,11 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let mut optimizer = SGD::new(0.1, loss.parameters()); + let mut optimizer = Adagrad::new(0.5, loss.parameters()); for _ in 0..num_epochs { let _x = arr2(&[[rand::random::()]]); - let _y = 3.0 * &_x + 5.0; + let _y = 0.5 * &_x + 0.2; x.set_value(&_x); y.set_value(&_y); @@ -1054,7 +1052,7 @@ mod tests { let y_hat = u_vec.vector_dot(&v_vec); let mut loss = (output.clone() - y_hat.clone()).square(); - let num_epochs = 100; + let num_epochs = 200; let mut optimizer = Adagrad::new(0.1, loss.parameters()); let mut loss_val = 0.0; @@ -1082,7 +1080,7 @@ mod tests { println!("Loss {}", loss_val) } - assert!(loss_val < 1e-3); + assert!(loss_val < 1e-2); } #[test] diff --git a/src/numerics.rs b/src/numerics.rs index 9e252b84..8ce4d72d 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -40,16 +40,12 @@ pub fn tanh(x: f32) -> f32 { #[inline(always)] pub fn sigmoid(x: f32) -> f32 { - 1.0 / (1.0 + exp(-x)) + 1.0 / (1.0 + (-x).exp()) } #[inline(always)] pub fn pow2(x: f32) -> f32 { - if cfg!(feature = "fast-math") { - fastpow2(x) - } else { - x.powi(2) - } + x.powi(2) } pub fn softmax_exp_sum(xs: &[f32], max: f32) -> f32 { From 6871e4544c85f9a6b9896569e7c254220b8b0f5a Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 28 Feb 2018 23:59:43 +0000 Subject: [PATCH 052/108] Fix hogwild param safety. Don't use borrow when parameters can be shared across threads. --- src/nodes.rs | 8 ++++---- src/numerics.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index c08b2ed1..c25fee02 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -537,12 +537,12 @@ impl HogwildParameter { } } - pub fn value(&self) -> Ref { - self.value.borrow() + pub fn value(&self) -> &Arr { + unsafe { & *(self.value.as_ptr()) } } - pub fn squared_gradients(&self) -> Ref { - self.squared_gradients.borrow() + pub fn squared_gradients(&self) -> &Arr { + unsafe { & *(self.squared_gradients.as_ptr()) } } pub(crate) unsafe fn value_mut(&self) -> &mut Arr { diff --git a/src/numerics.rs b/src/numerics.rs index 8ce4d72d..cba5212b 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -4,7 +4,7 @@ use stdsimd; use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; -use fast_approx::{fastexp, fastlog, tanhf_fast, fastpow2}; +use fast_approx::{fastexp, fastlog, tanhf_fast}; use super::Arr; From d06cb3520c061fdaa65f9ad105149325b6e46af6 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 4 Mar 2018 17:01:11 +0000 Subject: [PATCH 053/108] Sensibly clone LSTM parameter objects. --- src/lib.rs | 2 ++ src/nn/lstm.rs | 26 ++++++++++++++++++++++++++ src/nodes.rs | 6 +++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7a32d08e..6d4ee286 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -267,6 +267,8 @@ where } } +pub type BoxedNode = Rc>; + impl Variable where T: Node, diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 019d8d42..4d19e4b0 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -58,6 +58,7 @@ use {Arr, DataInput, Variable}; /// /// Construct this first, then use the `build` method to instantiate /// LSTM cell nodes. +#[derive(Debug)] pub struct Parameters { input_dim: usize, hidden_dim: usize, @@ -75,6 +76,31 @@ pub struct Parameters { output_gate_biases: Arc, } +impl Clone for Parameters { + /// Clones the parameter values. + /// + /// (This is in contrast to creating a shared reference to + /// the same paratmer object.) + fn clone(&self) -> Self { + Parameters { + input_dim: self.input_dim, + hidden_dim: self.hidden_dim, + + forget_weights: Arc::new(self.forget_weights.as_ref().clone()), + forget_biases: Arc::new(self.forget_biases.as_ref().clone()), + + update_gate_weights: Arc::new(self.update_gate_weights.as_ref().clone()), + update_gate_biases: Arc::new(self.update_gate_biases.as_ref().clone()), + + update_value_weights: Arc::new(self.update_gate_weights.as_ref().clone()), + update_value_biases: Arc::new(self.update_value_biases.as_ref().clone()), + + output_gate_weights: Arc::new(self.output_gate_weights.as_ref().clone()), + output_gate_biases: Arc::new(self.output_gate_biases.as_ref().clone()), + } + } +} + impl Parameters { /// Create a new LSTM parameters object. pub fn new(input_dim: usize, hidden_dim: usize) -> Self { diff --git a/src/nodes.rs b/src/nodes.rs index c25fee02..b56b77bb 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -520,7 +520,7 @@ unsafe impl Sync for HogwildParameter {} /// Struct used to hold parameters that need to be shared among /// multiple `ParameterNode`s for asynchronous, parallel optimization. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct HogwildParameter { pub value: RefCell, pub squared_gradients: RefCell, @@ -538,11 +538,11 @@ impl HogwildParameter { } pub fn value(&self) -> &Arr { - unsafe { & *(self.value.as_ptr()) } + unsafe { &*(self.value.as_ptr()) } } pub fn squared_gradients(&self) -> &Arr { - unsafe { & *(self.squared_gradients.as_ptr()) } + unsafe { &*(self.squared_gradients.as_ptr()) } } pub(crate) unsafe fn value_mut(&self) -> &mut Arr { From 1d318cab92065e53aa30a3de5237189fb251a7e9 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 19 Apr 2018 16:57:33 +0100 Subject: [PATCH 054/108] Start removing simd to make things run on stable. --- Cargo.toml | 1 - src/lib.rs | 36 +++++---- src/numerics.rs | 208 ++++++++++++++++-------------------------------- 3 files changed, 91 insertions(+), 154 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3b7927ac..7f853a82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ itertools = "0.7.3" rayon = "0.9.0" serde = { version = "1.0.24", features = ["rc"] } serde_derive = "1.0.24" -stdsimd = "0.0.3" [dev-dependencies] ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } diff --git a/src/lib.rs b/src/lib.rs index 6d4ee286..91c55bc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,7 +151,6 @@ extern crate ndarray; extern crate rand; extern crate rayon; extern crate smallvec; -extern crate stdsimd; extern crate test; #[macro_use] @@ -163,19 +162,19 @@ use ndarray::Axis; pub type Arr = ndarray::Array2; use std::cell::RefCell; +use std::clone::Clone; use std::ops::{Add, Deref, Div, Mul, Neg, Sub}; use std::rc::Rc; -use std::clone::Clone; -mod nodes; -mod numerics; mod fast_approx; pub mod nn; +mod nodes; +mod numerics; use nodes::*; -pub use numerics::simd_dot; pub use nodes::{Bor, HogwildParameter, IndexInputNode, InputNode, Node, ParameterNode}; +pub use numerics::simd_dot; fn clamp(x: f32, min: f32, max: f32) -> f32 { if x > max { @@ -598,6 +597,7 @@ impl SGD { /// accumulated gradients. pub struct Adagrad { learning_rate: f32, + l2: f32, parameters: Vec>, } @@ -606,10 +606,17 @@ impl Adagrad { pub fn new(learning_rate: f32, parameters: Vec>) -> Self { Adagrad { learning_rate: learning_rate, + l2: 0.0, parameters: parameters, } } + /// Set the L2 penalty. + pub fn l2_penalty(mut self, l2_penalty: f32) -> Self { + self.l2 = l2_penalty; + self + } + /// Perform a single SGD step. pub fn step(&mut self) { let learning_rate = self.learning_rate; @@ -625,7 +632,7 @@ impl Adagrad { sink.dense_gradient().as_slice().unwrap(), squared_gradient.as_slice_mut().unwrap() ) { - *value -= learning_rate * gradient / squared_gradient.sqrt(); + *value -= learning_rate * gradient / squared_gradient.sqrt() + *value * self.l2; *squared_gradient += numerics::pow2(gradient); } } @@ -642,7 +649,8 @@ impl Adagrad { grad_row.into_slice().unwrap(), squared_row.as_slice_mut().unwrap() ) { - *value -= learning_rate * gradient / squared_gradient.sqrt(); + *value -= learning_rate * gradient / squared_gradient.sqrt() + + *value * self.l2; *squared_gradient += numerics::pow2(gradient); } } @@ -721,9 +729,9 @@ fn assert_close(x: &Arr, y: &Arr, tol: f32) { #[cfg(test)] mod tests { - use std::sync::Arc; use ndarray::arr2; use rayon::prelude::*; + use std::sync::Arc; use super::*; @@ -998,13 +1006,11 @@ mod tests { let mut optimizer = SGD::new(0.1, loss.parameters()); for _ in 0..num_epochs { - let _x = arr2(&[ - [ - rand::random::(), - rand::random::(), - rand::random::(), - ], - ]); + let _x = arr2(&[[ + rand::random::(), + rand::random::(), + rand::random::(), + ]]); let _y = &_x.dot(&coefficients) + 5.0; x.set_value(&_x); diff --git a/src/numerics.rs b/src/numerics.rs index cba5212b..727fe29e 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -1,8 +1,7 @@ -use std::cmp; -use stdsimd; +use std; -use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; +use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; use fast_approx::{fastexp, fastlog, tanhf_fast}; @@ -115,23 +114,7 @@ fn vec_mat_mul( } fn saxpy(alpha: f32, xs: &[f32], beta: f32, outs: &mut [f32]) { - let stride = 8; - let simd_alpha = stdsimd::simd::f32x8::splat(alpha); - let simd_beta = stdsimd::simd::f32x8::splat(beta); - - let split_idx = xs.len() / stride * stride; - let (simd_xs, scalar_xs) = xs.split_at(split_idx); - let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); - - for (x, out) in izip!(simd_xs.chunks(stride), simd_outs.chunks_mut(stride)) { - unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) * simd_alpha - + stdsimd::simd::f32x8::load_unchecked(out, 0) * simd_beta; - elem.store_unchecked(out, 0); - } - } - - for (&x, out) in izip!(scalar_xs.iter(), scalar_outs.iter_mut()) { + for (&x, out) in izip!(xs.iter(), outs.iter_mut()) { *out = x * alpha + *out * beta; } } @@ -202,126 +185,95 @@ pub fn mat_mul( /// SIMD-enabled vector-vector dot product. pub fn simd_dot(xs: &[f32], ys: &[f32]) -> f32 { - let mut simd_result = stdsimd::simd::f32x8::splat(0.0); - let mut scalar_result = 0.0; - let stride = 8; - - let split_idx = cmp::min(xs.len(), ys.len()) / stride * stride; - let (simd_xs, scalar_xs) = xs.split_at(split_idx); - let (simd_ys, scalar_ys) = ys.split_at(split_idx); - - for (x, y) in simd_xs.chunks(stride).zip(simd_ys.chunks(stride)) { - unsafe { - simd_result = simd_result - + stdsimd::simd::f32x8::load_unchecked(x, 0) - * stdsimd::simd::f32x8::load_unchecked(y, 0); - } + let len = std::cmp::min(xs.len(), ys.len()); + let mut xs = &xs[..len]; + let mut ys = &ys[..len]; + + let mut s = 0.; + let (mut p0, mut p1, mut p2, mut p3, mut p4, mut p5, mut p6, mut p7) = + (0., 0., 0., 0., 0., 0., 0., 0.); + + while xs.len() >= 8 { + p0 += xs[0] * ys[0]; + p1 += xs[1] * ys[1]; + p2 += xs[2] * ys[2]; + p3 += xs[3] * ys[3]; + p4 += xs[4] * ys[4]; + p5 += xs[5] * ys[5]; + p6 += xs[6] * ys[6]; + p7 += xs[7] * ys[7]; + + xs = &xs[8..]; + ys = &ys[8..]; } + s += p0 + p4; + s += p1 + p5; + s += p2 + p6; + s += p3 + p7; - for (x_scalar, y_scalar) in scalar_xs.iter().zip(scalar_ys.iter()) { - scalar_result += x_scalar * y_scalar; + for i in 0..xs.len() { + s += xs[i] * ys[i]; } - scalar_result - + (0..stride as u32) - .map(|idx| simd_result.extract(idx)) - .sum::() + s } pub fn simd_sum(xs: &[f32]) -> f32 { - let mut simd_result = stdsimd::simd::f32x8::splat(0.0); - let mut scalar_result = 0.0; - let stride = 8; + let mut xs = xs; + + let mut s = 0.; + let (mut p0, mut p1, mut p2, mut p3, mut p4, mut p5, mut p6, mut p7) = + (0., 0., 0., 0., 0., 0., 0., 0.); - let split_idx = (xs.len() / stride) * stride; - let (simd_xs, scalar_xs) = xs.split_at(split_idx); + while xs.len() >= 8 { + p0 += xs[0]; + p1 += xs[1]; + p2 += xs[2]; + p3 += xs[3]; + p4 += xs[4]; + p5 += xs[5]; + p6 += xs[6]; + p7 += xs[7]; - for x in simd_xs.chunks(stride) { - unsafe { simd_result = simd_result + stdsimd::simd::f32x8::load_unchecked(x, 0) } + xs = &xs[8..]; } - for x_scalar in scalar_xs.iter() { - scalar_result += x_scalar; + s += p0 + p4; + s += p1 + p5; + s += p2 + p6; + s += p3 + p7; + + for i in 0..xs.len() { + s += xs[i]; } - scalar_result - + (0..stride as u32) - .map(|idx| simd_result.extract(idx)) - .sum::() + s } pub fn simd_scaled_assign(xs: &mut [f32], ys: &[f32], alpha: f32) { - let stride = 8; - let simd_alpha = stdsimd::simd::f32x8::splat(alpha); - - let split_idx = xs.len() / stride * stride; - let (simd_xs, scalar_xs) = xs.split_at_mut(split_idx); - let (simd_ys, scalar_ys) = ys.split_at(split_idx); - - for (x, y) in simd_xs.chunks_mut(stride).zip(simd_ys.chunks(stride)) { - unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(y, 0) * simd_alpha; - elem.store_unchecked(x, 0); - } - } - - for (x_scalar, y_scalar) in scalar_xs.iter_mut().zip(scalar_ys.iter()) { - *x_scalar = y_scalar * alpha; + for (x, y) in xs.iter_mut().zip(ys.iter()) { + *x = y * alpha; } } pub fn simd_scaled_add(xs: &mut [f32], ys: &[f32], alpha: f32) { - let stride = 8; - let simd_alpha = stdsimd::simd::f32x8::splat(alpha); - - let split_idx = xs.len() / stride * stride; - let (simd_xs, scalar_xs) = xs.split_at_mut(split_idx); - let (simd_ys, scalar_ys) = ys.split_at(split_idx); - - for (x, y) in simd_xs.chunks_mut(stride).zip(simd_ys.chunks(stride)) { - unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) - + stdsimd::simd::f32x8::load_unchecked(y, 0) * simd_alpha; - elem.store_unchecked(x, 0); - } - } - - for (x_scalar, y_scalar) in scalar_xs.iter_mut().zip(scalar_ys.iter()) { - *x_scalar += y_scalar * alpha; + for (x, y) in xs.iter_mut().zip(ys.iter()) { + *x += y * alpha; } } -macro_rules! simd_binary_op { - ( $name:ident, $simd_name:ident, - $increment_name:ident,$simd_increment_name:ident, $op:tt ) => { +macro_rules! slice_binary_op { + ( $name:ident, $slice_name:ident, + $increment_name:ident,$slice_increment_name:ident, $op:tt ) => { pub fn $name(xs: &Arr, ys: &Arr, out: &mut Arr) { - $simd_name(xs.as_slice().unwrap(), + $slice_name(xs.as_slice().unwrap(), ys.as_slice().unwrap(), out.as_slice_mut().unwrap()); } - fn $simd_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { - let stride = 8; - - let split_idx = xs.len() / stride * stride; - let (simd_xs, scalar_xs) = xs.split_at(split_idx); - let (simd_ys, scalar_ys) = ys.split_at(split_idx); - let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); - - for (x, y, out) in izip!( - simd_xs.chunks(stride), - simd_ys.chunks(stride), - simd_outs.chunks_mut(stride) - ) { - unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(x, 0) - $op stdsimd::simd::f32x8::load_unchecked(y, 0); - elem.store_unchecked(out, 0); - } - } - + fn $slice_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { for (&x_scalar, &y_scalar, out_scalar) in - izip!(scalar_xs.iter(), scalar_ys.iter(), scalar_outs.iter_mut()) + izip!(xs.iter(), ys.iter(), outs.iter_mut()) { *out_scalar = x_scalar $op y_scalar; } @@ -329,35 +281,15 @@ macro_rules! simd_binary_op { #[allow(dead_code)] pub fn $increment_name(xs: &Arr, ys: &Arr, out: &mut Arr) { - $simd_increment_name(xs.as_slice().unwrap(), + $slice_increment_name(xs.as_slice().unwrap(), ys.as_slice().unwrap(), out.as_slice_mut().unwrap()); } #[allow(dead_code)] - fn $simd_increment_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { - let stride = 8; - - let split_idx = xs.len() / stride * stride; - let (simd_xs, scalar_xs) = xs.split_at(split_idx); - let (simd_ys, scalar_ys) = ys.split_at(split_idx); - let (simd_outs, scalar_outs) = outs.split_at_mut(split_idx); - - for (x, y, out) in izip!( - simd_xs.chunks(stride), - simd_ys.chunks(stride), - simd_outs.chunks_mut(stride) - ) { - unsafe { - let elem = stdsimd::simd::f32x8::load_unchecked(out, 0) + - (stdsimd::simd::f32x8::load_unchecked(x, 0) - $op stdsimd::simd::f32x8::load_unchecked(y, 0)); - elem.store_unchecked(out, 0); - } - } - + fn $slice_increment_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { for (&x_scalar, &y_scalar, out_scalar) in - izip!(scalar_xs.iter(), scalar_ys.iter(), scalar_outs.iter_mut()) + izip!(xs.iter(), ys.iter(), outs.iter_mut()) { *out_scalar += x_scalar $op y_scalar; } @@ -365,10 +297,10 @@ macro_rules! simd_binary_op { } } -simd_binary_op!(add, simd_add, increment_add, increment_simd_add, +); -simd_binary_op!(sub, simd_sub, increment_sub, increment_simd_sub, -); -simd_binary_op!(mul, simd_mul, increment_mul, increment_simd_mul, *); -simd_binary_op!(div, simd_div, increment_div, increment_simd_div, /); +slice_binary_op!(add, slice_add, increment_add, increment_slice_add, +); +slice_binary_op!(sub, slice_sub, increment_sub, increment_slice_sub, -); +slice_binary_op!(mul, slice_mul, increment_mul, increment_slice_mul, *); +slice_binary_op!(div, slice_div, increment_div, increment_slice_div, /); pub fn slice_assign(xs: &mut [f32], ys: &[f32]) { for (x, &y) in xs.iter_mut().zip(ys.iter()) { From c91a6df64599a270d816933e3f6c5072ae04fb36 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 19 Apr 2018 19:01:47 +0100 Subject: [PATCH 055/108] Start working on moving benchmarks out. --- Cargo.toml | 5 + benches/benchmark.rs | 224 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 44 --------- src/nn/lstm.rs | 61 +----------- src/numerics.rs | 136 -------------------------- 5 files changed, 232 insertions(+), 238 deletions(-) create mode 100644 benches/benchmark.rs diff --git a/Cargo.toml b/Cargo.toml index 7f853a82..56b23226 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,11 @@ serde_derive = "1.0.24" ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } openblas-src = { version = "0.5.6", default-features = false, features = ["static"] } +criterion = "0.2.3" + +[[bench]] +name = "benchmark" +harness = false [profile.bench] lto = true diff --git a/benches/benchmark.rs b/benches/benchmark.rs new file mode 100644 index 00000000..539cef53 --- /dev/null +++ b/benches/benchmark.rs @@ -0,0 +1,224 @@ +#[macro_use] +extern crate criterion; + +use criterion::Criterion; + +fn bench_node_reuse(c: &mut Criterion) { + let dim = 128; + + let x = ParameterNode::new(random_matrix(1, dim)); + let y = ParameterNode::new(random_matrix(dim, 10)); + let v = x.dot(&y); + let z = v.clone() + v.clone() + v.clone() + v.clone(); + + c.bench_function( + "node_reuse", + b.iter(|| { + z.forward(); + z.zero_gradient(); + }), + ); +} + +fn bench_matrix_multiply(b: &mut Criterion) { + let dim = 64; + let num_epochs = 20; + + let x_data = Arc::new(HogwildParameter::new(random_matrix(1, dim))); + let y_data = Arc::new(HogwildParameter::new(random_matrix(dim, 10))); + + b.bench_function( + "bench_matrix_multiply", + b.iter(|| { + (0..rayon::current_num_threads()) + .into_par_iter() + .for_each(|_| { + let x = ParameterNode::shared(x_data.clone()); + let y = ParameterNode::shared(y_data.clone()); + + let v = x.dot(&y); + + for _ in 0..num_epochs { + v.forward(); + v.zero_gradient(); + } + }); + }), + ); +} + +// fn bench_sofmax_exp_sum(b: &mut Criterion) { +// let x = vec![1.0; 32]; +// let max = 1.0; + +// b.iter(|| x.iter().map(|&x| numerics::exp(x - max)).sum::().ln()) +// } + +// #[bench] +// fn bench_sofmax_exp_sum_unrolled(b: &mut Criterion) { +// let x = vec![1.0; 32]; +// let max = 1.0; + +// b.iter(|| softmax_exp_sum(&x, max).ln()) +// } + +// fn bench_exp(b: &mut Criterion) { +// let x: Vec = vec![1.0; 32]; + +// let mut v = 0.0; + +// b.iter(|| x.iter().for_each(|&y| v += y.exp())); +// } + +// fn bench_fastexp(b: &mut Criterion) { +// let x: Vec = vec![1.0; 32]; + +// let mut v = 0.0; + +// b.iter(|| x.iter().for_each(|&y| v += fastexp(y))); +// } + +// fn bench_dot(b: &mut Criterion) { +// let xs = vec![0.0; 256]; +// let ys = vec![0.0; 256]; + +// b.iter(|| dot(&xs[..], &ys[..])); +// } + +// fn bench_unrolled_dot(b: &mut Criterion) { +// let xs = vec![0.0; 256]; +// let ys = vec![0.0; 256]; + +// b.iter(|| unrolled_dot(&xs[..], &ys[..])); +// } + +// fn bench_simd_dot(b: &mut Criterion) { +// let xs = vec![0.0; 256]; +// let ys = vec![0.0; 256]; + +// b.iter(|| simd_dot(&xs[..], &ys[..])); +// } + +// fn bench_array_scaled_assign(b: &mut Criterion) { +// let mut xs = random_matrix(256, 1); +// let ys = random_matrix(256, 1); + +// b.iter(|| array_scaled_assign(&mut xs, &ys, 3.5)); +// } + +// fn bench_slice_scaled_assign(b: &mut Criterion) { +// let mut xs = random_matrix(256, 1); +// let ys = random_matrix(256, 1); + +// b.iter(|| scaled_assign(&mut xs, &ys, 3.5)); +// } + +// fn bench_array_assign(b: &mut Criterion) { +// let mut xs = random_matrix(256, 1); +// let ys = random_matrix(256, 1); + +// b.iter(|| array_assign(&mut xs, &ys)); +// } + +// fn bench_slice_assign(b: &mut Criterion) { +// let mut xs = random_matrix(256, 1); +// let ys = random_matrix(256, 1); + +// b.iter(|| assign(&mut xs, &ys)); +// } + +// fn dot_node_specializations_mm(b: &mut Criterion) { +// let x = random_matrix(64, 64); +// let y = random_matrix(64, 64); +// let mut z = random_matrix(64, 64); + +// b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); +// } + +// fn dot_node_general_vm(b: &mut Criterion) { +// let x = random_matrix(1, 64); +// let y = random_matrix(64, 64); +// let mut z = random_matrix(1, 64); + +// b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); +// } + +// fn dot_node_specializations_vm(b: &mut Criterion) { +// let x = random_matrix(1, 64); +// let y = random_matrix(64, 64); +// let mut z = random_matrix(1, 64); + +// b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); +// } + +// fn dot_node_specializations_mv(b: &mut Criterion) { +// let x = random_matrix(64, 64); +// let y = random_matrix(64, 1); +// let mut z = random_matrix(64, 1); + +// b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); +// } + +// fn dot_node_general_mv(b: &mut Criterion) { +// let x = random_matrix(64, 64); +// let y = random_matrix(64, 1); +// let mut z = random_matrix(64, 1); + +// b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); +// } + +// fn bench_lstm(b: &mut Criterion) { +// let sequence_length = 4; +// let num_digits = 10; +// let input_dim = 16; +// let hidden_dim = 32; + +// let lstm_params = Parameters::new(input_dim, hidden_dim); +// let lstm = lstm_params.build(); + +// let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); +// let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); +// let y = nodes::IndexInputNode::new(&vec![0]); + +// let inputs: Vec<_> = (0..sequence_length) +// .map(|_| nodes::IndexInputNode::new(&vec![0])) +// .collect(); +// let embeddings: Vec<_> = inputs +// .iter() +// .map(|input| embeddings.index(&input)) +// .collect(); + +// let hidden_states = lstm.forward(&embeddings); +// let hidden = hidden_states.last().unwrap(); + +// let prediction = hidden.dot(&final_layer); +// let mut loss = sparse_categorical_crossentropy(&prediction, &y); +// let mut optimizer = SGD::new(0.05, loss.parameters()); + +// let digits = pi_digits(100); + +// b.iter(|| { +// for i in 0..(digits.len() - sequence_length - 1) { +// let digit_chunk = &digits[i..(i + sequence_length + 1)]; +// if digit_chunk.len() < sequence_length + 1 { +// break; +// } + +// for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { +// input.set_value(digit); +// } + +// let target_digit = *digit_chunk.last().unwrap(); +// y.set_value(target_digit); + +// loss.forward(); +// loss.backward(1.0); + +// optimizer.step(); +// loss.zero_gradient(); +// } +// }); +// } + +criterion_group!(benches, bench_node_reuse, bench_matrix_multiply); +criterion_main!(benches); diff --git a/src/lib.rs b/src/lib.rs index 91c55bc0..6eda8f06 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(test)] //! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! //! # Features @@ -151,7 +150,6 @@ extern crate ndarray; extern crate rand; extern crate rayon; extern crate smallvec; -extern crate test; #[macro_use] extern crate itertools; @@ -1158,46 +1156,4 @@ mod tests { assert!(sum_loss / (losses.len() as f32) < 1e-3); } - - use test::Bencher; - - #[bench] - fn bench_node_reuse(b: &mut Bencher) { - let dim = 128; - - let x = ParameterNode::new(random_matrix(1, dim)); - let y = ParameterNode::new(random_matrix(dim, 10)); - let v = x.dot(&y); - let z = v.clone() + v.clone() + v.clone() + v.clone(); - - b.iter(|| { - z.forward(); - z.zero_gradient(); - }); - } - - #[bench] - fn bench_matrix_multiply(b: &mut Bencher) { - let dim = 64; - let num_epochs = 20; - - let x_data = Arc::new(HogwildParameter::new(random_matrix(1, dim))); - let y_data = Arc::new(HogwildParameter::new(random_matrix(dim, 10))); - - b.iter(|| { - (0..rayon::current_num_threads()) - .into_par_iter() - .for_each(|_| { - let x = ParameterNode::shared(x_data.clone()); - let y = ParameterNode::shared(y_data.clone()); - - let v = x.dot(&y); - - for _ in 0..num_epochs { - v.forward(); - v.zero_gradient(); - } - }); - }); - } } diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 4d19e4b0..9b9707a3 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -42,8 +42,8 @@ //! lstm.reset_state(); //! # } //! ``` -use std::sync::Arc; use std::rc::Rc; +use std::sync::Arc; use ndarray; @@ -280,13 +280,12 @@ impl Layer { mod tests { use std::ops::Deref; - use test::Bencher; use super::*; + use nn::losses::sparse_categorical_crossentropy; + use nodes::InputNode; use DataInput; use SGD; - use nodes::InputNode; - use nn::losses::sparse_categorical_crossentropy; fn pi_digits(num: usize) -> Vec { let pi_str = include_str!("pi.txt"); @@ -419,58 +418,4 @@ mod tests { assert!((correct as f32 / total as f32) > 0.75); } - - #[bench] - fn bench_lstm(b: &mut Bencher) { - let sequence_length = 4; - let num_digits = 10; - let input_dim = 16; - let hidden_dim = 32; - - let lstm_params = Parameters::new(input_dim, hidden_dim); - let lstm = lstm_params.build(); - - let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); - let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); - let y = nodes::IndexInputNode::new(&vec![0]); - - let inputs: Vec<_> = (0..sequence_length) - .map(|_| nodes::IndexInputNode::new(&vec![0])) - .collect(); - let embeddings: Vec<_> = inputs - .iter() - .map(|input| embeddings.index(&input)) - .collect(); - - let hidden_states = lstm.forward(&embeddings); - let hidden = hidden_states.last().unwrap(); - - let prediction = hidden.dot(&final_layer); - let mut loss = sparse_categorical_crossentropy(&prediction, &y); - let mut optimizer = SGD::new(0.05, loss.parameters()); - - let digits = pi_digits(100); - - b.iter(|| { - for i in 0..(digits.len() - sequence_length - 1) { - let digit_chunk = &digits[i..(i + sequence_length + 1)]; - if digit_chunk.len() < sequence_length + 1 { - break; - } - - for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { - input.set_value(digit); - } - - let target_digit = *digit_chunk.last().unwrap(); - y.set_value(target_digit); - - loss.forward(); - loss.backward(1.0); - - optimizer.step(); - loss.zero_gradient(); - } - }); - } } diff --git a/src/numerics.rs b/src/numerics.rs index 727fe29e..63835410 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -381,7 +381,6 @@ mod tests { use rand; use rand::Rng; - use test::Bencher; use numerics; @@ -579,139 +578,4 @@ mod tests { assert_close(&result, &expected, 0.001); } - - #[bench] - fn bench_sofmax_exp_sum(b: &mut Bencher) { - let x = vec![1.0; 32]; - let max = 1.0; - - b.iter(|| x.iter().map(|&x| numerics::exp(x - max)).sum::().ln()) - } - - #[bench] - fn bench_sofmax_exp_sum_unrolled(b: &mut Bencher) { - let x = vec![1.0; 32]; - let max = 1.0; - - b.iter(|| softmax_exp_sum(&x, max).ln()) - } - - #[bench] - fn bench_exp(b: &mut Bencher) { - let x: Vec = vec![1.0; 32]; - - let mut v = 0.0; - - b.iter(|| x.iter().for_each(|&y| v += y.exp())); - } - - #[bench] - fn bench_fastexp(b: &mut Bencher) { - let x: Vec = vec![1.0; 32]; - - let mut v = 0.0; - - b.iter(|| x.iter().for_each(|&y| v += fastexp(y))); - } - - #[bench] - fn bench_dot(b: &mut Bencher) { - let xs = vec![0.0; 256]; - let ys = vec![0.0; 256]; - - b.iter(|| dot(&xs[..], &ys[..])); - } - - #[bench] - fn bench_unrolled_dot(b: &mut Bencher) { - let xs = vec![0.0; 256]; - let ys = vec![0.0; 256]; - - b.iter(|| unrolled_dot(&xs[..], &ys[..])); - } - - #[bench] - fn bench_simd_dot(b: &mut Bencher) { - let xs = vec![0.0; 256]; - let ys = vec![0.0; 256]; - - b.iter(|| simd_dot(&xs[..], &ys[..])); - } - - #[bench] - fn bench_array_scaled_assign(b: &mut Bencher) { - let mut xs = random_matrix(256, 1); - let ys = random_matrix(256, 1); - - b.iter(|| array_scaled_assign(&mut xs, &ys, 3.5)); - } - - #[bench] - fn bench_slice_scaled_assign(b: &mut Bencher) { - let mut xs = random_matrix(256, 1); - let ys = random_matrix(256, 1); - - b.iter(|| scaled_assign(&mut xs, &ys, 3.5)); - } - - #[bench] - fn bench_array_assign(b: &mut Bencher) { - let mut xs = random_matrix(256, 1); - let ys = random_matrix(256, 1); - - b.iter(|| array_assign(&mut xs, &ys)); - } - - #[bench] - fn bench_slice_assign(b: &mut Bencher) { - let mut xs = random_matrix(256, 1); - let ys = random_matrix(256, 1); - - b.iter(|| assign(&mut xs, &ys)); - } - - #[bench] - fn dot_node_specializations_mm(b: &mut Bencher) { - let x = random_matrix(64, 64); - let y = random_matrix(64, 64); - let mut z = random_matrix(64, 64); - - b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); - } - - #[bench] - fn dot_node_general_vm(b: &mut Bencher) { - let x = random_matrix(1, 64); - let y = random_matrix(64, 64); - let mut z = random_matrix(1, 64); - - b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); - } - - #[bench] - fn dot_node_specializations_vm(b: &mut Bencher) { - let x = random_matrix(1, 64); - let y = random_matrix(64, 64); - let mut z = random_matrix(1, 64); - - b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); - } - - #[bench] - fn dot_node_specializations_mv(b: &mut Bencher) { - let x = random_matrix(64, 64); - let y = random_matrix(64, 1); - let mut z = random_matrix(64, 1); - - b.iter(|| mat_mul(1.0, &x, &y, 0.0, &mut z)); - } - - #[bench] - fn dot_node_general_mv(b: &mut Bencher) { - let x = random_matrix(64, 64); - let y = random_matrix(64, 1); - let mut z = random_matrix(64, 1); - - b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); - } } From e1ae5a8fa51825ed8791cce9e4fd62362b57870a Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 21 Apr 2018 23:38:43 +0100 Subject: [PATCH 056/108] Continue with benches, add relu, add arith ops for float constants. --- benches/benchmark.rs | 154 ++++++++++++++++++++++++------------------- src/lib.rs | 121 ++++++++++++++++++++-------------- src/nodes.rs | 115 +++++++++++++++++++++++++++++--- 3 files changed, 265 insertions(+), 125 deletions(-) diff --git a/benches/benchmark.rs b/benches/benchmark.rs index 539cef53..f0efc133 100644 --- a/benches/benchmark.rs +++ b/benches/benchmark.rs @@ -1,34 +1,42 @@ +use std::sync::Arc; + #[macro_use] extern crate criterion; +extern crate rayon; +extern crate wyrm; + +use rayon::prelude::*; use criterion::Criterion; +use wyrm::nn::lstm; +use wyrm::nn::xavier_normal; +use wyrm::{DataInput, HogwildParameter, ParameterNode, SGD}; + fn bench_node_reuse(c: &mut Criterion) { - let dim = 128; + c.bench_function("node_reuse", |b| { + let dim = 128; - let x = ParameterNode::new(random_matrix(1, dim)); - let y = ParameterNode::new(random_matrix(dim, 10)); - let v = x.dot(&y); - let z = v.clone() + v.clone() + v.clone() + v.clone(); + let x = ParameterNode::new(xavier_normal(1, dim)); + let y = ParameterNode::new(xavier_normal(dim, 10)); + let v = x.dot(&y); + let z = v.clone() + v.clone() + v.clone() + v.clone(); - c.bench_function( - "node_reuse", b.iter(|| { z.forward(); z.zero_gradient(); - }), - ); + }) + }); } -fn bench_matrix_multiply(b: &mut Criterion) { - let dim = 64; - let num_epochs = 20; +fn bench_matrix_multiply(c: &mut Criterion) { + c.bench_function("bench_matrix_multiply", |b| { + let dim = 64; + let num_epochs = 20; - let x_data = Arc::new(HogwildParameter::new(random_matrix(1, dim))); - let y_data = Arc::new(HogwildParameter::new(random_matrix(dim, 10))); + let x_data = Arc::new(HogwildParameter::new(xavier_normal(1, dim))); + let y_data = Arc::new(HogwildParameter::new(xavier_normal(dim, 10))); - b.bench_function( - "bench_matrix_multiply", b.iter(|| { (0..rayon::current_num_threads()) .into_par_iter() @@ -43,15 +51,17 @@ fn bench_matrix_multiply(b: &mut Criterion) { v.zero_gradient(); } }); - }), - ); + }) + }); } // fn bench_sofmax_exp_sum(b: &mut Criterion) { -// let x = vec![1.0; 32]; -// let max = 1.0; +// c.bench_function("bench_softmax_exp_sum", |b| { +// let x = vec![1.0; 32]; +// let max = 1.0; -// b.iter(|| x.iter().map(|&x| numerics::exp(x - max)).sum::().ln()) +// b.iter(|| x.iter().map(|&x| wyrm::exp(x - max)).sum::().ln()) +// }) // } // #[bench] @@ -167,58 +177,70 @@ fn bench_matrix_multiply(b: &mut Criterion) { // b.iter(|| general_mat_mul(1.0, &x, &y, 0.0, &mut z)); // } -// fn bench_lstm(b: &mut Criterion) { -// let sequence_length = 4; -// let num_digits = 10; -// let input_dim = 16; -// let hidden_dim = 32; - -// let lstm_params = Parameters::new(input_dim, hidden_dim); -// let lstm = lstm_params.build(); - -// let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); -// let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); -// let y = nodes::IndexInputNode::new(&vec![0]); - -// let inputs: Vec<_> = (0..sequence_length) -// .map(|_| nodes::IndexInputNode::new(&vec![0])) -// .collect(); -// let embeddings: Vec<_> = inputs -// .iter() -// .map(|input| embeddings.index(&input)) -// .collect(); +fn pi_digits(num: usize) -> Vec { + let pi_str = include_str!("../src/nn/pi.txt"); + pi_str + .chars() + .filter_map(|x| x.to_digit(10)) + .map(|x| x as usize) + .take(num) + .collect() +} -// let hidden_states = lstm.forward(&embeddings); -// let hidden = hidden_states.last().unwrap(); +fn bench_lstm(c: &mut Criterion) { + c.bench_function("bench_lstm", |b| { + let sequence_length = 4; + let num_digits = 10; + let input_dim = 16; + let hidden_dim = 32; -// let prediction = hidden.dot(&final_layer); -// let mut loss = sparse_categorical_crossentropy(&prediction, &y); -// let mut optimizer = SGD::new(0.05, loss.parameters()); + let lstm_params = lstm::Parameters::new(input_dim, hidden_dim); + let lstm = lstm_params.build(); -// let digits = pi_digits(100); + let final_layer = wyrm::ParameterNode::new(xavier_normal(hidden_dim, num_digits)); + let embeddings = wyrm::ParameterNode::new(xavier_normal(num_digits, input_dim)); + let y = wyrm::IndexInputNode::new(&vec![0]); -// b.iter(|| { -// for i in 0..(digits.len() - sequence_length - 1) { -// let digit_chunk = &digits[i..(i + sequence_length + 1)]; -// if digit_chunk.len() < sequence_length + 1 { -// break; -// } + let inputs: Vec<_> = (0..sequence_length) + .map(|_| wyrm::IndexInputNode::new(&vec![0])) + .collect(); + let embeddings: Vec<_> = inputs + .iter() + .map(|input| embeddings.index(&input)) + .collect(); -// for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { -// input.set_value(digit); -// } + let hidden_states = lstm.forward(&embeddings); + let hidden = hidden_states.last().unwrap(); -// let target_digit = *digit_chunk.last().unwrap(); -// y.set_value(target_digit); + let prediction = hidden.dot(&final_layer); + let mut loss = wyrm::nn::losses::sparse_categorical_crossentropy(&prediction, &y); + let mut optimizer = SGD::new(0.05, loss.parameters()); -// loss.forward(); -// loss.backward(1.0); + let digits = pi_digits(100); -// optimizer.step(); -// loss.zero_gradient(); -// } -// }); -// } + b.iter(|| { + for i in 0..(digits.len() - sequence_length - 1) { + let digit_chunk = &digits[i..(i + sequence_length + 1)]; + if digit_chunk.len() < sequence_length + 1 { + break; + } + + for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { + input.set_value(digit); + } + + let target_digit = *digit_chunk.last().unwrap(); + y.set_value(target_digit); + + loss.forward(); + loss.backward(1.0); + + optimizer.step(); + loss.zero_gradient(); + } + }) + }); +} -criterion_group!(benches, bench_node_reuse, bench_matrix_multiply); +criterion_group!(benches, bench_node_reuse, bench_matrix_multiply, bench_lstm); criterion_main!(benches); diff --git a/src/lib.rs b/src/lib.rs index 6eda8f06..42847a26 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -364,6 +364,14 @@ where ) } + /// Compute the ReLU of this variable. + pub fn relu(&self) -> Variable> { + Variable::new( + Rc::new(ReluNode::new(Rc::clone(&self.node))), + self.parameters.clone(), + ) + } + /// Compute the row-wise vector dot product of LHS and RHS. pub fn vector_dot(&self, other: &Variable) -> Variable> where @@ -481,61 +489,64 @@ impl DataInput for Variable { } } -impl Add> for Variable -where - RHS: Node, - LHS: Node, -{ - type Output = Variable>; - fn add(self, other: Variable) -> Self::Output { - Variable::new( - Rc::new(AddNode::new(self.node, other.node)), - merge_parameters(&self.parameters, &other.parameters), - ) - } -} +macro_rules! impl_arithmetic_op { + ($trait:ident, $fn:ident, $node:ident) => { + impl $trait> for Variable + where + RHS: Node, + LHS: Node, + { + type Output = Variable<$node>; + fn $fn(self, other: Variable) -> Self::Output { + Variable::new( + Rc::new($node::new(self.node, other.node)), + merge_parameters(&self.parameters, &other.parameters), + ) + } + } -impl Sub> for Variable -where - RHS: Node, - LHS: Node, -{ - type Output = Variable>; - fn sub(self, other: Variable) -> Self::Output { - Variable::new( - Rc::new(SubNode::new(self.node, other.node)), - merge_parameters(&self.parameters, &other.parameters), - ) - } -} + /// The constant will be broadcast to have the same shape + /// as the LHS. + impl $trait for Variable + where + LHS: Node, + { + type Output = Variable<$node>; + fn $fn(self, other: f32) -> Self::Output { + let constant = InputNode::new(self.value().deref() * 0.0); + constant.set_value(other); + + Variable::new( + Rc::new($node::new(self.node, constant.node)), + merge_parameters(&self.parameters, &constant.parameters), + ) + } + } -impl Mul> for Variable -where - RHS: Node, - LHS: Node, -{ - type Output = Variable>; - fn mul(self, other: Variable) -> Self::Output { - Variable::new( - Rc::new(MulNode::new(self.node, other.node)), - merge_parameters(&self.parameters, &other.parameters), - ) - } + /// The constant will be broadcast to have the same shape + /// as the RHS. + impl $trait> for f32 + where + RHS: Node, + { + type Output = Variable<$node>; + fn $fn(self, other: Variable) -> Self::Output { + let constant = InputNode::new(other.value().deref() * 0.0); + constant.set_value(self); + + Variable::new( + Rc::new($node::new(constant.node, other.node)), + merge_parameters(&constant.parameters, &other.parameters), + ) + } + } + }; } -impl Div> for Variable -where - RHS: Node, - LHS: Node, -{ - type Output = Variable>; - fn div(self, other: Variable) -> Self::Output { - Variable::new( - Rc::new(DivNode::new(self.node, other.node)), - merge_parameters(&self.parameters, &other.parameters), - ) - } -} +impl_arithmetic_op!(Add, add, AddNode); +impl_arithmetic_op!(Sub, sub, SubNode); +impl_arithmetic_op!(Mul, mul, MulNode); +impl_arithmetic_op!(Div, div, DivNode); impl Neg for Variable where @@ -874,6 +885,14 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn relu_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = (x.clone() + x.clone()).relu(); + + let (finite_difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&finite_difference, &gradient, TOLERANCE); + } + #[test] fn neg_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut z = -(x.clone() + x.clone()); diff --git a/src/nodes.rs b/src/nodes.rs index b56b77bb..54e573ef 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1,9 +1,9 @@ use std; +use std::cell::{Cell, Ref, RefCell}; use std::fmt; use std::ops::{AddAssign, Deref, DerefMut}; -use std::cell::{Cell, Ref, RefCell}; -use std::sync::Arc; use std::rc::Rc; +use std::sync::Arc; use ndarray; use ndarray::Axis; @@ -1605,6 +1605,106 @@ where } } +#[derive(Debug)] +pub struct ReluNode { + value: RefCell, + operand_gradient: RefCell, + operand: Rc, + needs_gradient: bool, + counter: PassCounter, +} + +impl ReluNode +where + T: Node, +{ + pub fn new(operand: Rc) -> Self { + let value = operand + .value() + .deref() + .map(|&x| if x < 0.0 { 0.0 } else { x }); + let gradient = &value * 0.0; + let needs_gradient = operand.needs_gradient(); + + ReluNode { + value: RefCell::new(value), + operand_gradient: RefCell::new(gradient), + operand: operand, + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } +} + +impl Node for ReluNode +where + T: Node, +{ + type Value = Arr; + type InputGradient = Arr; + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + + self.operand.forward(); + + let mut dest = self.value.borrow_mut(); + + numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { + if x < 0.0 { + 0.0 + } else { + x + } + }); + } + + fn backward(&self, gradient: &Ref) { + match self.counter.backward() { + BackwardAction::Set => { + let mut operand_gradient = self.operand_gradient.borrow_mut(); + + numerics::map_assign_binary( + &mut operand_gradient, + self.value.borrow().deref(), + gradient, + |x, grad| if x < 0.0 { 0.0 } else { grad }, + ); + } + BackwardAction::Increment => { + let mut operand_gradient = self.operand_gradient.borrow_mut(); + + numerics::map_inplace_assign_binary( + &mut operand_gradient, + self.value.borrow().deref(), + gradient, + |dest, x, grad| *dest += if x < 0.0 { 0.0 } else { grad }, + ); + } + } + + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()) + } + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + + fn zero_gradient(&self) { + if !self.counter.is_zero() { + self.operand.zero_gradient(); + self.counter.clear(); + } + } +} + #[derive(Debug)] pub struct NegNode { value: RefCell, @@ -1981,12 +2081,11 @@ where let operand_slice = operand_value.deref().as_slice().unwrap(); let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); - let denominator = max - + operand_slice - .iter() - .map(|&x| numerics::exp(x - max)) - .sum::() - .ln(); + let denominator = max + operand_slice + .iter() + .map(|&x| numerics::exp(x - max)) + .sum::() + .ln(); operand_value.deref() - denominator }; From 2b375eb84af6913c5cc53bcf636b5adede3216b7 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 22 Apr 2018 16:12:28 +0100 Subject: [PATCH 057/108] Add optional gradient clipping. --- src/lib.rs | 27 ++++++++++++++++++++++++++- src/nodes.rs | 23 ++++++++++++++++++----- src/numerics.rs | 10 +++++++++- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 42847a26..a4d080b8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -562,6 +562,7 @@ where pub struct SGD { learning_rate: f32, parameters: Vec>, + clamp: Option<(f32, f32)>, } impl SGD { @@ -570,9 +571,16 @@ impl SGD { SGD { learning_rate: learning_rate, parameters: parameters, + clamp: None, } } + /// Set the clamp bounds. + pub fn clamp(mut self, min: f32, max: f32) -> Self { + self.clamp = Some((min, max)); + self + } + /// Perform a single SGD step. pub fn step(&mut self) { let learning_rate = self.learning_rate; @@ -580,6 +588,10 @@ impl SGD { let mut sink = parameter.node.gradient.borrow_mut(); let mut param_value = unsafe { parameter.node.value.value_mut() }; + if let Some((min, max)) = self.clamp { + sink.clamp(min, max); + } + if sink.has_dense { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); } @@ -593,7 +605,7 @@ impl SGD { numerics::map_add_assign_slice( param_row.into_slice().unwrap(), grad_row.into_slice().unwrap(), - |x| -learning_rate * clamp(x, -10.0, 10.0), + |x| -learning_rate * x, ); } } @@ -608,6 +620,7 @@ pub struct Adagrad { learning_rate: f32, l2: f32, parameters: Vec>, + clamp: Option<(f32, f32)>, } impl Adagrad { @@ -617,9 +630,16 @@ impl Adagrad { learning_rate: learning_rate, l2: 0.0, parameters: parameters, + clamp: None, } } + /// Set the clamp bounds. + pub fn clamp(mut self, min: f32, max: f32) -> Self { + self.clamp = Some((min, max)); + self + } + /// Set the L2 penalty. pub fn l2_penalty(mut self, l2_penalty: f32) -> Self { self.l2 = l2_penalty; @@ -632,6 +652,11 @@ impl Adagrad { for parameter in &self.parameters { let mut sink = parameter.node.gradient.borrow_mut(); + + if let Some((min, max)) = self.clamp { + sink.clamp(min, max); + } + let mut param_value = unsafe { parameter.node.value.value_mut() }; let mut squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; diff --git a/src/nodes.rs b/src/nodes.rs index 54e573ef..564f989b 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -12,7 +12,7 @@ use smallvec::SmallVec; use numerics; -use super::{Arr, Variable}; +use super::{clamp, Arr, Variable}; #[derive(Debug, PartialEq)] pub enum ForwardAction { @@ -483,6 +483,22 @@ impl GradientAccumulator { self.has_dense = false; self.has_sparse = false; } + + pub fn clamp(&mut self, min: f32, max: f32) { + self.dense_gradient() + .as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x = clamp(*x, min, max)); + self.sparse_gradient + .iter_mut() + .for_each(|(_, ref mut grad)| { + grad.as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x = clamp(*x, min, max)) + }); + } } pub trait GradientSink { @@ -1523,10 +1539,7 @@ where T: Node, { pub fn new(operand: Rc) -> Self { - let value = operand - .value() - .deref() - .map(|x| 1.0 / (1.0 + numerics::exp(-x))); + let value = operand.value().deref().map(|&x| numerics::sigmoid(x)); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); diff --git a/src/numerics.rs b/src/numerics.rs index 63835410..a60d7451 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -39,7 +39,15 @@ pub fn tanh(x: f32) -> f32 { #[inline(always)] pub fn sigmoid(x: f32) -> f32 { - 1.0 / (1.0 + (-x).exp()) + let critical_value = 10.0; + + if x > critical_value { + 1.0 + } else if x < -critical_value { + 0.0 + } else { + 1.0 / (1.0 + exp(-x)) + } } #[inline(always)] From bb2060cf76297872438f8b9a65b6b4c5f790d87a Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 24 Apr 2018 17:21:04 +0100 Subject: [PATCH 058/108] Conform to standard agagrad impls. --- src/lib.rs | 10 +++++++--- src/nodes.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a4d080b8..c17cc409 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -621,6 +621,7 @@ pub struct Adagrad { l2: f32, parameters: Vec>, clamp: Option<(f32, f32)>, + eps: f32, } impl Adagrad { @@ -631,6 +632,7 @@ impl Adagrad { l2: 0.0, parameters: parameters, clamp: None, + eps: 1e-6, } } @@ -666,8 +668,9 @@ impl Adagrad { sink.dense_gradient().as_slice().unwrap(), squared_gradient.as_slice_mut().unwrap() ) { - *value -= learning_rate * gradient / squared_gradient.sqrt() + *value * self.l2; *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate * gradient / (self.eps + squared_gradient.sqrt()) + + *value * self.l2; } } @@ -683,9 +686,10 @@ impl Adagrad { grad_row.into_slice().unwrap(), squared_row.as_slice_mut().unwrap() ) { - *value -= learning_rate * gradient / squared_gradient.sqrt() - + *value * self.l2; *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate * gradient + / (self.eps + squared_gradient.sqrt()) + + *value * self.l2; } } } diff --git a/src/nodes.rs b/src/nodes.rs index 564f989b..18f8ddd6 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -545,7 +545,7 @@ pub struct HogwildParameter { impl HogwildParameter { /// Create a new parameter object. pub fn new(value: Arr) -> Self { - let squared_gradients = &value * 0.0 + 1.0; + let squared_gradients = &value * 0.0; HogwildParameter { value: RefCell::new(value), From 5563b7047c676c4bf92787d80e66f4bee94c6cf2 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 25 Apr 2018 18:30:06 +0100 Subject: [PATCH 059/108] Fix L2 regularization. --- src/lib.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c17cc409..abc47f49 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -648,6 +648,19 @@ impl Adagrad { self } + /// Decay weights. + pub fn decay_weights(&mut self, penalty: f32) { + for parameter in &self.parameters { + let mut param_value = unsafe { parameter.node.value.value_mut() }; + + param_value + .as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x -= x.signum() * penalty * numerics::pow2(*x)); + } + } + /// Perform a single SGD step. pub fn step(&mut self) { let learning_rate = self.learning_rate; @@ -670,7 +683,7 @@ impl Adagrad { ) { *squared_gradient += numerics::pow2(gradient); *value -= learning_rate * gradient / (self.eps + squared_gradient.sqrt()) - + *value * self.l2; + + value.signum() * numerics::pow2(*value) * self.l2; } } @@ -689,7 +702,7 @@ impl Adagrad { *squared_gradient += numerics::pow2(gradient); *value -= learning_rate * gradient / (self.eps + squared_gradient.sqrt()) - + *value * self.l2; + + value.signum() * numerics::pow2(*value) * self.l2 } } } From d88c597d10a25a118e01ef006b52d66d7d93455e Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 27 Apr 2018 17:39:24 +0100 Subject: [PATCH 060/108] Fix sub operations. --- src/lib.rs | 17 +++++++++++++---- src/nodes.rs | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index abc47f49..1b42b024 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -513,8 +513,7 @@ macro_rules! impl_arithmetic_op { { type Output = Variable<$node>; fn $fn(self, other: f32) -> Self::Output { - let constant = InputNode::new(self.value().deref() * 0.0); - constant.set_value(other); + let constant = InputNode::new(self.value().deref() * 0.0 + other); Variable::new( Rc::new($node::new(self.node, constant.node)), @@ -531,8 +530,7 @@ macro_rules! impl_arithmetic_op { { type Output = Variable<$node>; fn $fn(self, other: Variable) -> Self::Output { - let constant = InputNode::new(other.value().deref() * 0.0); - constant.set_value(self); + let constant = InputNode::new(other.value().deref() * 0.0 + self); Variable::new( Rc::new($node::new(constant.node, other.node)), @@ -792,6 +790,17 @@ mod tests { Arr::zeros((rows, cols)).map(|_| rand::random::()) } + #[test] + fn test_constant_sub() { + let x = ParameterNode::new(Arr::zeros((10, 10)) + 1.0); + let y = 1.0 - x; + + assert_eq!(y.value().scalar_sum(), 0.0); + y.zero_gradient(); + y.forward(); + assert_eq!(y.value().scalar_sum(), 0.0); + } + #[test] fn parameter_deduplication() { let x = ParameterNode::new(random_matrix(1, 1)); diff --git a/src/nodes.rs b/src/nodes.rs index 18f8ddd6..3cdcaf72 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -656,7 +656,7 @@ where { pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = lhs.value().deref() + rhs.value().deref(); + let value = lhs.value().deref() - rhs.value().deref(); let rhs_gradient = rhs.value().deref() * 0.0; From a48b5d2f468f015935fbaf30860c1ad52122105d Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 29 Apr 2018 09:44:54 +0100 Subject: [PATCH 061/108] Speed up tanh layer. --- src/nodes.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index 3cdcaf72..6182b412 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1489,16 +1489,16 @@ where fn backward(&self, gradient: &Ref) { match self.counter.backward() { BackwardAction::Set => for (dest, value, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.value().iter(), - gradient.iter() + self.operand_gradient.borrow_mut().as_slice_mut().unwrap(), + self.value().as_slice().unwrap(), + gradient.as_slice().unwrap() ) { *dest = grad_val * (1.0 - value.powi(2)); }, BackwardAction::Increment => for (dest, value, grad_val) in izip!( - self.operand_gradient.borrow_mut().iter_mut(), - self.value().iter(), - gradient.iter() + self.operand_gradient.borrow_mut().as_slice_mut().unwrap(), + self.value().as_slice().unwrap(), + gradient.as_slice().unwrap() ) { *dest += grad_val * (1.0 - value.powi(2)); }, From 5b499a0e4830596b92bc6f79e8e2b3a97d2f9462 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 29 Apr 2018 09:45:07 +0100 Subject: [PATCH 062/108] Fix L2 reg in adagrad. --- src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1b42b024..770a8921 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -679,9 +679,9 @@ impl Adagrad { sink.dense_gradient().as_slice().unwrap(), squared_gradient.as_slice_mut().unwrap() ) { + let gradient = gradient + *value * self.l2; *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate * gradient / (self.eps + squared_gradient.sqrt()) - + value.signum() * numerics::pow2(*value) * self.l2; + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; } } @@ -697,10 +697,10 @@ impl Adagrad { grad_row.into_slice().unwrap(), squared_row.as_slice_mut().unwrap() ) { + let gradient = gradient + *value * self.l2; *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate * gradient - / (self.eps + squared_gradient.sqrt()) - + value.signum() * numerics::pow2(*value) * self.l2 + *value -= + learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; } } } From dc4507da3f78212309b7a7c685b2ec4439d5ddbf Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 29 Apr 2018 09:45:21 +0100 Subject: [PATCH 063/108] Switch LSTM init to uniform. --- src/nn/lstm.rs | 35 ++++++++++++++++++++++++----------- src/nn/mod.rs | 8 +++++++- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 9b9707a3..540f84ee 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -50,7 +50,7 @@ use ndarray; use nodes; use nodes::{HogwildParameter, Node, ParameterNode}; -use nn::xavier_normal; +use nn::{uniform, xavier_normal}; use {Arr, DataInput, Variable}; @@ -104,33 +104,44 @@ impl Clone for Parameters { impl Parameters { /// Create a new LSTM parameters object. pub fn new(input_dim: usize, hidden_dim: usize) -> Self { + let min = -1.0 / (hidden_dim as f32).sqrt(); + let max = 1.0 / (hidden_dim as f32).sqrt(); + Self { input_dim: input_dim, hidden_dim: hidden_dim, - forget_weights: Arc::new(HogwildParameter::new(xavier_normal( + forget_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, + min, + max, ))), - forget_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), + forget_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), - update_gate_weights: Arc::new(HogwildParameter::new(xavier_normal( + update_gate_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, + min, + max, ))), - update_gate_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), + update_gate_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), - update_value_weights: Arc::new(HogwildParameter::new(xavier_normal( + update_value_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, + min, + max, ))), - update_value_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), + update_value_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), - output_gate_weights: Arc::new(HogwildParameter::new(xavier_normal( + output_gate_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, + min, + max, ))), - output_gate_biases: Arc::new(HogwildParameter::new(Arr::zeros((1, hidden_dim)))), + output_gate_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), } } @@ -284,8 +295,8 @@ mod tests { use super::*; use nn::losses::sparse_categorical_crossentropy; use nodes::InputNode; + use Adagrad; use DataInput; - use SGD; fn pi_digits(num: usize) -> Vec { let pi_str = include_str!("pi.txt"); @@ -368,7 +379,7 @@ mod tests { let prediction = hidden.dot(&final_layer); let mut loss = sparse_categorical_crossentropy(&prediction, &y); - let mut optimizer = SGD::new(0.05, loss.parameters()); + let mut optimizer = Adagrad::new(0.05, loss.parameters()).l2_penalty(1e-3); let digits = pi_digits(100); @@ -399,6 +410,8 @@ mod tests { loss_val += loss.value().scalar_sum(); + // println!("Hiddent {:#?}", hidden.value()); + optimizer.step(); loss.zero_gradient(); diff --git a/src/nn/mod.rs b/src/nn/mod.rs index c05b9a7c..318cade4 100644 --- a/src/nn/mod.rs +++ b/src/nn/mod.rs @@ -4,7 +4,7 @@ pub mod losses; pub mod lstm; use rand; -use rand::distributions::{IndependentSample, Normal}; +use rand::distributions::{IndependentSample, Normal, Range}; use Arr; @@ -13,3 +13,9 @@ pub fn xavier_normal(rows: usize, cols: usize) -> Arr { let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); Arr::zeros((rows, cols)).map(|_| normal.ind_sample(&mut rand::thread_rng()) as f32) } + +/// Return a random matrix with values drawn uniformly from `(min, max)`. +pub fn uniform(rows: usize, cols: usize, min: f32, max: f32) -> Arr { + let dist = Range::new(min, max); + Arr::zeros((rows, cols)).map(|_| dist.ind_sample(&mut rand::thread_rng()) as f32) +} From d0919c6cefca80ce7c69a8c551136f7a8abf470d Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 30 Apr 2018 09:06:55 +0100 Subject: [PATCH 064/108] Add synchronization barrier. --- src/barrier.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 92 ++++++++++++++++++++++++++++++++++++++++++++++- src/nn/lstm.rs | 2 +- 3 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 src/barrier.rs diff --git a/src/barrier.rs b/src/barrier.rs new file mode 100644 index 00000000..94dc3e37 --- /dev/null +++ b/src/barrier.rs @@ -0,0 +1,98 @@ +// A modification of the stdlib barrier to allow resizing. + +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::fmt; +use std::sync::{Condvar, Mutex}; + +pub struct Barrier { + lock: Mutex, + cvar: Condvar, +} + +struct BarrierState { + count: usize, + generation_id: usize, + num_threads: usize, +} + +pub struct BarrierWaitResult(bool); + +impl fmt::Debug for Barrier { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.pad("Barrier { .. }") + } +} + +impl Barrier { + pub fn new(n: usize) -> Barrier { + Barrier { + lock: Mutex::new(BarrierState { + count: 0, + generation_id: 0, + num_threads: n, + }), + cvar: Condvar::new(), + } + } + + pub fn increment_num_threads(&self) { + let mut lock = self.lock.lock().unwrap(); + lock.num_threads += 1; + // println!("INC Now thread count {}", lock.num_threads); + } + + pub fn decrement_num_threads(&self) { + let mut lock = self.lock.lock().unwrap(); + lock.num_threads = lock.num_threads.saturating_sub(1); + // println!("DEC Now thread count {}", lock.num_threads); + + // Notify if deregistering makes the barrier conditions met. + if lock.count >= lock.num_threads { + lock.count = 0; + lock.generation_id = lock.generation_id.wrapping_add(1); + self.cvar.notify_all(); + } + } + + pub fn wait(&self) -> BarrierWaitResult { + let mut lock = self.lock.lock().unwrap(); + let local_gen = lock.generation_id; + lock.count += 1; + if lock.count < lock.num_threads { + // We need a while loop to guard against spurious wakeups. + // http://en.wikipedia.org/wiki/Spurious_wakeup + while local_gen == lock.generation_id && lock.count < lock.num_threads { + lock = self.cvar.wait(lock).unwrap(); + } + BarrierWaitResult(false) + } else { + lock.count = 0; + lock.generation_id = lock.generation_id.wrapping_add(1); + self.cvar.notify_all(); + BarrierWaitResult(true) + } + } +} + +impl fmt::Debug for BarrierWaitResult { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("BarrierWaitResult") + .field("is_leader", &self.is_leader()) + .finish() + } +} + +impl BarrierWaitResult { + pub fn is_leader(&self) -> bool { + self.0 + } +} diff --git a/src/lib.rs b/src/lib.rs index 770a8921..9b0699d1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -163,7 +163,9 @@ use std::cell::RefCell; use std::clone::Clone; use std::ops::{Add, Deref, Div, Mul, Neg, Sub}; use std::rc::Rc; +use std::sync::{Arc, Mutex, MutexGuard}; +mod barrier; mod fast_approx; pub mod nn; mod nodes; @@ -612,6 +614,73 @@ impl SGD { } } +pub struct SynchronizationBarrier { + core: Arc, +} + +impl SynchronizationBarrier { + pub fn new() -> Self { + SynchronizationBarrier { + core: Arc::new(SynchronizationBarrierCore::new()), + } + } + + fn register_thread(&self) -> SynchronizationBarrierGuard { + self.core.register_thread(); + SynchronizationBarrierGuard { + barrier: Arc::clone(&self.core), + } + } +} + +struct SynchronizationBarrierCore { + barrier: barrier::Barrier, + parameter_lock: Mutex<()>, +} + +impl SynchronizationBarrierCore { + fn new() -> Self { + Self { + barrier: barrier::Barrier::new(0), + parameter_lock: Mutex::default(), + } + } + + fn register_thread(&self) { + self.barrier.increment_num_threads(); + } + + fn deregister_thread(&self) { + self.barrier.decrement_num_threads(); + // println!("Deregistered"); + } + + fn wait(&self) { + self.barrier.wait(); + } +} + +pub struct SynchronizationBarrierGuard { + barrier: Arc, +} + +impl SynchronizationBarrierGuard { + fn wait(&self) { + self.barrier.wait(); + } + + fn lock(&self) -> MutexGuard<()> { + self.barrier.parameter_lock.lock().unwrap() + } +} + +impl Drop for SynchronizationBarrierGuard { + fn drop(&mut self) { + // println!("Deregistering"); + self.barrier.deregister_thread(); + } +} + /// Adagrad optimizer, scaled the learning rate by the inverse of previously /// accumulated gradients. pub struct Adagrad { @@ -620,6 +689,7 @@ pub struct Adagrad { parameters: Vec>, clamp: Option<(f32, f32)>, eps: f32, + sync_barrier: Option, } impl Adagrad { @@ -631,9 +701,15 @@ impl Adagrad { parameters: parameters, clamp: None, eps: 1e-6, + sync_barrier: None, } } + pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { + self.sync_barrier = Some(barrier.register_thread()); + self + } + /// Set the clamp bounds. pub fn clamp(mut self, min: f32, max: f32) -> Self { self.clamp = Some((min, max)); @@ -660,7 +736,7 @@ impl Adagrad { } /// Perform a single SGD step. - pub fn step(&mut self) { + pub fn step(&self) { let learning_rate = self.learning_rate; for parameter in &self.parameters { @@ -707,6 +783,20 @@ impl Adagrad { } } } + + pub fn synchronized_step(&self) { + if let Some(ref barrier) = self.sync_barrier { + { + // println!("Locking params"); + let _ = barrier.lock(); + self.step(); + } + + // println!("Acquiring end barrier."); + barrier.wait(); + // println!("Done."); + } + } } /// Compute finite difference gradient estimates of the output variable diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 540f84ee..20e393ac 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -104,8 +104,8 @@ impl Clone for Parameters { impl Parameters { /// Create a new LSTM parameters object. pub fn new(input_dim: usize, hidden_dim: usize) -> Self { - let min = -1.0 / (hidden_dim as f32).sqrt(); let max = 1.0 / (hidden_dim as f32).sqrt(); + let min = -max; Self { input_dim: input_dim, From 178aa50e6a7b47c8f6572e3136d2d25c11446b68 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 30 Apr 2018 09:39:13 +0100 Subject: [PATCH 065/108] Integrate Adagrad barrier lock into step. --- src/lib.rs | 85 +++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9b0699d1..4dce0d8b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -735,66 +735,67 @@ impl Adagrad { } } - /// Perform a single SGD step. - pub fn step(&self) { + fn do_step(&self, parameter: &Variable) { let learning_rate = self.learning_rate; - for parameter in &self.parameters { - let mut sink = parameter.node.gradient.borrow_mut(); + let mut sink = parameter.node.gradient.borrow_mut(); - if let Some((min, max)) = self.clamp { - sink.clamp(min, max); - } - - let mut param_value = unsafe { parameter.node.value.value_mut() }; - let mut squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; + if let Some((min, max)) = self.clamp { + sink.clamp(min, max); + } - if sink.has_dense { - for (value, &gradient, squared_gradient) in izip!( - param_value.as_slice_mut().unwrap(), - sink.dense_gradient().as_slice().unwrap(), - squared_gradient.as_slice_mut().unwrap() - ) { - let gradient = gradient + *value * self.l2; - *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; - } + let param_value = unsafe { parameter.node.value.value_mut() }; + let squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; + + if sink.has_dense { + for (value, &gradient, squared_gradient) in izip!( + param_value.as_slice_mut().unwrap(), + sink.dense_gradient().as_slice().unwrap(), + squared_gradient.as_slice_mut().unwrap() + ) { + let gradient = gradient + *value * self.l2; + *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; } + } - if sink.has_sparse { - for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let grad_row = grad.subview(Axis(0), grad_idx); - let mut param_row = param_value.subview_mut(Axis(0), param_idx); - let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); - - for (value, &gradient, squared_gradient) in izip!( - param_row.as_slice_mut().unwrap(), - grad_row.into_slice().unwrap(), - squared_row.as_slice_mut().unwrap() - ) { - let gradient = gradient + *value * self.l2; - *squared_gradient += numerics::pow2(gradient); - *value -= - learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; - } + if sink.has_sparse { + for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let grad_row = grad.subview(Axis(0), grad_idx); + let mut param_row = param_value.subview_mut(Axis(0), param_idx); + let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); + + for (value, &gradient, squared_gradient) in izip!( + param_row.as_slice_mut().unwrap(), + grad_row.into_slice().unwrap(), + squared_row.as_slice_mut().unwrap() + ) { + let gradient = gradient + *value * self.l2; + *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; } } } } } - pub fn synchronized_step(&self) { + /// Perform a single SGD step. + pub fn step(&self) { if let Some(ref barrier) = self.sync_barrier { { - // println!("Locking params"); let _ = barrier.lock(); - self.step(); + + for parameter in &self.parameters { + self.do_step(parameter); + } } - // println!("Acquiring end barrier."); barrier.wait(); - // println!("Done."); + } else { + for parameter in &self.parameters { + self.do_step(parameter); + } } } } From b9b01df032f4d9ffb70f0bd767f1d09c858cd3ae Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 4 May 2018 21:43:29 +0100 Subject: [PATCH 066/108] Add Adam. Fix backprop for sub and add. Add synchronous training. --- src/lib.rs | 88 ++++++++++++++++++++-------- src/nn/lstm.rs | 48 +++++++++++++-- src/nodes.rs | 75 +++++++++++++++++++++--- src/numerics.rs | 21 ------- src/optim.rs | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 327 insertions(+), 56 deletions(-) create mode 100644 src/optim.rs diff --git a/src/lib.rs b/src/lib.rs index 4dce0d8b..d6db211e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -170,6 +170,7 @@ mod fast_approx; pub mod nn; mod nodes; mod numerics; +pub mod optim; use nodes::*; @@ -822,6 +823,7 @@ where changed_input[idx] += 0.5 * delta_x; input.set_value(&changed_input); output.forward(); + output.backward(1.0); output.value().clone() }; @@ -831,6 +833,7 @@ where changed_input[idx] -= 0.5 * delta_x; input.set_value(&changed_input); output.forward(); + output.backward(1.0); output.value().clone() }; @@ -883,13 +886,17 @@ mod tests { #[test] fn test_constant_sub() { - let x = ParameterNode::new(Arr::zeros((10, 10)) + 1.0); - let y = 1.0 - x; + let mut x = ParameterNode::new(Arr::zeros((10, 10)) + 1.0); + let mut y = (1.0 - x.clone()) * 2.0; assert_eq!(y.value().scalar_sum(), 0.0); y.zero_gradient(); y.forward(); + y.backward(1.0); assert_eq!(y.value().scalar_sum(), 0.0); + + let (difference, gradient) = finite_difference(&mut x, &mut y); + assert_close(&difference, &gradient, TOLERANCE); } #[test] @@ -906,20 +913,37 @@ mod tests { #[test] fn add_finite_difference() { let mut x = ParameterNode::new(random_matrix(1, 1)); - let y = ParameterNode::new(random_matrix(1, 1)); - let mut z = x.clone() + y.clone(); + let mut y = ParameterNode::new(random_matrix(1, 1)); + let mut z = x.clone() + y.clone() + x.clone() + x.clone(); - let (finite_difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&finite_difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); } #[test] - fn mul_finite_difference() { + fn sub_finite_difference() { let mut x = ParameterNode::new(random_matrix(1, 1)); - let y = ParameterNode::new(random_matrix(1, 1)); - let mut z = x.clone() * y.clone(); + let mut y = ParameterNode::new(random_matrix(1, 1)); + let z = x.clone() - (y.clone() - x.clone()); + let mut z = z.clone() * 2.0 + z.clone().sigmoid(); - let (finite_difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&finite_difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] + fn mul_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 10)); + let mut y = ParameterNode::new(random_matrix(10, 10)); + let z = x.clone() * y.clone(); + let mut z = z.clone() + z.clone(); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); } #[test] fn div_finite_difference() { @@ -933,20 +957,27 @@ mod tests { #[test] fn vector_dot_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); - let y = ParameterNode::new(random_matrix(10, 5)); - let mut z = x.vector_dot(&y); + let mut y = ParameterNode::new(random_matrix(10, 5)); + let z = x.vector_dot(&y); + let mut z = z.clone() + z.clone(); - let (finite_difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&finite_difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); } #[test] fn dot_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); - let y = ParameterNode::new(random_matrix(5, 10)); + let mut y = ParameterNode::new(random_matrix(5, 10)); let mut z = (x.clone() + x.clone()).dot(&y); - let (finite_difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&finite_difference, &gradient, TOLERANCE); + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + + let (difference, gradient) = finite_difference(&mut y, &mut z); + assert_close(&difference, &gradient, TOLERANCE); } #[test] fn dot_accumulation_finite_difference() { @@ -994,6 +1025,14 @@ mod tests { assert_close(&finite_difference, &gradient, TOLERANCE); } #[test] + fn squared_sum_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let mut z = x.square().scalar_sum(); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] fn transpose_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut z = (x.clone() + x.clone()).t(); @@ -1021,7 +1060,8 @@ mod tests { #[test] fn sigmoid_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); - let mut z = (x.clone() + x.clone()).sigmoid(); + let z = (x.clone() + x.clone()).sigmoid(); + let mut z = z.clone() + z.clone(); let (finite_difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&finite_difference, &gradient, TOLERANCE); @@ -1029,7 +1069,8 @@ mod tests { #[test] fn relu_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); - let mut z = (x.clone() + x.clone()).relu(); + let z = (x.clone() + x.clone()).relu(); + let mut z = z * 3.0; let (finite_difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&finite_difference, &gradient, TOLERANCE); @@ -1077,7 +1118,8 @@ mod tests { let mut y = ParameterNode::new(random_matrix(10, 5)); let v = x.clone() + y.clone(); - let mut z = v.stack(&v, ndarray::Axis(0)).sigmoid(); + let z = v.stack(&v, ndarray::Axis(0)); + let mut z = z.clone().sigmoid() * z.clone().relu(); assert_eq!(z.value().rows(), 20); assert_eq!(z.value().cols(), 5); @@ -1119,7 +1161,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let mut optimizer = Adagrad::new(0.5, loss.parameters()); + let optimizer = Adagrad::new(0.5, loss.parameters()); for _ in 0..num_epochs { let _x = arr2(&[[rand::random::()]]); @@ -1220,7 +1262,7 @@ mod tests { let mut loss = (output.clone() - y_hat.clone()).square(); let num_epochs = 200; - let mut optimizer = Adagrad::new(0.1, loss.parameters()); + let optimizer = Adagrad::new(0.1, loss.parameters()); let mut loss_val = 0.0; diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 20e393ac..cc4d0f49 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -50,7 +50,7 @@ use ndarray; use nodes; use nodes::{HogwildParameter, Node, ParameterNode}; -use nn::{uniform, xavier_normal}; +use nn::uniform; use {Arr, DataInput, Variable}; @@ -58,7 +58,7 @@ use {Arr, DataInput, Variable}; /// /// Construct this first, then use the `build` method to instantiate /// LSTM cell nodes. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct Parameters { input_dim: usize, hidden_dim: usize, @@ -293,11 +293,25 @@ mod tests { use std::ops::Deref; use super::*; + use finite_difference; use nn::losses::sparse_categorical_crossentropy; + use nn::xavier_normal; use nodes::InputNode; - use Adagrad; + use optim::Adam; use DataInput; + const TOLERANCE: f32 = 0.2; + + fn assert_close(x: &Arr, y: &Arr, tol: f32) { + assert!( + x.all_close(y, tol), + "{:#?} not within {} of {:#?}", + x, + tol, + y + ); + } + fn pi_digits(num: usize) -> Vec { let pi_str = include_str!("pi.txt"); pi_str @@ -308,6 +322,32 @@ mod tests { .collect() } + #[test] + fn lstm_finite_difference() { + let num_steps = 10; + let dim = 10; + + let mut xs: Vec<_> = (0..num_steps) + .map(|_| ParameterNode::new(xavier_normal(1, dim))) + .collect(); + + let lstm_params = Parameters::new(dim, dim); + let lstm = lstm_params.build(); + + let mut hidden_states = lstm.forward(&xs); + let mut hidden = hidden_states.last_mut().unwrap(); + + for x in &mut xs { + let (difference, gradient) = finite_difference(x, &mut hidden); + assert_close(&difference, &gradient, TOLERANCE); + } + + for x in hidden.parameters().iter_mut() { + let (difference, gradient) = finite_difference(x, &mut hidden); + assert_close(&difference, &gradient, TOLERANCE); + } + } + #[test] fn test_basic_lstm() { let input_dim = 10; @@ -379,7 +419,7 @@ mod tests { let prediction = hidden.dot(&final_layer); let mut loss = sparse_categorical_crossentropy(&prediction, &y); - let mut optimizer = Adagrad::new(0.05, loss.parameters()).l2_penalty(1e-3); + let optimizer = Adam::new(loss.parameters()).learning_rate(0.01); let digits = pi_digits(100); diff --git a/src/nodes.rs b/src/nodes.rs index 6182b412..38725f52 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -38,12 +38,16 @@ impl PassCounter { self.backward_count.set(0); } pub fn is_zero(&self) -> bool { + debug_assert!(self.recurse_backward(), "Not fully backpropagated."); + self.forward_count.get() == 0 } pub fn recurse_backward(&self) -> bool { let backward_count = self.backward_count.get(); let forward_count = self.forward_count.get(); + assert!(backward_count <= forward_count); + backward_count == forward_count } pub fn forward(&self) -> ForwardAction { @@ -137,6 +141,7 @@ impl Node for Rc> { #[derive(Debug)] pub struct AddNode { value: RefCell, + gradient: RefCell, lhs: Rc, rhs: Rc, needs_gradient: bool, @@ -151,9 +156,11 @@ where pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); let value = lhs.value().deref() + rhs.value().deref(); + let gradient = rhs.value().deref() * 0.0; AddNode { value: RefCell::new(value), + gradient: RefCell::new(gradient), lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, @@ -196,8 +203,27 @@ where numerics::add(lhs_value.deref(), rhs_value.deref(), self_value.deref_mut()); } fn backward(&self, gradient: &Ref) { - self.lhs.backward(gradient); - self.rhs.backward(gradient); + match self.counter.backward() { + BackwardAction::Set => { + let mut operand_gradient = self.gradient.borrow_mut(); + + numerics::slice_assign( + operand_gradient.as_slice_mut().unwrap(), + gradient.as_slice().unwrap(), + ); + } + BackwardAction::Increment => { + let mut operand_gradient = self.gradient.borrow_mut(); + + operand_gradient.scaled_add(1.0, gradient); + } + } + + if self.counter.recurse_backward() { + let gradient = self.gradient.borrow(); + self.lhs.backward(&gradient); + self.rhs.backward(&gradient); + } } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) @@ -524,6 +550,9 @@ impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { index_vec.extend_from_slice(&index[..]); grad.assign(value); return; + } else if &index_vec[..] == index { + grad.add_assign(value); + return; } } @@ -540,16 +569,22 @@ unsafe impl Sync for HogwildParameter {} pub struct HogwildParameter { pub value: RefCell, pub squared_gradients: RefCell, + pub moments: RefCell, + num_updates: RefCell, } impl HogwildParameter { /// Create a new parameter object. pub fn new(value: Arr) -> Self { let squared_gradients = &value * 0.0; + let moments = &value * 0.0; + let num_updates = &value * 0.0; HogwildParameter { value: RefCell::new(value), squared_gradients: RefCell::new(squared_gradients), + moments: RefCell::new(moments), + num_updates: RefCell::new(num_updates), } } @@ -568,6 +603,14 @@ impl HogwildParameter { pub(crate) unsafe fn squared_gradient_mut(&self) -> &mut Arr { &mut *(self.squared_gradients.as_ptr()) } + + pub(crate) unsafe fn moments_mut(&self) -> &mut Arr { + &mut *(self.moments.as_ptr()) + } + + pub(crate) unsafe fn num_updates_mut(&self) -> &mut Arr { + &mut *(self.num_updates.as_ptr()) + } } /// Parameter node, holds the optimizable parameters of the model. @@ -642,6 +685,7 @@ where RHS: Node, { value: RefCell, + lhs_gradient: RefCell, rhs_gradient: RefCell, lhs: Rc, rhs: Rc, @@ -659,10 +703,12 @@ where let value = lhs.value().deref() - rhs.value().deref(); let rhs_gradient = rhs.value().deref() * 0.0; + let lhs_gradient = lhs.value().deref() * 0.0; SubNode { value: RefCell::new(value), rhs_gradient: RefCell::new(rhs_gradient), + lhs_gradient: RefCell::new(lhs_gradient), lhs: lhs, rhs: rhs, needs_gradient: needs_gradient, @@ -705,16 +751,26 @@ where gradient.as_slice().unwrap(), -1.0, ); + + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + + numerics::simd_scaled_assign( + lhs_gradient.as_slice_mut().unwrap(), + gradient.as_slice().unwrap(), + 1.0, + ); } BackwardAction::Increment => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - rhs_gradient.scaled_add(-1.0, gradient); + + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + lhs_gradient.scaled_add(1.0, gradient); } } if self.counter.recurse_backward() { - self.lhs.backward(gradient); + self.lhs.backward(&self.lhs_gradient.borrow()); self.rhs.backward(&self.rhs_gradient.borrow()); } } @@ -1566,11 +1622,13 @@ where self.operand.forward(); - let mut dest = self.value.borrow_mut(); + { + let mut dest = self.value.borrow_mut(); - numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { - numerics::sigmoid(x) - }); + numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { + numerics::sigmoid(x) + }); + } } fn backward(&self, gradient: &Ref) { @@ -2359,6 +2417,7 @@ impl Node for IndexNode { } fn backward(&self, gradient: &Ref) { + self.counter.backward(); self.operand .gradient .borrow_mut() diff --git a/src/numerics.rs b/src/numerics.rs index a60d7451..103f6af8 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -390,8 +390,6 @@ mod tests { use rand; use rand::Rng; - use numerics; - fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) } @@ -415,25 +413,6 @@ mod tests { lhs.iter().zip(rhs.iter()).map(|(x, y)| x * y).sum() } - fn array_assign(xs: &mut Arr, ys: &Arr) { - xs.assign(ys); - } - - fn assign(xs: &mut Arr, ys: &Arr) { - assert_eq!( - xs.shape(), - ys.shape(), - "Operands do not have the same shape." - ); - - let xs = xs.as_slice_mut().expect("Unable to convert LHS to slice."); - let ys = ys.as_slice().expect("Unable to convert RHS to slice."); - - for (x, &y) in xs.iter_mut().zip(ys.iter()) { - *x = y; - } - } - fn unrolled_dot(xs: &[f32], ys: &[f32]) -> f32 { let len = std::cmp::min(xs.len(), ys.len()); let mut xs = &xs[..len]; diff --git a/src/optim.rs b/src/optim.rs new file mode 100644 index 00000000..0acda42b --- /dev/null +++ b/src/optim.rs @@ -0,0 +1,151 @@ +use super::{numerics, Arr, ParameterNode, SynchronizationBarrier, SynchronizationBarrierGuard, + Variable}; + +use ndarray::Axis; + +struct AdamParameters<'params> { + value: &'params mut Arr, + m: &'params mut Arr, + v: &'params mut Arr, + t: &'params mut Arr, +} + +pub struct Adam { + learning_rate: f32, + l2: f32, + beta_m: f32, + beta_v: f32, + eps: f32, + parameters: Vec>, + clamp: Option<(f32, f32)>, + sync_barrier: Option, +} + +impl Adam { + pub fn new(parameters: Vec>) -> Self { + Self { + learning_rate: 0.05, + l2: 0.0, + beta_m: 0.9, + beta_v: 0.999, + eps: 1.0e-8, + parameters: parameters, + clamp: None, + sync_barrier: None, + } + } + + /// Set the learning rate. + pub fn learning_rate(mut self, learning_rate: f32) -> Self { + self.learning_rate = learning_rate; + self + } + + /// Use synchronous parallel training. + pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { + self.sync_barrier = Some(barrier.register_thread()); + self + } + + /// Set the clamp bounds. + pub fn clamp(mut self, min: f32, max: f32) -> Self { + self.clamp = Some((min, max)); + self + } + + /// Set the L2 penalty. + pub fn l2_penalty(mut self, l2_penalty: f32) -> Self { + self.l2 = l2_penalty; + self + } + + fn param_fields<'par>(&self, parameter: &'par Variable) -> AdamParameters<'par> { + AdamParameters { + value: unsafe { parameter.node.value.value_mut() }, + m: unsafe { parameter.node.value.squared_gradient_mut() }, + v: unsafe { parameter.node.value.moments_mut() }, + t: unsafe { parameter.node.value.num_updates_mut() }, + } + } + + #[inline(always)] + fn update(&self, value: &mut f32, gradient: f32, m: &mut f32, v: &mut f32, t: &mut f32) { + // Apply L2 to gradient. + let gradient = gradient + *value * self.l2; + + // Update t. I _think_ this overflows to saturate at std::f32::MAX; + *t += 1.0; + + // Update m and v. + *m = self.beta_m * *m + (1.0 - self.beta_m) * gradient; + *v = self.beta_v * *v + (1.0 - self.beta_v) * numerics::pow2(gradient); + + let m_hat = *m / (1.0 - self.beta_m.powf(*t)); + let v_hat = *v / (1.0 - self.beta_v.powf(*t)); + + *value -= self.learning_rate / (v_hat.sqrt() + self.eps) * m_hat; + } + + fn do_step(&self, parameter: &Variable) { + let mut sink = parameter.node.gradient.borrow_mut(); + + if let Some((min, max)) = self.clamp { + sink.clamp(min, max); + } + + let param = self.param_fields(parameter); + + if sink.has_dense { + for (value, &gradient, m, v, t) in izip!( + param.value.as_slice_mut().unwrap(), + sink.dense_gradient().as_slice().unwrap(), + param.m.as_slice_mut().unwrap(), + param.v.as_slice_mut().unwrap(), + param.t.as_slice_mut().unwrap(), + ) { + self.update(value, gradient, m, v, t); + } + } + + if sink.has_sparse { + for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let mut value_row = param.value.subview_mut(Axis(0), param_idx); + let grad_row = grad.subview(Axis(0), grad_idx); + let mut m_row = param.m.subview_mut(Axis(0), param_idx); + let mut v_row = param.v.subview_mut(Axis(0), param_idx); + let mut t_row = param.t.subview_mut(Axis(0), param_idx); + + for (value, &gradient, m, v, t) in izip!( + value_row.as_slice_mut().unwrap(), + grad_row.into_slice().unwrap(), + m_row.as_slice_mut().unwrap(), + v_row.as_slice_mut().unwrap(), + t_row.as_slice_mut().unwrap(), + ) { + self.update(value, gradient, m, v, t); + } + } + } + } + } + + /// Perform a single SGD step. + pub fn step(&self) { + if let Some(ref barrier) = self.sync_barrier { + { + let _ = barrier.lock(); + + for parameter in &self.parameters { + self.do_step(parameter); + } + } + + barrier.wait(); + } else { + for parameter in &self.parameters { + self.do_step(parameter); + } + } + } +} From 8fd7ebde31629f56a46daa4015c569073a8fd6df Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 5 May 2018 16:07:26 +0100 Subject: [PATCH 067/108] Add sparse gradient finite difference checking. --- src/lib.rs | 51 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d6db211e..017cafb6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,6 +151,8 @@ extern crate rand; extern crate rayon; extern crate smallvec; +use smallvec::SmallVec; + #[macro_use] extern crate itertools; @@ -425,11 +427,16 @@ impl Variable { /// Return the (dense) gradient value of this node. pub fn dense_gradient(&self) -> Option { match self.node.gradient.borrow().dense_gradient { - Some(ref arr) => Some(arr.clone()), + Some(ref gradients) => Some(gradients.clone()), None => None, } } + /// Return the (dense) gradient value of this node. + fn sparse_gradient(&self) -> SmallVec<[(SmallVec<[usize; 4]>, Arr); 4]> { + self.node.gradient.borrow().sparse_gradient.clone() + } + /// Row-wise indexing of this parameter node. Primiarily used /// to implement embedding layers. pub fn index(&self, index: &Variable) -> Variable> { @@ -848,9 +855,17 @@ where output.forward(); output.backward(1.0); - input - .dense_gradient() - .expect("Expecting a gradient but gradient not present.") + let mut gradient = input.dense_gradient().unwrap_or(initial_input * 0.0); + + for (indices, grad) in input.sparse_gradient().iter() { + for &row_idx in indices.iter() { + for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { + *dest += orig; + } + } + } + + gradient }; output.zero_gradient(); @@ -858,8 +873,8 @@ where (central_difference, gradient) } -#[allow(dead_code)] -fn assert_close(x: &Arr, y: &Arr, tol: f32) { +/// Assert two arrays are within `tol` of each other. +pub fn assert_close(x: &Arr, y: &Arr, tol: f32) { assert!( x.all_close(y, tol), "{:#?} not within {} of {:#?}", @@ -873,6 +888,7 @@ fn assert_close(x: &Arr, y: &Arr, tol: f32) { mod tests { use ndarray::arr2; + use rand::distributions::{IndependentSample, Range}; use rayon::prelude::*; use std::sync::Arc; @@ -884,6 +900,10 @@ mod tests { Arr::zeros((rows, cols)).map(|_| rand::random::()) } + fn random_index(rows: usize) -> usize { + Range::new(0, rows).ind_sample(&mut rand::thread_rng()) + } + #[test] fn test_constant_sub() { let mut x = ParameterNode::new(Arr::zeros((10, 10)) + 1.0); @@ -1116,9 +1136,9 @@ mod tests { fn rowwise_stack_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut y = ParameterNode::new(random_matrix(10, 5)); - let v = x.clone() + y.clone(); + //let v = x.clone() + y.clone(); - let z = v.stack(&v, ndarray::Axis(0)); + let z = x.stack(&y, ndarray::Axis(0)); let mut z = z.clone().sigmoid() * z.clone().relu(); assert_eq!(z.value().rows(), 20); @@ -1134,9 +1154,9 @@ mod tests { fn columnwise_stack_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut y = ParameterNode::new(random_matrix(10, 5)); - let v = x.clone() + y.clone(); + //let v = x.clone() + y.clone(); - let mut z = v.stack(&v, ndarray::Axis(1)).sigmoid(); + let mut z = x.stack(&y, ndarray::Axis(1)).sigmoid(); assert_eq!(z.value().rows(), 10); assert_eq!(z.value().cols(), 10); @@ -1148,6 +1168,17 @@ mod tests { assert_close(&difference, &gradient, TOLERANCE); } #[test] + fn sparse_index_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 5)); + let idx_0 = IndexInputNode::new(&[random_index(10)]); + let idx_1 = IndexInputNode::new(&[random_index(10)]); + + let mut z = (x.index(&idx_0).tanh() * x.index(&idx_1)).square(); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] fn univariate_regression() { let slope = ParameterNode::new(random_matrix(1, 1)); let intercept = ParameterNode::new(random_matrix(1, 1)); From 7adb6710f34833f8318621238a334bf264364f2d Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 5 May 2018 16:56:51 +0100 Subject: [PATCH 068/108] Make finite diff tolerance tighter. --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 017cafb6..579f6250 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -894,7 +894,7 @@ mod tests { use super::*; - const TOLERANCE: f32 = 0.2; + const TOLERANCE: f32 = 0.05; fn random_matrix(rows: usize, cols: usize) -> Arr { Arr::zeros((rows, cols)).map(|_| rand::random::()) @@ -1042,7 +1042,7 @@ mod tests { let mut z = (x.clone() + x.clone()).scalar_sum(); let (finite_difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&finite_difference, &gradient, TOLERANCE); + assert_close(&finite_difference, &gradient, TOLERANCE * 2.0); } #[test] fn squared_sum_finite_difference() { From ac9e7aebeb80b5920a8a0caad56da63a32f9d856 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 10 May 2018 16:56:13 +0100 Subject: [PATCH 069/108] Decrease eps for adagrad. --- src/lib.rs | 6 ++++-- src/optim.rs | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 579f6250..b492d749 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -708,7 +708,7 @@ impl Adagrad { l2: 0.0, parameters: parameters, clamp: None, - eps: 1e-6, + eps: 1e-10, sync_barrier: None, } } @@ -787,9 +787,11 @@ impl Adagrad { } } } +} +impl optim::Optimizer for Adagrad { /// Perform a single SGD step. - pub fn step(&self) { + fn step(&self) { if let Some(ref barrier) = self.sync_barrier { { let _ = barrier.lock(); diff --git a/src/optim.rs b/src/optim.rs index 0acda42b..56d1a0a4 100644 --- a/src/optim.rs +++ b/src/optim.rs @@ -3,6 +3,10 @@ use super::{numerics, Arr, ParameterNode, SynchronizationBarrier, Synchronizatio use ndarray::Axis; +pub trait Optimizer { + fn step(&self); +} + struct AdamParameters<'params> { value: &'params mut Arr, m: &'params mut Arr, @@ -129,9 +133,11 @@ impl Adam { } } } +} +impl Optimizer for Adam { /// Perform a single SGD step. - pub fn step(&self) { + fn step(&self) { if let Some(ref barrier) = self.sync_barrier { { let _ = barrier.lock(); From 361b36a04f74a6502c350fe631c08aa901b52dee Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 12 May 2018 21:13:36 +0100 Subject: [PATCH 070/108] Number of updates in adam now global. Corresponds to decent results in wheedle. --- src/lib.rs | 27 +++++++++++++++++++++++---- src/nn/lstm.rs | 2 +- src/nodes.rs | 16 +++++++++------- src/optim.rs | 25 +++++++++++-------------- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b492d749..3708f770 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,6 +175,7 @@ mod numerics; pub mod optim; use nodes::*; +use optim::Optimizer; pub use nodes::{Bor, HogwildParameter, IndexInputNode, InputNode, Node, ParameterNode}; pub use numerics::simd_dot; @@ -288,15 +289,33 @@ where /// The weight parameter scales the gradients. pub fn backward(&mut self, weight: f32) { self.grad - .get_or_insert(RefCell::new(self.node.value().map(|_| weight))); + .get_or_insert(RefCell::new(self.node.value().map(|_| weight))) + .borrow_mut() + .as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x = weight); + if let Some(ref grad) = self.grad { - { - grad.borrow_mut().map_inplace(|x| *x = weight); - } self.node.backward(&grad.borrow()); } } + /// Clip the value. Useful for clipping losses. + pub fn clip(&self, min: f32, max: f32) { + let bor_value = self.node.value(); + let value: &Arr = bor_value.deref(); + let value = unsafe { &mut *(value as *const Arr as *mut Arr) }; + + value + .as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x = 100.0 * clamp(*x, min, max)); + + println!("value {:#?}", value); + } + /// Square this variable. pub fn square(&self) -> Variable> { Variable::new( diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index cc4d0f49..2ef199e0 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -297,7 +297,7 @@ mod tests { use nn::losses::sparse_categorical_crossentropy; use nn::xavier_normal; use nodes::InputNode; - use optim::Adam; + use optim::{Adam, Optimizer}; use DataInput; const TOLERANCE: f32 = 0.2; diff --git a/src/nodes.rs b/src/nodes.rs index 38725f52..da679d4b 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -570,7 +570,7 @@ pub struct HogwildParameter { pub value: RefCell, pub squared_gradients: RefCell, pub moments: RefCell, - num_updates: RefCell, + num_updates: Cell, } impl HogwildParameter { @@ -578,13 +578,12 @@ impl HogwildParameter { pub fn new(value: Arr) -> Self { let squared_gradients = &value * 0.0; let moments = &value * 0.0; - let num_updates = &value * 0.0; HogwildParameter { value: RefCell::new(value), squared_gradients: RefCell::new(squared_gradients), moments: RefCell::new(moments), - num_updates: RefCell::new(num_updates), + num_updates: Cell::new(0), } } @@ -608,7 +607,7 @@ impl HogwildParameter { &mut *(self.moments.as_ptr()) } - pub(crate) unsafe fn num_updates_mut(&self) -> &mut Arr { + pub(crate) unsafe fn num_updates_mut(&self) -> &mut i32 { &mut *(self.num_updates.as_ptr()) } } @@ -1740,7 +1739,7 @@ where &mut operand_gradient, self.value.borrow().deref(), gradient, - |x, grad| if x < 0.0 { 0.0 } else { grad }, + |x, grad| if x <= 0.0 { 0.0 } else { grad }, ); } BackwardAction::Increment => { @@ -1750,12 +1749,13 @@ where &mut operand_gradient, self.value.borrow().deref(), gradient, - |dest, x, grad| *dest += if x < 0.0 { 0.0 } else { grad }, + |dest, x, grad| *dest += if x <= 0.0 { 0.0 } else { grad }, ); } } if self.counter.recurse_backward() { + // println!("bacwarding with {:#?}", self.operand_gradient.borrow()); self.operand.backward(&self.operand_gradient.borrow()) } } @@ -1956,7 +1956,9 @@ where { pub fn new(operand: Rc) -> Self { let needs_gradient = operand.needs_gradient(); - let value = RefCell::new(&operand.value().t() * 1.0); + let mut value = Arr::zeros((operand.value().cols(), operand.value().rows())); + value.assign(&operand.value().t()); + let value = RefCell::new(value); let gradient = RefCell::new(operand.value().deref() * 0.0); TransposeNode { diff --git a/src/optim.rs b/src/optim.rs index 56d1a0a4..b2c6dd31 100644 --- a/src/optim.rs +++ b/src/optim.rs @@ -11,7 +11,7 @@ struct AdamParameters<'params> { value: &'params mut Arr, m: &'params mut Arr, v: &'params mut Arr, - t: &'params mut Arr, + t: &'params mut i32, } pub struct Adam { @@ -73,19 +73,16 @@ impl Adam { } #[inline(always)] - fn update(&self, value: &mut f32, gradient: f32, m: &mut f32, v: &mut f32, t: &mut f32) { + fn update(&self, value: &mut f32, gradient: f32, m: &mut f32, v: &mut f32, t: &i32) { // Apply L2 to gradient. let gradient = gradient + *value * self.l2; - // Update t. I _think_ this overflows to saturate at std::f32::MAX; - *t += 1.0; - // Update m and v. *m = self.beta_m * *m + (1.0 - self.beta_m) * gradient; *v = self.beta_v * *v + (1.0 - self.beta_v) * numerics::pow2(gradient); - let m_hat = *m / (1.0 - self.beta_m.powf(*t)); - let v_hat = *v / (1.0 - self.beta_v.powf(*t)); + let m_hat = *m / (1.0 - self.beta_m.powi(*t)); + let v_hat = *v / (1.0 - self.beta_v.powi(*t)); *value -= self.learning_rate / (v_hat.sqrt() + self.eps) * m_hat; } @@ -99,15 +96,17 @@ impl Adam { let param = self.param_fields(parameter); + // Increment number of updates + *param.t = param.t.saturating_add(1); + if sink.has_dense { - for (value, &gradient, m, v, t) in izip!( + for (value, &gradient, m, v) in izip!( param.value.as_slice_mut().unwrap(), sink.dense_gradient().as_slice().unwrap(), param.m.as_slice_mut().unwrap(), param.v.as_slice_mut().unwrap(), - param.t.as_slice_mut().unwrap(), ) { - self.update(value, gradient, m, v, t); + self.update(value, gradient, m, v, param.t); } } @@ -118,16 +117,14 @@ impl Adam { let grad_row = grad.subview(Axis(0), grad_idx); let mut m_row = param.m.subview_mut(Axis(0), param_idx); let mut v_row = param.v.subview_mut(Axis(0), param_idx); - let mut t_row = param.t.subview_mut(Axis(0), param_idx); - for (value, &gradient, m, v, t) in izip!( + for (value, &gradient, m, v) in izip!( value_row.as_slice_mut().unwrap(), grad_row.into_slice().unwrap(), m_row.as_slice_mut().unwrap(), v_row.as_slice_mut().unwrap(), - t_row.as_slice_mut().unwrap(), ) { - self.update(value, gradient, m, v, t); + self.update(value, gradient, m, v, param.t); } } } From 7fac7667541c6bb04f20c22903bcaead6bb5622f Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 13 May 2018 10:09:18 +0100 Subject: [PATCH 071/108] Bump to rand 0.5. --- Cargo.toml | 2 +- src/lib.rs | 18 ++++++++++-------- src/nn/lstm.rs | 29 ++++++++++++++++++++--------- src/nn/mod.rs | 10 +++++----- src/nodes.rs | 8 ++------ src/numerics.rs | 4 +++- 6 files changed, 41 insertions(+), 30 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 56b23226..efc991a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ fast-math = [] [dependencies] ndarray = { version = "0.11.0", features = ["serde-1"] } -rand = "0.3.18" +rand = { version = "0.5.0-pre.1", features = ["serde1"] } smallvec = { version = "0.5.0", features = ["serde"] } itertools = "0.7.3" rayon = "0.9.0" diff --git a/src/lib.rs b/src/lib.rs index 3708f770..c8f7c24e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -909,7 +909,9 @@ pub fn assert_close(x: &Arr, y: &Arr, tol: f32) { mod tests { use ndarray::arr2; - use rand::distributions::{IndependentSample, Range}; + + use rand::distributions::{Distribution, Uniform}; + use rand::Rng; use rayon::prelude::*; use std::sync::Arc; @@ -918,11 +920,11 @@ mod tests { const TOLERANCE: f32 = 0.05; fn random_matrix(rows: usize, cols: usize) -> Arr { - Arr::zeros((rows, cols)).map(|_| rand::random::()) + nn::xavier_normal(rows, cols) } fn random_index(rows: usize) -> usize { - Range::new(0, rows).ind_sample(&mut rand::thread_rng()) + Uniform::new(0, rows).sample(&mut rand::thread_rng()) } #[test] @@ -1044,7 +1046,7 @@ mod tests { #[test] fn ln_finite_difference() { let mut x = ParameterNode::new(random_matrix(2, 2)); - let mut z = (x.clone() + x.clone()).ln(); + let mut z = (x.clone() + x.clone()).exp().ln(); let (finite_difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&finite_difference, &gradient, TOLERANCE); @@ -1216,7 +1218,7 @@ mod tests { let optimizer = Adagrad::new(0.5, loss.parameters()); for _ in 0..num_epochs { - let _x = arr2(&[[rand::random::()]]); + let _x = arr2(&[[rand::thread_rng().gen()]]); let _y = 0.5 * &_x + 0.2; x.set_value(&_x); @@ -1260,9 +1262,9 @@ mod tests { for _ in 0..num_epochs { let _x = arr2(&[[ - rand::random::(), - rand::random::(), - rand::random::(), + rand::thread_rng().gen(), + rand::thread_rng().gen(), + rand::thread_rng().gen(), ]]); let _y = &_x.dot(&coefficients) + 5.0; diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 2ef199e0..f860c882 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -21,7 +21,7 @@ //! let hidden_dim = 5; //! //! // Initialize the parameters. -//! let parameters = lstm::Parameters::new(input_dim, hidden_dim); +//! let parameters = lstm::Parameters::new(input_dim, hidden_dim, &mut rand::thread_rng()); //! let lstm = parameters.build(); //! //! // Construct the input nodes. @@ -46,6 +46,7 @@ use std::rc::Rc; use std::sync::Arc; use ndarray; +use rand; use nodes; use nodes::{HogwildParameter, Node, ParameterNode}; @@ -103,7 +104,7 @@ impl Clone for Parameters { impl Parameters { /// Create a new LSTM parameters object. - pub fn new(input_dim: usize, hidden_dim: usize) -> Self { + pub fn new(input_dim: usize, hidden_dim: usize, rng: &mut R) -> Self { let max = 1.0 / (hidden_dim as f32).sqrt(); let min = -max; @@ -116,32 +117,42 @@ impl Parameters { hidden_dim, min, max, + rng, ))), - forget_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), + forget_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max, rng))), update_gate_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, min, max, + rng, + ))), + update_gate_biases: Arc::new(HogwildParameter::new(uniform( + 1, hidden_dim, min, max, rng, ))), - update_gate_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), update_value_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, min, max, + rng, + ))), + update_value_biases: Arc::new(HogwildParameter::new(uniform( + 1, hidden_dim, min, max, rng, ))), - update_value_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), output_gate_weights: Arc::new(HogwildParameter::new(uniform( input_dim + hidden_dim, hidden_dim, min, max, + rng, + ))), + output_gate_biases: Arc::new(HogwildParameter::new(uniform( + 1, hidden_dim, min, max, rng, ))), - output_gate_biases: Arc::new(HogwildParameter::new(uniform(1, hidden_dim, min, max))), } } @@ -331,7 +342,7 @@ mod tests { .map(|_| ParameterNode::new(xavier_normal(1, dim))) .collect(); - let lstm_params = Parameters::new(dim, dim); + let lstm_params = Parameters::new(dim, dim, &mut rand::thread_rng()); let lstm = lstm_params.build(); let mut hidden_states = lstm.forward(&xs); @@ -354,7 +365,7 @@ mod tests { let hidden_dim = 5; // Initialize the parameters. - let lstm_params = Parameters::new(input_dim, hidden_dim); + let lstm_params = Parameters::new(input_dim, hidden_dim, &mut rand::thread_rng()); let lstm = lstm_params.build_cell(); // Initialize the cell state and hidden state. @@ -399,7 +410,7 @@ mod tests { let input_dim = 16; let hidden_dim = 32; - let lstm_params = Parameters::new(input_dim, hidden_dim); + let lstm_params = Parameters::new(input_dim, hidden_dim, &mut rand::thread_rng()); let lstm = lstm_params.build(); let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); diff --git a/src/nn/mod.rs b/src/nn/mod.rs index 318cade4..7ebe059a 100644 --- a/src/nn/mod.rs +++ b/src/nn/mod.rs @@ -4,18 +4,18 @@ pub mod losses; pub mod lstm; use rand; -use rand::distributions::{IndependentSample, Normal, Range}; +use rand::distributions::{Distribution, Normal, Uniform}; use Arr; /// Return a Xavier-normal initialised random array. pub fn xavier_normal(rows: usize, cols: usize) -> Arr { let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); - Arr::zeros((rows, cols)).map(|_| normal.ind_sample(&mut rand::thread_rng()) as f32) + Arr::zeros((rows, cols)).map(|_| normal.sample(&mut rand::thread_rng()) as f32) } /// Return a random matrix with values drawn uniformly from `(min, max)`. -pub fn uniform(rows: usize, cols: usize, min: f32, max: f32) -> Arr { - let dist = Range::new(min, max); - Arr::zeros((rows, cols)).map(|_| dist.ind_sample(&mut rand::thread_rng()) as f32) +pub fn uniform(rows: usize, cols: usize, min: f32, max: f32, rng: &mut R) -> Arr { + let dist = Uniform::new(min, max); + Arr::zeros((rows, cols)).map(|_| dist.sample(rng) as f32) } diff --git a/src/nodes.rs b/src/nodes.rs index da679d4b..5e383243 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -2443,17 +2443,13 @@ impl Node for IndexNode { #[cfg(test)] mod tests { - use rand; + use nn; use super::*; - fn random_matrix(rows: usize, cols: usize) -> Arr { - Arr::zeros((rows, cols)).map(|_| rand::random::()) - } - #[test] fn test_sub_counter() { - let x = ParameterNode::new(random_matrix(1, 1)); + let x = ParameterNode::new(nn::xavier_normal(1, 1)); let y = x.clone() - x.clone(); let mut z = y.clone() + y.clone() + y.clone(); diff --git a/src/numerics.rs b/src/numerics.rs index 103f6af8..a927826b 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -390,8 +390,10 @@ mod tests { use rand; use rand::Rng; + use nn; + fn random_matrix(rows: usize, cols: usize) -> Arr { - Arr::zeros((rows, cols)).map(|_| rand::random::()) + nn::xavier_normal(rows, cols) } fn array_scaled_assign(xs: &mut Arr, ys: &Arr, alpha: f32) { From 8b06d564d985a132fc508fd1127c5205d6338d14 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 13 May 2018 21:21:02 +0100 Subject: [PATCH 072/108] Make sure to use slice-based ops on arrays. --- src/nodes.rs | 11 ++++++----- src/numerics.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index 5e383243..1ebcd3b3 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -11,6 +11,7 @@ use ndarray::Axis; use smallvec::SmallVec; use numerics; +use numerics::ArraySliceOps; use super::{clamp, Arr, Variable}; @@ -533,7 +534,7 @@ pub trait GradientSink { impl<'a, 'b> GradientSink<&'a Ref<'b, Arr>> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: &Ref) { - self.dense_gradient().add_assign(gradient.deref()); + self.dense_gradient().slice_add_assign(gradient.deref()); self.has_dense = true; } } @@ -548,10 +549,10 @@ impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { for &mut (ref mut index_vec, ref mut grad) in gradients.iter_mut() { if index_vec.is_empty() { index_vec.extend_from_slice(&index[..]); - grad.assign(value); + grad.slice_assign(value); return; } else if &index_vec[..] == index { - grad.add_assign(value); + grad.slice_add_assign(value); return; } } @@ -1991,7 +1992,7 @@ where self.gradient.borrow_mut().assign(&gradient.t()); } BackwardAction::Increment => { - self.gradient.borrow_mut().add_assign(&gradient.t()); + self.gradient.borrow_mut().slice_add_assign(&gradient.t()); } } @@ -2300,7 +2301,7 @@ where BackwardAction::Increment => { self.operand_gradient .borrow_mut() - .add_assign(gradient[(0, 0)]); + .slice_add_assign(gradient[(0, 0)]); } } diff --git a/src/numerics.rs b/src/numerics.rs index a927826b..4a73eb27 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -7,6 +7,46 @@ use fast_approx::{fastexp, fastlog, tanhf_fast}; use super::Arr; +pub trait ArraySliceOps { + fn slice_assign(&mut self, RHS); + fn slice_add_assign(&mut self, RHS); +} + +impl<'a, T> ArraySliceOps<&'a ArrayBase> for Arr +where + T: Data, +{ + fn slice_assign(&mut self, other: &ArrayBase) { + for (lhs, &rhs) in izip!( + self.as_slice_mut().unwrap().iter_mut(), + other.as_slice().unwrap().iter() + ) { + *lhs = rhs; + } + } + fn slice_add_assign(&mut self, other: &ArrayBase) { + for (lhs, &rhs) in izip!( + self.as_slice_mut().unwrap().iter_mut(), + other.as_slice().unwrap().iter() + ) { + *lhs += rhs; + } + } +} + +impl ArraySliceOps for Arr { + fn slice_assign(&mut self, rhs: f32) { + for lhs in self.as_slice_mut().unwrap().iter_mut() { + *lhs = rhs; + } + } + fn slice_add_assign(&mut self, rhs: f32) { + for lhs in self.as_slice_mut().unwrap().iter_mut() { + *lhs += rhs; + } + } +} + /// Uses approximate e^x when the fast-math feature is enabled. #[inline(always)] pub fn exp(x: f32) -> f32 { From 42f08120ab9d132b700beae47ff2e56aa5696e72 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 14 May 2018 13:18:40 +0100 Subject: [PATCH 073/108] Get rid of homegrown vec-mat mul impl. --- src/numerics.rs | 85 +++++-------------------------------------------- 1 file changed, 8 insertions(+), 77 deletions(-) diff --git a/src/numerics.rs b/src/numerics.rs index 4a73eb27..0da941f4 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -1,7 +1,7 @@ use std; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; -use ndarray::{ArrayBase, Axis, Data, DataMut, Ix1, Ix2}; +use ndarray::{ArrayBase, Axis, Data, DataMut, Ix2}; use fast_approx::{fastexp, fastlog, tanhf_fast}; @@ -126,59 +126,6 @@ pub fn softmax_exp_sum(xs: &[f32], max: f32) -> f32 { s } -fn vec_mat_mul( - alpha: f32, - lhs: &ArrayBase, - rhs: &ArrayBase, - beta: f32, - out: &mut ArrayBase, -) where - S1: Data, - S2: Data, - S3: DataMut, -{ - if lhs.len() != rhs.rows() || rhs.cols() != out.len() { - panic!( - "Shapes are incompatible for a vec-mat mul: LHS: {} vs RHS ({} x {}) vs out {}", - lhs.len(), - rhs.rows(), - rhs.cols(), - out.len() - ) - } - - let out_slice = out.as_slice_mut().expect("Vec-mat result not contiguous."); - let lhs_slice = lhs.as_slice().expect("LHS not contiguous"); - - for (row_idx, (&x, row)) in lhs_slice.iter().zip(rhs.genrows()).enumerate() { - let row_slice = row.as_slice().expect("RHS row not C-contiguous."); - - if row_idx == 0 { - saxpy(x * alpha, row_slice, beta, out_slice); - } else { - saxpy(x * alpha, row_slice, 1.0, out_slice); - } - } -} - -fn saxpy(alpha: f32, xs: &[f32], beta: f32, outs: &mut [f32]) { - for (&x, out) in izip!(xs.iter(), outs.iter_mut()) { - *out = x * alpha + *out * beta; - } -} - -enum MatrixLayout { - RowMajor, - ColumnMajor, -} - -fn layout>(matrix: &ArrayBase) -> MatrixLayout { - match matrix.strides()[0] { - 1 => MatrixLayout::ColumnMajor, - _ => MatrixLayout::RowMajor, - } -} - pub fn mat_mul( alpha: f32, lhs: &ArrayBase, @@ -201,29 +148,13 @@ pub fn mat_mul( ); } (1, _) => { - // general_mat_vec_mul( - // alpha, - // &rhs, - // &lhs.subview(Axis(0), 0), - // beta, - // &mut out.subview_mut(Axis(0), 0), - // ); - match layout(rhs) { - MatrixLayout::RowMajor => vec_mat_mul( - alpha, - &lhs.subview(Axis(0), 0), - rhs, - beta, - &mut out.subview_mut(Axis(0), 0), - ), - MatrixLayout::ColumnMajor => general_mat_vec_mul( - alpha, - &rhs.t(), - &lhs.subview(Axis(0), 0), - beta, - &mut out.subview_mut(Axis(0), 0), - ), - } + general_mat_vec_mul( + alpha, + &rhs.t(), + &lhs.subview(Axis(0), 0), + beta, + &mut out.subview_mut(Axis(0), 0), + ); } _ => { general_mat_mul(alpha, lhs, rhs, beta, out); From 1eeb298c452e31053025dd1ed1cd29a5b7fb8fba Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 14 May 2018 16:09:54 +0100 Subject: [PATCH 074/108] Modify sparse gradient store. --- src/lib.rs | 37 +++++++++++------------ src/nodes.rs | 83 ++++++++++++++++++++++++++++++++-------------------- src/optim.rs | 30 +++++++++---------- 3 files changed, 84 insertions(+), 66 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c8f7c24e..429accf6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,6 +151,7 @@ extern crate rand; extern crate rayon; extern crate smallvec; +use rayon::prelude::*; use smallvec::SmallVec; #[macro_use] @@ -452,7 +453,7 @@ impl Variable { } /// Return the (dense) gradient value of this node. - fn sparse_gradient(&self) -> SmallVec<[(SmallVec<[usize; 4]>, Arr); 4]> { + fn sparse_gradient(&self) -> SparseGradientStore { self.node.gradient.borrow().sparse_gradient.clone() } @@ -623,18 +624,16 @@ impl SGD { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); } - if sink.has_sparse { - for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let grad_row = grad.subview(Axis(0), grad_idx); - let mut param_row = param_value.subview_mut(Axis(0), param_idx); - - numerics::map_add_assign_slice( - param_row.into_slice().unwrap(), - grad_row.into_slice().unwrap(), - |x| -learning_rate * x, - ); - } + for (ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let grad_row = grad.subview(Axis(0), grad_idx); + let mut param_row = param_value.subview_mut(Axis(0), param_idx); + + numerics::map_add_assign_slice( + param_row.into_slice().unwrap(), + grad_row.into_slice().unwrap(), + |x| -learning_rate * x, + ); } } } @@ -786,8 +785,9 @@ impl Adagrad { } } - if sink.has_sparse { - for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + sink.sparse_gradient + .iter() + .for_each(|(ref index_vec, ref grad)| { for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { let grad_row = grad.subview(Axis(0), grad_idx); let mut param_row = param_value.subview_mut(Axis(0), param_idx); @@ -803,8 +803,7 @@ impl Adagrad { *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; } } - } - } + }); } } @@ -878,7 +877,9 @@ where let mut gradient = input.dense_gradient().unwrap_or(initial_input * 0.0); - for (indices, grad) in input.sparse_gradient().iter() { + let sparse_gradient = input.sparse_gradient(); + + for (indices, grad) in sparse_gradient.iter() { for &row_idx in indices.iter() { for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { *dest += orig; diff --git a/src/nodes.rs b/src/nodes.rs index 1ebcd3b3..0db34cd4 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1,7 +1,7 @@ use std; use std::cell::{Cell, Ref, RefCell}; use std::fmt; -use std::ops::{AddAssign, Deref, DerefMut}; +use std::ops::{Deref, DerefMut}; use std::rc::Rc; use std::sync::Arc; @@ -472,13 +472,57 @@ impl Node for InputNode { fn zero_gradient(&self) {} } +#[derive(Debug, Clone)] +pub(crate) struct SparseGradientStore { + len: usize, + data: Vec<(Vec, Arr)>, +} + +impl SparseGradientStore { + pub fn new() -> Self { + SparseGradientStore { + len: 0, + data: Vec::new(), + } + } + + pub fn push(&mut self, gradient: (&[usize], &Arr)) { + let (index, value) = gradient; + + if self.len < self.data.len() { + let (index_vec, grad) = &mut self.data[self.len]; + index_vec.clear(); + index_vec.extend_from_slice(&index[..]); + grad.slice_assign(value); + self.len += 1; + } else { + self.data.push((Vec::from(&index[..]), value.clone())); + } + } + + pub fn iter(&self) -> impl Iterator, Arr)> { + self.data.iter().take(self.len) + } + + pub fn iter_mut(&mut self) -> impl Iterator, Arr)> { + self.data.iter_mut().take(self.len) + } + + pub fn clear(&mut self) { + self.len = 0; + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + #[derive(Debug)] -pub struct GradientAccumulator { +pub(crate) struct GradientAccumulator { pub dense_shape: (usize, usize), pub dense_gradient: Option, - pub sparse_gradient: SmallVec<[(SmallVec<[usize; 4]>, Arr); 4]>, + pub sparse_gradient: SparseGradientStore, pub has_dense: bool, - pub has_sparse: bool, } impl GradientAccumulator { @@ -486,9 +530,8 @@ impl GradientAccumulator { GradientAccumulator { dense_shape: dense_shape, dense_gradient: None, - sparse_gradient: SmallVec::new(), + sparse_gradient: SparseGradientStore::new(), has_dense: false, - has_sparse: false, } } pub fn dense_gradient(&mut self) -> &mut Arr { @@ -500,15 +543,8 @@ impl GradientAccumulator { self.dense_gradient().fill(0.0); } - if self.has_sparse { - for &mut (ref mut index_vec, ref mut grad) in self.sparse_gradient.iter_mut() { - index_vec.clear(); - grad.fill(0.0) - } - } - + self.sparse_gradient.clear(); self.has_dense = false; - self.has_sparse = false; } pub fn clamp(&mut self, min: f32, max: f32) { @@ -541,24 +577,7 @@ impl<'a, 'b> GradientSink<&'a Ref<'b, Arr>> for GradientAccumulator { impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: (&'a [usize], &'a Arr)) { - let (index, value) = gradient; - self.has_sparse = true; - let gradients = &mut self.sparse_gradient; - - // Check if we can reuse one of the gradient accumulators - for &mut (ref mut index_vec, ref mut grad) in gradients.iter_mut() { - if index_vec.is_empty() { - index_vec.extend_from_slice(&index[..]); - grad.slice_assign(value); - return; - } else if &index_vec[..] == index { - grad.slice_add_assign(value); - return; - } - } - - // Otherwise create one - gradients.push((SmallVec::from(&index[..]), value.clone())); + self.sparse_gradient.push(gradient); } } diff --git a/src/optim.rs b/src/optim.rs index b2c6dd31..70aa380b 100644 --- a/src/optim.rs +++ b/src/optim.rs @@ -110,22 +110,20 @@ impl Adam { } } - if sink.has_sparse { - for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let mut value_row = param.value.subview_mut(Axis(0), param_idx); - let grad_row = grad.subview(Axis(0), grad_idx); - let mut m_row = param.m.subview_mut(Axis(0), param_idx); - let mut v_row = param.v.subview_mut(Axis(0), param_idx); - - for (value, &gradient, m, v) in izip!( - value_row.as_slice_mut().unwrap(), - grad_row.into_slice().unwrap(), - m_row.as_slice_mut().unwrap(), - v_row.as_slice_mut().unwrap(), - ) { - self.update(value, gradient, m, v, param.t); - } + for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let mut value_row = param.value.subview_mut(Axis(0), param_idx); + let grad_row = grad.subview(Axis(0), grad_idx); + let mut m_row = param.m.subview_mut(Axis(0), param_idx); + let mut v_row = param.v.subview_mut(Axis(0), param_idx); + + for (value, &gradient, m, v) in izip!( + value_row.as_slice_mut().unwrap(), + grad_row.into_slice().unwrap(), + m_row.as_slice_mut().unwrap(), + v_row.as_slice_mut().unwrap(), + ) { + self.update(value, gradient, m, v, param.t); } } } From 297749095cfb28bdf4792722b75f502f4ff118d5 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 15 May 2018 15:09:36 +0100 Subject: [PATCH 075/108] Use get_or_insert_with for dense gradients and grad weights in backward. --- src/lib.rs | 9 +++++--- src/nodes.rs | 22 +++++++++--------- src/numerics.rs | 59 +++++++++++++++++++++++++++++++++---------------- src/optim.rs | 2 +- 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 429accf6..9aaca2aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -289,8 +289,10 @@ where /// Run the backward pass through the subgraph terminating at this node. /// The weight parameter scales the gradients. pub fn backward(&mut self, weight: f32) { + let val = self.node.value(); + self.grad - .get_or_insert(RefCell::new(self.node.value().map(|_| weight))) + .get_or_insert_with(|| RefCell::new(val.map(|_| weight))) .borrow_mut() .as_slice_mut() .unwrap() @@ -624,7 +626,7 @@ impl SGD { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); } - for (ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for (ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { let grad_row = grad.subview(Axis(0), grad_idx); let mut param_row = param_value.subview_mut(Axis(0), param_idx); @@ -786,6 +788,7 @@ impl Adagrad { } sink.sparse_gradient + .as_slice() .iter() .for_each(|(ref index_vec, ref grad)| { for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { @@ -879,7 +882,7 @@ where let sparse_gradient = input.sparse_gradient(); - for (indices, grad) in sparse_gradient.iter() { + for (indices, grad) in sparse_gradient.as_slice() { for &row_idx in indices.iter() { for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { *dest += orig; diff --git a/src/nodes.rs b/src/nodes.rs index 0db34cd4..62260efe 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -216,7 +216,7 @@ where BackwardAction::Increment => { let mut operand_gradient = self.gradient.borrow_mut(); - operand_gradient.scaled_add(1.0, gradient); + operand_gradient.slice_add_assign(gradient.deref()); } } @@ -500,12 +500,12 @@ impl SparseGradientStore { } } - pub fn iter(&self) -> impl Iterator, Arr)> { - self.data.iter().take(self.len) + pub fn as_slice(&self) -> &[(Vec, Arr)] { + &self.data[..self.len] } - pub fn iter_mut(&mut self) -> impl Iterator, Arr)> { - self.data.iter_mut().take(self.len) + pub fn as_slice_mut(&mut self) -> &mut [(Vec, Arr)] { + &mut self.data[..self.len] } pub fn clear(&mut self) { @@ -535,8 +535,9 @@ impl GradientAccumulator { } } pub fn dense_gradient(&mut self) -> &mut Arr { - self.dense_gradient - .get_or_insert(Arr::zeros(self.dense_shape)) + let shape = self.dense_shape; + + self.dense_gradient.get_or_insert_with(|| Arr::zeros(shape)) } fn zero_gradient(&mut self) { if self.has_dense { @@ -554,6 +555,7 @@ impl GradientAccumulator { .iter_mut() .for_each(|x| *x = clamp(*x, min, max)); self.sparse_gradient + .as_slice_mut() .iter_mut() .for_each(|(_, ref mut grad)| { grad.as_slice_mut() @@ -781,10 +783,10 @@ where } BackwardAction::Increment => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - rhs_gradient.scaled_add(-1.0, gradient); + rhs_gradient.slice_sub_assign(gradient.deref()); let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - lhs_gradient.scaled_add(1.0, gradient); + lhs_gradient.slice_add_assign(gradient.deref()); } } @@ -2434,7 +2436,7 @@ impl Node for IndexNode { for (&idx, mut row) in idx_value.iter().zip(arr_value.genrows_mut()) { let new_val = operand_value.subview(Axis(0), idx); - numerics::slice_assign(row.into_slice().unwrap(), new_val.as_slice().unwrap()) + row.slice_assign(&new_val); } } diff --git a/src/numerics.rs b/src/numerics.rs index 0da941f4..efa721d8 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -1,7 +1,7 @@ use std; use ndarray::linalg::{general_mat_mul, general_mat_vec_mul}; -use ndarray::{ArrayBase, Axis, Data, DataMut, Ix2}; +use ndarray::{ArrayBase, ArrayViewMut, Axis, Data, DataMut, Ix1, Ix2}; use fast_approx::{fastexp, fastlog, tanhf_fast}; @@ -10,30 +10,46 @@ use super::Arr; pub trait ArraySliceOps { fn slice_assign(&mut self, RHS); fn slice_add_assign(&mut self, RHS); + fn slice_sub_assign(&mut self, RHS); } -impl<'a, T> ArraySliceOps<&'a ArrayBase> for Arr -where - T: Data, -{ - fn slice_assign(&mut self, other: &ArrayBase) { - for (lhs, &rhs) in izip!( - self.as_slice_mut().unwrap().iter_mut(), - other.as_slice().unwrap().iter() - ) { - *lhs = rhs; - } - } - fn slice_add_assign(&mut self, other: &ArrayBase) { - for (lhs, &rhs) in izip!( - self.as_slice_mut().unwrap().iter_mut(), - other.as_slice().unwrap().iter() - ) { - *lhs += rhs; +macro_rules! slice_op { + ($lhs:ty, $($rhs:ty),*) => { + $( + impl<'a, 'b, T> ArraySliceOps<&'a $rhs> for $lhs + where + T: Data, + { + fn slice_assign(&mut self, other: &$rhs) { + let lhs_slice = self.as_slice_mut().unwrap(); + let rhs_slice = other.as_slice().unwrap(); + + lhs_slice.copy_from_slice(rhs_slice); + } + fn slice_add_assign(&mut self, other: &$rhs) { + let lhs_slice = self.as_slice_mut().unwrap(); + let rhs_slice = other.as_slice().unwrap(); + + for (lhs, &rhs) in lhs_slice.iter_mut().zip(rhs_slice.iter()) { + *lhs += rhs; + } + } + fn slice_sub_assign(&mut self, other: &$rhs) { + let lhs_slice = self.as_slice_mut().unwrap(); + let rhs_slice = other.as_slice().unwrap(); + + for (lhs, &rhs) in lhs_slice.iter_mut().zip(rhs_slice.iter()) { + *lhs -= rhs; + } + } } + )* } } +slice_op!(Arr, ArrayBase); +slice_op!(ArrayViewMut<'b, f32, Ix1>, ArrayBase); + impl ArraySliceOps for Arr { fn slice_assign(&mut self, rhs: f32) { for lhs in self.as_slice_mut().unwrap().iter_mut() { @@ -45,6 +61,11 @@ impl ArraySliceOps for Arr { *lhs += rhs; } } + fn slice_sub_assign(&mut self, rhs: f32) { + for lhs in self.as_slice_mut().unwrap().iter_mut() { + *lhs -= rhs; + } + } } /// Uses approximate e^x when the fast-math feature is enabled. diff --git a/src/optim.rs b/src/optim.rs index 70aa380b..eb4dd181 100644 --- a/src/optim.rs +++ b/src/optim.rs @@ -110,7 +110,7 @@ impl Adam { } } - for &(ref index_vec, ref grad) in sink.sparse_gradient.iter() { + for &(ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { let mut value_row = param.value.subview_mut(Axis(0), param_idx); let grad_row = grad.subview(Axis(0), grad_idx); From 3ad42dddf34bacf1e46f8e243a540daa8f0e60c6 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 15 May 2018 18:08:09 +0100 Subject: [PATCH 076/108] Add faster array slicing. --- src/lib.rs | 14 ++++---- src/nodes.rs | 7 ++-- src/numerics.rs | 93 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9aaca2aa..fa0b8edd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,9 +151,6 @@ extern crate rand; extern crate rayon; extern crate smallvec; -use rayon::prelude::*; -use smallvec::SmallVec; - #[macro_use] extern crate itertools; @@ -180,6 +177,7 @@ use optim::Optimizer; pub use nodes::{Bor, HogwildParameter, IndexInputNode, InputNode, Node, ParameterNode}; pub use numerics::simd_dot; +use numerics::{ArraySlice, ArraySliceMut}; fn clamp(x: f32, min: f32, max: f32) -> f32 { if x > max { @@ -777,9 +775,9 @@ impl Adagrad { if sink.has_dense { for (value, &gradient, squared_gradient) in izip!( - param_value.as_slice_mut().unwrap(), - sink.dense_gradient().as_slice().unwrap(), - squared_gradient.as_slice_mut().unwrap() + param_value.fast_slice_mut(), + sink.dense_gradient().fast_slice(), + squared_gradient.fast_slice_mut() ) { let gradient = gradient + *value * self.l2; *squared_gradient += numerics::pow2(gradient); @@ -797,9 +795,9 @@ impl Adagrad { let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); for (value, &gradient, squared_gradient) in izip!( - param_row.as_slice_mut().unwrap(), + param_row.fast_slice_mut(), grad_row.into_slice().unwrap(), - squared_row.as_slice_mut().unwrap() + squared_row.fast_slice_mut() ) { let gradient = gradient + *value * self.l2; *squared_gradient += numerics::pow2(gradient); diff --git a/src/nodes.rs b/src/nodes.rs index 62260efe..ecccf3c7 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -38,6 +38,7 @@ impl PassCounter { self.forward_count.set(0); self.backward_count.set(0); } + #[inline(always)] pub fn is_zero(&self) -> bool { debug_assert!(self.recurse_backward(), "Not fully backpropagated."); @@ -51,6 +52,7 @@ impl PassCounter { backward_count == forward_count } + #[inline(always)] pub fn forward(&self) -> ForwardAction { let count = self.forward_count.get(); self.forward_count.set(count + 1); @@ -60,6 +62,7 @@ impl PassCounter { _ => ForwardAction::Cached, } } + #[inline(always)] pub fn backward(&self) -> BackwardAction { let backward_count = self.backward_count.get(); @@ -511,10 +514,6 @@ impl SparseGradientStore { pub fn clear(&mut self) { self.len = 0; } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } } #[derive(Debug)] diff --git a/src/numerics.rs b/src/numerics.rs index efa721d8..67526037 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -7,6 +7,46 @@ use fast_approx::{fastexp, fastlog, tanhf_fast}; use super::Arr; +pub trait ArraySlice { + fn fast_slice(&self) -> &[f32]; +} + +pub trait ArraySliceMut { + fn fast_slice_mut(&mut self) -> &mut [f32]; +} + +macro_rules! fast_slice { + ($x:ty) => { + impl ArraySlice for $x + where + T: Data, + { + fn fast_slice(&self) -> &[f32] { + if cfg!(debug_assertions) { + self.as_slice().unwrap() + } else { + self.as_slice_memory_order().unwrap() + } + } + } + impl ArraySliceMut for $x + where + T: DataMut, + { + fn fast_slice_mut(&mut self) -> &mut [f32] { + if cfg!(debug_assertions) { + self.as_slice_mut().unwrap() + } else { + self.as_slice_memory_order_mut().unwrap() + } + } + } + }; +} + +fast_slice!(ArrayBase); +fast_slice!(ArrayBase); + pub trait ArraySliceOps { fn slice_assign(&mut self, RHS); fn slice_add_assign(&mut self, RHS); @@ -21,22 +61,22 @@ macro_rules! slice_op { T: Data, { fn slice_assign(&mut self, other: &$rhs) { - let lhs_slice = self.as_slice_mut().unwrap(); - let rhs_slice = other.as_slice().unwrap(); + let lhs_slice = self.fast_slice_mut(); + let rhs_slice = other.fast_slice(); lhs_slice.copy_from_slice(rhs_slice); } fn slice_add_assign(&mut self, other: &$rhs) { - let lhs_slice = self.as_slice_mut().unwrap(); - let rhs_slice = other.as_slice().unwrap(); + let lhs_slice = self.fast_slice_mut(); + let rhs_slice = other.fast_slice(); for (lhs, &rhs) in lhs_slice.iter_mut().zip(rhs_slice.iter()) { *lhs += rhs; } } fn slice_sub_assign(&mut self, other: &$rhs) { - let lhs_slice = self.as_slice_mut().unwrap(); - let rhs_slice = other.as_slice().unwrap(); + let lhs_slice = self.fast_slice_mut(); + let rhs_slice = other.fast_slice(); for (lhs, &rhs) in lhs_slice.iter_mut().zip(rhs_slice.iter()) { *lhs -= rhs; @@ -52,17 +92,17 @@ slice_op!(ArrayViewMut<'b, f32, Ix1>, ArrayBase); impl ArraySliceOps for Arr { fn slice_assign(&mut self, rhs: f32) { - for lhs in self.as_slice_mut().unwrap().iter_mut() { + for lhs in self.fast_slice_mut().iter_mut() { *lhs = rhs; } } fn slice_add_assign(&mut self, rhs: f32) { - for lhs in self.as_slice_mut().unwrap().iter_mut() { + for lhs in self.fast_slice_mut().iter_mut() { *lhs += rhs; } } fn slice_sub_assign(&mut self, rhs: f32) { - for lhs in self.as_slice_mut().unwrap().iter_mut() { + for lhs in self.fast_slice_mut().iter_mut() { *lhs -= rhs; } } @@ -266,9 +306,9 @@ macro_rules! slice_binary_op { ( $name:ident, $slice_name:ident, $increment_name:ident,$slice_increment_name:ident, $op:tt ) => { pub fn $name(xs: &Arr, ys: &Arr, out: &mut Arr) { - $slice_name(xs.as_slice().unwrap(), - ys.as_slice().unwrap(), - out.as_slice_mut().unwrap()); + $slice_name(xs.fast_slice(), + ys.fast_slice(), + out.fast_slice_mut()); } fn $slice_name(xs: &[f32], ys: &[f32], outs: &mut [f32]) { @@ -281,9 +321,9 @@ macro_rules! slice_binary_op { #[allow(dead_code)] pub fn $increment_name(xs: &Arr, ys: &Arr, out: &mut Arr) { - $slice_increment_name(xs.as_slice().unwrap(), - ys.as_slice().unwrap(), - out.as_slice_mut().unwrap()); + $slice_increment_name(xs.fast_slice(), + ys.fast_slice(), + out.fast_slice_mut()); } #[allow(dead_code)] @@ -312,8 +352,8 @@ pub fn map_assign(xs: &mut Arr, ys: &Arr, func: F) where F: Fn(f32) -> f32, { - let xs = xs.as_slice_mut().expect("Unable to convert LHS to slice."); - let ys = ys.as_slice().expect("Unable to convert RHS to slice."); + let xs = xs.fast_slice_mut(); + let ys = ys.fast_slice(); for (x, &y) in xs.iter_mut().zip(ys.iter()) { *x = func(y); @@ -333,10 +373,9 @@ pub fn map_assign_binary(xs: &mut Arr, ys: &Arr, zs: &Arr, func: F) where F: Fn(f32, f32) -> f32, { - let xs = xs.as_slice_mut() - .expect("Unable to convert operand to slice."); - let ys = ys.as_slice().expect("Unable to convert operand to slice."); - let zs = zs.as_slice().expect("Unable to convert operand to slice."); + let xs = xs.fast_slice_mut(); + let ys = ys.fast_slice(); + let zs = zs.fast_slice(); for (x, &y, &z) in izip!(xs.iter_mut(), ys.iter(), zs.iter()) { *x = func(y, z); @@ -348,9 +387,8 @@ pub fn map_inplace_assign(xs: &mut Arr, ys: &Arr, func: F) where F: Fn(&mut f32, f32), { - let xs = xs.as_slice_mut() - .expect("Unable to convert operand to slice."); - let ys = ys.as_slice().expect("Unable to convert operand to slice."); + let xs = xs.fast_slice_mut(); + let ys = ys.fast_slice(); for (x, &y) in izip!(xs.iter_mut(), ys.iter()) { func(x, y); @@ -362,10 +400,9 @@ pub fn map_inplace_assign_binary(xs: &mut Arr, ys: &Arr, zs: &Arr, func: F) where F: Fn(&mut f32, f32, f32), { - let xs = xs.as_slice_mut() - .expect("Unable to convert operand to slice."); - let ys = ys.as_slice().expect("Unable to convert operand to slice."); - let zs = zs.as_slice().expect("Unable to convert operand to slice."); + let xs = xs.fast_slice_mut(); + let ys = ys.fast_slice(); + let zs = zs.fast_slice(); for (x, &y, &z) in izip!(xs.iter_mut(), ys.iter(), zs.iter()) { func(x, y, z); From 4721f1d8dd084b61cc2f944dbf9c1e42c3d90527 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 17 May 2018 18:41:08 +0100 Subject: [PATCH 077/108] Barrier on entry and exit to parameter update section. --- src/barrier.rs | 2 -- src/lib.rs | 33 ++++++++++++++++++++++----------- src/nn/lstm.rs | 2 -- src/nodes.rs | 41 ++++++++++++++++++++--------------------- src/optim.rs | 3 ++- 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/src/barrier.rs b/src/barrier.rs index 94dc3e37..b6f70d66 100644 --- a/src/barrier.rs +++ b/src/barrier.rs @@ -47,13 +47,11 @@ impl Barrier { pub fn increment_num_threads(&self) { let mut lock = self.lock.lock().unwrap(); lock.num_threads += 1; - // println!("INC Now thread count {}", lock.num_threads); } pub fn decrement_num_threads(&self) { let mut lock = self.lock.lock().unwrap(); lock.num_threads = lock.num_threads.saturating_sub(1); - // println!("DEC Now thread count {}", lock.num_threads); // Notify if deregistering makes the barrier conditions met. if lock.count >= lock.num_threads { diff --git a/src/lib.rs b/src/lib.rs index fa0b8edd..4562a059 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -660,29 +660,36 @@ impl SynchronizationBarrier { } struct SynchronizationBarrierCore { - barrier: barrier::Barrier, + start_barrier: barrier::Barrier, + end_barrier: barrier::Barrier, parameter_lock: Mutex<()>, } impl SynchronizationBarrierCore { fn new() -> Self { Self { - barrier: barrier::Barrier::new(0), + start_barrier: barrier::Barrier::new(0), + end_barrier: barrier::Barrier::new(0), parameter_lock: Mutex::default(), } } fn register_thread(&self) { - self.barrier.increment_num_threads(); + self.start_barrier.increment_num_threads(); + self.end_barrier.increment_num_threads(); } fn deregister_thread(&self) { - self.barrier.decrement_num_threads(); - // println!("Deregistered"); + self.start_barrier.decrement_num_threads(); + self.end_barrier.decrement_num_threads(); } - fn wait(&self) { - self.barrier.wait(); + fn start_wait(&self) { + self.start_barrier.wait(); + } + + fn end_wait(&self) { + self.end_barrier.wait(); } } @@ -691,8 +698,12 @@ pub struct SynchronizationBarrierGuard { } impl SynchronizationBarrierGuard { - fn wait(&self) { - self.barrier.wait(); + fn start_wait(&self) { + self.barrier.start_wait(); + } + + fn end_wait(&self) { + self.barrier.end_wait(); } fn lock(&self) -> MutexGuard<()> { @@ -702,7 +713,6 @@ impl SynchronizationBarrierGuard { impl Drop for SynchronizationBarrierGuard { fn drop(&mut self) { - // println!("Deregistering"); self.barrier.deregister_thread(); } } @@ -812,6 +822,7 @@ impl optim::Optimizer for Adagrad { /// Perform a single SGD step. fn step(&self) { if let Some(ref barrier) = self.sync_barrier { + barrier.start_wait(); { let _ = barrier.lock(); @@ -820,7 +831,7 @@ impl optim::Optimizer for Adagrad { } } - barrier.wait(); + barrier.end_wait(); } else { for parameter in &self.parameters { self.do_step(parameter); diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index f860c882..1774d261 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -461,8 +461,6 @@ mod tests { loss_val += loss.value().scalar_sum(); - // println!("Hiddent {:#?}", hidden.value()); - optimizer.step(); loss.zero_gradient(); diff --git a/src/nodes.rs b/src/nodes.rs index ecccf3c7..0658a3dd 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -11,7 +11,7 @@ use ndarray::Axis; use smallvec::SmallVec; use numerics; -use numerics::ArraySliceOps; +use numerics::{ArraySlice, ArraySliceMut, ArraySliceOps}; use super::{clamp, Arr, Variable}; @@ -204,21 +204,22 @@ where let mut self_value = self.value.borrow_mut(); - numerics::add(lhs_value.deref(), rhs_value.deref(), self_value.deref_mut()); + for (v, &lhs, &rhs) in izip!( + self_value.fast_slice_mut(), + lhs_value.fast_slice(), + rhs_value.fast_slice() + ) { + *v = lhs + rhs; + } } fn backward(&self, gradient: &Ref) { match self.counter.backward() { BackwardAction::Set => { let mut operand_gradient = self.gradient.borrow_mut(); - - numerics::slice_assign( - operand_gradient.as_slice_mut().unwrap(), - gradient.as_slice().unwrap(), - ); + operand_gradient.slice_assign(gradient.deref()); } BackwardAction::Increment => { let mut operand_gradient = self.gradient.borrow_mut(); - operand_gradient.slice_add_assign(gradient.deref()); } } @@ -245,7 +246,8 @@ where } fn row_wise_stack(dest: &mut Arr, lhs: &Arr, rhs: &Arr) { - for (mut dest_row, source_row) in dest.genrows_mut() + for (mut dest_row, source_row) in dest + .genrows_mut() .into_iter() .zip(lhs.genrows().into_iter().chain(rhs.genrows())) { @@ -278,9 +280,9 @@ fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: lhs.genrows_mut().into_iter(), rhs.genrows_mut().into_iter() ) { - let grad_row = grad_row.as_slice().unwrap(); - let lhs_row = lhs_row.as_slice_mut().unwrap(); - let rhs_row = rhs_row.as_slice_mut().unwrap(); + let grad_row = grad_row.fast_slice(); + let lhs_row = lhs_row.fast_slice_mut(); + let rhs_row = rhs_row.fast_slice_mut(); let (left, right) = grad_row.split_at(lhs_row.len()); @@ -294,12 +296,8 @@ fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: } } BackwardAction::Set => { - for (x, &y) in lhs_row.iter_mut().zip(left.iter()) { - *x = y; - } - for (x, &y) in rhs_row.iter_mut().zip(right.iter()) { - *x = y; - } + lhs_row.copy_from_slice(left); + rhs_row.copy_from_slice(right); } } } @@ -1776,7 +1774,6 @@ where } if self.counter.recurse_backward() { - // println!("bacwarding with {:#?}", self.operand_gradient.borrow()); self.operand.backward(&self.operand_gradient.borrow()) } } @@ -2096,7 +2093,8 @@ where let mut dest = self.value.borrow_mut(); dest.assign(self.operand.value().deref()); - let max = self.operand + let max = self + .operand .value() .deref() .iter() @@ -2116,7 +2114,8 @@ where .zip(value.iter()) .enumerate() { - for (col_idx, (grad, col_val)) in row.as_slice_mut() + for (col_idx, (grad, col_val)) in row + .as_slice_mut() .unwrap() .iter_mut() .zip(value.as_slice().unwrap()) diff --git a/src/optim.rs b/src/optim.rs index eb4dd181..ce95b17e 100644 --- a/src/optim.rs +++ b/src/optim.rs @@ -134,6 +134,7 @@ impl Optimizer for Adam { /// Perform a single SGD step. fn step(&self) { if let Some(ref barrier) = self.sync_barrier { + barrier.start_wait(); { let _ = barrier.lock(); @@ -142,7 +143,7 @@ impl Optimizer for Adam { } } - barrier.wait(); + barrier.end_wait(); } else { for parameter in &self.parameters { self.do_step(parameter); From 6bc720380a719ce75e663730afe50ba3875ac642 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 20 May 2018 15:05:49 +0100 Subject: [PATCH 078/108] Move all optimizers into the `optim` module. --- src/lib.rs | 280 +------------------------------- src/numerics.rs | 1 - src/optim/adagrad.rs | 136 ++++++++++++++++ src/{optim.rs => optim/adam.rs} | 11 +- src/{ => optim}/barrier.rs | 79 ++++++++- src/optim/mod.rs | 18 ++ src/optim/sgd.rs | 92 +++++++++++ 7 files changed, 338 insertions(+), 279 deletions(-) create mode 100644 src/optim/adagrad.rs rename src/{optim.rs => optim/adam.rs} (95%) rename src/{ => optim}/barrier.rs (60%) create mode 100644 src/optim/mod.rs create mode 100644 src/optim/sgd.rs diff --git a/src/lib.rs b/src/lib.rs index 4562a059..441df177 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,7 @@ //! # extern crate rand; //! # extern crate wyrm; //! # use wyrm::*; +//! # use wyrm::optim::*; //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } @@ -56,7 +57,7 @@ //! # let y_hat = slope.clone() * x.clone() + intercept.clone(); //! # let mut loss = (y.clone() - y_hat).square(); //! # let num_epochs = 10; -//! let mut optimizer = SGD::new(0.1, loss.parameters()); +//! let mut optimizer = SGD::new(loss.parameters()).learning_rate(0.1); //! //! for _ in 0..num_epochs { //! let x_value: f32 = rand::random(); @@ -86,6 +87,7 @@ //! # use std::sync::Arc; //! # use rayon::prelude::*; //! # use wyrm::*; +//! # use wyrm::optim::*; //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } @@ -104,7 +106,7 @@ //! let y_hat = slope.clone() * x.clone() + intercept.clone(); //! let mut loss = (y.clone() - y_hat).square(); //! -//! let mut optimizer = SGD::new(0.1, loss.parameters()); +//! let optimizer = SGD::new(loss.parameters()).learning_rate(0.1); //! //! for _ in 0..num_epochs { //! let x_value: f32 = rand::random(); @@ -138,9 +140,6 @@ //! Enable the `fast-math` option to use fast approximations to transcendental functions. //! This should give substantial speed gains in networks that are `exp`, `ln`, or `tanh`-heavy. -// TODO: pass through of parent values in .value(), -// optimizations in forward -// check for needs gradient #[macro_use] extern crate serde_derive; @@ -154,8 +153,6 @@ extern crate smallvec; #[macro_use] extern crate itertools; -use ndarray::Axis; - /// Alias for a `f32` `ndarray` matrix. pub type Arr = ndarray::Array2; @@ -163,9 +160,7 @@ use std::cell::RefCell; use std::clone::Clone; use std::ops::{Add, Deref, Div, Mul, Neg, Sub}; use std::rc::Rc; -use std::sync::{Arc, Mutex, MutexGuard}; -mod barrier; mod fast_approx; pub mod nn; mod nodes; @@ -173,11 +168,9 @@ mod numerics; pub mod optim; use nodes::*; -use optim::Optimizer; pub use nodes::{Bor, HogwildParameter, IndexInputNode, InputNode, Node, ParameterNode}; pub use numerics::simd_dot; -use numerics::{ArraySlice, ArraySliceMut}; fn clamp(x: f32, min: f32, max: f32) -> f32 { if x > max { @@ -313,8 +306,6 @@ where .unwrap() .iter_mut() .for_each(|x| *x = 100.0 * clamp(*x, min, max)); - - println!("value {:#?}", value); } /// Square this variable. @@ -586,260 +577,6 @@ where } } -/// Standard stochastic gradient descent optimizer with a fixed learning rate. -pub struct SGD { - learning_rate: f32, - parameters: Vec>, - clamp: Option<(f32, f32)>, -} - -impl SGD { - /// Create a new optimizer instance with a given set of parameters. - pub fn new(learning_rate: f32, parameters: Vec>) -> Self { - SGD { - learning_rate: learning_rate, - parameters: parameters, - clamp: None, - } - } - - /// Set the clamp bounds. - pub fn clamp(mut self, min: f32, max: f32) -> Self { - self.clamp = Some((min, max)); - self - } - - /// Perform a single SGD step. - pub fn step(&mut self) { - let learning_rate = self.learning_rate; - for parameter in &self.parameters { - let mut sink = parameter.node.gradient.borrow_mut(); - let mut param_value = unsafe { parameter.node.value.value_mut() }; - - if let Some((min, max)) = self.clamp { - sink.clamp(min, max); - } - - if sink.has_dense { - param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); - } - - for (ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let grad_row = grad.subview(Axis(0), grad_idx); - let mut param_row = param_value.subview_mut(Axis(0), param_idx); - - numerics::map_add_assign_slice( - param_row.into_slice().unwrap(), - grad_row.into_slice().unwrap(), - |x| -learning_rate * x, - ); - } - } - } - } -} - -pub struct SynchronizationBarrier { - core: Arc, -} - -impl SynchronizationBarrier { - pub fn new() -> Self { - SynchronizationBarrier { - core: Arc::new(SynchronizationBarrierCore::new()), - } - } - - fn register_thread(&self) -> SynchronizationBarrierGuard { - self.core.register_thread(); - SynchronizationBarrierGuard { - barrier: Arc::clone(&self.core), - } - } -} - -struct SynchronizationBarrierCore { - start_barrier: barrier::Barrier, - end_barrier: barrier::Barrier, - parameter_lock: Mutex<()>, -} - -impl SynchronizationBarrierCore { - fn new() -> Self { - Self { - start_barrier: barrier::Barrier::new(0), - end_barrier: barrier::Barrier::new(0), - parameter_lock: Mutex::default(), - } - } - - fn register_thread(&self) { - self.start_barrier.increment_num_threads(); - self.end_barrier.increment_num_threads(); - } - - fn deregister_thread(&self) { - self.start_barrier.decrement_num_threads(); - self.end_barrier.decrement_num_threads(); - } - - fn start_wait(&self) { - self.start_barrier.wait(); - } - - fn end_wait(&self) { - self.end_barrier.wait(); - } -} - -pub struct SynchronizationBarrierGuard { - barrier: Arc, -} - -impl SynchronizationBarrierGuard { - fn start_wait(&self) { - self.barrier.start_wait(); - } - - fn end_wait(&self) { - self.barrier.end_wait(); - } - - fn lock(&self) -> MutexGuard<()> { - self.barrier.parameter_lock.lock().unwrap() - } -} - -impl Drop for SynchronizationBarrierGuard { - fn drop(&mut self) { - self.barrier.deregister_thread(); - } -} - -/// Adagrad optimizer, scaled the learning rate by the inverse of previously -/// accumulated gradients. -pub struct Adagrad { - learning_rate: f32, - l2: f32, - parameters: Vec>, - clamp: Option<(f32, f32)>, - eps: f32, - sync_barrier: Option, -} - -impl Adagrad { - /// Create a new optimizer instance with a given set of parameters. - pub fn new(learning_rate: f32, parameters: Vec>) -> Self { - Adagrad { - learning_rate: learning_rate, - l2: 0.0, - parameters: parameters, - clamp: None, - eps: 1e-10, - sync_barrier: None, - } - } - - pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { - self.sync_barrier = Some(barrier.register_thread()); - self - } - - /// Set the clamp bounds. - pub fn clamp(mut self, min: f32, max: f32) -> Self { - self.clamp = Some((min, max)); - self - } - - /// Set the L2 penalty. - pub fn l2_penalty(mut self, l2_penalty: f32) -> Self { - self.l2 = l2_penalty; - self - } - - /// Decay weights. - pub fn decay_weights(&mut self, penalty: f32) { - for parameter in &self.parameters { - let mut param_value = unsafe { parameter.node.value.value_mut() }; - - param_value - .as_slice_mut() - .unwrap() - .iter_mut() - .for_each(|x| *x -= x.signum() * penalty * numerics::pow2(*x)); - } - } - - fn do_step(&self, parameter: &Variable) { - let learning_rate = self.learning_rate; - - let mut sink = parameter.node.gradient.borrow_mut(); - - if let Some((min, max)) = self.clamp { - sink.clamp(min, max); - } - - let param_value = unsafe { parameter.node.value.value_mut() }; - let squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; - - if sink.has_dense { - for (value, &gradient, squared_gradient) in izip!( - param_value.fast_slice_mut(), - sink.dense_gradient().fast_slice(), - squared_gradient.fast_slice_mut() - ) { - let gradient = gradient + *value * self.l2; - *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; - } - } - - sink.sparse_gradient - .as_slice() - .iter() - .for_each(|(ref index_vec, ref grad)| { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let grad_row = grad.subview(Axis(0), grad_idx); - let mut param_row = param_value.subview_mut(Axis(0), param_idx); - let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); - - for (value, &gradient, squared_gradient) in izip!( - param_row.fast_slice_mut(), - grad_row.into_slice().unwrap(), - squared_row.fast_slice_mut() - ) { - let gradient = gradient + *value * self.l2; - *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; - } - } - }); - } -} - -impl optim::Optimizer for Adagrad { - /// Perform a single SGD step. - fn step(&self) { - if let Some(ref barrier) = self.sync_barrier { - barrier.start_wait(); - { - let _ = barrier.lock(); - - for parameter in &self.parameters { - self.do_step(parameter); - } - } - - barrier.end_wait(); - } else { - for parameter in &self.parameters { - self.do_step(parameter); - } - } - } -} - /// Compute finite difference gradient estimates of the output variable /// with respect to the input. Use to verify correctness of gradient /// computations. @@ -923,6 +660,7 @@ mod tests { use ndarray::arr2; + use optim::{Adagrad, Optimizer, SGD}; use rand::distributions::{Distribution, Uniform}; use rand::Rng; use rayon::prelude::*; @@ -1228,7 +966,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let optimizer = Adagrad::new(0.5, loss.parameters()); + let optimizer = Adagrad::new(loss.parameters()).learning_rate(0.5); for _ in 0..num_epochs { let _x = arr2(&[[rand::thread_rng().gen()]]); @@ -1271,7 +1009,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let mut optimizer = SGD::new(0.1, loss.parameters()); + let optimizer = SGD::new(loss.parameters()).learning_rate(0.1); for _ in 0..num_epochs { let _x = arr2(&[[ @@ -1329,7 +1067,7 @@ mod tests { let mut loss = (output.clone() - y_hat.clone()).square(); let num_epochs = 200; - let optimizer = Adagrad::new(0.1, loss.parameters()); + let optimizer = Adagrad::new(loss.parameters()).learning_rate(0.1); let mut loss_val = 0.0; @@ -1391,7 +1129,7 @@ mod tests { let num_epochs = 100; - let mut optimizer = SGD::new(0.1, loss.parameters()); + let optimizer = SGD::new(loss.parameters()); let mut loss_val = 0.0; diff --git a/src/numerics.rs b/src/numerics.rs index 67526037..092a92e0 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -337,7 +337,6 @@ macro_rules! slice_binary_op { } } -slice_binary_op!(add, slice_add, increment_add, increment_slice_add, +); slice_binary_op!(sub, slice_sub, increment_sub, increment_slice_sub, -); slice_binary_op!(mul, slice_mul, increment_mul, increment_slice_mul, *); slice_binary_op!(div, slice_div, increment_div, increment_slice_div, /); diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs new file mode 100644 index 00000000..990c5cdd --- /dev/null +++ b/src/optim/adagrad.rs @@ -0,0 +1,136 @@ +use super::barrier::{SynchronizationBarrier, SynchronizationBarrierGuard}; +use super::Optimizer; +use numerics::{ArraySlice, ArraySliceMut}; +use {numerics, ParameterNode, Variable}; + +use ndarray::Axis; + +/// Adagrad optimizer, scaled the learning rate by the inverse of previously +/// accumulated gradients. +pub struct Adagrad { + learning_rate: f32, + l2: f32, + parameters: Vec>, + clamp: Option<(f32, f32)>, + eps: f32, + sync_barrier: Option, +} + +impl Adagrad { + /// Create a new optimizer instance with a given set of parameters. + pub fn new(parameters: Vec>) -> Self { + Adagrad { + learning_rate: 0.05, + l2: 0.0, + parameters: parameters, + clamp: None, + eps: 1e-10, + sync_barrier: None, + } + } + + /// Set the learning rate. + pub fn learning_rate(mut self, learning_rate: f32) -> Self { + self.learning_rate = learning_rate; + self + } + + /// Use the optimizer in synchrnous mode. + pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { + self.sync_barrier = Some(barrier.register_thread()); + self + } + + /// Set the clamp bounds. + pub fn clamp(mut self, min: f32, max: f32) -> Self { + self.clamp = Some((min, max)); + self + } + + /// Set the L2 penalty. + pub fn l2_penalty(mut self, l2_penalty: f32) -> Self { + self.l2 = l2_penalty; + self + } + + /// Decay weights. + pub fn decay_weights(&mut self, penalty: f32) { + for parameter in &self.parameters { + let mut param_value = unsafe { parameter.node.value.value_mut() }; + + param_value + .as_slice_mut() + .unwrap() + .iter_mut() + .for_each(|x| *x -= x.signum() * penalty * numerics::pow2(*x)); + } + } + + fn do_step(&self, parameter: &Variable) { + let learning_rate = self.learning_rate; + + let mut sink = parameter.node.gradient.borrow_mut(); + + if let Some((min, max)) = self.clamp { + sink.clamp(min, max); + } + + let param_value = unsafe { parameter.node.value.value_mut() }; + let squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; + + if sink.has_dense { + for (value, &gradient, squared_gradient) in izip!( + param_value.fast_slice_mut(), + sink.dense_gradient().fast_slice(), + squared_gradient.fast_slice_mut() + ) { + let gradient = gradient + *value * self.l2; + *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; + } + } + + sink.sparse_gradient + .as_slice() + .iter() + .for_each(|(ref index_vec, ref grad)| { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let grad_row = grad.subview(Axis(0), grad_idx); + let mut param_row = param_value.subview_mut(Axis(0), param_idx); + let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); + + for (value, &gradient, squared_gradient) in izip!( + param_row.fast_slice_mut(), + grad_row.into_slice().unwrap(), + squared_row.fast_slice_mut() + ) { + let gradient = gradient + *value * self.l2; + *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; + } + } + }); + } +} + +impl Optimizer for Adagrad { + /// Perform a single SGD step. + fn step(&self) { + if let Some(ref barrier) = self.sync_barrier { + barrier.start_wait(); + { + let _ = barrier.lock(); + + for parameter in &self.parameters { + self.do_step(parameter); + } + } + + barrier.end_wait(); + } else { + for parameter in &self.parameters { + self.do_step(parameter); + } + } + } +} diff --git a/src/optim.rs b/src/optim/adam.rs similarity index 95% rename from src/optim.rs rename to src/optim/adam.rs index ce95b17e..0a9435da 100644 --- a/src/optim.rs +++ b/src/optim/adam.rs @@ -1,12 +1,9 @@ -use super::{numerics, Arr, ParameterNode, SynchronizationBarrier, SynchronizationBarrierGuard, - Variable}; +use super::barrier::{SynchronizationBarrier, SynchronizationBarrierGuard}; +use super::Optimizer; +use {numerics, Arr, ParameterNode, Variable}; use ndarray::Axis; -pub trait Optimizer { - fn step(&self); -} - struct AdamParameters<'params> { value: &'params mut Arr, m: &'params mut Arr, @@ -14,6 +11,7 @@ struct AdamParameters<'params> { t: &'params mut i32, } +/// ADAM optimizer. pub struct Adam { learning_rate: f32, l2: f32, @@ -26,6 +24,7 @@ pub struct Adam { } impl Adam { + /// Build new optimizer object. pub fn new(parameters: Vec>) -> Self { Self { learning_rate: 0.05, diff --git a/src/barrier.rs b/src/optim/barrier.rs similarity index 60% rename from src/barrier.rs rename to src/optim/barrier.rs index b6f70d66..1628f64b 100644 --- a/src/barrier.rs +++ b/src/optim/barrier.rs @@ -11,7 +11,7 @@ // except according to those terms. use std::fmt; -use std::sync::{Condvar, Mutex}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; pub struct Barrier { lock: Mutex, @@ -94,3 +94,80 @@ impl BarrierWaitResult { self.0 } } + +pub struct SynchronizationBarrier { + core: Arc, +} + +impl SynchronizationBarrier { + pub fn new() -> Self { + SynchronizationBarrier { + core: Arc::new(SynchronizationBarrierCore::new()), + } + } + + pub(crate) fn register_thread(&self) -> SynchronizationBarrierGuard { + self.core.register_thread(); + SynchronizationBarrierGuard { + barrier: Arc::clone(&self.core), + } + } +} + +struct SynchronizationBarrierCore { + start_barrier: Barrier, + end_barrier: Barrier, + parameter_lock: Mutex<()>, +} + +impl SynchronizationBarrierCore { + fn new() -> Self { + Self { + start_barrier: Barrier::new(0), + end_barrier: Barrier::new(0), + parameter_lock: Mutex::default(), + } + } + + fn register_thread(&self) { + self.start_barrier.increment_num_threads(); + self.end_barrier.increment_num_threads(); + } + + fn deregister_thread(&self) { + self.start_barrier.decrement_num_threads(); + self.end_barrier.decrement_num_threads(); + } + + pub(crate) fn start_wait(&self) { + self.start_barrier.wait(); + } + + pub(crate) fn end_wait(&self) { + self.end_barrier.wait(); + } +} + +pub struct SynchronizationBarrierGuard { + barrier: Arc, +} + +impl SynchronizationBarrierGuard { + pub fn start_wait(&self) { + self.barrier.start_wait(); + } + + pub fn end_wait(&self) { + self.barrier.end_wait(); + } + + pub fn lock(&self) -> MutexGuard<()> { + self.barrier.parameter_lock.lock().unwrap() + } +} + +impl Drop for SynchronizationBarrierGuard { + fn drop(&mut self) { + self.barrier.deregister_thread(); + } +} diff --git a/src/optim/mod.rs b/src/optim/mod.rs new file mode 100644 index 00000000..48a7ca2c --- /dev/null +++ b/src/optim/mod.rs @@ -0,0 +1,18 @@ +//! Optimization module. +//! +//! Contains a number of optimizers. +mod adagrad; +mod adam; +mod barrier; +mod sgd; + +/// Core trait implemented by all optimizer methods. +pub trait Optimizer { + /// Perform a single SGD step. + fn step(&self); +} + +pub use self::adagrad::Adagrad; +pub use self::adam::Adam; +pub use self::barrier::SynchronizationBarrier; +pub use self::sgd::SGD; diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs new file mode 100644 index 00000000..daa2e8a9 --- /dev/null +++ b/src/optim/sgd.rs @@ -0,0 +1,92 @@ +use super::barrier::{SynchronizationBarrier, SynchronizationBarrierGuard}; +use super::Optimizer; +use {numerics, ParameterNode, Variable}; + +use ndarray::Axis; + +/// Standard stochastic gradient descent optimizer with a fixed learning rate. +pub struct SGD { + learning_rate: f32, + parameters: Vec>, + clamp: Option<(f32, f32)>, + sync_barrier: Option, +} + +impl SGD { + /// Create a new optimizer instance with a given set of parameters. + pub fn new(parameters: Vec>) -> Self { + SGD { + learning_rate: 0.05, + parameters: parameters, + clamp: None, + sync_barrier: None, + } + } + + /// Set the learning rate. + pub fn learning_rate(mut self, learning_rate: f32) -> Self { + self.learning_rate = learning_rate; + self + } + + /// Use the optimizer in synchrnous mode. + pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { + self.sync_barrier = Some(barrier.register_thread()); + self + } + + /// Set the clamp bounds. + pub fn clamp(mut self, min: f32, max: f32) -> Self { + self.clamp = Some((min, max)); + self + } + + /// Perform a single SGD step. + fn do_step(&self, parameter: &Variable) { + let learning_rate = self.learning_rate; + let mut sink = parameter.node.gradient.borrow_mut(); + let param_value = unsafe { parameter.node.value.value_mut() }; + + if let Some((min, max)) = self.clamp { + sink.clamp(min, max); + } + + if sink.has_dense { + param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); + } + + for (ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { + for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { + let grad_row = grad.subview(Axis(0), grad_idx); + let mut param_row = param_value.subview_mut(Axis(0), param_idx); + + numerics::map_add_assign_slice( + param_row.into_slice().unwrap(), + grad_row.into_slice().unwrap(), + |x| -learning_rate * x, + ); + } + } + } +} + +impl Optimizer for SGD { + fn step(&self) { + if let Some(ref barrier) = self.sync_barrier { + barrier.start_wait(); + { + let _ = barrier.lock(); + + for parameter in &self.parameters { + self.do_step(parameter); + } + } + + barrier.end_wait(); + } else { + for parameter in &self.parameters { + self.do_step(parameter); + } + } + } +} From 2cd882aa2adc6fe68e0ae8e2cd0ce689ff1bf87e Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 20 May 2018 15:09:49 +0100 Subject: [PATCH 079/108] Bump version. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index efc991a3..5b057328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.7.2" +version = "0.8.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 6c89cf37564f573c92c40c6af479aba0e2696287 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 20 May 2018 15:10:51 +0100 Subject: [PATCH 080/108] Remove mentions of nightly compiler from readme. --- src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 441df177..48767efe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,8 +12,6 @@ //! 3. Define-by-run. //! 4. Trivial Hogwild-style parallelisation, scaling linearly with the number of CPU cores available. //! -//! Requires the nightly compiler due to use of SIMD intrinsics. -//! //! # Quickstart //! //! The following defines a univariate linear regression model, then From 87c7e60cd8918fa2aedaf17ca7d54dc42736950c Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 20 May 2018 15:44:27 +0100 Subject: [PATCH 081/108] Fix all instances of non-full backprop. --- src/nn/losses.rs | 25 ++++++++++++++++++------- src/nodes.rs | 34 +++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/nn/losses.rs b/src/nn/losses.rs index 5d14c245..22a82826 100644 --- a/src/nn/losses.rs +++ b/src/nn/losses.rs @@ -1,12 +1,11 @@ //! Loss functions. - -use std::rc::Rc; use std::cell::{Ref, RefCell}; use std::ops::Deref; +use std::rc::Rc; -use {Arr, Node, Variable}; -use nodes::{Bor, ForwardAction, IndexInputNode, LogSoftmaxNode, PassCounter}; +use nodes::{BackwardAction, Bor, ForwardAction, IndexInputNode, LogSoftmaxNode, PassCounter}; use numerics; +use {Arr, Node, Variable}; /// Sparse categorical cross entropy loss. /// @@ -111,8 +110,16 @@ where self.loss_value.borrow_mut().fill(loss_value); } + /// The backpropagation mechanics for this node are a little strange, + /// because it uses the log-softmax node for the forward pass but not + /// for the backward pass. fn backward(&self, _: &Ref) { // TODO: actually use the input gradient + let beta = match self.counter.backward() { + BackwardAction::Set => 0.0, + BackwardAction::Increment => 1.0, + }; + { let mut gradient = self.gradient.borrow_mut(); let gradient_slice = gradient.as_slice_mut().unwrap(); @@ -121,7 +128,7 @@ where let value_slice = value.as_slice().unwrap(); for (grad, &val) in izip!(gradient_slice.iter_mut(), value_slice.iter()) { - *grad = numerics::exp(val); + *grad = beta * *grad + numerics::exp(val); } for &idx in self.y.value().iter() { @@ -129,7 +136,9 @@ where } } - self.operand.backward(&self.gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.gradient.borrow()); + } } fn value(&self) -> Bor { Bor::RefGuard(self.loss_value.borrow()) @@ -139,7 +148,9 @@ where } fn zero_gradient(&self) { if !self.counter.is_zero() { - self.log_softmax.zero_gradient(); + self.operand.zero_gradient(); + self.log_softmax.zero_counter(); + self.y.zero_gradient(); self.counter.clear(); } } diff --git a/src/nodes.rs b/src/nodes.rs index 0658a3dd..cb2c1290 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -2091,12 +2091,12 @@ where self.operand.forward(); let mut dest = self.value.borrow_mut(); - dest.assign(self.operand.value().deref()); + dest.slice_assign(self.operand.value().deref()); let max = self .operand .value() - .deref() + .fast_slice() .iter() .fold(std::f32::MIN, |x, y| x.max(*y)); dest.map_inplace(|x| *x = numerics::exp(*x - max)); @@ -2108,6 +2108,11 @@ where let value = self.value.borrow(); let mut jacobian = self.jacobian.borrow_mut(); + let beta = match self.counter.backward() { + BackwardAction::Set => 0.0, + BackwardAction::Increment => 1.0, + }; + for (row_idx, (mut row, row_val)) in jacobian .genrows_mut() .into_iter() @@ -2134,12 +2139,14 @@ where 1.0, gradient, jacobian.deref_mut(), - 0.0, + beta, self.operand_gradient.borrow_mut().deref_mut(), ); } - self.operand.backward(&self.operand_gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) @@ -2194,6 +2201,13 @@ where counter: PassCounter::default(), } } + + /// An additional method for zeroing the counter for use in the + /// log-softmax loss, where the actuall log-softmax layer is skipped + /// when backpropagating. + pub fn zero_counter(&self) { + self.counter.clear(); + } } impl Node for LogSoftmaxNode @@ -2223,7 +2237,11 @@ where .for_each(|x| *x -= denominator); } fn backward(&self, gradient: &Ref) { - // TODO: accumulate gradients + let beta = match self.counter.backward() { + BackwardAction::Set => 0.0, + BackwardAction::Increment => 1.0, + }; + { let value = self.value.borrow(); let value_slice = value.as_slice().expect("Can't get value slice."); @@ -2241,11 +2259,13 @@ where for (out_grad, in_grad, &val) in izip!(downstream_gradient_slice, gradient_slice, value_slice) { - *out_grad = in_grad - numerics::exp(val) * gradient_sum; + *out_grad = beta * *out_grad + in_grad - numerics::exp(val) * gradient_sum; } } - self.operand.backward(&self.operand_gradient.borrow()); + if self.counter.recurse_backward() { + self.operand.backward(&self.operand_gradient.borrow()); + } } fn value(&self) -> Bor { Bor::RefGuard(self.value.borrow()) From fdfae27306a1b927c3fbb782656d3bbf44cb90fc Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 21 May 2018 18:02:50 +0100 Subject: [PATCH 082/108] Incorporate clippy suggestions. --- Cargo.toml | 2 +- src/fast_approx.rs | 12 +++++++----- src/lib.rs | 2 +- src/nn/lstm.rs | 1 + src/nodes.rs | 9 +++++---- src/numerics.rs | 2 ++ src/optim/barrier.rs | 6 ++++++ 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5b057328..fa300dab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ ndarray = { version = "0.11.0", features = ["serde-1"] } rand = { version = "0.5.0-pre.1", features = ["serde1"] } smallvec = { version = "0.5.0", features = ["serde"] } itertools = "0.7.3" -rayon = "0.9.0" +rayon = "1.0.0" serde = { version = "1.0.24", features = ["rc"] } serde_derive = "1.0.24" diff --git a/src/fast_approx.rs b/src/fast_approx.rs index abf59239..aac19916 100644 --- a/src/fast_approx.rs +++ b/src/fast_approx.rs @@ -41,6 +41,8 @@ * Contact: Paul Mineiro * *=====================================================================*/ +use std::f32; + #[repr(C)] union FloatUint { f: f32, @@ -61,15 +63,15 @@ pub fn fastlog2(x: f32) -> f32 { i: (vx.i & 0x007FFFFF) | 0x3f000000, }; let mut y = vx.i as f32; - y *= 1.1920928955078125e-7; + y *= 1.192_092_9e-7; - y - 124.22551499 - 1.498030302 * mx.f - 1.72587999 / (0.3520887068 + mx.f) + y - 124.225_52 - 1.498_030_3 * mx.f - 1.725_88 / (0.352_088_72 + mx.f) } } #[inline(always)] pub fn fastlog(x: f32) -> f32 { - 0.69314718 * fastlog2(x) + f32::consts::LN_2 * fastlog2(x) } #[inline(always)] @@ -92,7 +94,7 @@ pub fn fastpow2(x: f32) -> f32 { let v = UintFloat { i: ((1 << 23) as f32 - * (clip + 121.2740575 + 27.7280233 / (4.84252568 - z) - 1.49012907 * z)) + * (clip + 121.274_055 + 27.728_024 / (4.842_525_5 - z) - 1.490_129_1 * z)) as u32, }; @@ -101,5 +103,5 @@ pub fn fastpow2(x: f32) -> f32 { #[inline(always)] pub fn fastexp(x: f32) -> f32 { - fastpow2(1.442695040 * x) + fastpow2(f32::consts::LOG2_E * x) } diff --git a/src/lib.rs b/src/lib.rs index 48767efe..499188d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -137,7 +137,7 @@ //! //! Enable the `fast-math` option to use fast approximations to transcendental functions. //! This should give substantial speed gains in networks that are `exp`, `ln`, or `tanh`-heavy. - +#![cfg_attr(feature = "cargo-clippy", allow(unreadable_literal, redundant_field_names))] #[macro_use] extern crate serde_derive; diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 1774d261..9ec953c9 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -206,6 +206,7 @@ impl Cell { /// /// If this is the first cell, initialize the cell state and the hidden state; /// otherwise pass the cell and hidden states from previous iterations. + #[cfg_attr(feature = "cargo-clippy", allow(needless_pass_by_value, type_complexity))] pub fn forward( &self, state: (Variable, Variable), diff --git a/src/nodes.rs b/src/nodes.rs index cb2c1290..11a32424 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -274,7 +274,7 @@ fn column_wise_stack(dest: &mut Arr, lhs: &Arr, rhs: &Arr) { } } -fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: BackwardAction) { +fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: &BackwardAction) { for (grad_row, mut lhs_row, mut rhs_row) in izip!( gradient.genrows().into_iter(), lhs.genrows_mut().into_iter(), @@ -303,7 +303,7 @@ fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: } } -fn row_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: BackwardAction) { +fn row_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: &BackwardAction) { for (grad_row, mut dest_row) in gradient .genrows() .into_iter() @@ -407,13 +407,13 @@ where gradient, lhs_grad.deref_mut(), rhs_grad.deref_mut(), - self.counter.backward(), + &self.counter.backward(), ), ndarray::Axis(1) => column_wise_stack_gradient( gradient, lhs_grad.deref_mut(), rhs_grad.deref_mut(), - self.counter.backward(), + &self.counter.backward(), ), _ => panic!("Stacking tensors not allowed."), } @@ -592,6 +592,7 @@ pub struct HogwildParameter { num_updates: Cell, } +#[cfg_attr(feature = "cargo-clippy", allow(mut_from_ref))] impl HogwildParameter { /// Create a new parameter object. pub fn new(value: Arr) -> Self { diff --git a/src/numerics.rs b/src/numerics.rs index 092a92e0..194245b0 100644 --- a/src/numerics.rs +++ b/src/numerics.rs @@ -156,6 +156,7 @@ pub fn pow2(x: f32) -> f32 { x.powi(2) } +#[cfg_attr(feature = "cargo-clippy", allow(needless_range_loop))] pub fn softmax_exp_sum(xs: &[f32], max: f32) -> f32 { let mut xs = xs; let mut s = 0.; @@ -258,6 +259,7 @@ pub fn simd_dot(xs: &[f32], ys: &[f32]) -> f32 { s } +#[cfg_attr(feature = "cargo-clippy", allow(needless_range_loop))] pub fn simd_sum(xs: &[f32]) -> f32 { let mut xs = xs; diff --git a/src/optim/barrier.rs b/src/optim/barrier.rs index 1628f64b..4e30a57f 100644 --- a/src/optim/barrier.rs +++ b/src/optim/barrier.rs @@ -99,6 +99,12 @@ pub struct SynchronizationBarrier { core: Arc, } +impl Default for SynchronizationBarrier { + fn default() -> Self { + Self::new() + } +} + impl SynchronizationBarrier { pub fn new() -> Self { SynchronizationBarrier { From bad114b254df326692a13a317fa2b6e5a338d52b Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 22 May 2018 18:58:40 +0100 Subject: [PATCH 083/108] Bump rand to 0.5.0. --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fa300dab..6cf45fc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,12 @@ fast-math = [] [dependencies] ndarray = { version = "0.11.0", features = ["serde-1"] } -rand = { version = "0.5.0-pre.1", features = ["serde1"] } +rand = { version = "0.5.0", features = ["serde1"] } smallvec = { version = "0.5.0", features = ["serde"] } itertools = "0.7.3" rayon = "1.0.0" -serde = { version = "1.0.24", features = ["rc"] } -serde_derive = "1.0.24" +serde = { version = "1.0.0", features = ["rc"] } +serde_derive = "1.0.0" [dev-dependencies] ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } From 6e2dcff5287428eca582f978e8e621bbc3d70aee Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 27 May 2018 11:41:12 +0100 Subject: [PATCH 084/108] Add slice nodes. --- src/lib.rs | 32 ++++++++++++++++++ src/nodes.rs | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 499188d5..1cf6c69b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -143,6 +143,7 @@ extern crate serde_derive; extern crate serde; +#[macro_use] extern crate ndarray; extern crate rand; extern crate rayon; @@ -430,6 +431,17 @@ where merge_parameters(&self.parameters, &other.parameters), ) } + + /// Slice the node according to the `ndarray` slice syntax. + pub fn slice( + &self, + slice: &ndarray::SliceInfo<[ndarray::SliceOrIndex; 2], ndarray::Ix2>, + ) -> Variable> { + Variable::new( + Rc::new(SliceNode::new(Rc::clone(&self.node), slice)), + self.parameters.clone(), + ) + } } impl Variable { @@ -940,6 +952,26 @@ mod tests { assert_close(&difference, &gradient, TOLERANCE); } #[test] + fn columnwise_view_finite_difference() { + let mut x = ParameterNode::new(random_matrix(10, 30)); + + let x_0 = x.slice(s![.., 0..10]); + let x_1 = x.slice(s![.., 10..20]); + let x_2 = x.slice(s![.., 20..30]); + + assert_eq!(x_0.value().rows(), 10); + assert_eq!(x_0.value().cols(), 10); + assert_eq!(x_1.value().rows(), 10); + assert_eq!(x_1.value().cols(), 10); + assert_eq!(x_2.value().rows(), 10); + assert_eq!(x_2.value().cols(), 10); + + let mut z = (x_0 + x_1 + x_2).sigmoid(); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } + #[test] fn sparse_index_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let idx_0 = IndexInputNode::new(&[random_index(10)]); diff --git a/src/nodes.rs b/src/nodes.rs index 11a32424..21b899f5 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1,7 +1,7 @@ use std; use std::cell::{Cell, Ref, RefCell}; use std::fmt; -use std::ops::{Deref, DerefMut}; +use std::ops::{AddAssign, Deref, DerefMut}; use std::rc::Rc; use std::sync::Arc; @@ -440,6 +440,100 @@ where } } +#[derive(Debug)] +pub struct SliceNode { + slice: ndarray::SliceInfo<[ndarray::SliceOrIndex; 2], ndarray::Ix2>, + value: RefCell, + lhs_gradient: RefCell, + lhs: Rc, + needs_gradient: bool, + counter: PassCounter, +} + +impl SliceNode +where + LHS: Node, +{ + pub fn new( + lhs: Rc, + slice: &ndarray::SliceInfo<[ndarray::SliceOrIndex; 2], ndarray::Ix2>, + ) -> Self { + let needs_gradient = lhs.needs_gradient(); + + let value = { + let val = lhs.value(); + let sliced = val.slice(slice); + let mut value = Arr::zeros((sliced.rows(), sliced.cols())); + value.assign(&sliced); + + value + }; + + let lhs_gradient = lhs.value().deref() * 0.0; + + SliceNode { + slice: *slice, + value: RefCell::new(value), + lhs_gradient: RefCell::new(lhs_gradient), + lhs: lhs, + needs_gradient: needs_gradient, + counter: PassCounter::default(), + } + } +} + +impl Node for SliceNode +where + LHS: Node, +{ + type Value = Arr; + type InputGradient = Arr; + fn forward(&self) { + if self.counter.forward() == ForwardAction::Cached { + return; + } + + self.lhs.forward(); + + let lhs_value = self.lhs.value(); + let mut self_value = self.value.borrow_mut(); + self_value.assign(&lhs_value.slice(&self.slice)); + } + fn backward(&self, gradient: &Ref) { + match self.counter.backward() { + BackwardAction::Set => { + self.lhs_gradient + .borrow_mut() + .slice_mut(&self.slice) + .assign(gradient.deref()); + } + BackwardAction::Increment => { + self.lhs_gradient + .borrow_mut() + .slice_mut(&self.slice) + .add_assign(gradient.deref()); + } + } + + if self.counter.recurse_backward() { + self.lhs.backward(&self.lhs_gradient.borrow()); + } + } + + fn value(&self) -> Bor { + Bor::RefGuard(self.value.borrow()) + } + fn needs_gradient(&self) -> bool { + self.needs_gradient + } + fn zero_gradient(&self) { + if !self.counter.is_zero() { + self.lhs.zero_gradient(); + self.counter.clear(); + } + } +} + /// Input node for the graph. #[derive(Debug)] pub struct InputNode { From d11a5c659e706a1c753eac2d20469f693d60fad6 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 27 May 2018 12:22:33 +0100 Subject: [PATCH 085/108] Fix compile on 1.25.0. --- src/lib.rs | 2 +- src/nodes.rs | 12 ++++++------ src/optim/adagrad.rs | 2 +- src/optim/sgd.rs | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1cf6c69b..a1c609de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -638,7 +638,7 @@ where let sparse_gradient = input.sparse_gradient(); - for (indices, grad) in sparse_gradient.as_slice() { + for &(ref indices, ref grad) in sparse_gradient.as_slice() { for &row_idx in indices.iter() { for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { *dest += orig; diff --git a/src/nodes.rs b/src/nodes.rs index 21b899f5..5bdcd172 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -287,7 +287,7 @@ fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: let (left, right) = grad_row.split_at(lhs_row.len()); match op { - BackwardAction::Increment => { + &BackwardAction::Increment => { for (x, y) in lhs_row.iter_mut().zip(left.iter()) { *x += y; } @@ -295,7 +295,7 @@ fn column_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: *x += y; } } - BackwardAction::Set => { + &BackwardAction::Set => { lhs_row.copy_from_slice(left); rhs_row.copy_from_slice(right); } @@ -313,10 +313,10 @@ fn row_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: &Ba let dest_row = dest_row.as_slice_mut().unwrap(); match op { - BackwardAction::Increment => for (x, y) in dest_row.iter_mut().zip(grad_row.iter()) { + &BackwardAction::Increment => for (x, y) in dest_row.iter_mut().zip(grad_row.iter()) { *x += y; }, - BackwardAction::Set => for (x, &y) in dest_row.iter_mut().zip(grad_row.iter()) { + &BackwardAction::Set => for (x, &y) in dest_row.iter_mut().zip(grad_row.iter()) { *x = y; }, } @@ -585,7 +585,7 @@ impl SparseGradientStore { let (index, value) = gradient; if self.len < self.data.len() { - let (index_vec, grad) = &mut self.data[self.len]; + let &mut (ref mut index_vec, ref mut grad) = &mut self.data[self.len]; index_vec.clear(); index_vec.extend_from_slice(&index[..]); grad.slice_assign(value); @@ -648,7 +648,7 @@ impl GradientAccumulator { self.sparse_gradient .as_slice_mut() .iter_mut() - .for_each(|(_, ref mut grad)| { + .for_each(|&mut (_, ref mut grad)| { grad.as_slice_mut() .unwrap() .iter_mut() diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index 990c5cdd..90edba83 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -93,7 +93,7 @@ impl Adagrad { sink.sparse_gradient .as_slice() .iter() - .for_each(|(ref index_vec, ref grad)| { + .for_each(|&(ref index_vec, ref grad)| { for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { let grad_row = grad.subview(Axis(0), grad_idx); let mut param_row = param_value.subview_mut(Axis(0), param_idx); diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index daa2e8a9..30c6dfd2 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -55,7 +55,7 @@ impl SGD { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); } - for (ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { + for &(ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { let grad_row = grad.subview(Axis(0), grad_idx); let mut param_row = param_value.subview_mut(Axis(0), param_idx); From 465eeb325e9e9df84278b955d6281b0e040c3eca Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sun, 27 May 2018 13:17:31 +0100 Subject: [PATCH 086/108] Bump version to 0.8.1. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6cf45fc7..d5d173da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.8.0" +version = "0.8.1" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From fcfdc786e1cf1ef184e3f2e0b3ecc8be5612a64d Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 28 May 2018 14:04:28 +0100 Subject: [PATCH 087/108] Make sure sparse indices are order-independent. --- src/nodes.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index 5bdcd172..8481bc99 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -584,15 +584,26 @@ impl SparseGradientStore { pub fn push(&mut self, gradient: (&[usize], &Arr)) { let (index, value) = gradient; - if self.len < self.data.len() { - let &mut (ref mut index_vec, ref mut grad) = &mut self.data[self.len]; + // Increment gradients when indices match. + for &mut (ref mut index_vec, ref mut grad) in self.data[..self.len].iter_mut() { + if &index_vec[..] == index { + grad.slice_add_assign(value); + return; + } + } + + // Set gradients on existing objects for new indices. + for &mut (ref mut index_vec, ref mut grad) in self.data[self.len..].iter_mut() { index_vec.clear(); - index_vec.extend_from_slice(&index[..]); + index_vec.extend_from_slice(index); grad.slice_assign(value); self.len += 1; - } else { - self.data.push((Vec::from(&index[..]), value.clone())); + return; } + + // Increase index capacity. + self.data.push((Vec::from(&index[..]), value.clone())); + self.len += 1; } pub fn as_slice(&self) -> &[(Vec, Arr)] { From ab17d2d618e6011841abe0eba28d40055e403088 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Mon, 28 May 2018 18:34:12 +0100 Subject: [PATCH 088/108] Major overhaul of optimizers to support true sync parallelism. --- src/lib.rs | 129 ++++++++++++++++++++++++++++++++++--------- src/nn/lstm.rs | 10 ++-- src/nodes.rs | 84 ++++++++++++++++++++++++---- src/optim/adagrad.rs | 69 ++++++++--------------- src/optim/adam.rs | 68 ++++++++++------------- src/optim/barrier.rs | 4 ++ src/optim/mod.rs | 80 +++++++++++++++++++++++++-- src/optim/sgd.rs | 59 +++++++++----------- 8 files changed, 339 insertions(+), 164 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a1c609de..345934f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,7 +55,7 @@ //! # let y_hat = slope.clone() * x.clone() + intercept.clone(); //! # let mut loss = (y.clone() - y_hat).square(); //! # let num_epochs = 10; -//! let mut optimizer = SGD::new(loss.parameters()).learning_rate(0.1); +//! let mut optimizer = SGD::new().learning_rate(0.1); //! //! for _ in 0..num_epochs { //! let x_value: f32 = rand::random(); @@ -69,7 +69,7 @@ //! loss.forward(); //! loss.backward(1.0); //! -//! optimizer.step(); +//! optimizer.step(loss.parameters()); //! loss.zero_gradient(); //! } //! # } @@ -104,7 +104,7 @@ //! let y_hat = slope.clone() * x.clone() + intercept.clone(); //! let mut loss = (y.clone() - y_hat).square(); //! -//! let optimizer = SGD::new(loss.parameters()).learning_rate(0.1); +//! let optimizer = SGD::new().learning_rate(0.1); //! //! for _ in 0..num_epochs { //! let x_value: f32 = rand::random(); @@ -116,7 +116,7 @@ //! loss.forward(); //! loss.backward(1.0); //! -//! optimizer.step(); +//! optimizer.step(loss.parameters()); //! loss.zero_gradient(); //! } //! }); @@ -187,11 +187,14 @@ pub trait DataInput { fn set_value(&self, T); } -fn merge_parameters(xs: &[Rc], ys: &[Rc]) -> Vec> { +fn merge_parameters( + xs: &[Variable], + ys: &[Variable], +) -> Vec> { let mut unique_params: Vec<_> = xs.iter().chain(ys.iter()).cloned().collect(); - unique_params.sort_unstable_by_key(|x| x.deref() as *const ParameterNode); - unique_params.dedup_by_key(|x| (*x).deref() as *const ParameterNode); + unique_params.sort_unstable_by_key(|x| x.as_ptr()); + unique_params.dedup_by_key(|x| (*x).as_ptr()); unique_params } @@ -206,7 +209,7 @@ where { node: Rc, grad: Option>, - parameters: Vec>, + parameters: Vec>, } impl Clone for Variable { @@ -223,7 +226,7 @@ impl Variable where T: Node, { - fn new(node: Rc, parameters: Vec>) -> Self { + fn new(node: Rc, parameters: Vec>) -> Self { Variable { node: node, grad: None, @@ -249,15 +252,13 @@ where } /// Return the parameters of the graph. - pub fn parameters(&self) -> Vec> { - let mut unique_params = self.parameters.clone(); - unique_params.sort_unstable_by_key(|x| x.deref() as *const ParameterNode); - unique_params.dedup_by_key(|x| (*x).deref() as *const ParameterNode); + pub fn parameters(&self) -> &[Variable] { + &self.parameters[..] + } - unique_params - .iter() - .map(|x| Variable::new(Rc::clone(x), Vec::new())) - .collect() + /// Mutably return the parameters of the graph. + pub fn parameters_mut(&mut self) -> &mut [Variable] { + &mut self.parameters[..] } } @@ -458,6 +459,10 @@ impl Variable { self.node.gradient.borrow().sparse_gradient.clone() } + fn as_ptr(&self) -> *const ParameterNode { + self.node.deref() as *const ParameterNode + } + /// Row-wise indexing of this parameter node. Primiarily used /// to implement embedding layers. pub fn index(&self, index: &Variable) -> Variable> { @@ -469,6 +474,10 @@ impl Variable { merge_parameters(&self.parameters, &index.parameters), ) } + + pub(crate) fn gradient_push_down(&self) { + self.node.gradient_push_down(); + } } impl Variable> @@ -996,7 +1005,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let optimizer = Adagrad::new(loss.parameters()).learning_rate(0.5); + let optimizer = Adagrad::new().learning_rate(0.5); for _ in 0..num_epochs { let _x = arr2(&[[rand::thread_rng().gen()]]); @@ -1008,7 +1017,7 @@ mod tests { loss.forward(); loss.backward(1.0); - optimizer.step(); + optimizer.step(loss.parameters()); loss.zero_gradient(); } @@ -1039,7 +1048,7 @@ mod tests { let diff = y.clone() - y_hat.clone(); let mut loss = diff.square(); - let optimizer = SGD::new(loss.parameters()).learning_rate(0.1); + let optimizer = SGD::new().learning_rate(0.1); for _ in 0..num_epochs { let _x = arr2(&[[ @@ -1055,7 +1064,7 @@ mod tests { loss.forward(); loss.backward(1.0); - optimizer.step(); + optimizer.step(loss.parameters()); loss.zero_gradient(); } @@ -1097,7 +1106,7 @@ mod tests { let mut loss = (output.clone() - y_hat.clone()).square(); let num_epochs = 200; - let optimizer = Adagrad::new(loss.parameters()).learning_rate(0.1); + let optimizer = Adagrad::new().learning_rate(0.1); let mut loss_val = 0.0; @@ -1116,7 +1125,7 @@ mod tests { loss_val += loss.value().scalar_sum(); - optimizer.step(); + optimizer.step(loss.parameters()); loss.zero_gradient(); } } @@ -1159,7 +1168,77 @@ mod tests { let num_epochs = 100; - let optimizer = SGD::new(loss.parameters()); + let optimizer = SGD::new(); + + let mut loss_val = 0.0; + + for _ in 0..num_epochs { + loss_val = 0.0; + + for row_idx in 0..rows { + for col_idx in 0..cols { + u_index.set_value(row_idx); + v_index.set_value(col_idx); + + output.set_value(x[(row_idx, col_idx)]); + + loss.forward(); + loss.backward(1.0); + + loss_val += loss.value().scalar_sum(); + + optimizer.step(loss.parameters()); + loss.zero_gradient(); + } + } + } + + println!("Loss val {}", loss_val); + + loss_val + }) + .collect(); + + let sum_loss: f32 = losses.iter().sum(); + + assert!(sum_loss / (losses.len() as f32) < 1e-3); + } + + #[test] + fn synchronized_embedding_factorization() { + let (rows, cols) = (10, 4); + + let true_u = random_matrix(rows, 10); + let true_v = random_matrix(cols, 10); + let x = true_u.dot(&true_v.t()); + + let u_input = vec![0]; + let v_input = vec![0]; + + let u_parameters = Arc::new(HogwildParameter::new(random_matrix(rows, 10))); + let v_parameters = Arc::new(HogwildParameter::new(random_matrix(cols, 10))); + + let optimizer = SGD::new(); + + let losses: Vec = (0..rayon::current_num_threads()) + .into_par_iter() + .map(|_| { + let u_embedding = ParameterNode::shared(u_parameters.clone()); + let v_embedding = ParameterNode::shared(v_parameters.clone()); + + let u_index = IndexInputNode::new(&u_input); + let v_index = IndexInputNode::new(&v_input); + let output = InputNode::new(random_matrix(1, 1)); + + let optimizer = optimizer.synchronized(); + + let u_vec = u_embedding.index(&u_index); + let v_vec = v_embedding.index(&v_index); + + let y_hat = u_vec.vector_dot(&v_vec); + let mut loss = (output.clone() - y_hat.clone()).square(); + + let num_epochs = 100; let mut loss_val = 0.0; @@ -1178,7 +1257,7 @@ mod tests { loss_val += loss.value().scalar_sum(); - optimizer.step(); + optimizer.step(loss.parameters()); loss.zero_gradient(); } } diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 9ec953c9..5a46d5de 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -354,8 +354,10 @@ mod tests { assert_close(&difference, &gradient, TOLERANCE); } - for x in hidden.parameters().iter_mut() { - let (difference, gradient) = finite_difference(x, &mut hidden); + let mut params = hidden.parameters().to_owned(); + + for x in params.iter_mut() { + let (difference, gradient) = finite_difference(x, hidden); assert_close(&difference, &gradient, TOLERANCE); } } @@ -431,7 +433,7 @@ mod tests { let prediction = hidden.dot(&final_layer); let mut loss = sparse_categorical_crossentropy(&prediction, &y); - let optimizer = Adam::new(loss.parameters()).learning_rate(0.01); + let optimizer = Adam::new().learning_rate(0.01); let digits = pi_digits(100); @@ -462,7 +464,7 @@ mod tests { loss_val += loss.value().scalar_sum(); - optimizer.step(); + optimizer.step(loss.parameters()); loss.zero_gradient(); if target_digit == predicted_label(prediction.value().deref()) { diff --git a/src/nodes.rs b/src/nodes.rs index 8481bc99..1459046a 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -3,7 +3,7 @@ use std::cell::{Cell, Ref, RefCell}; use std::fmt; use std::ops::{AddAssign, Deref, DerefMut}; use std::rc::Rc; -use std::sync::Arc; +use std::sync::{Arc, Mutex, MutexGuard, TryLockResult}; use ndarray; use ndarray::Axis; @@ -567,7 +567,7 @@ impl Node for InputNode { fn zero_gradient(&self) {} } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct SparseGradientStore { len: usize, data: Vec<(Vec, Arr)>, @@ -619,7 +619,7 @@ impl SparseGradientStore { } } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub(crate) struct GradientAccumulator { pub dense_shape: (usize, usize), pub dense_gradient: Option, @@ -641,7 +641,7 @@ impl GradientAccumulator { self.dense_gradient.get_or_insert_with(|| Arr::zeros(shape)) } - fn zero_gradient(&mut self) { + pub(crate) fn zero_gradient(&mut self) { if self.has_dense { self.dense_gradient().fill(0.0); } @@ -679,6 +679,20 @@ impl<'a, 'b> GradientSink<&'a Ref<'b, Arr>> for GradientAccumulator { } } +impl<'a> GradientSink<&'a Arr> for GradientAccumulator { + fn accumulate_gradient(&mut self, gradient: &'a Arr) { + self.dense_gradient().slice_add_assign(gradient.deref()); + self.has_dense = true; + } +} + +impl<'a> GradientSink<&'a mut Arr> for GradientAccumulator { + fn accumulate_gradient(&mut self, gradient: &'a mut Arr) { + self.dense_gradient().slice_add_assign(gradient.deref()); + self.has_dense = true; + } +} + impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: (&'a [usize], &'a Arr)) { self.sparse_gradient.push(gradient); @@ -689,12 +703,28 @@ unsafe impl Sync for HogwildParameter {} /// Struct used to hold parameters that need to be shared among /// multiple `ParameterNode`s for asynchronous, parallel optimization. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct HogwildParameter { + shape: (usize, usize), pub value: RefCell, pub squared_gradients: RefCell, pub moments: RefCell, num_updates: Cell, + #[serde(skip)] + gradient: Mutex>, +} + +impl Clone for HogwildParameter { + fn clone(&self) -> HogwildParameter { + HogwildParameter { + shape: self.shape.clone(), + value: self.value.clone(), + squared_gradients: self.squared_gradients.clone(), + moments: self.moments.clone(), + num_updates: self.num_updates.clone(), + gradient: Mutex::new(None), + } + } } #[cfg_attr(feature = "cargo-clippy", allow(mut_from_ref))] @@ -703,15 +733,35 @@ impl HogwildParameter { pub fn new(value: Arr) -> Self { let squared_gradients = &value * 0.0; let moments = &value * 0.0; + let shape = (value.rows(), value.cols()); HogwildParameter { + shape: shape, value: RefCell::new(value), squared_gradients: RefCell::new(squared_gradients), moments: RefCell::new(moments), num_updates: Cell::new(0), + gradient: Mutex::new(None), } } + pub(crate) fn gradient_accumulator(&self) -> MutexGuard> { + let mut accumulator = self + .gradient + .lock() + .expect("Unable to lock gradients for gradient push-down."); + + accumulator.get_or_insert_with(|| GradientAccumulator::new(self.shape)); + + accumulator + } + + pub(crate) fn try_gradient_accumulator( + &self, + ) -> TryLockResult>> { + self.gradient.try_lock() + } + pub fn value(&self) -> &Arr { unsafe { &*(self.value.as_ptr()) } } @@ -761,7 +811,7 @@ impl ParameterNode { value: value, gradient: RefCell::new(GradientAccumulator::new(shape)), }); - let params = vec![Rc::clone(&node)]; + let params = vec![Variable::new(Rc::clone(&node), Vec::new())]; Variable::new(node, params) } @@ -774,14 +824,26 @@ impl ParameterNode { value: Arc::new(HogwildParameter::new(value)), gradient: RefCell::new(GradientAccumulator::new(shape)), }); - let params = vec![Rc::clone(&node)]; + let params = vec![Variable::new(Rc::clone(&node), Vec::new())]; Variable::new(node, params) } - // /// Zero the accumulated gradients of this node. - // pub fn zero_gradient(&self) { - // //self.gradient.borrow_mut().zero_gradient(); - // } + + pub(crate) fn gradient_push_down(&self) { + self.value + .gradient_accumulator() + .iter_mut() + .for_each(|accum| { + let mut self_accum = self.gradient.borrow_mut(); + if self_accum.has_dense { + accum.accumulate_gradient(self_accum.dense_gradient()); + } + + for &(ref index_vec, ref grad) in self_accum.sparse_gradient.as_slice() { + accum.accumulate_gradient((&index_vec[..], grad)); + } + }); + } } impl Node for ParameterNode { diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index 90edba83..f6621e2c 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -1,7 +1,9 @@ -use super::barrier::{SynchronizationBarrier, SynchronizationBarrierGuard}; -use super::Optimizer; +use super::barrier::SynchronizationBarrier; +use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; use numerics::{ArraySlice, ArraySliceMut}; -use {numerics, ParameterNode, Variable}; +use std::ops::DerefMut; +use std::sync::Arc; +use {numerics, HogwildParameter, ParameterNode, Variable}; use ndarray::Axis; @@ -10,22 +12,20 @@ use ndarray::Axis; pub struct Adagrad { learning_rate: f32, l2: f32, - parameters: Vec>, clamp: Option<(f32, f32)>, eps: f32, - sync_barrier: Option, + sync_barrier: SynchronizationBarrier, } impl Adagrad { /// Create a new optimizer instance with a given set of parameters. - pub fn new(parameters: Vec>) -> Self { + pub fn new() -> Self { Adagrad { learning_rate: 0.05, l2: 0.0, - parameters: parameters, clamp: None, eps: 1e-10, - sync_barrier: None, + sync_barrier: SynchronizationBarrier::default(), } } @@ -35,12 +35,6 @@ impl Adagrad { self } - /// Use the optimizer in synchrnous mode. - pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { - self.sync_barrier = Some(barrier.register_thread()); - self - } - /// Set the clamp bounds. pub fn clamp(mut self, min: f32, max: f32) -> Self { self.clamp = Some((min, max)); @@ -53,30 +47,26 @@ impl Adagrad { self } - /// Decay weights. - pub fn decay_weights(&mut self, penalty: f32) { - for parameter in &self.parameters { - let mut param_value = unsafe { parameter.node.value.value_mut() }; - - param_value - .as_slice_mut() - .unwrap() - .iter_mut() - .for_each(|x| *x -= x.signum() * penalty * numerics::pow2(*x)); - } + /// Return a synchoronised wrapper for this optimizer. + pub fn synchronized(&self) -> SynchronizedOptimizer { + SynchronizedOptimizer::new(self, self.sync_barrier.register_thread()) } +} - fn do_step(&self, parameter: &Variable) { +impl InnerOptimizer for Adagrad { + fn inner_step>( + &self, + param: &Arc, + mut sink: T, + ) { let learning_rate = self.learning_rate; - let mut sink = parameter.node.gradient.borrow_mut(); - if let Some((min, max)) = self.clamp { sink.clamp(min, max); } - let param_value = unsafe { parameter.node.value.value_mut() }; - let squared_gradient = unsafe { parameter.node.value.squared_gradient_mut() }; + let param_value = unsafe { param.value_mut() }; + let squared_gradient = unsafe { param.squared_gradient_mut() }; if sink.has_dense { for (value, &gradient, squared_gradient) in izip!( @@ -115,22 +105,9 @@ impl Adagrad { impl Optimizer for Adagrad { /// Perform a single SGD step. - fn step(&self) { - if let Some(ref barrier) = self.sync_barrier { - barrier.start_wait(); - { - let _ = barrier.lock(); - - for parameter in &self.parameters { - self.do_step(parameter); - } - } - - barrier.end_wait(); - } else { - for parameter in &self.parameters { - self.do_step(parameter); - } + fn step(&self, parameters: &[Variable]) { + for parameter in parameters { + self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()) } } } diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 0a9435da..1f5373a7 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -1,6 +1,8 @@ -use super::barrier::{SynchronizationBarrier, SynchronizationBarrierGuard}; -use super::Optimizer; -use {numerics, Arr, ParameterNode, Variable}; +use super::barrier::SynchronizationBarrier; +use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; +use std::ops::DerefMut; +use std::sync::Arc; +use {numerics, Arr, HogwildParameter, ParameterNode, Variable}; use ndarray::Axis; @@ -18,38 +20,35 @@ pub struct Adam { beta_m: f32, beta_v: f32, eps: f32, - parameters: Vec>, clamp: Option<(f32, f32)>, - sync_barrier: Option, + sync_barrier: SynchronizationBarrier, } impl Adam { /// Build new optimizer object. - pub fn new(parameters: Vec>) -> Self { + pub fn new() -> Self { Self { learning_rate: 0.05, l2: 0.0, beta_m: 0.9, beta_v: 0.999, eps: 1.0e-8, - parameters: parameters, clamp: None, - sync_barrier: None, + sync_barrier: SynchronizationBarrier::default(), } } + /// Return a synchoronised wrapper for this optimizer. + pub fn synchronized(&self) -> SynchronizedOptimizer { + SynchronizedOptimizer::new(self, self.sync_barrier.register_thread()) + } + /// Set the learning rate. pub fn learning_rate(mut self, learning_rate: f32) -> Self { self.learning_rate = learning_rate; self } - /// Use synchronous parallel training. - pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { - self.sync_barrier = Some(barrier.register_thread()); - self - } - /// Set the clamp bounds. pub fn clamp(mut self, min: f32, max: f32) -> Self { self.clamp = Some((min, max)); @@ -62,12 +61,12 @@ impl Adam { self } - fn param_fields<'par>(&self, parameter: &'par Variable) -> AdamParameters<'par> { + fn param_fields<'par>(&self, value: &'par Arc) -> AdamParameters<'par> { AdamParameters { - value: unsafe { parameter.node.value.value_mut() }, - m: unsafe { parameter.node.value.squared_gradient_mut() }, - v: unsafe { parameter.node.value.moments_mut() }, - t: unsafe { parameter.node.value.num_updates_mut() }, + value: unsafe { value.value_mut() }, + m: unsafe { value.squared_gradient_mut() }, + v: unsafe { value.moments_mut() }, + t: unsafe { value.num_updates_mut() }, } } @@ -85,15 +84,19 @@ impl Adam { *value -= self.learning_rate / (v_hat.sqrt() + self.eps) * m_hat; } +} - fn do_step(&self, parameter: &Variable) { - let mut sink = parameter.node.gradient.borrow_mut(); - +impl InnerOptimizer for Adam { + fn inner_step>( + &self, + param: &Arc, + mut sink: T, + ) { if let Some((min, max)) = self.clamp { sink.clamp(min, max); } - let param = self.param_fields(parameter); + let param = self.param_fields(param); // Increment number of updates *param.t = param.t.saturating_add(1); @@ -131,22 +134,9 @@ impl Adam { impl Optimizer for Adam { /// Perform a single SGD step. - fn step(&self) { - if let Some(ref barrier) = self.sync_barrier { - barrier.start_wait(); - { - let _ = barrier.lock(); - - for parameter in &self.parameters { - self.do_step(parameter); - } - } - - barrier.end_wait(); - } else { - for parameter in &self.parameters { - self.do_step(parameter); - } + fn step(&self, parameters: &[Variable]) { + for parameter in parameters { + self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()) } } } diff --git a/src/optim/barrier.rs b/src/optim/barrier.rs index 4e30a57f..e851a9ee 100644 --- a/src/optim/barrier.rs +++ b/src/optim/barrier.rs @@ -46,6 +46,10 @@ impl Barrier { pub fn increment_num_threads(&self) { let mut lock = self.lock.lock().unwrap(); + if lock.generation_id != 0 { + panic!("Can't register more threads after the first generation."); + } + lock.num_threads += 1; } diff --git a/src/optim/mod.rs b/src/optim/mod.rs index 48a7ca2c..63142eca 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -1,18 +1,88 @@ //! Optimization module. //! //! Contains a number of optimizers. +use std::ops::DerefMut; +use std::sync::Arc; +use {HogwildParameter, ParameterNode, Variable}; mod adagrad; mod adam; mod barrier; mod sgd; +pub use self::adagrad::Adagrad; +pub use self::adam::Adam; +pub use self::barrier::SynchronizationBarrier; +pub use self::sgd::SGD; + /// Core trait implemented by all optimizer methods. pub trait Optimizer { /// Perform a single SGD step. - fn step(&self); + fn step(&self, parameters: &[Variable]); } -pub use self::adagrad::Adagrad; -pub use self::adam::Adam; -pub use self::barrier::SynchronizationBarrier; -pub use self::sgd::SGD; +pub(crate) trait InnerOptimizer { + fn inner_step>( + &self, + parameter: &Arc, + sink: T, + ); +} + +pub struct SynchronizedOptimizer<'a, T: 'a> { + optimizer: &'a T, + barrier_guard: barrier::SynchronizationBarrierGuard, +} + +impl<'a, T: 'a> SynchronizedOptimizer<'a, T> { + pub fn new(optimizer: &'a T, barrier_guard: barrier::SynchronizationBarrierGuard) -> Self { + ::std::thread::sleep_ms(5); + SynchronizedOptimizer { + optimizer: optimizer, + barrier_guard: barrier_guard, + } + } +} + +macro_rules! impl_sync_optimizer { + ($type:ty) => { + impl<'a> Optimizer for SynchronizedOptimizer<'a, $type> { + fn step(&self, parameters: &[Variable]) { + // Push down this thread's gradients to the shared + // accumulators. + for parameter in parameters { + parameter.gradient_push_down(); + } + + // Wait until all the other threads have finished push down. + self.barrier_guard.start_wait(); + + // Start. + for parameter in parameters { + // Attempt to get the lock. If unsuccessful, skip this parameter: + // another thread is taking care of it already. + if let Ok(mut lock) = parameter.node.value.try_gradient_accumulator() { + if let Some(ref mut sink) = lock.deref_mut() { + self.optimizer.inner_step(¶meter.node.value, sink); + } else { + panic!("Sink should always be instantiated.") + } + + if let Some(ref mut sink) = lock.deref_mut() { + sink.zero_gradient(); + } else { + panic!("Sink should always be instantiated.") + } + } + } + + // Wait until all the other threads have finished updating. + self.barrier_guard.end_wait(); + } + } + }; +} + +// TODO: impl using generics, atm it's painful because of private type leakage +impl_sync_optimizer!(SGD); +impl_sync_optimizer!(Adagrad); +impl_sync_optimizer!(Adam); diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index 30c6dfd2..4a074884 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -1,25 +1,25 @@ -use super::barrier::{SynchronizationBarrier, SynchronizationBarrierGuard}; -use super::Optimizer; -use {numerics, ParameterNode, Variable}; +use super::barrier::SynchronizationBarrier; +use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; +use std::ops::DerefMut; +use std::sync::Arc; +use {numerics, HogwildParameter, ParameterNode, Variable}; use ndarray::Axis; /// Standard stochastic gradient descent optimizer with a fixed learning rate. pub struct SGD { learning_rate: f32, - parameters: Vec>, clamp: Option<(f32, f32)>, - sync_barrier: Option, + sync_barrier: SynchronizationBarrier, } impl SGD { /// Create a new optimizer instance with a given set of parameters. - pub fn new(parameters: Vec>) -> Self { + pub fn new() -> Self { SGD { learning_rate: 0.05, - parameters: parameters, clamp: None, - sync_barrier: None, + sync_barrier: SynchronizationBarrier::default(), } } @@ -29,23 +29,27 @@ impl SGD { self } - /// Use the optimizer in synchrnous mode. - pub fn synchronized(mut self, barrier: &SynchronizationBarrier) -> Self { - self.sync_barrier = Some(barrier.register_thread()); - self - } - /// Set the clamp bounds. pub fn clamp(mut self, min: f32, max: f32) -> Self { self.clamp = Some((min, max)); self } - /// Perform a single SGD step. - fn do_step(&self, parameter: &Variable) { + /// Return a synchoronised wrapper for this optimizer. + pub fn synchronized(&self) -> SynchronizedOptimizer { + SynchronizedOptimizer::new(self, self.sync_barrier.register_thread()) + } +} + +impl InnerOptimizer for SGD { + fn inner_step>( + &self, + param: &Arc, + mut sink: T, + ) { + let param_value = unsafe { param.value_mut() }; let learning_rate = self.learning_rate; - let mut sink = parameter.node.gradient.borrow_mut(); - let param_value = unsafe { parameter.node.value.value_mut() }; + let sink = sink.deref_mut(); if let Some((min, max)) = self.clamp { sink.clamp(min, max); @@ -71,22 +75,9 @@ impl SGD { } impl Optimizer for SGD { - fn step(&self) { - if let Some(ref barrier) = self.sync_barrier { - barrier.start_wait(); - { - let _ = barrier.lock(); - - for parameter in &self.parameters { - self.do_step(parameter); - } - } - - barrier.end_wait(); - } else { - for parameter in &self.parameters { - self.do_step(parameter); - } + fn step(&self, parameters: &[Variable]) { + for parameter in parameters { + self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()) } } } From 6e13189105e7aa1659f25ecc433e4cd2a3c7166c Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 29 May 2018 11:36:49 +0100 Subject: [PATCH 089/108] Ergonomics fixes for synchronized optimizers. --- src/nodes.rs | 2 +- src/optim/adagrad.rs | 2 +- src/optim/adam.rs | 2 +- src/optim/barrier.rs | 6 +++++- src/optim/mod.rs | 34 ++++++++++++++++++++++++++++++++++ src/optim/sgd.rs | 2 +- 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index 1459046a..9d06a25b 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -595,7 +595,7 @@ impl SparseGradientStore { // Set gradients on existing objects for new indices. for &mut (ref mut index_vec, ref mut grad) in self.data[self.len..].iter_mut() { index_vec.clear(); - index_vec.extend_from_slice(index); + index_vec.extend_from_slice(&index[..]); grad.slice_assign(value); self.len += 1; return; diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index f6621e2c..f36753dc 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -14,7 +14,7 @@ pub struct Adagrad { l2: f32, clamp: Option<(f32, f32)>, eps: f32, - sync_barrier: SynchronizationBarrier, + pub(in optim) sync_barrier: SynchronizationBarrier, } impl Adagrad { diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 1f5373a7..60d58c91 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -21,7 +21,7 @@ pub struct Adam { beta_v: f32, eps: f32, clamp: Option<(f32, f32)>, - sync_barrier: SynchronizationBarrier, + pub(in optim) sync_barrier: SynchronizationBarrier, } impl Adam { diff --git a/src/optim/barrier.rs b/src/optim/barrier.rs index e851a9ee..df30f1a8 100644 --- a/src/optim/barrier.rs +++ b/src/optim/barrier.rs @@ -11,7 +11,7 @@ // except according to those terms. use std::fmt; -use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, TryLockResult}; pub struct Barrier { lock: Mutex, @@ -174,6 +174,10 @@ impl SynchronizationBarrierGuard { pub fn lock(&self) -> MutexGuard<()> { self.barrier.parameter_lock.lock().unwrap() } + + pub fn try_lock(&self) -> TryLockResult> { + self.barrier.parameter_lock.try_lock() + } } impl Drop for SynchronizationBarrierGuard { diff --git a/src/optim/mod.rs b/src/optim/mod.rs index 63142eca..be4acdaf 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -20,6 +20,12 @@ pub trait Optimizer { fn step(&self, parameters: &[Variable]); } +pub trait Synchronizable { + fn synchronized(&self) -> SynchronizedOptimizer + where + Self: Sized; +} + pub(crate) trait InnerOptimizer { fn inner_step>( &self, @@ -79,6 +85,12 @@ macro_rules! impl_sync_optimizer { self.barrier_guard.end_wait(); } } + + impl Synchronizable for $type { + fn synchronized(&self) -> SynchronizedOptimizer { + self.synchronized() + } + } }; } @@ -86,3 +98,25 @@ macro_rules! impl_sync_optimizer { impl_sync_optimizer!(SGD); impl_sync_optimizer!(Adagrad); impl_sync_optimizer!(Adam); + +macro_rules! impl_optimizer_enum { + ($(($tag:ident, $type:ty)),*) => { + pub enum Optimizers { + $( + $tag($type), + )* + } + + impl Optimizer for Optimizers { + fn step(&self, parameters: &[Variable]) { + match self { + $( + Optimizers::$tag(val) => val.step(parameters), + )* + } + } + } + } +} + +impl_optimizer_enum!((SGD, SGD), (Adagrad, Adagrad), (Adam, Adam)); diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index 4a074884..199f9304 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -10,7 +10,7 @@ use ndarray::Axis; pub struct SGD { learning_rate: f32, clamp: Option<(f32, f32)>, - sync_barrier: SynchronizationBarrier, + pub(in optim) sync_barrier: SynchronizationBarrier, } impl SGD { From 239c378d4b7c1fc7ca24332acaff2a8b5600806e Mon Sep 17 00:00:00 2001 From: maciejkula Date: Tue, 29 May 2018 14:55:22 +0100 Subject: [PATCH 090/108] Hashmap-based sparse accumulators. --- Cargo.toml | 2 + src/lib.rs | 11 ++-- src/nodes.rs | 138 +++++++++++++++++++++++++++++++------------ src/optim/adagrad.rs | 32 ++++------ src/optim/adam.rs | 27 ++++----- src/optim/sgd.rs | 15 ++--- 6 files changed, 139 insertions(+), 86 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d5d173da..a5764317 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,8 @@ itertools = "0.7.3" rayon = "1.0.0" serde = { version = "1.0.0", features = ["rc"] } serde_derive = "1.0.0" +indexmap = { version = "1.0.1", features = ["serde-1"] } +fnv = "1.0.6" [dev-dependencies] ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } diff --git a/src/lib.rs b/src/lib.rs index 345934f8..202144c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -152,6 +152,9 @@ extern crate smallvec; #[macro_use] extern crate itertools; +extern crate fnv; +extern crate indexmap; + /// Alias for a `f32` `ndarray` matrix. pub type Arr = ndarray::Array2; @@ -647,11 +650,9 @@ where let sparse_gradient = input.sparse_gradient(); - for &(ref indices, ref grad) in sparse_gradient.as_slice() { - for &row_idx in indices.iter() { - for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { - *dest += orig; - } + for (row_idx, grad) in sparse_gradient.iter() { + for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { + *dest += orig; } } diff --git a/src/nodes.rs b/src/nodes.rs index 9d06a25b..92de3b3d 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -10,6 +10,10 @@ use ndarray::Axis; use smallvec::SmallVec; +use fnv::FnvBuildHasher; +use indexmap::map::Entry; +use indexmap::IndexMap; + use numerics; use numerics::{ArraySlice, ArraySliceMut, ArraySliceOps}; @@ -568,54 +572,111 @@ impl Node for InputNode { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct SparseGradientStore { +struct Slab { + cols: usize, len: usize, - data: Vec<(Vec, Arr)>, + data: Vec, } -impl SparseGradientStore { - pub fn new() -> Self { - SparseGradientStore { +impl Slab { + fn new(cols: usize) -> Self { + Slab { + cols: cols, len: 0, data: Vec::new(), } } - pub fn push(&mut self, gradient: (&[usize], &Arr)) { - let (index, value) = gradient; + fn get(&self, idx: usize) -> &[f32] { + let start = idx * self.cols; + let stop = start + self.cols; + &self.data[start..stop] + } - // Increment gradients when indices match. - for &mut (ref mut index_vec, ref mut grad) in self.data[..self.len].iter_mut() { - if &index_vec[..] == index { - grad.slice_add_assign(value); - return; + fn get_mut(&mut self, idx: usize) -> &mut [f32] { + if idx >= self.len { + for _ in 0..self.cols { + self.data.push(0.0); } + self.len += 1; } - // Set gradients on existing objects for new indices. - for &mut (ref mut index_vec, ref mut grad) in self.data[self.len..].iter_mut() { - index_vec.clear(); - index_vec.extend_from_slice(&index[..]); - grad.slice_assign(value); - self.len += 1; - return; + let start = idx * self.cols; + let stop = start + self.cols; + &mut self.data[start..stop] + } + + fn len(&self) -> usize { + self.len + } + + fn clear(&mut self) { + self.len = 0; + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct SparseGradientStore { + slab: Slab, + map: IndexMap, +} + +impl SparseGradientStore { + pub fn new(cols: usize) -> Self { + SparseGradientStore { + slab: Slab::new(cols), + map: IndexMap::default(), + } + } + + pub fn push(&mut self, gradient: (&[usize], &Arr)) { + let (indices, value) = gradient; + + for (&idx, row) in izip!(indices.iter(), value.genrows().into_iter()) { + self.scalar_push(idx, row.as_slice().unwrap()); } + } + + fn scalar_push(&mut self, idx: usize, row: &[f32]) { + match self.map.entry(idx) { + Entry::Occupied(entry) => { + let slab_idx = entry.get(); + let slab_value = self.slab.get_mut(*slab_idx); + + for (slab_val, row_val) in izip!(slab_value.iter_mut(), row.iter()) { + *slab_val += row_val; + } + } + Entry::Vacant(entry) => { + let slab_idx = self.slab.len(); + entry.insert(slab_idx); - // Increase index capacity. - self.data.push((Vec::from(&index[..]), value.clone())); - self.len += 1; + let slab_value = self.slab.get_mut(slab_idx); + + for (slab_val, &row_val) in izip!(slab_value.iter_mut(), row.iter()) { + *slab_val = row_val; + } + } + } } - pub fn as_slice(&self) -> &[(Vec, Arr)] { - &self.data[..self.len] + pub fn iter(&self) -> impl Iterator { + self.map + .keys() + .cloned() + .zip(self.slab.data.chunks(self.slab.cols)) } - pub fn as_slice_mut(&mut self) -> &mut [(Vec, Arr)] { - &mut self.data[..self.len] + pub fn iter_mut(&mut self) -> impl Iterator { + self.map + .keys() + .cloned() + .zip(self.slab.data.chunks_mut(self.slab.cols)) } pub fn clear(&mut self) { - self.len = 0; + self.slab.clear(); + self.map.clear(); } } @@ -629,10 +690,12 @@ pub(crate) struct GradientAccumulator { impl GradientAccumulator { fn new(dense_shape: (usize, usize)) -> Self { + let cols = dense_shape.1; + GradientAccumulator { dense_shape: dense_shape, dense_gradient: None, - sparse_gradient: SparseGradientStore::new(), + sparse_gradient: SparseGradientStore::new(cols), has_dense: false, } } @@ -657,14 +720,8 @@ impl GradientAccumulator { .iter_mut() .for_each(|x| *x = clamp(*x, min, max)); self.sparse_gradient - .as_slice_mut() .iter_mut() - .for_each(|&mut (_, ref mut grad)| { - grad.as_slice_mut() - .unwrap() - .iter_mut() - .for_each(|x| *x = clamp(*x, min, max)) - }); + .for_each(|(_, grad)| grad.iter_mut().for_each(|x| *x = clamp(*x, min, max))); } } @@ -699,6 +756,13 @@ impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { } } +impl<'a> GradientSink<(usize, &'a [f32])> for GradientAccumulator { + fn accumulate_gradient(&mut self, gradient: (usize, &'a [f32])) { + let (idx, grad) = gradient; + self.sparse_gradient.scalar_push(idx, grad); + } +} + unsafe impl Sync for HogwildParameter {} /// Struct used to hold parameters that need to be shared among @@ -839,8 +903,8 @@ impl ParameterNode { accum.accumulate_gradient(self_accum.dense_gradient()); } - for &(ref index_vec, ref grad) in self_accum.sparse_gradient.as_slice() { - accum.accumulate_gradient((&index_vec[..], grad)); + for (row_idx, grad) in self_accum.sparse_gradient.iter() { + accum.accumulate_gradient((row_idx, grad)); } }); } diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index f36753dc..f5d26d30 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -80,26 +80,20 @@ impl InnerOptimizer for Adagrad { } } - sink.sparse_gradient - .as_slice() - .iter() - .for_each(|&(ref index_vec, ref grad)| { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let grad_row = grad.subview(Axis(0), grad_idx); - let mut param_row = param_value.subview_mut(Axis(0), param_idx); - let mut squared_row = squared_gradient.subview_mut(Axis(0), param_idx); + for (row_idx, grad) in sink.sparse_gradient.iter() { + let mut param_row = param_value.subview_mut(Axis(0), row_idx); + let mut squared_row = squared_gradient.subview_mut(Axis(0), row_idx); - for (value, &gradient, squared_gradient) in izip!( - param_row.fast_slice_mut(), - grad_row.into_slice().unwrap(), - squared_row.fast_slice_mut() - ) { - let gradient = gradient + *value * self.l2; - *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; - } - } - }); + for (value, &gradient, squared_gradient) in izip!( + param_row.fast_slice_mut(), + grad.iter(), + squared_row.fast_slice_mut() + ) { + let gradient = gradient + *value * self.l2; + *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; + } + } } } diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 60d58c91..08843334 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -112,21 +112,18 @@ impl InnerOptimizer for Adam { } } - for &(ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let mut value_row = param.value.subview_mut(Axis(0), param_idx); - let grad_row = grad.subview(Axis(0), grad_idx); - let mut m_row = param.m.subview_mut(Axis(0), param_idx); - let mut v_row = param.v.subview_mut(Axis(0), param_idx); - - for (value, &gradient, m, v) in izip!( - value_row.as_slice_mut().unwrap(), - grad_row.into_slice().unwrap(), - m_row.as_slice_mut().unwrap(), - v_row.as_slice_mut().unwrap(), - ) { - self.update(value, gradient, m, v, param.t); - } + for (row_idx, ref grad) in sink.sparse_gradient.iter() { + let mut value_row = param.value.subview_mut(Axis(0), row_idx); + let mut m_row = param.m.subview_mut(Axis(0), row_idx); + let mut v_row = param.v.subview_mut(Axis(0), row_idx); + + for (value, &gradient, m, v) in izip!( + value_row.as_slice_mut().unwrap(), + grad.iter(), + m_row.as_slice_mut().unwrap(), + v_row.as_slice_mut().unwrap(), + ) { + self.update(value, gradient, m, v, param.t); } } } diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index 199f9304..c4c9936e 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -59,17 +59,12 @@ impl InnerOptimizer for SGD { param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); } - for &(ref index_vec, ref grad) in sink.sparse_gradient.as_slice() { - for (grad_idx, ¶m_idx) in index_vec.iter().enumerate() { - let grad_row = grad.subview(Axis(0), grad_idx); - let mut param_row = param_value.subview_mut(Axis(0), param_idx); + for (row_idx, grad) in sink.sparse_gradient.iter() { + let mut param_row = param_value.subview_mut(Axis(0), row_idx); - numerics::map_add_assign_slice( - param_row.into_slice().unwrap(), - grad_row.into_slice().unwrap(), - |x| -learning_rate * x, - ); - } + numerics::map_add_assign_slice(param_row.into_slice().unwrap(), grad, |x| { + -learning_rate * x + }); } } } From 6c2e0a920790d1e064c49184c3622b9262ef277a Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 30 May 2018 13:28:15 +0100 Subject: [PATCH 091/108] Add hibitset accumulator. --- Cargo.toml | 1 + src/lib.rs | 1 + src/nodes.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index a5764317..6ac79ff2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ serde = { version = "1.0.0", features = ["rc"] } serde_derive = "1.0.0" indexmap = { version = "1.0.1", features = ["serde-1"] } fnv = "1.0.6" +hibitset = "0.5.0" [dev-dependencies] ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } diff --git a/src/lib.rs b/src/lib.rs index 202144c1..293c6d53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,6 +153,7 @@ extern crate smallvec; extern crate itertools; extern crate fnv; +extern crate hibitset; extern crate indexmap; /// Alias for a `f32` `ndarray` matrix. diff --git a/src/nodes.rs b/src/nodes.rs index 92de3b3d..370691e9 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -11,6 +11,7 @@ use ndarray::Axis; use smallvec::SmallVec; use fnv::FnvBuildHasher; +use hibitset::BitSet; use indexmap::map::Entry; use indexmap::IndexMap; @@ -680,6 +681,73 @@ impl SparseGradientStore { } } +struct GradientAccumulatorV2 { + gradient: Arr, + sparse_index: BitSet, + dense: bool, + sparse: bool, +} + +impl GradientAccumulatorV2 { + pub fn new(shape: (usize, usize)) -> Self { + Self { + gradient: Arr::zeros(shape), + sparse_index: BitSet::with_capacity(100), + dense: false, + sparse: false, + } + } + + pub fn add_dense(&mut self, grad: &Arr) { + if !self.dense { + self.gradient.slice_assign(grad); + } else { + self.gradient.slice_add_assign(grad); + } + + self.dense = true; + } + + pub fn add_sparse(&mut self, indices: &[usize], grad: &Arr) { + for (&idx, row) in izip!(indices.iter(), grad.genrows().into_iter()) { + self.add_sparse_row(idx, &row); + } + } + + pub fn add_sparse_row(&mut self, idx: usize, grad: &ndarray::ArrayView) { + if self.sparse_index.add(idx as u32) { + self.gradient + .subview_mut(Axis(0), idx) + .slice_add_assign(grad); + } else { + self.gradient.subview_mut(Axis(0), idx).slice_assign(grad); + } + + self.sparse = true; + } + + pub fn sparse_iter( + &self, + ) -> impl Iterator)> { + let idx = &self.sparse_index; + let grad = &self.gradient; + + idx.into_iter().map(move |idx| { + let idx = idx as usize; + (idx, grad.subview(Axis(0), idx)) + }) + } + + pub fn clear(&mut self) { + if self.sparse { + self.sparse_index.clear() + } + + self.dense = false; + self.sparse = false; + } +} + #[derive(Debug, Serialize, Deserialize)] pub(crate) struct GradientAccumulator { pub dense_shape: (usize, usize), From 220bb17bb0439c78b940b6ece554e872069d7ca7 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Wed, 30 May 2018 14:26:18 +0100 Subject: [PATCH 092/108] Use hierarchical bitsets for sparse gradient accumulation. --- Cargo.toml | 2 - src/lib.rs | 41 +++----- src/nodes.rs | 220 +++++++++++-------------------------------- src/optim/adagrad.rs | 30 +++--- src/optim/adam.rs | 32 +++---- src/optim/sgd.rs | 21 +++-- 6 files changed, 109 insertions(+), 237 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6ac79ff2..c2efb7ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,8 +22,6 @@ itertools = "0.7.3" rayon = "1.0.0" serde = { version = "1.0.0", features = ["rc"] } serde_derive = "1.0.0" -indexmap = { version = "1.0.1", features = ["serde-1"] } -fnv = "1.0.6" hibitset = "0.5.0" [dev-dependencies] diff --git a/src/lib.rs b/src/lib.rs index 293c6d53..1ee495e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -152,9 +152,7 @@ extern crate smallvec; #[macro_use] extern crate itertools; -extern crate fnv; extern crate hibitset; -extern crate indexmap; /// Alias for a `f32` `ndarray` matrix. pub type Arr = ndarray::Array2; @@ -451,16 +449,8 @@ where impl Variable { /// Return the (dense) gradient value of this node. - pub fn dense_gradient(&self) -> Option { - match self.node.gradient.borrow().dense_gradient { - Some(ref gradients) => Some(gradients.clone()), - None => None, - } - } - - /// Return the (dense) gradient value of this node. - fn sparse_gradient(&self) -> SparseGradientStore { - self.node.gradient.borrow().sparse_gradient.clone() + pub fn gradient(&self) -> Arr { + self.node.gradient.borrow().materialized_gradient() } fn as_ptr(&self) -> *const ParameterNode { @@ -647,17 +637,7 @@ where output.forward(); output.backward(1.0); - let mut gradient = input.dense_gradient().unwrap_or(initial_input * 0.0); - - let sparse_gradient = input.sparse_gradient(); - - for (row_idx, grad) in sparse_gradient.iter() { - for (dest, orig) in gradient.row_mut(row_idx).iter_mut().zip(grad.iter()) { - *dest += orig; - } - } - - gradient + input.gradient() }; output.zero_gradient(); @@ -984,14 +964,17 @@ mod tests { } #[test] fn sparse_index_finite_difference() { - let mut x = ParameterNode::new(random_matrix(10, 5)); - let idx_0 = IndexInputNode::new(&[random_index(10)]); - let idx_1 = IndexInputNode::new(&[random_index(10)]); + let mut x = ParameterNode::new(random_matrix(100, 5)); - let mut z = (x.index(&idx_0).tanh() * x.index(&idx_1)).square(); + for _ in 0..10 { + let idx_0 = IndexInputNode::new(&[random_index(10)]); + let idx_1 = IndexInputNode::new(&[random_index(10)]); - let (difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&difference, &gradient, TOLERANCE); + let mut z = (x.index(&idx_0).tanh() * x.index(&idx_1)).square(); + + let (difference, gradient) = finite_difference(&mut x, &mut z); + assert_close(&difference, &gradient, TOLERANCE); + } } #[test] fn univariate_regression() { diff --git a/src/nodes.rs b/src/nodes.rs index 370691e9..9dab8a7e 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -10,10 +10,7 @@ use ndarray::Axis; use smallvec::SmallVec; -use fnv::FnvBuildHasher; use hibitset::BitSet; -use indexmap::map::Entry; -use indexmap::IndexMap; use numerics; use numerics::{ArraySlice, ArraySliceMut, ArraySliceOps}; @@ -572,123 +569,15 @@ impl Node for InputNode { fn zero_gradient(&self) {} } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Slab { - cols: usize, - len: usize, - data: Vec, -} - -impl Slab { - fn new(cols: usize) -> Self { - Slab { - cols: cols, - len: 0, - data: Vec::new(), - } - } - - fn get(&self, idx: usize) -> &[f32] { - let start = idx * self.cols; - let stop = start + self.cols; - &self.data[start..stop] - } - - fn get_mut(&mut self, idx: usize) -> &mut [f32] { - if idx >= self.len { - for _ in 0..self.cols { - self.data.push(0.0); - } - self.len += 1; - } - - let start = idx * self.cols; - let stop = start + self.cols; - &mut self.data[start..stop] - } - - fn len(&self) -> usize { - self.len - } - - fn clear(&mut self) { - self.len = 0; - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct SparseGradientStore { - slab: Slab, - map: IndexMap, -} - -impl SparseGradientStore { - pub fn new(cols: usize) -> Self { - SparseGradientStore { - slab: Slab::new(cols), - map: IndexMap::default(), - } - } - - pub fn push(&mut self, gradient: (&[usize], &Arr)) { - let (indices, value) = gradient; - - for (&idx, row) in izip!(indices.iter(), value.genrows().into_iter()) { - self.scalar_push(idx, row.as_slice().unwrap()); - } - } - - fn scalar_push(&mut self, idx: usize, row: &[f32]) { - match self.map.entry(idx) { - Entry::Occupied(entry) => { - let slab_idx = entry.get(); - let slab_value = self.slab.get_mut(*slab_idx); - - for (slab_val, row_val) in izip!(slab_value.iter_mut(), row.iter()) { - *slab_val += row_val; - } - } - Entry::Vacant(entry) => { - let slab_idx = self.slab.len(); - entry.insert(slab_idx); - - let slab_value = self.slab.get_mut(slab_idx); - - for (slab_val, &row_val) in izip!(slab_value.iter_mut(), row.iter()) { - *slab_val = row_val; - } - } - } - } - - pub fn iter(&self) -> impl Iterator { - self.map - .keys() - .cloned() - .zip(self.slab.data.chunks(self.slab.cols)) - } - - pub fn iter_mut(&mut self) -> impl Iterator { - self.map - .keys() - .cloned() - .zip(self.slab.data.chunks_mut(self.slab.cols)) - } - - pub fn clear(&mut self) { - self.slab.clear(); - self.map.clear(); - } -} - -struct GradientAccumulatorV2 { +#[derive(Debug)] +pub(crate) struct GradientAccumulator { gradient: Arr, sparse_index: BitSet, dense: bool, sparse: bool, } -impl GradientAccumulatorV2 { +impl GradientAccumulator { pub fn new(shape: (usize, usize)) -> Self { Self { gradient: Arr::zeros(shape), @@ -738,7 +627,7 @@ impl GradientAccumulatorV2 { }) } - pub fn clear(&mut self) { + pub fn zero_gradient(&mut self) { if self.sparse { self.sparse_index.clear() } @@ -746,50 +635,46 @@ impl GradientAccumulatorV2 { self.dense = false; self.sparse = false; } -} - -#[derive(Debug, Serialize, Deserialize)] -pub(crate) struct GradientAccumulator { - pub dense_shape: (usize, usize), - pub dense_gradient: Option, - pub sparse_gradient: SparseGradientStore, - pub has_dense: bool, -} -impl GradientAccumulator { - fn new(dense_shape: (usize, usize)) -> Self { - let cols = dense_shape.1; - - GradientAccumulator { - dense_shape: dense_shape, - dense_gradient: None, - sparse_gradient: SparseGradientStore::new(cols), - has_dense: false, - } + pub fn gradient(&self) -> &Arr { + &self.gradient } - pub fn dense_gradient(&mut self) -> &mut Arr { - let shape = self.dense_shape; - self.dense_gradient.get_or_insert_with(|| Arr::zeros(shape)) - } - pub(crate) fn zero_gradient(&mut self) { - if self.has_dense { - self.dense_gradient().fill(0.0); + /// With sparse gradients we don't reset to zero, so we + /// need this to provide correct dense gradients to + /// finite difference methods. + pub fn materialized_gradient(&self) -> Arr { + if self.has_dense() { + self.gradient.clone() + } else { + let mut grad = &self.gradient * 0.0; + for (idx, row) in self.sparse_iter() { + grad.subview_mut(Axis(0), idx).slice_assign(&row); + } + grad } + } - self.sparse_gradient.clear(); - self.has_dense = false; + pub fn has_dense(&self) -> bool { + self.dense } pub fn clamp(&mut self, min: f32, max: f32) { - self.dense_gradient() - .as_slice_mut() - .unwrap() - .iter_mut() - .for_each(|x| *x = clamp(*x, min, max)); - self.sparse_gradient - .iter_mut() - .for_each(|(_, grad)| grad.iter_mut().for_each(|x| *x = clamp(*x, min, max))); + if self.has_dense() { + self.gradient + .fast_slice_mut() + .iter_mut() + .for_each(|x| *x = clamp(*x, min, max)); + } else { + unimplemented!(); + // for (idx, row) in self.sparse_iter() { + // self.gradient + // .subview_mut(Axis(0), idx) + // .fast_slice_mut() + // .iter_mut() + // .for_each(|x| *x = clamp(*x, min, max)); + // } + } } } @@ -799,35 +684,38 @@ pub trait GradientSink { impl<'a, 'b> GradientSink<&'a Ref<'b, Arr>> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: &Ref) { - self.dense_gradient().slice_add_assign(gradient.deref()); - self.has_dense = true; + self.add_dense(gradient.deref()); } } impl<'a> GradientSink<&'a Arr> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: &'a Arr) { - self.dense_gradient().slice_add_assign(gradient.deref()); - self.has_dense = true; + self.add_dense(gradient); } } impl<'a> GradientSink<&'a mut Arr> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: &'a mut Arr) { - self.dense_gradient().slice_add_assign(gradient.deref()); - self.has_dense = true; + self.add_dense(gradient.deref()); } } impl<'a> GradientSink<(&'a [usize], &'a Arr)> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: (&'a [usize], &'a Arr)) { - self.sparse_gradient.push(gradient); + let (idx, grad) = gradient; + self.add_sparse(idx, grad); } } -impl<'a> GradientSink<(usize, &'a [f32])> for GradientAccumulator { - fn accumulate_gradient(&mut self, gradient: (usize, &'a [f32])) { +impl<'a, 'b: 'a> GradientSink<(usize, &'a ndarray::ArrayView<'b, f32, ndarray::Ix1>)> + for GradientAccumulator +{ + fn accumulate_gradient( + &mut self, + gradient: (usize, &'a ndarray::ArrayView<'b, f32, ndarray::Ix1>), + ) { let (idx, grad) = gradient; - self.sparse_gradient.scalar_push(idx, grad); + self.add_sparse_row(idx, grad); } } @@ -966,13 +854,13 @@ impl ParameterNode { .gradient_accumulator() .iter_mut() .for_each(|accum| { - let mut self_accum = self.gradient.borrow_mut(); - if self_accum.has_dense { - accum.accumulate_gradient(self_accum.dense_gradient()); + let self_accum = self.gradient.borrow(); + if self_accum.has_dense() { + accum.accumulate_gradient(self_accum.gradient()); } - for (row_idx, grad) in self_accum.sparse_gradient.iter() { - accum.accumulate_gradient((row_idx, grad)); + for (row_idx, grad) in self_accum.sparse_iter() { + accum.accumulate_gradient((row_idx, &grad)); } }); } diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index f5d26d30..0a00e1b9 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -68,30 +68,30 @@ impl InnerOptimizer for Adagrad { let param_value = unsafe { param.value_mut() }; let squared_gradient = unsafe { param.squared_gradient_mut() }; - if sink.has_dense { + if sink.has_dense() { for (value, &gradient, squared_gradient) in izip!( param_value.fast_slice_mut(), - sink.dense_gradient().fast_slice(), + sink.gradient().fast_slice(), squared_gradient.fast_slice_mut() ) { let gradient = gradient + *value * self.l2; *squared_gradient += numerics::pow2(gradient); *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; } - } - - for (row_idx, grad) in sink.sparse_gradient.iter() { - let mut param_row = param_value.subview_mut(Axis(0), row_idx); - let mut squared_row = squared_gradient.subview_mut(Axis(0), row_idx); + } else { + for (row_idx, grad) in sink.sparse_iter() { + let mut param_row = param_value.subview_mut(Axis(0), row_idx); + let mut squared_row = squared_gradient.subview_mut(Axis(0), row_idx); - for (value, &gradient, squared_gradient) in izip!( - param_row.fast_slice_mut(), - grad.iter(), - squared_row.fast_slice_mut() - ) { - let gradient = gradient + *value * self.l2; - *squared_gradient += numerics::pow2(gradient); - *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; + for (value, &gradient, squared_gradient) in izip!( + param_row.fast_slice_mut(), + grad.iter(), + squared_row.fast_slice_mut() + ) { + let gradient = gradient + *value * self.l2; + *squared_gradient += numerics::pow2(gradient); + *value -= learning_rate / (self.eps + squared_gradient.sqrt()) * gradient; + } } } } diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 08843334..b5b7ca56 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -101,29 +101,29 @@ impl InnerOptimizer for Adam { // Increment number of updates *param.t = param.t.saturating_add(1); - if sink.has_dense { + if sink.has_dense() { for (value, &gradient, m, v) in izip!( param.value.as_slice_mut().unwrap(), - sink.dense_gradient().as_slice().unwrap(), + sink.gradient().as_slice().unwrap(), param.m.as_slice_mut().unwrap(), param.v.as_slice_mut().unwrap(), ) { self.update(value, gradient, m, v, param.t); } - } - - for (row_idx, ref grad) in sink.sparse_gradient.iter() { - let mut value_row = param.value.subview_mut(Axis(0), row_idx); - let mut m_row = param.m.subview_mut(Axis(0), row_idx); - let mut v_row = param.v.subview_mut(Axis(0), row_idx); - - for (value, &gradient, m, v) in izip!( - value_row.as_slice_mut().unwrap(), - grad.iter(), - m_row.as_slice_mut().unwrap(), - v_row.as_slice_mut().unwrap(), - ) { - self.update(value, gradient, m, v, param.t); + } else { + for (row_idx, ref grad) in sink.sparse_iter() { + let mut value_row = param.value.subview_mut(Axis(0), row_idx); + let mut m_row = param.m.subview_mut(Axis(0), row_idx); + let mut v_row = param.v.subview_mut(Axis(0), row_idx); + + for (value, &gradient, m, v) in izip!( + value_row.as_slice_mut().unwrap(), + grad.iter(), + m_row.as_slice_mut().unwrap(), + v_row.as_slice_mut().unwrap(), + ) { + self.update(value, gradient, m, v, param.t); + } } } } diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index c4c9936e..38362218 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -1,5 +1,6 @@ use super::barrier::SynchronizationBarrier; use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; +use numerics::ArraySlice; use std::ops::DerefMut; use std::sync::Arc; use {numerics, HogwildParameter, ParameterNode, Variable}; @@ -55,16 +56,18 @@ impl InnerOptimizer for SGD { sink.clamp(min, max); } - if sink.has_dense { - param_value.scaled_add(-self.learning_rate, sink.dense_gradient()); - } - - for (row_idx, grad) in sink.sparse_gradient.iter() { - let mut param_row = param_value.subview_mut(Axis(0), row_idx); + if sink.has_dense() { + param_value.scaled_add(-self.learning_rate, sink.gradient()); + } else { + for (row_idx, grad) in sink.sparse_iter() { + let mut param_row = param_value.subview_mut(Axis(0), row_idx); - numerics::map_add_assign_slice(param_row.into_slice().unwrap(), grad, |x| { - -learning_rate * x - }); + numerics::map_add_assign_slice( + param_row.into_slice().unwrap(), + grad.fast_slice(), + |x| -learning_rate * x, + ); + } } } } From cb77c9a1da46402ce1dfacb75e75ba01fefb5313 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 31 May 2018 11:56:28 +0100 Subject: [PATCH 093/108] Slight cleanup. --- src/lib.rs | 17 +++++++++++------ src/nodes.rs | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1ee495e4..ecd6f277 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -162,6 +162,8 @@ use std::clone::Clone; use std::ops::{Add, Deref, Div, Mul, Neg, Sub}; use std::rc::Rc; +use itertools::Itertools; + mod fast_approx; pub mod nn; mod nodes; @@ -193,12 +195,15 @@ fn merge_parameters( xs: &[Variable], ys: &[Variable], ) -> Vec> { - let mut unique_params: Vec<_> = xs.iter().chain(ys.iter()).cloned().collect(); - - unique_params.sort_unstable_by_key(|x| x.as_ptr()); - unique_params.dedup_by_key(|x| (*x).as_ptr()); - - unique_params + xs.iter() + .merge_join_by(ys.iter(), |x, y| x.as_ptr().cmp(&y.as_ptr())) + .map(|either| match either { + itertools::EitherOrBoth::Left(x) => x, + itertools::EitherOrBoth::Right(x) => x, + itertools::EitherOrBoth::Both(x, _) => x, + }) + .cloned() + .collect() } /// Handle to a node in the computation graph. The underlying nodes diff --git a/src/nodes.rs b/src/nodes.rs index 9dab8a7e..8ad5212d 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -857,10 +857,10 @@ impl ParameterNode { let self_accum = self.gradient.borrow(); if self_accum.has_dense() { accum.accumulate_gradient(self_accum.gradient()); - } - - for (row_idx, grad) in self_accum.sparse_iter() { - accum.accumulate_gradient((row_idx, &grad)); + } else { + for (row_idx, grad) in self_accum.sparse_iter() { + accum.accumulate_gradient((row_idx, &grad)); + } } }); } From c9cd44d2a76b54489a29cd8ad8768531960b15c7 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 31 May 2018 13:01:25 +0100 Subject: [PATCH 094/108] First step to removing gradient zeroing. --- src/lib.rs | 20 ++++--- src/nn/losses.rs | 9 ++-- src/nodes.rs | 124 +++++++++++++++++++++++-------------------- src/optim/adagrad.rs | 8 +-- src/optim/adam.rs | 6 ++- src/optim/mod.rs | 6 +-- src/optim/sgd.rs | 6 ++- 7 files changed, 102 insertions(+), 77 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ecd6f277..b658cbc8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -249,9 +249,17 @@ where pub fn forward(&self) { self.node.forward() } - /// Zero the gradients. Must be called after a backward step or whenever inputs change. + /// Clear the graph caches. Must be called whenever inputs change and [backward] is not + /// called. + pub fn clear(&self) { + self.node.clear(); + } + + /// Zero the accumulated gradients for the parameter nodes in this graph. pub fn zero_gradient(&self) { - self.node.zero_gradient(); + for param in self.parameters() { + param.node.zero_gradient(); + } } pub fn needs_gradient(&self) -> bool { @@ -1210,9 +1218,10 @@ mod tests { let optimizer = SGD::new(); - let losses: Vec = (0..rayon::current_num_threads()) + let losses: Vec = optimizer + .synchronized(rayon::current_num_threads()) .into_par_iter() - .map(|_| { + .map(|optimizer| { let u_embedding = ParameterNode::shared(u_parameters.clone()); let v_embedding = ParameterNode::shared(v_parameters.clone()); @@ -1220,8 +1229,6 @@ mod tests { let v_index = IndexInputNode::new(&v_input); let output = InputNode::new(random_matrix(1, 1)); - let optimizer = optimizer.synchronized(); - let u_vec = u_embedding.index(&u_index); let v_vec = v_embedding.index(&v_index); @@ -1263,4 +1270,5 @@ mod tests { assert!(sum_loss / (losses.len() as f32) < 1e-3); } + } diff --git a/src/nn/losses.rs b/src/nn/losses.rs index 22a82826..82dba611 100644 --- a/src/nn/losses.rs +++ b/src/nn/losses.rs @@ -137,6 +137,7 @@ where } if self.counter.recurse_backward() { + self.log_softmax.zero_counter(); self.operand.backward(&self.gradient.borrow()); } } @@ -146,11 +147,11 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); - self.log_softmax.zero_counter(); - self.y.zero_gradient(); + self.operand.clear(); + self.log_softmax.clear(); + self.y.clear(); self.counter.clear(); } } diff --git a/src/nodes.rs b/src/nodes.rs index 8ad5212d..7f784255 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -52,7 +52,13 @@ impl PassCounter { assert!(backward_count <= forward_count); - backward_count == forward_count + if backward_count == forward_count { + self.clear(); + + true + } else { + false + } } #[inline(always)] pub fn forward(&self) -> ForwardAction { @@ -121,7 +127,7 @@ pub trait Node: fmt::Debug + 'static { fn value(&self) -> Bor; /// If the node needs to be used in the backward step. fn needs_gradient(&self) -> bool; - fn zero_gradient(&self); + fn clear(&self); } impl Node for Rc> { @@ -139,8 +145,8 @@ impl Node for Rc> { fn needs_gradient(&self) -> bool { self.deref().needs_gradient() } - fn zero_gradient(&self) { - self.deref().zero_gradient() + fn clear(&self) { + self.deref().clear() } } @@ -238,10 +244,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -433,10 +439,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -528,9 +534,9 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); + self.lhs.clear(); self.counter.clear(); } } @@ -566,7 +572,7 @@ impl Node for InputNode { fn needs_gradient(&self) -> bool { false } - fn zero_gradient(&self) {} + fn clear(&self) {} } #[derive(Debug)] @@ -849,6 +855,10 @@ impl ParameterNode { Variable::new(node, params) } + pub(crate) fn zero_gradient(&self) { + self.gradient.borrow_mut().zero_gradient(); + } + pub(crate) fn gradient_push_down(&self) { self.value .gradient_accumulator() @@ -879,9 +889,7 @@ impl Node for ParameterNode { fn needs_gradient(&self) -> bool { true } - fn zero_gradient(&self) { - self.gradient.borrow_mut().zero_gradient(); - } + fn clear(&self) {} } #[derive(Debug)] @@ -986,10 +994,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -1100,10 +1108,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -1227,10 +1235,10 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -1332,10 +1340,10 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -1521,10 +1529,10 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.lhs.zero_gradient(); - self.rhs.zero_gradient(); + self.lhs.clear(); + self.rhs.clear(); self.counter.clear(); } } @@ -1607,9 +1615,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -1693,9 +1701,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -1779,9 +1787,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -1874,9 +1882,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -1974,9 +1982,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2058,9 +2066,9 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2139,9 +2147,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2214,9 +2222,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2342,9 +2350,9 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2461,9 +2469,9 @@ where fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2543,9 +2551,9 @@ where self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2581,7 +2589,7 @@ impl Node for IndexInputNode { fn needs_gradient(&self) -> bool { false } - fn zero_gradient(&self) {} + fn clear(&self) {} } #[derive(Debug)] @@ -2652,6 +2660,7 @@ impl Node for IndexNode { .gradient .borrow_mut() .accumulate_gradient((&self.index_value.borrow()[..], gradient.deref())); + self.counter.recurse_backward(); } fn value(&self) -> Bor { @@ -2661,9 +2670,9 @@ impl Node for IndexNode { fn needs_gradient(&self) -> bool { self.needs_gradient } - fn zero_gradient(&self) { + fn clear(&self) { if !self.counter.is_zero() { - self.operand.zero_gradient(); + self.operand.clear(); self.counter.clear(); } } @@ -2685,6 +2694,7 @@ mod tests { z.forward(); assert_eq!(y.node.counter.forward_count.get(), 3); z.backward(1.0); - assert_eq!(y.node.counter.backward_count.get(), 3); + assert_eq!(y.node.counter.backward_count.get(), 0); + assert_eq!(y.node.counter.forward_count.get(), 0); } } diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index 0a00e1b9..37cd8b91 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -48,8 +48,10 @@ impl Adagrad { } /// Return a synchoronised wrapper for this optimizer. - pub fn synchronized(&self) -> SynchronizedOptimizer { - SynchronizedOptimizer::new(self, self.sync_barrier.register_thread()) + pub fn synchronized(&self, num_threads: usize) -> Vec> { + (0..num_threads) + .map(|_| SynchronizedOptimizer::new(self, self.sync_barrier.register_thread())) + .collect() } } @@ -101,7 +103,7 @@ impl Optimizer for Adagrad { /// Perform a single SGD step. fn step(&self, parameters: &[Variable]) { for parameter in parameters { - self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()) + self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()); } } } diff --git a/src/optim/adam.rs b/src/optim/adam.rs index b5b7ca56..575816a2 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -39,8 +39,10 @@ impl Adam { } /// Return a synchoronised wrapper for this optimizer. - pub fn synchronized(&self) -> SynchronizedOptimizer { - SynchronizedOptimizer::new(self, self.sync_barrier.register_thread()) + pub fn synchronized(&self, num_threads: usize) -> Vec> { + (0..num_threads) + .map(|_| SynchronizedOptimizer::new(self, self.sync_barrier.register_thread())) + .collect() } /// Set the learning rate. diff --git a/src/optim/mod.rs b/src/optim/mod.rs index be4acdaf..e0e9bfe7 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -21,7 +21,7 @@ pub trait Optimizer { } pub trait Synchronizable { - fn synchronized(&self) -> SynchronizedOptimizer + fn synchronized(&self, num_threads: usize) -> Vec> where Self: Sized; } @@ -87,8 +87,8 @@ macro_rules! impl_sync_optimizer { } impl Synchronizable for $type { - fn synchronized(&self) -> SynchronizedOptimizer { - self.synchronized() + fn synchronized(&self, num_threads: usize) -> Vec> { + self.synchronized(num_threads) } } }; diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index 38362218..b2c517c8 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -37,8 +37,10 @@ impl SGD { } /// Return a synchoronised wrapper for this optimizer. - pub fn synchronized(&self) -> SynchronizedOptimizer { - SynchronizedOptimizer::new(self, self.sync_barrier.register_thread()) + pub fn synchronized(&self, num_threads: usize) -> Vec> { + (0..num_threads) + .map(|_| SynchronizedOptimizer::new(self, self.sync_barrier.register_thread())) + .collect() } } From 6845d08f8edb0f4a424e28e907be829c2be8cea4 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 31 May 2018 13:07:54 +0100 Subject: [PATCH 095/108] Eliminate manual zero_gradient calls. --- src/lib.rs | 7 ------- src/optim/adagrad.rs | 1 + src/optim/adam.rs | 3 ++- src/optim/mod.rs | 1 + src/optim/sgd.rs | 3 ++- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b658cbc8..cf097784 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,7 +70,6 @@ //! loss.backward(1.0); //! //! optimizer.step(loss.parameters()); -//! loss.zero_gradient(); //! } //! # } //! ``` @@ -117,7 +116,6 @@ //! loss.backward(1.0); //! //! optimizer.step(loss.parameters()); -//! loss.zero_gradient(); //! } //! }); //! # } @@ -1016,7 +1014,6 @@ mod tests { loss.backward(1.0); optimizer.step(loss.parameters()); - loss.zero_gradient(); } println!( @@ -1063,7 +1060,6 @@ mod tests { loss.backward(1.0); optimizer.step(loss.parameters()); - loss.zero_gradient(); } println!( @@ -1124,7 +1120,6 @@ mod tests { loss_val += loss.value().scalar_sum(); optimizer.step(loss.parameters()); - loss.zero_gradient(); } } @@ -1186,7 +1181,6 @@ mod tests { loss_val += loss.value().scalar_sum(); optimizer.step(loss.parameters()); - loss.zero_gradient(); } } } @@ -1255,7 +1249,6 @@ mod tests { loss_val += loss.value().scalar_sum(); optimizer.step(loss.parameters()); - loss.zero_gradient(); } } } diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index 37cd8b91..d702217d 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -104,6 +104,7 @@ impl Optimizer for Adagrad { fn step(&self, parameters: &[Variable]) { for parameter in parameters { self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()); + parameter.node.zero_gradient(); } } } diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 575816a2..04d07fcf 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -135,7 +135,8 @@ impl Optimizer for Adam { /// Perform a single SGD step. fn step(&self, parameters: &[Variable]) { for parameter in parameters { - self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()) + self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()); + parameter.node.zero_gradient(); } } } diff --git a/src/optim/mod.rs b/src/optim/mod.rs index e0e9bfe7..505c5fd2 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -57,6 +57,7 @@ macro_rules! impl_sync_optimizer { // accumulators. for parameter in parameters { parameter.gradient_push_down(); + parameter.node.zero_gradient(); } // Wait until all the other threads have finished push down. diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index b2c517c8..77a4465b 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -77,7 +77,8 @@ impl InnerOptimizer for SGD { impl Optimizer for SGD { fn step(&self, parameters: &[Variable]) { for parameter in parameters { - self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()) + self.inner_step(¶meter.node.value, parameter.node.gradient.borrow_mut()); + parameter.node.zero_gradient(); } } } From 7ee4bfa8eeee2e7e71d3657d3a00895c65543554 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 31 May 2018 18:43:09 +0100 Subject: [PATCH 096/108] Working deterministic update order. --- src/lib.rs | 4 -- src/nodes.rs | 39 +------------------- src/optim/barrier.rs | 88 +++++++++++++++++++++++++++++++++----------- src/optim/mod.rs | 62 ++++++++++++++----------------- 4 files changed, 94 insertions(+), 99 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cf097784..63b8a04f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -479,10 +479,6 @@ impl Variable { merge_parameters(&self.parameters, &index.parameters), ) } - - pub(crate) fn gradient_push_down(&self) { - self.node.gradient_push_down(); - } } impl Variable> diff --git a/src/nodes.rs b/src/nodes.rs index 7f784255..68760f7c 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -3,7 +3,7 @@ use std::cell::{Cell, Ref, RefCell}; use std::fmt; use std::ops::{AddAssign, Deref, DerefMut}; use std::rc::Rc; -use std::sync::{Arc, Mutex, MutexGuard, TryLockResult}; +use std::sync::Arc; use ndarray; use ndarray::Axis; @@ -736,8 +736,6 @@ pub struct HogwildParameter { pub squared_gradients: RefCell, pub moments: RefCell, num_updates: Cell, - #[serde(skip)] - gradient: Mutex>, } impl Clone for HogwildParameter { @@ -748,7 +746,6 @@ impl Clone for HogwildParameter { squared_gradients: self.squared_gradients.clone(), moments: self.moments.clone(), num_updates: self.num_updates.clone(), - gradient: Mutex::new(None), } } } @@ -767,27 +764,9 @@ impl HogwildParameter { squared_gradients: RefCell::new(squared_gradients), moments: RefCell::new(moments), num_updates: Cell::new(0), - gradient: Mutex::new(None), } } - pub(crate) fn gradient_accumulator(&self) -> MutexGuard> { - let mut accumulator = self - .gradient - .lock() - .expect("Unable to lock gradients for gradient push-down."); - - accumulator.get_or_insert_with(|| GradientAccumulator::new(self.shape)); - - accumulator - } - - pub(crate) fn try_gradient_accumulator( - &self, - ) -> TryLockResult>> { - self.gradient.try_lock() - } - pub fn value(&self) -> &Arr { unsafe { &*(self.value.as_ptr()) } } @@ -858,22 +837,6 @@ impl ParameterNode { pub(crate) fn zero_gradient(&self) { self.gradient.borrow_mut().zero_gradient(); } - - pub(crate) fn gradient_push_down(&self) { - self.value - .gradient_accumulator() - .iter_mut() - .for_each(|accum| { - let self_accum = self.gradient.borrow(); - if self_accum.has_dense() { - accum.accumulate_gradient(self_accum.gradient()); - } else { - for (row_idx, grad) in self_accum.sparse_iter() { - accum.accumulate_gradient((row_idx, &grad)); - } - } - }); - } } impl Node for ParameterNode { diff --git a/src/optim/barrier.rs b/src/optim/barrier.rs index df30f1a8..a61d651f 100644 --- a/src/optim/barrier.rs +++ b/src/optim/barrier.rs @@ -9,9 +9,9 @@ // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. - +use std::collections::VecDeque; use std::fmt; -use std::sync::{Arc, Condvar, Mutex, MutexGuard, TryLockResult}; +use std::sync::{Arc, Condvar, Mutex}; pub struct Barrier { lock: Mutex, @@ -44,16 +44,18 @@ impl Barrier { } } - pub fn increment_num_threads(&self) { + pub fn increment_num_threads(&self) -> usize { let mut lock = self.lock.lock().unwrap(); if lock.generation_id != 0 { panic!("Can't register more threads after the first generation."); } lock.num_threads += 1; + + lock.num_threads } - pub fn decrement_num_threads(&self) { + pub fn decrement_num_threads(&self) -> usize { let mut lock = self.lock.lock().unwrap(); lock.num_threads = lock.num_threads.saturating_sub(1); @@ -63,6 +65,8 @@ impl Barrier { lock.generation_id = lock.generation_id.wrapping_add(1); self.cvar.notify_all(); } + + lock.num_threads } pub fn wait(&self) -> BarrierWaitResult { @@ -117,8 +121,8 @@ impl SynchronizationBarrier { } pub(crate) fn register_thread(&self) -> SynchronizationBarrierGuard { - self.core.register_thread(); SynchronizationBarrierGuard { + thread_num: self.core.register_thread(), barrier: Arc::clone(&self.core), } } @@ -127,7 +131,7 @@ impl SynchronizationBarrier { struct SynchronizationBarrierCore { start_barrier: Barrier, end_barrier: Barrier, - parameter_lock: Mutex<()>, + thread_queue: Mutex>, } impl SynchronizationBarrierCore { @@ -135,53 +139,93 @@ impl SynchronizationBarrierCore { Self { start_barrier: Barrier::new(0), end_barrier: Barrier::new(0), - parameter_lock: Mutex::default(), + thread_queue: Mutex::new(VecDeque::default()), } } - fn register_thread(&self) { + fn register_thread(&self) -> usize { self.start_barrier.increment_num_threads(); - self.end_barrier.increment_num_threads(); + let thread_num = self.end_barrier.increment_num_threads(); + + self.thread_queue.lock().unwrap().push_back(thread_num); + + thread_num } - fn deregister_thread(&self) { + fn deregister_thread(&self, thread_id: usize) { self.start_barrier.decrement_num_threads(); self.end_barrier.decrement_num_threads(); + + self.thread_queue + .lock() + .unwrap() + .retain(|&x| x != thread_id); + } + + fn should_update(&self, thread_id: usize) -> bool { + self.thread_queue.lock().unwrap().front().unwrap() == &thread_id + } + + fn advance_queue(&self) { + let mut queue_lock = self.thread_queue.lock().unwrap(); + let thread_id = queue_lock.pop_front().expect("Thread queue empty.").clone(); + + // Requeue at the end + queue_lock.push_back(thread_id); } - pub(crate) fn start_wait(&self) { + fn start_wait(&self) { self.start_barrier.wait(); } - pub(crate) fn end_wait(&self) { + fn end_wait(&self) { self.end_barrier.wait(); } } +/// Synchronizes parameter updates so that no two threads can update +/// variables at the same time, and that all updates always occur in +/// the same order, making results reproducible even when multiple +/// threads are used. pub struct SynchronizationBarrierGuard { + thread_num: usize, barrier: Arc, } impl SynchronizationBarrierGuard { - pub fn start_wait(&self) { + /// Returns a synchronization guard, guaranteeing that no other thread + /// will be active at the same time. + /// + /// The result of this _must_ be assigned; when dropped, it + /// allows the next thread to proceed. + pub fn synchronize(&self) -> Result { self.barrier.start_wait(); - } + while !self.barrier.should_update(self.thread_num) {} - pub fn end_wait(&self) { - self.barrier.end_wait(); + Ok(ThreadQueueGuard { guard: self }) } - pub fn lock(&self) -> MutexGuard<()> { - self.barrier.parameter_lock.lock().unwrap() + /// Return the id of the current thread. + /// Threads always update parameters in the + /// same (ascending) sequence of thread ids. + pub fn thread_id(&self) -> usize { + self.thread_num } +} - pub fn try_lock(&self) -> TryLockResult> { - self.barrier.parameter_lock.try_lock() +impl Drop for SynchronizationBarrierGuard { + fn drop(&mut self) { + self.barrier.deregister_thread(self.thread_num); } } -impl Drop for SynchronizationBarrierGuard { +pub struct ThreadQueueGuard<'a> { + guard: &'a SynchronizationBarrierGuard, +} + +impl<'a> Drop for ThreadQueueGuard<'a> { fn drop(&mut self) { - self.barrier.deregister_thread(); + self.guard.barrier.advance_queue(); + self.guard.barrier.end_wait(); } } diff --git a/src/optim/mod.rs b/src/optim/mod.rs index 505c5fd2..e8374129 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -1,6 +1,7 @@ //! Optimization module. //! //! Contains a number of optimizers. +use std::cell::Cell; use std::ops::DerefMut; use std::sync::Arc; use {HogwildParameter, ParameterNode, Variable}; @@ -35,14 +36,15 @@ pub(crate) trait InnerOptimizer { } pub struct SynchronizedOptimizer<'a, T: 'a> { + num_updates: Cell, optimizer: &'a T, barrier_guard: barrier::SynchronizationBarrierGuard, } impl<'a, T: 'a> SynchronizedOptimizer<'a, T> { pub fn new(optimizer: &'a T, barrier_guard: barrier::SynchronizationBarrierGuard) -> Self { - ::std::thread::sleep_ms(5); SynchronizedOptimizer { + num_updates: Cell::new(0), optimizer: optimizer, barrier_guard: barrier_guard, } @@ -53,43 +55,14 @@ macro_rules! impl_sync_optimizer { ($type:ty) => { impl<'a> Optimizer for SynchronizedOptimizer<'a, $type> { fn step(&self, parameters: &[Variable]) { - // Push down this thread's gradients to the shared - // accumulators. - for parameter in parameters { - parameter.gradient_push_down(); - parameter.node.zero_gradient(); - } - - // Wait until all the other threads have finished push down. - self.barrier_guard.start_wait(); + self.num_updates.set(self.num_updates.get() + 1); - // Start. - for parameter in parameters { - // Attempt to get the lock. If unsuccessful, skip this parameter: - // another thread is taking care of it already. - if let Ok(mut lock) = parameter.node.value.try_gradient_accumulator() { - if let Some(ref mut sink) = lock.deref_mut() { - self.optimizer.inner_step(¶meter.node.value, sink); - } else { - panic!("Sink should always be instantiated.") - } + if self.num_updates.get() == 8 { + let _barrier = self.barrier_guard.synchronize(); + self.optimizer.step(parameters); - if let Some(ref mut sink) = lock.deref_mut() { - sink.zero_gradient(); - } else { - panic!("Sink should always be instantiated.") - } - } + self.num_updates.set(0); } - - // Wait until all the other threads have finished updating. - self.barrier_guard.end_wait(); - } - } - - impl Synchronizable for $type { - fn synchronized(&self, num_threads: usize) -> Vec> { - self.synchronized(num_threads) } } }; @@ -99,6 +72,7 @@ macro_rules! impl_sync_optimizer { impl_sync_optimizer!(SGD); impl_sync_optimizer!(Adagrad); impl_sync_optimizer!(Adam); +impl_sync_optimizer!(Optimizers); macro_rules! impl_optimizer_enum { ($(($tag:ident, $type:ty)),*) => { @@ -108,6 +82,16 @@ macro_rules! impl_optimizer_enum { )* } + impl Optimizers { + fn register_thread(&self) -> barrier::SynchronizationBarrierGuard { + match self { + $( + Optimizers::$tag(val) => val.sync_barrier.register_thread(), + )* + } + } + } + impl Optimizer for Optimizers { fn step(&self, parameters: &[Variable]) { match self { @@ -117,6 +101,14 @@ macro_rules! impl_optimizer_enum { } } } + + impl Synchronizable for Optimizers { + fn synchronized(&self, num_threads: usize) -> Vec> { + (0..num_threads) + .map(|_| SynchronizedOptimizer::new(self, self.register_thread())) + .collect() + } + } } } From 43b7817641acadd51a56ce83eb70170df72efa7b Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 31 May 2018 21:05:08 +0100 Subject: [PATCH 097/108] Clean up optimizer implementation. --- src/lib.rs | 1 + src/optim/adagrad.rs | 14 +----- src/optim/adam.rs | 14 +----- src/optim/barrier.rs | 1 + src/optim/mod.rs | 104 ++++++++++++++++++++++++------------------- src/optim/sgd.rs | 14 +----- 6 files changed, 62 insertions(+), 86 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 63b8a04f..d3d3abf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -674,6 +674,7 @@ mod tests { use rayon::prelude::*; use std::sync::Arc; + use super::optim::Synchronizable; use super::*; const TOLERANCE: f32 = 0.05; diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index d702217d..ef4b5c34 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -1,5 +1,4 @@ -use super::barrier::SynchronizationBarrier; -use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; +use super::Optimizer; use numerics::{ArraySlice, ArraySliceMut}; use std::ops::DerefMut; use std::sync::Arc; @@ -14,7 +13,6 @@ pub struct Adagrad { l2: f32, clamp: Option<(f32, f32)>, eps: f32, - pub(in optim) sync_barrier: SynchronizationBarrier, } impl Adagrad { @@ -25,7 +23,6 @@ impl Adagrad { l2: 0.0, clamp: None, eps: 1e-10, - sync_barrier: SynchronizationBarrier::default(), } } @@ -47,15 +44,6 @@ impl Adagrad { self } - /// Return a synchoronised wrapper for this optimizer. - pub fn synchronized(&self, num_threads: usize) -> Vec> { - (0..num_threads) - .map(|_| SynchronizedOptimizer::new(self, self.sync_barrier.register_thread())) - .collect() - } -} - -impl InnerOptimizer for Adagrad { fn inner_step>( &self, param: &Arc, diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 04d07fcf..ff3d6b19 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -1,5 +1,4 @@ -use super::barrier::SynchronizationBarrier; -use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; +use super::Optimizer; use std::ops::DerefMut; use std::sync::Arc; use {numerics, Arr, HogwildParameter, ParameterNode, Variable}; @@ -21,7 +20,6 @@ pub struct Adam { beta_v: f32, eps: f32, clamp: Option<(f32, f32)>, - pub(in optim) sync_barrier: SynchronizationBarrier, } impl Adam { @@ -34,17 +32,9 @@ impl Adam { beta_v: 0.999, eps: 1.0e-8, clamp: None, - sync_barrier: SynchronizationBarrier::default(), } } - /// Return a synchoronised wrapper for this optimizer. - pub fn synchronized(&self, num_threads: usize) -> Vec> { - (0..num_threads) - .map(|_| SynchronizedOptimizer::new(self, self.sync_barrier.register_thread())) - .collect() - } - /// Set the learning rate. pub fn learning_rate(mut self, learning_rate: f32) -> Self { self.learning_rate = learning_rate; @@ -86,9 +76,7 @@ impl Adam { *value -= self.learning_rate / (v_hat.sqrt() + self.eps) * m_hat; } -} -impl InnerOptimizer for Adam { fn inner_step>( &self, param: &Arc, diff --git a/src/optim/barrier.rs b/src/optim/barrier.rs index a61d651f..6307be7e 100644 --- a/src/optim/barrier.rs +++ b/src/optim/barrier.rs @@ -208,6 +208,7 @@ impl SynchronizationBarrierGuard { /// Return the id of the current thread. /// Threads always update parameters in the /// same (ascending) sequence of thread ids. + #[allow(dead_code)] pub fn thread_id(&self) -> usize { self.thread_num } diff --git a/src/optim/mod.rs b/src/optim/mod.rs index e8374129..e9112b29 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -2,9 +2,7 @@ //! //! Contains a number of optimizers. use std::cell::Cell; -use std::ops::DerefMut; -use std::sync::Arc; -use {HogwildParameter, ParameterNode, Variable}; +use {ParameterNode, Variable}; mod adagrad; mod adam; mod barrier; @@ -21,29 +19,49 @@ pub trait Optimizer { fn step(&self, parameters: &[Variable]); } +/// Trait implemented by synchronizable optimizers. +/// +/// Using a set of synchronized optimizers guarantees that parameter +/// updates will always happen in the same order, guaranteeing reproducible +/// results at the price of some performance relative to asynchronous parallel +/// optimization. pub trait Synchronizable { + /// Synchronize this optimizer, producing a set of synchronized optimimzers + /// to be used by individual fitting threads. fn synchronized(&self, num_threads: usize) -> Vec> where - Self: Sized; -} - -pub(crate) trait InnerOptimizer { - fn inner_step>( + Self: Sized, + { + self.synchronized_with_step(num_threads, 8) + } + /// Synchronize this optimizer, producing a set of synchronized optimimzers + /// to be used by individual fitting threads. The threads will synchonize + /// their updates every `step_size` steps. + fn synchronized_with_step( &self, - parameter: &Arc, - sink: T, - ); + num_threads: usize, + step_size: usize, + ) -> Vec> + where + Self: Sized; } +/// Synchronized optimizer wrapper. pub struct SynchronizedOptimizer<'a, T: 'a> { + step_size: usize, num_updates: Cell, optimizer: &'a T, barrier_guard: barrier::SynchronizationBarrierGuard, } impl<'a, T: 'a> SynchronizedOptimizer<'a, T> { - pub fn new(optimizer: &'a T, barrier_guard: barrier::SynchronizationBarrierGuard) -> Self { + fn new( + optimizer: &'a T, + barrier_guard: barrier::SynchronizationBarrierGuard, + step_size: usize, + ) -> Self { SynchronizedOptimizer { + step_size: step_size, num_updates: Cell::new(0), optimizer: optimizer, barrier_guard: barrier_guard, @@ -51,28 +69,38 @@ impl<'a, T: 'a> SynchronizedOptimizer<'a, T> { } } -macro_rules! impl_sync_optimizer { - ($type:ty) => { - impl<'a> Optimizer for SynchronizedOptimizer<'a, $type> { - fn step(&self, parameters: &[Variable]) { - self.num_updates.set(self.num_updates.get() + 1); +impl<'a, T> Optimizer for SynchronizedOptimizer<'a, T> +where + T: Optimizer, +{ + fn step(&self, parameters: &[Variable]) { + self.num_updates.set(self.num_updates.get() + 1); - if self.num_updates.get() == 8 { - let _barrier = self.barrier_guard.synchronize(); - self.optimizer.step(parameters); + if self.num_updates.get() == self.step_size { + let _barrier = self.barrier_guard.synchronize(); + self.optimizer.step(parameters); - self.num_updates.set(0); - } - } + self.num_updates.set(0); } - }; + } } -// TODO: impl using generics, atm it's painful because of private type leakage -impl_sync_optimizer!(SGD); -impl_sync_optimizer!(Adagrad); -impl_sync_optimizer!(Adam); -impl_sync_optimizer!(Optimizers); +impl Synchronizable for T +where + T: Optimizer + Sized, +{ + fn synchronized_with_step( + &self, + num_threads: usize, + step_size: usize, + ) -> Vec> { + let barrier = SynchronizationBarrier::default(); + + (0..num_threads) + .map(|_| SynchronizedOptimizer::new(self, barrier.register_thread(), step_size)) + .collect() + } +} macro_rules! impl_optimizer_enum { ($(($tag:ident, $type:ty)),*) => { @@ -82,16 +110,6 @@ macro_rules! impl_optimizer_enum { )* } - impl Optimizers { - fn register_thread(&self) -> barrier::SynchronizationBarrierGuard { - match self { - $( - Optimizers::$tag(val) => val.sync_barrier.register_thread(), - )* - } - } - } - impl Optimizer for Optimizers { fn step(&self, parameters: &[Variable]) { match self { @@ -101,14 +119,6 @@ macro_rules! impl_optimizer_enum { } } } - - impl Synchronizable for Optimizers { - fn synchronized(&self, num_threads: usize) -> Vec> { - (0..num_threads) - .map(|_| SynchronizedOptimizer::new(self, self.register_thread())) - .collect() - } - } } } diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index 77a4465b..44d8087b 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -1,5 +1,4 @@ -use super::barrier::SynchronizationBarrier; -use super::{InnerOptimizer, Optimizer, SynchronizedOptimizer}; +use super::Optimizer; use numerics::ArraySlice; use std::ops::DerefMut; use std::sync::Arc; @@ -11,7 +10,6 @@ use ndarray::Axis; pub struct SGD { learning_rate: f32, clamp: Option<(f32, f32)>, - pub(in optim) sync_barrier: SynchronizationBarrier, } impl SGD { @@ -20,7 +18,6 @@ impl SGD { SGD { learning_rate: 0.05, clamp: None, - sync_barrier: SynchronizationBarrier::default(), } } @@ -36,15 +33,6 @@ impl SGD { self } - /// Return a synchoronised wrapper for this optimizer. - pub fn synchronized(&self, num_threads: usize) -> Vec> { - (0..num_threads) - .map(|_| SynchronizedOptimizer::new(self, self.sync_barrier.register_thread())) - .collect() - } -} - -impl InnerOptimizer for SGD { fn inner_step>( &self, param: &Arc, From 629f94d941c91a3a41d85967ed7b48aca26f9289 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 1 Jun 2018 10:57:56 +0100 Subject: [PATCH 098/108] Add default impls for optimizers. Add deny_missing_docs. --- src/lib.rs | 7 +++---- src/nn/losses.rs | 4 +++- src/nodes.rs | 21 ++++++++------------- src/optim/adagrad.rs | 6 ++++++ src/optim/adam.rs | 6 ++++++ src/optim/mod.rs | 14 +++++++++----- src/optim/sgd.rs | 6 ++++++ 7 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d3d3abf4..1b32a230 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![deny(missing_docs)] //! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! //! # Features @@ -186,6 +187,7 @@ fn clamp(x: f32, min: f32, max: f32) -> f32 { /// Trait describing nodes that can accept new values once /// the graph has been defined. pub trait DataInput { + /// Set the value of this node. fn set_value(&self, T); } @@ -260,10 +262,6 @@ where } } - pub fn needs_gradient(&self) -> bool { - self.node.needs_gradient() - } - /// Return the parameters of the graph. pub fn parameters(&self) -> &[Variable] { &self.parameters[..] @@ -275,6 +273,7 @@ where } } +/// An alias for a node whose concrete type has been erased. pub type BoxedNode = Rc>; impl Variable diff --git a/src/nn/losses.rs b/src/nn/losses.rs index 82dba611..51226b09 100644 --- a/src/nn/losses.rs +++ b/src/nn/losses.rs @@ -24,6 +24,7 @@ where Variable::new(Rc::new(node), x.parameters.clone()) } +/// Sparse categorical cross-entropy loss node. #[derive(Debug)] pub struct SparseCategoricalCrossentropyNode { operand: Rc, @@ -39,7 +40,7 @@ impl SparseCategoricalCrossentropyNode where LHS: Node, { - pub fn new(operand: Rc, y: Rc) -> Self { + pub(crate) fn new(operand: Rc, y: Rc) -> Self { assert!( operand.value().rows() == 1, "Minibatches not supported: rows must be 1." @@ -75,6 +76,7 @@ where } } + /// Return the predictions made by this layer. pub fn predictions(&self) -> Bor { self.log_softmax.value() } diff --git a/src/nodes.rs b/src/nodes.rs index 68760f7c..b2cc5ebd 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -89,7 +89,9 @@ impl PassCounter { /// and simple references. #[derive(Debug)] pub enum Bor<'value, T: 'value> { + /// Ref from a `RefCell`. RefGuard(Ref<'value, T>), + /// Plain reference. Reference(&'value T), } @@ -127,6 +129,7 @@ pub trait Node: fmt::Debug + 'static { fn value(&self) -> Bor; /// If the node needs to be used in the backward step. fn needs_gradient(&self) -> bool; + /// Reset the caches of this node and its parents. fn clear(&self); } @@ -545,7 +548,7 @@ where /// Input node for the graph. #[derive(Debug)] pub struct InputNode { - pub value: RefCell, + pub(crate) value: RefCell, } impl InputNode { @@ -732,9 +735,9 @@ unsafe impl Sync for HogwildParameter {} #[derive(Debug, Serialize, Deserialize)] pub struct HogwildParameter { shape: (usize, usize), - pub value: RefCell, - pub squared_gradients: RefCell, - pub moments: RefCell, + pub(crate) value: RefCell, + pub(crate) squared_gradients: RefCell, + pub(crate) moments: RefCell, num_updates: Cell, } @@ -767,14 +770,6 @@ impl HogwildParameter { } } - pub fn value(&self) -> &Arr { - unsafe { &*(self.value.as_ptr()) } - } - - pub fn squared_gradients(&self) -> &Arr { - unsafe { &*(self.squared_gradients.as_ptr()) } - } - pub(crate) unsafe fn value_mut(&self) -> &mut Arr { &mut *(self.value.as_ptr()) } @@ -2526,7 +2521,7 @@ where /// for implementing indexable embedding layers. #[derive(Debug)] pub struct IndexInputNode { - pub value: RefCell>, + pub(crate) value: RefCell>, } impl IndexInputNode { diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index ef4b5c34..25fa9c4a 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -15,6 +15,12 @@ pub struct Adagrad { eps: f32, } +impl Default for Adagrad { + fn default() -> Self { + Self::new() + } +} + impl Adagrad { /// Create a new optimizer instance with a given set of parameters. pub fn new() -> Self { diff --git a/src/optim/adam.rs b/src/optim/adam.rs index ff3d6b19..1e77b13f 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -22,6 +22,12 @@ pub struct Adam { clamp: Option<(f32, f32)>, } +impl Default for Adam { + fn default() -> Self { + Self::new() + } +} + impl Adam { /// Build new optimizer object. pub fn new() -> Self { diff --git a/src/optim/mod.rs b/src/optim/mod.rs index e9112b29..00ab31ad 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -10,7 +10,7 @@ mod sgd; pub use self::adagrad::Adagrad; pub use self::adam::Adam; -pub use self::barrier::SynchronizationBarrier; +use self::barrier::SynchronizationBarrier; pub use self::sgd::SGD; /// Core trait implemented by all optimizer methods. @@ -104,10 +104,14 @@ where macro_rules! impl_optimizer_enum { ($(($tag:ident, $type:ty)),*) => { + /// Enum containing all optimizers. + /// + /// Makes runtime switching between optimizers slightly more ergonomic. pub enum Optimizers { - $( - $tag($type), - )* + $( + #[allow(missing_docs)] + $tag($type), + )* } impl Optimizer for Optimizers { @@ -115,7 +119,7 @@ macro_rules! impl_optimizer_enum { match self { $( Optimizers::$tag(val) => val.step(parameters), - )* + )* } } } diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index 44d8087b..c812aea7 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -12,6 +12,12 @@ pub struct SGD { clamp: Option<(f32, f32)>, } +impl Default for SGD { + fn default() -> Self { + Self::new() + } +} + impl SGD { /// Create a new optimizer instance with a given set of parameters. pub fn new() -> Self { From 751ab7e70a2e76837d385d2a1166f7063c180d85 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 1 Jun 2018 12:43:53 +0100 Subject: [PATCH 099/108] Add coupled update-forget gates to LSTM. --- src/nn/lstm.rs | 44 +++++++++++++++++++++++++++++++++----------- src/nodes.rs | 5 +++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/nn/lstm.rs b/src/nn/lstm.rs index 5a46d5de..98226bd5 100644 --- a/src/nn/lstm.rs +++ b/src/nn/lstm.rs @@ -156,16 +156,11 @@ impl Parameters { } } - /// Build an LSTM layer. - pub fn build(&self) -> Layer { - Layer::new(self.build_cell()) - } - - /// Build an LSTM cell. - pub fn build_cell(&self) -> Cell { + fn inner_build_cell(&self, coupled: bool) -> Cell { Cell { input_dim: self.input_dim, hidden_dim: self.hidden_dim, + coupled_input: coupled, forget_weights: ParameterNode::shared(self.forget_weights.clone()), forget_biases: ParameterNode::shared(self.forget_biases.clone()), @@ -180,6 +175,28 @@ impl Parameters { output_gate_biases: ParameterNode::shared(self.output_gate_biases.clone()), } } + + /// Build an LSTM layer. + pub fn build(&self) -> Layer { + Layer::new(self.build_cell()) + } + + /// Build an LSTM layer where the input and forget gates + /// are coupled. + pub fn build_coupled(&self) -> Layer { + Layer::new(self.build_coupled_cell()) + } + + /// Build an LSTM cell. + pub fn build_cell(&self) -> Cell { + self.inner_build_cell(false) + } + + /// Build an LSTM cell where the input and forget gates + /// are coupled. + pub fn build_coupled_cell(&self) -> Cell { + self.inner_build_cell(true) + } } /// An LSTM cell. @@ -187,6 +204,7 @@ impl Parameters { pub struct Cell { input_dim: usize, hidden_dim: usize, + coupled_input: bool, forget_weights: Variable, forget_biases: Variable, @@ -227,12 +245,16 @@ impl Cell { // Forget part of the cell state let forget_gate = (stacked_input.dot(&self.forget_weights) + self.forget_biases.clone()).sigmoid(); - let cell = forget_gate * cell; + let cell = forget_gate.clone() * cell; // Update the cell state with new input - let update_gate = (stacked_input.dot(&self.update_gate_weights) - + self.update_gate_biases.clone()) - .sigmoid(); + let update_gate = if self.coupled_input { + (1.0 - forget_gate).boxed() + } else { + (stacked_input.dot(&self.update_gate_weights) + self.update_gate_biases.clone()) + .sigmoid() + .boxed() + }; let update_value = (stacked_input.dot(&self.update_value_weights) + self.update_value_biases.clone()) .tanh(); diff --git a/src/nodes.rs b/src/nodes.rs index b2cc5ebd..659a3f3a 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -770,6 +770,11 @@ impl HogwildParameter { } } + /// Get the parameter value. + pub fn value(&self) -> &Arr { + unsafe { &*(self.value.as_ptr()) } + } + pub(crate) unsafe fn value_mut(&self) -> &mut Arr { &mut *(self.value.as_ptr()) } From 7bb1b474667417254b13cd50fe7d0c31a26604b6 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Fri, 1 Jun 2018 17:29:39 +0100 Subject: [PATCH 100/108] Better gradient accumulation in dot nodes. --- src/nodes.rs | 55 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/nodes.rs b/src/nodes.rs index 659a3f3a..9c99dd55 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -1210,6 +1210,7 @@ where #[derive(Debug)] pub struct DotNode { value: RefCell, + gradient: RefCell, lhs_gradient: RefCell, rhs_gradient: RefCell, lhs: Rc, @@ -1226,12 +1227,14 @@ where pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); let value = lhs.value().dot(rhs.value().deref()); + let gradient = &value * 0.0; let lhs_gradient = lhs.value().deref() * 0.0; let rhs_gradient = rhs.value().deref() * 0.0; DotNode { value: RefCell::new(value), + gradient: RefCell::new(gradient), lhs_gradient: RefCell::new(lhs_gradient), rhs_gradient: RefCell::new(rhs_gradient), lhs: lhs, @@ -1268,29 +1271,43 @@ where } fn backward(&self, gradient: &Ref) { - let beta = match self.counter.backward() { - BackwardAction::Set => 0.0, - BackwardAction::Increment => 1.0, - }; + match self.counter.backward() { + BackwardAction::Set => { + self.gradient.borrow_mut().slice_assign(gradient.deref()); + } + BackwardAction::Increment => { + self.gradient + .borrow_mut() + .slice_add_assign(gradient.deref()); + } + } - { - let rhs_value = self.rhs.value(); - let lhs_value = self.lhs.value(); + if self.counter.recurse_backward() { + { + let rhs_value = self.rhs.value(); + let lhs_value = self.lhs.value(); - let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + let gradient = self.gradient.borrow(); - numerics::mat_mul(1.0, gradient, &rhs_value.t(), beta, &mut lhs_gradient); - numerics::mat_mul( - 1.0, - &lhs_value.t(), - gradient.deref(), - beta, - &mut rhs_gradient, - ); - } + let mut lhs_gradient = self.lhs_gradient.borrow_mut(); + let mut rhs_gradient = self.rhs_gradient.borrow_mut(); + + numerics::mat_mul( + 1.0, + gradient.deref(), + &rhs_value.t(), + 0.0, + &mut lhs_gradient, + ); + numerics::mat_mul( + 1.0, + &lhs_value.t(), + gradient.deref(), + 0.0, + &mut rhs_gradient, + ); + } - if self.counter.recurse_backward() { self.lhs.backward(&self.lhs_gradient.borrow()); self.rhs.backward(&self.rhs_gradient.borrow()); } From b2ac5a00b42799da0c8a8a700fa3879dd4374898 Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 2 Jun 2018 19:12:31 +0100 Subject: [PATCH 101/108] Bump to v0.9.0. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c2efb7ff..beaa8f78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.8.1" +version = "0.9.0" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." From 6272ad83b2fff32eb1e731289611f8cbd3f3997f Mon Sep 17 00:00:00 2001 From: maciejkula Date: Sat, 2 Jun 2018 19:19:32 +0100 Subject: [PATCH 102/108] Fix warnings and bump to 0.9.1. --- Cargo.toml | 2 +- src/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index beaa8f78..07f90ea7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wyrm" -version = "0.9.0" +version = "0.9.1" authors = ["Maciej Kula"] license = "MIT" description = "A low-overhead, define-by-run autodifferentiation library." diff --git a/src/lib.rs b/src/lib.rs index 1b32a230..abb8acf3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -142,6 +142,7 @@ extern crate serde_derive; extern crate serde; +#[allow(unused_imports)] #[macro_use] extern crate ndarray; extern crate rand; From fed37e9394aa2b3116c0b10e435adcbd17bf875a Mon Sep 17 00:00:00 2001 From: maciejkula Date: Thu, 21 Jun 2018 15:07:21 +0100 Subject: [PATCH 103/108] Add missing debug impls. --- src/lib.rs | 2 +- src/optim/adagrad.rs | 1 + src/optim/adam.rs | 1 + src/optim/barrier.rs | 2 ++ src/optim/mod.rs | 2 ++ src/optim/sgd.rs | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index abb8acf3..8e1cc0ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![deny(missing_docs)] +#![deny(missing_docs, missing_debug_implementations)] //! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! //! # Features diff --git a/src/optim/adagrad.rs b/src/optim/adagrad.rs index 25fa9c4a..c544701c 100644 --- a/src/optim/adagrad.rs +++ b/src/optim/adagrad.rs @@ -8,6 +8,7 @@ use ndarray::Axis; /// Adagrad optimizer, scaled the learning rate by the inverse of previously /// accumulated gradients. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Adagrad { learning_rate: f32, l2: f32, diff --git a/src/optim/adam.rs b/src/optim/adam.rs index 1e77b13f..b006c770 100644 --- a/src/optim/adam.rs +++ b/src/optim/adam.rs @@ -13,6 +13,7 @@ struct AdamParameters<'params> { } /// ADAM optimizer. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Adam { learning_rate: f32, l2: f32, diff --git a/src/optim/barrier.rs b/src/optim/barrier.rs index 6307be7e..8bb9f38f 100644 --- a/src/optim/barrier.rs +++ b/src/optim/barrier.rs @@ -128,6 +128,7 @@ impl SynchronizationBarrier { } } +#[derive(Debug)] struct SynchronizationBarrierCore { start_barrier: Barrier, end_barrier: Barrier, @@ -187,6 +188,7 @@ impl SynchronizationBarrierCore { /// variables at the same time, and that all updates always occur in /// the same order, making results reproducible even when multiple /// threads are used. +#[derive(Debug)] pub struct SynchronizationBarrierGuard { thread_num: usize, barrier: Arc, diff --git a/src/optim/mod.rs b/src/optim/mod.rs index 00ab31ad..b650fe59 100644 --- a/src/optim/mod.rs +++ b/src/optim/mod.rs @@ -47,6 +47,7 @@ pub trait Synchronizable { } /// Synchronized optimizer wrapper. +#[derive(Debug)] pub struct SynchronizedOptimizer<'a, T: 'a> { step_size: usize, num_updates: Cell, @@ -107,6 +108,7 @@ macro_rules! impl_optimizer_enum { /// Enum containing all optimizers. /// /// Makes runtime switching between optimizers slightly more ergonomic. + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Optimizers { $( #[allow(missing_docs)] diff --git a/src/optim/sgd.rs b/src/optim/sgd.rs index c812aea7..d967092e 100644 --- a/src/optim/sgd.rs +++ b/src/optim/sgd.rs @@ -7,6 +7,7 @@ use {numerics, HogwildParameter, ParameterNode, Variable}; use ndarray::Axis; /// Standard stochastic gradient descent optimizer with a fixed learning rate. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct SGD { learning_rate: f32, clamp: Option<(f32, f32)>, From ec8cf1fb2e0c8c7de2f5ac1dcabba8263b4c3a68 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Fri, 7 Aug 2020 15:01:49 +0100 Subject: [PATCH 104/108] move into amadeus-ml --- Cargo.toml => amadeus-ml/Cargo.toml | 0 {benches => amadeus-ml/benches}/benchmark.rs | 0 {src => amadeus-ml/src}/fast_approx.rs | 0 {src => amadeus-ml/src}/lib.rs | 0 {src => amadeus-ml/src}/nn/losses.rs | 0 {src => amadeus-ml/src}/nn/lstm.rs | 0 {src => amadeus-ml/src}/nn/mod.rs | 0 {src => amadeus-ml/src}/nn/pi.txt | 0 {src => amadeus-ml/src}/nodes.rs | 0 {src => amadeus-ml/src}/numerics.rs | 0 {src => amadeus-ml/src}/optim/adagrad.rs | 0 {src => amadeus-ml/src}/optim/adam.rs | 0 {src => amadeus-ml/src}/optim/barrier.rs | 0 {src => amadeus-ml/src}/optim/mod.rs | 0 {src => amadeus-ml/src}/optim/sgd.rs | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename Cargo.toml => amadeus-ml/Cargo.toml (100%) rename {benches => amadeus-ml/benches}/benchmark.rs (100%) rename {src => amadeus-ml/src}/fast_approx.rs (100%) rename {src => amadeus-ml/src}/lib.rs (100%) rename {src => amadeus-ml/src}/nn/losses.rs (100%) rename {src => amadeus-ml/src}/nn/lstm.rs (100%) rename {src => amadeus-ml/src}/nn/mod.rs (100%) rename {src => amadeus-ml/src}/nn/pi.txt (100%) rename {src => amadeus-ml/src}/nodes.rs (100%) rename {src => amadeus-ml/src}/numerics.rs (100%) rename {src => amadeus-ml/src}/optim/adagrad.rs (100%) rename {src => amadeus-ml/src}/optim/adam.rs (100%) rename {src => amadeus-ml/src}/optim/barrier.rs (100%) rename {src => amadeus-ml/src}/optim/mod.rs (100%) rename {src => amadeus-ml/src}/optim/sgd.rs (100%) diff --git a/Cargo.toml b/amadeus-ml/Cargo.toml similarity index 100% rename from Cargo.toml rename to amadeus-ml/Cargo.toml diff --git a/benches/benchmark.rs b/amadeus-ml/benches/benchmark.rs similarity index 100% rename from benches/benchmark.rs rename to amadeus-ml/benches/benchmark.rs diff --git a/src/fast_approx.rs b/amadeus-ml/src/fast_approx.rs similarity index 100% rename from src/fast_approx.rs rename to amadeus-ml/src/fast_approx.rs diff --git a/src/lib.rs b/amadeus-ml/src/lib.rs similarity index 100% rename from src/lib.rs rename to amadeus-ml/src/lib.rs diff --git a/src/nn/losses.rs b/amadeus-ml/src/nn/losses.rs similarity index 100% rename from src/nn/losses.rs rename to amadeus-ml/src/nn/losses.rs diff --git a/src/nn/lstm.rs b/amadeus-ml/src/nn/lstm.rs similarity index 100% rename from src/nn/lstm.rs rename to amadeus-ml/src/nn/lstm.rs diff --git a/src/nn/mod.rs b/amadeus-ml/src/nn/mod.rs similarity index 100% rename from src/nn/mod.rs rename to amadeus-ml/src/nn/mod.rs diff --git a/src/nn/pi.txt b/amadeus-ml/src/nn/pi.txt similarity index 100% rename from src/nn/pi.txt rename to amadeus-ml/src/nn/pi.txt diff --git a/src/nodes.rs b/amadeus-ml/src/nodes.rs similarity index 100% rename from src/nodes.rs rename to amadeus-ml/src/nodes.rs diff --git a/src/numerics.rs b/amadeus-ml/src/numerics.rs similarity index 100% rename from src/numerics.rs rename to amadeus-ml/src/numerics.rs diff --git a/src/optim/adagrad.rs b/amadeus-ml/src/optim/adagrad.rs similarity index 100% rename from src/optim/adagrad.rs rename to amadeus-ml/src/optim/adagrad.rs diff --git a/src/optim/adam.rs b/amadeus-ml/src/optim/adam.rs similarity index 100% rename from src/optim/adam.rs rename to amadeus-ml/src/optim/adam.rs diff --git a/src/optim/barrier.rs b/amadeus-ml/src/optim/barrier.rs similarity index 100% rename from src/optim/barrier.rs rename to amadeus-ml/src/optim/barrier.rs diff --git a/src/optim/mod.rs b/amadeus-ml/src/optim/mod.rs similarity index 100% rename from src/optim/mod.rs rename to amadeus-ml/src/optim/mod.rs diff --git a/src/optim/sgd.rs b/amadeus-ml/src/optim/sgd.rs similarity index 100% rename from src/optim/sgd.rs rename to amadeus-ml/src/optim/sgd.rs From 6334be02e3f062cef6ac352f203f2fd81cd3ed0d Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sat, 8 Aug 2020 09:56:45 +0100 Subject: [PATCH 105/108] update deps & clippy clean --- amadeus-ml/Cargo.toml | 20 ++- amadeus-ml/benches/benchmark.rs | 35 ++-- amadeus-ml/src/fast_approx.rs | 6 +- amadeus-ml/src/lib.rs | 138 ++++++++------- amadeus-ml/src/nn/losses.rs | 10 +- amadeus-ml/src/nn/lstm.rs | 66 +++----- amadeus-ml/src/nn/mod.rs | 9 +- amadeus-ml/src/nodes.rs | 287 +++++++++++++------------------- amadeus-ml/src/numerics.rs | 25 +-- amadeus-ml/src/optim/adagrad.rs | 4 +- amadeus-ml/src/optim/adam.rs | 8 +- amadeus-ml/src/optim/barrier.rs | 10 +- amadeus-ml/src/optim/sgd.rs | 4 +- 13 files changed, 281 insertions(+), 341 deletions(-) diff --git a/amadeus-ml/Cargo.toml b/amadeus-ml/Cargo.toml index b894fcd6..e2f7f1ff 100644 --- a/amadeus-ml/Cargo.toml +++ b/amadeus-ml/Cargo.toml @@ -22,21 +22,23 @@ maintenance = { status = "actively-developed" } fast-math = [] [dependencies] -ndarray = { version = "0.11.0", features = ["serde-1"] } -rand = { version = "0.5.0", features = ["serde1"] } -smallvec = { version = "0.5.0", features = ["serde"] } -itertools = "0.7.3" -rayon = "1.0.0" +approx = "0.3" +hibitset = "0.6" +itertools = "0.9" +ndarray = { version = "0.13", features = ["approx", "serde"] } +rand = { version = "0.7", features = ["serde1"] } +rand_distr = "0.2" +rayon = "1.3" serde = { version = "1", features = ["derive", "rc"] } -hibitset = "0.5.0" +smallvec = { version = "1.4", features = ["serde"] } [build-dependencies] rustversion = "1.0" [dev-dependencies] -# ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } -# blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } -# openblas-src = { version = "0.5.6", default-features = false, features = ["static"] } +# ndarray = { version = "0.13", features = ["blas"] } +# blas-src = { version = "0.2", default-features = false, features = ["openblas"] } +# openblas-src = { version = "0.6", default-features = false, features = ["static"] } criterion = "0.2.3" [[bench]] diff --git a/amadeus-ml/benches/benchmark.rs b/amadeus-ml/benches/benchmark.rs index 9d7944f0..e1fc97c0 100644 --- a/amadeus-ml/benches/benchmark.rs +++ b/amadeus-ml/benches/benchmark.rs @@ -1,16 +1,11 @@ -use std::sync::Arc; - -#[macro_use] -extern crate criterion; -extern crate rayon; -extern crate wyrm; +#![allow(clippy::many_single_char_names)] +use criterion::{criterion_group, criterion_main, Criterion}; use rayon::prelude::*; +use std::sync::Arc; -use criterion::Criterion; - -use wyrm::{ - nn::{lstm, xavier_normal}, DataInput, HogwildParameter, ParameterNode, SGD +use amadeus_ml::{ + nn::{lstm, xavier_normal}, optim::{Optimizer, SGD}, DataInput, HogwildParameter, ParameterNode }; fn bench_node_reuse(c: &mut Criterion) { @@ -20,7 +15,7 @@ fn bench_node_reuse(c: &mut Criterion) { let x = ParameterNode::new(xavier_normal(1, dim)); let y = ParameterNode::new(xavier_normal(dim, 10)); let v = x.dot(&y); - let z = v.clone() + v.clone() + v.clone() + v.clone(); + let z = v.clone() + v.clone() + v.clone() + v; b.iter(|| { z.forward(); @@ -60,7 +55,7 @@ fn bench_matrix_multiply(c: &mut Criterion) { // let x = vec![1.0; 32]; // let max = 1.0; -// b.iter(|| x.iter().map(|&x| wyrm::exp(x - max)).sum::().ln()) +// b.iter(|| x.iter().map(|&x| amadeus_ml::exp(x - max)).sum::().ln()) // }) // } @@ -194,15 +189,15 @@ fn bench_lstm(c: &mut Criterion) { let input_dim = 16; let hidden_dim = 32; - let lstm_params = lstm::Parameters::new(input_dim, hidden_dim); + let lstm_params = lstm::Parameters::new(input_dim, hidden_dim, &mut rand::thread_rng()); let lstm = lstm_params.build(); - let final_layer = wyrm::ParameterNode::new(xavier_normal(hidden_dim, num_digits)); - let embeddings = wyrm::ParameterNode::new(xavier_normal(num_digits, input_dim)); - let y = wyrm::IndexInputNode::new(&vec![0]); + let final_layer = amadeus_ml::ParameterNode::new(xavier_normal(hidden_dim, num_digits)); + let embeddings = amadeus_ml::ParameterNode::new(xavier_normal(num_digits, input_dim)); + let y = amadeus_ml::IndexInputNode::new(&[0]); let inputs: Vec<_> = (0..sequence_length) - .map(|_| wyrm::IndexInputNode::new(&vec![0])) + .map(|_| amadeus_ml::IndexInputNode::new(&[0])) .collect(); let embeddings: Vec<_> = inputs .iter() @@ -213,8 +208,8 @@ fn bench_lstm(c: &mut Criterion) { let hidden = hidden_states.last().unwrap(); let prediction = hidden.dot(&final_layer); - let mut loss = wyrm::nn::losses::sparse_categorical_crossentropy(&prediction, &y); - let mut optimizer = SGD::new(0.05, loss.parameters()); + let mut loss = amadeus_ml::nn::losses::sparse_categorical_crossentropy(&prediction, &y); + let optimizer = SGD::new(); let digits = pi_digits(100); @@ -235,7 +230,7 @@ fn bench_lstm(c: &mut Criterion) { loss.forward(); loss.backward(1.0); - optimizer.step(); + optimizer.step(loss.parameters()); loss.zero_gradient(); } }) diff --git a/amadeus-ml/src/fast_approx.rs b/amadeus-ml/src/fast_approx.rs index 660f30ca..ce5c9526 100644 --- a/amadeus-ml/src/fast_approx.rs +++ b/amadeus-ml/src/fast_approx.rs @@ -1,5 +1,5 @@ // Direct translations from -// https://github.com/etheory/fastapprox/blob/master/fastapprox/src/fastlog.h +// https://github.com/romeric/fastapprox/blob/master/fastapprox/src/fastlog.h /*=====================================================================* * Copyright (C) 2011 Paul Mineiro * @@ -41,6 +41,8 @@ * Contact: Paul Mineiro * *=====================================================================*/ +#![allow(clippy::cast_possible_truncation, clippy::cast_precision_loss, clippy::cast_sign_loss)] + use std::f32; #[repr(C)] @@ -60,7 +62,7 @@ pub fn fastlog2(x: f32) -> f32 { unsafe { let vx = FloatUint { f: x }; let mx = UintFloat { - i: (vx.i & 0x007FFFFF) | 0x3f000000, + i: (vx.i & 0x007F_FFFF) | 0x3f00_0000, }; let mut y = vx.i as f32; y *= 1.192_092_9e-7; diff --git a/amadeus-ml/src/lib.rs b/amadeus-ml/src/lib.rs index 67ccf192..12bb376b 100644 --- a/amadeus-ml/src/lib.rs +++ b/amadeus-ml/src/lib.rs @@ -18,9 +18,7 @@ //! backpropagates through it. //! //! ```rust -//! # extern crate rand; -//! # extern crate wyrm; -//! # use wyrm::*; +//! # use amadeus_ml::*; //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } @@ -40,10 +38,8 @@ //! go through several epochs of learning: //! //! ```rust -//! # extern crate rand; -//! # extern crate wyrm; -//! # use wyrm::*; -//! # use wyrm::optim::*; +//! # use amadeus_ml::*; +//! # use amadeus_ml::optim::*; //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } @@ -78,13 +74,10 @@ //! parameters, then building a per-thread copy of the model: //! //! ```rust -//! # extern crate rand; -//! # extern crate wyrm; -//! # extern crate rayon; //! # use std::sync::Arc; //! # use rayon::prelude::*; -//! # use wyrm::*; -//! # use wyrm::optim::*; +//! # use amadeus_ml::*; +//! # use amadeus_ml::optim::*; //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } @@ -135,8 +128,29 @@ //! //! Enable the `fast-math` option to use fast approximations to transcendental functions. //! This should give substantial speed gains in networks that are `exp`, `ln`, or `tanh`-heavy. -#![deny(missing_docs, missing_debug_implementations)] -#![allow(clippy::unreadable_literal, clippy::redundant_field_names)] + +#![doc(html_root_url = "https://docs.rs/amadeus-ml/0.4.0")] +#![warn( + missing_debug_implementations, + missing_docs, + trivial_casts, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + clippy::pedantic +)] +// from https://github.com/rust-unofficial/patterns/blob/master/anti_patterns/deny-warnings.md +#![allow( + clippy::cast_precision_loss, + clippy::doc_markdown, + clippy::float_cmp, + clippy::if_not_else, + clippy::inline_always, + clippy::module_name_repetitions, + clippy::must_use_candidate, + clippy::unsafe_derive_deserialize +)] mod fast_approx; pub mod nn; @@ -144,12 +158,15 @@ mod nodes; mod numerics; pub mod optim; +use approx::AbsDiffEq; use itertools::Itertools; use std::{ - cell::RefCell, clone::Clone, ops::{Add, Deref, Div, Mul, Neg, Sub}, rc::Rc + cell::RefCell, clone::Clone, ops::{Add, Div, Mul, Neg, Sub}, rc::Rc }; -use nodes::*; +use nodes::{ + AddNode, ConcatenateNode, DivNode, DotNode, ExpNode, IndexNode, LogNode, LogSoftmaxNode, MulNode, NegNode, ReluNode, SigmoidNode, SliceNode, SoftmaxNode, SquareNode, SubNode, SumNode, TanhNode, TransposeNode, VectorDotNode +}; pub use nodes::{Bor, HogwildParameter, IndexInputNode, InputNode, Node, ParameterNode}; pub use numerics::simd_dot; @@ -180,9 +197,9 @@ fn merge_parameters( xs.iter() .merge_join_by(ys.iter(), |x, y| x.as_ptr().cmp(&y.as_ptr())) .map(|either| match either { - itertools::EitherOrBoth::Left(x) => x, - itertools::EitherOrBoth::Right(x) => x, - itertools::EitherOrBoth::Both(x, _) => x, + itertools::EitherOrBoth::Left(x) + | itertools::EitherOrBoth::Right(x) + | itertools::EitherOrBoth::Both(x, _) => x, }) .cloned() .collect() @@ -265,10 +282,7 @@ where /// Box the variable, erasing its specific type. Use to manage the complexity /// of variable types in deep computation graphs. pub fn boxed(&self) -> Variable>> { - Variable::new( - Rc::new(self.node.clone() as Rc>), - self.parameters.clone(), - ) + Variable::new(Rc::new(self.node.clone()), self.parameters.clone()) } /// Run the backward pass through the subgraph terminating at this node. @@ -292,9 +306,9 @@ where /// Clip the value. Useful for clipping losses. pub fn clip(&mut self, min: f32, max: f32) { let bor_value = self.node.value(); - let value: &Arr = bor_value.deref(); + let value: *const Arr = &*bor_value; #[allow(clippy::cast_ref_to_mut)] - let value = unsafe { &mut *(value as *const Arr as *mut Arr) }; + let value = unsafe { &mut *(value as *mut Arr) }; value .as_slice_mut() @@ -444,7 +458,7 @@ impl Variable { } fn as_ptr(&self) -> *const ParameterNode { - self.node.deref() as *const ParameterNode + &*self.node } /// Row-wise indexing of this parameter node. Primiarily used @@ -476,7 +490,7 @@ where impl<'value> DataInput<&'value Arr> for Variable { fn set_value(&self, value: &Arr) { - let param_value = unsafe { &mut *(self.node.value.deref().value.as_ptr()) }; + let param_value = unsafe { &mut *(self.node.value.value.as_ptr()) }; param_value.assign(value) } } @@ -533,7 +547,7 @@ macro_rules! impl_arithmetic_op { { type Output = Variable<$node>; fn $fn(self, other: f32) -> Self::Output { - let constant = InputNode::new(self.value().deref() * 0.0 + other); + let constant = InputNode::new(&*self.value() * 0.0 + other); Variable::new( Rc::new($node::new(self.node, constant.node)), @@ -550,7 +564,7 @@ macro_rules! impl_arithmetic_op { { type Output = Variable<$node>; fn $fn(self, other: Variable) -> Self::Output { - let constant = InputNode::new(other.value().deref() * 0.0 + self); + let constant = InputNode::new(&*other.value() * 0.0 + self); Variable::new( Rc::new($node::new(constant.node, other.node)), @@ -633,7 +647,7 @@ where /// Assert two arrays are within `tol` of each other. pub fn assert_close(x: &Arr, y: &Arr, tol: f32) { assert!( - x.all_close(y, tol), + x.abs_diff_eq(y, tol), "{:#?} not within {} of {:#?}", x, tol, @@ -684,7 +698,7 @@ mod tests { let y = ParameterNode::new(random_matrix(1, 1)); let z = x + y; - let z = z.clone() + z.clone(); + let z = z.clone() + z; assert_eq!(z.parameters().len(), 2); } @@ -705,7 +719,7 @@ mod tests { let mut x = ParameterNode::new(random_matrix(1, 1)); let mut y = ParameterNode::new(random_matrix(1, 1)); let z = x.clone() - (y.clone() - x.clone()); - let mut z = z.clone() * 2.0 + z.clone().sigmoid(); + let mut z = z.clone() * 2.0 + z.sigmoid(); let (difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&difference, &gradient, TOLERANCE); @@ -717,7 +731,7 @@ mod tests { let mut x = ParameterNode::new(random_matrix(10, 10)); let mut y = ParameterNode::new(random_matrix(10, 10)); let z = x.clone() * y.clone(); - let mut z = z.clone() + z.clone(); + let mut z = z.clone() + z; let (difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&difference, &gradient, TOLERANCE); @@ -728,7 +742,7 @@ mod tests { fn div_finite_difference() { let mut x = ParameterNode::new(random_matrix(1, 1)); let y = ParameterNode::new(random_matrix(1, 1)); - let mut z = (x.clone() + x.clone()) / y.clone(); + let mut z = (x.clone() + x.clone()) / y; let (finite_difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&finite_difference, &gradient, TOLERANCE); @@ -738,7 +752,7 @@ mod tests { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut y = ParameterNode::new(random_matrix(10, 5)); let z = x.vector_dot(&y); - let mut z = z.clone() + z.clone(); + let mut z = z.clone() + z; let (difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&difference, &gradient, TOLERANCE); @@ -762,8 +776,8 @@ mod tests { fn dot_accumulation_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let mut y = ParameterNode::new(random_matrix(5, 10)); - let z = x.clone().dot(&y); - let mut v = z.clone() * z.clone(); + let z = x.dot(&y); + let mut v = z.clone() * z; let (difference, gradient) = finite_difference(&mut x, &mut v); assert_close(&difference, &gradient, TOLERANCE); @@ -840,7 +854,7 @@ mod tests { fn sigmoid_finite_difference() { let mut x = ParameterNode::new(random_matrix(10, 5)); let z = (x.clone() + x.clone()).sigmoid(); - let mut z = z.clone() + z.clone(); + let mut z = z.clone() + z; let (finite_difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&finite_difference, &gradient, TOLERANCE); @@ -876,7 +890,7 @@ mod tests { let mut z = (x.clone() + x.clone()).log_softmax(); let v = (x.clone() + x.clone()).softmax().ln(); - assert_close(v.value().deref(), z.value().deref(), TOLERANCE); + assert_close(&*v.value(), &*z.value(), TOLERANCE); let (finite_difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&finite_difference, &gradient, TOLERANCE); @@ -885,7 +899,7 @@ mod tests { fn sparse_categorical_cross_entropy_finite_difference() { let mut x = ParameterNode::new(random_matrix(1, 10)); let z = x.clone() + x.clone(); - let idx = IndexInputNode::new(&vec![0][..]); + let idx = IndexInputNode::new(&[0][..]); let mut loss = nn::losses::sparse_categorical_crossentropy(&z, &idx); let (finite_difference, gradient) = finite_difference(&mut x, &mut loss); @@ -898,10 +912,10 @@ mod tests { //let v = x.clone() + y.clone(); let z = x.stack(&y, ndarray::Axis(0)); - let mut z = z.clone().sigmoid() * z.clone().relu(); + let mut z = z.sigmoid() * z.relu(); - assert_eq!(z.value().rows(), 20); - assert_eq!(z.value().cols(), 5); + assert_eq!(z.value().nrows(), 20); + assert_eq!(z.value().ncols(), 5); let (difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&difference, &gradient, TOLERANCE); @@ -917,8 +931,8 @@ mod tests { let mut z = x.stack(&y, ndarray::Axis(1)).sigmoid(); - assert_eq!(z.value().rows(), 10); - assert_eq!(z.value().cols(), 10); + assert_eq!(z.value().nrows(), 10); + assert_eq!(z.value().ncols(), 10); let (difference, gradient) = finite_difference(&mut x, &mut z); assert_close(&difference, &gradient, TOLERANCE); @@ -934,12 +948,12 @@ mod tests { let x_1 = x.slice(s![.., 10..20]); let x_2 = x.slice(s![.., 20..30]); - assert_eq!(x_0.value().rows(), 10); - assert_eq!(x_0.value().cols(), 10); - assert_eq!(x_1.value().rows(), 10); - assert_eq!(x_1.value().cols(), 10); - assert_eq!(x_2.value().rows(), 10); - assert_eq!(x_2.value().cols(), 10); + assert_eq!(x_0.value().nrows(), 10); + assert_eq!(x_0.value().ncols(), 10); + assert_eq!(x_1.value().nrows(), 10); + assert_eq!(x_1.value().ncols(), 10); + assert_eq!(x_2.value().nrows(), 10); + assert_eq!(x_2.value().ncols(), 10); let mut z = (x_0 + x_1 + x_2).sigmoid(); @@ -977,11 +991,11 @@ mod tests { let optimizer = Adagrad::new().learning_rate(0.5); for _ in 0..num_epochs { - let _x = arr2(&[[rand::thread_rng().gen()]]); - let _y = 0.5 * &_x + 0.2; + let x_ = arr2(&[[rand::thread_rng().gen()]]); + let y_ = 0.5 * &x_ + 0.2; - x.set_value(&_x); - y.set_value(&_y); + x.set_value(&x_); + y.set_value(&y_); loss.forward(); loss.backward(1.0); @@ -1019,15 +1033,15 @@ mod tests { let optimizer = SGD::new().learning_rate(0.1); for _ in 0..num_epochs { - let _x = arr2(&[[ + let x_ = arr2(&[[ rand::thread_rng().gen(), rand::thread_rng().gen(), rand::thread_rng().gen(), ]]); - let _y = &_x.dot(&coefficients) + 5.0; + let y_ = &x_.dot(&coefficients) + 5.0; - x.set_value(&_x); - y.set_value(&_y); + x.set_value(&x_); + y.set_value(&y_); loss.forward(); loss.backward(1.0); @@ -1070,7 +1084,7 @@ mod tests { let v_vec = v_embedding.index(&v_index); let y_hat = u_vec.vector_dot(&v_vec); - let mut loss = (output.clone() - y_hat.clone()).square(); + let mut loss = (output.clone() - y_hat).square(); let num_epochs = 200; let optimizer = Adagrad::new().learning_rate(0.1); @@ -1130,7 +1144,7 @@ mod tests { let v_vec = v_embedding.index(&v_index); let y_hat = u_vec.vector_dot(&v_vec); - let mut loss = (output.clone() - y_hat.clone()).square(); + let mut loss = (output.clone() - y_hat).square(); let num_epochs = 100; @@ -1200,7 +1214,7 @@ mod tests { let v_vec = v_embedding.index(&v_index); let y_hat = u_vec.vector_dot(&v_vec); - let mut loss = (output.clone() - y_hat.clone()).square(); + let mut loss = (output.clone() - y_hat).square(); let num_epochs = 100; diff --git a/amadeus-ml/src/nn/losses.rs b/amadeus-ml/src/nn/losses.rs index 545a0c76..3d4b7535 100644 --- a/amadeus-ml/src/nn/losses.rs +++ b/amadeus-ml/src/nn/losses.rs @@ -2,7 +2,7 @@ use itertools::izip; use std::{ - cell::{Ref, RefCell}, ops::Deref, rc::Rc + cell::{Ref, RefCell}, rc::Rc }; use crate::{ @@ -43,7 +43,7 @@ where { pub(crate) fn new(operand: Rc, y: Rc) -> Self { assert!( - operand.value().rows() == 1, + operand.value().nrows() == 1, "Minibatches not supported: rows must be 1." ); @@ -63,7 +63,7 @@ where let mut loss_value = Arr::zeros((1, 1)); loss_value.fill(scalar_loss); - let gradient = operand.value().deref() * 0.0; + let gradient = &*operand.value() * 0.0; let needs_gradient = operand.needs_gradient(); SparseCategoricalCrossentropyNode { @@ -100,10 +100,10 @@ where let softmax_value = self.log_softmax.value(); debug_assert!( - softmax_value.rows() == 1, + softmax_value.nrows() == 1, "Minibatches not supported: rows must be 1." ); - let softmax_slice = softmax_value.deref().as_slice().unwrap(); + let softmax_slice = softmax_value.as_slice().unwrap(); let mut loss_value = 0.0; diff --git a/amadeus-ml/src/nn/lstm.rs b/amadeus-ml/src/nn/lstm.rs index bcbf2fa5..391d55ce 100644 --- a/amadeus-ml/src/nn/lstm.rs +++ b/amadeus-ml/src/nn/lstm.rs @@ -4,19 +4,15 @@ //! then applying it to your inputs: //! //! ```rust -//! # extern crate rand; -//! # extern crate wyrm; -//! # extern crate ndarray; //! # use std::sync::Arc; //! # use std::rc::Rc; //! # -//! # use wyrm::{HogwildParameter, InputNode, Node, ParameterNode}; +//! # use amadeus_ml::{HogwildParameter, InputNode, Node, ParameterNode}; //! # -//! # use wyrm::nn::xavier_normal; -//! # use wyrm::nn::lstm; +//! # use amadeus_ml::nn::xavier_normal; +//! # use amadeus_ml::nn::lstm; //! # -//! # use wyrm::{Arr, Variable}; -//! # fn main() { +//! # use amadeus_ml::{Arr, Variable}; //! let input_dim = 10; //! let hidden_dim = 5; //! @@ -40,7 +36,6 @@ //! //! // Reset the hidden state between sequences //! lstm.reset_state(); -//! # } //! ``` use serde::{Deserialize, Serialize}; @@ -48,7 +43,7 @@ use std::{rc::Rc, sync::Arc}; use super::uniform; use crate::{ - nodes::{self, HogwildParameter, Node, ParameterNode}, Arr, DataInput, Variable + nodes::{HogwildParameter, InputNode, Node, ParameterNode}, Arr, DataInput, Variable }; /// Holds shared parameters for an LSTM cell. @@ -60,17 +55,17 @@ pub struct Parameters { input_dim: usize, hidden_dim: usize, - forget_weights: Arc, - forget_biases: Arc, + forget_weights: Arc, + forget_biases: Arc, - update_gate_weights: Arc, - update_gate_biases: Arc, + update_gate_weights: Arc, + update_gate_biases: Arc, - update_value_weights: Arc, - update_value_biases: Arc, + update_value_weights: Arc, + update_value_biases: Arc, - output_gate_weights: Arc, - output_gate_biases: Arc, + output_gate_weights: Arc, + output_gate_biases: Arc, } impl Clone for Parameters { @@ -270,8 +265,8 @@ impl Cell { #[derive(Debug)] pub struct Layer { cell: Cell, - state: Variable, - hidden: Variable, + state: Variable, + hidden: Variable, } impl Layer { @@ -280,8 +275,8 @@ impl Layer { Layer { cell, - state: nodes::InputNode::new(Arr::zeros((1, hidden_dim))), - hidden: nodes::InputNode::new(Arr::zeros((1, hidden_dim))), + state: InputNode::new(Arr::zeros((1, hidden_dim))), + hidden: InputNode::new(Arr::zeros((1, hidden_dim))), } } /// Construct an LSTM layer over given inputs, returning the emitted @@ -316,21 +311,18 @@ impl Layer { #[cfg(test)] mod tests { - use std::ops::Deref; + use approx::AbsDiffEq; use super::*; use crate::{ - finite_difference, nn::{losses::sparse_categorical_crossentropy, xavier_normal}, optim::{Adam, Optimizer} + finite_difference, nn::{losses::sparse_categorical_crossentropy, xavier_normal}, nodes::{IndexInputNode, InputNode}, optim::{Adam, Optimizer} }; - use nodes::InputNode; - use DataInput; - const TOLERANCE: f32 = 0.2; fn assert_close(x: &Arr, y: &Arr, tol: f32) { assert!( - x.all_close(y, tol), + x.abs_diff_eq(y, tol), "{:#?} not within {} of {:#?}", x, tol, @@ -342,8 +334,7 @@ mod tests { let pi_str = include_str!("pi.txt"); pi_str .chars() - .filter_map(|x| x.to_digit(10)) - .map(|x| x as usize) + .map(|x| x.to_digit(10).unwrap() as usize) .take(num) .collect() } @@ -370,7 +361,7 @@ mod tests { let mut params = hidden.parameters().to_owned(); - for x in params.iter_mut() { + for x in &mut params { let (difference, gradient) = finite_difference(x, hidden); assert_close(&difference, &gradient, TOLERANCE); } @@ -432,10 +423,10 @@ mod tests { let final_layer = ParameterNode::new(xavier_normal(hidden_dim, num_digits)); let embeddings = ParameterNode::new(xavier_normal(num_digits, input_dim)); - let y = nodes::IndexInputNode::new(&vec![0]); + let y = IndexInputNode::new(&[0]); let inputs: Vec<_> = (0..sequence_length) - .map(|_| nodes::IndexInputNode::new(&vec![0])) + .map(|_| IndexInputNode::new(&[0])) .collect(); let embeddings: Vec<_> = inputs .iter() @@ -460,12 +451,7 @@ mod tests { correct = 0; total = 0; - for i in 0..(digits.len() - sequence_length - 1) { - let digit_chunk = &digits[i..(i + sequence_length + 1)]; - if digit_chunk.len() < sequence_length + 1 { - break; - } - + for digit_chunk in digits.windows(sequence_length + 1) { for (&digit, input) in digit_chunk[..digit_chunk.len() - 1].iter().zip(&inputs) { input.set_value(digit); } @@ -481,7 +467,7 @@ mod tests { optimizer.step(loss.parameters()); loss.zero_gradient(); - if target_digit == predicted_label(prediction.value().deref()) { + if target_digit == predicted_label(&*prediction.value()) { correct += 1; } diff --git a/amadeus-ml/src/nn/mod.rs b/amadeus-ml/src/nn/mod.rs index 31684062..90be2272 100644 --- a/amadeus-ml/src/nn/mod.rs +++ b/amadeus-ml/src/nn/mod.rs @@ -3,18 +3,19 @@ pub mod losses; pub mod lstm; -use rand::distributions::{Distribution, Normal, Uniform}; +use rand::distributions::{Distribution, Uniform}; +use rand_distr::Normal; use crate::Arr; /// Return a Xavier-normal initialised random array. pub fn xavier_normal(rows: usize, cols: usize) -> Arr { - let normal = Normal::new(0.0, 1.0 / (rows as f64).sqrt()); - Arr::zeros((rows, cols)).map(|_| normal.sample(&mut rand::thread_rng()) as f32) + let normal = Normal::::new(0.0, 1.0 / (rows as f32).sqrt()).unwrap(); + Arr::zeros((rows, cols)).map(|_| normal.sample(&mut rand::thread_rng())) } /// Return a random matrix with values drawn uniformly from `(min, max)`. pub fn uniform(rows: usize, cols: usize, min: f32, max: f32, rng: &mut R) -> Arr { let dist = Uniform::new(min, max); - Arr::zeros((rows, cols)).map(|_| dist.sample(rng) as f32) + Arr::zeros((rows, cols)).map(|_| dist.sample(rng)) } diff --git a/amadeus-ml/src/nodes.rs b/amadeus-ml/src/nodes.rs index 2a17870a..b7a75476 100644 --- a/amadeus-ml/src/nodes.rs +++ b/amadeus-ml/src/nodes.rs @@ -4,7 +4,7 @@ use ndarray::Axis; use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use std::{ - cell::{Cell, Ref, RefCell}, fmt, ops::{AddAssign, Deref, DerefMut}, rc::Rc, sync::Arc + cell::{Cell, Ref, RefCell}, convert::TryInto, fmt, ops::{AddAssign, Deref}, rc::Rc, sync::Arc }; use super::{clamp, Arr, Variable}; @@ -94,15 +94,15 @@ impl<'value, T: 'value> Deref for Bor<'value, T> { type Target = T; fn deref(&self) -> &T { match *self { - Bor::RefGuard(ref val) => val.deref(), - Bor::Reference(ref val) => val.deref(), + Bor::RefGuard(ref val) => &*val, + Bor::Reference(ref val) => &*val, } } } impl<'value, T: 'value + fmt::Display> fmt::Display for Bor<'value, T> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.deref()) + write!(f, "{}", &**self) } } @@ -132,19 +132,19 @@ impl Node for Rc> { type Value = Arr; type InputGradient = Arr; fn forward(&self) { - self.deref().forward() + (**self).forward() } fn backward(&self, gradient: &Ref) { - self.deref().backward(gradient) + (**self).backward(gradient) } fn value(&self) -> Bor { - self.deref().value() + (**self).value() } fn needs_gradient(&self) -> bool { - self.deref().needs_gradient() + (**self).needs_gradient() } fn clear(&self) { - self.deref().clear() + (**self).clear() } } @@ -165,8 +165,8 @@ where { pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = lhs.value().deref() + rhs.value().deref(); - let gradient = rhs.value().deref() * 0.0; + let value = &*lhs.value() + &*rhs.value(); + let gradient = &*rhs.value() * 0.0; AddNode { value: RefCell::new(value), @@ -222,11 +222,11 @@ where match self.counter.backward() { BackwardAction::Set => { let mut operand_gradient = self.gradient.borrow_mut(); - operand_gradient.slice_assign(gradient.deref()); + operand_gradient.slice_assign(&**gradient); } BackwardAction::Increment => { let mut operand_gradient = self.gradient.borrow_mut(); - operand_gradient.slice_add_assign(gradient.deref()); + operand_gradient.slice_add_assign(&**gradient); } } @@ -335,7 +335,7 @@ fn row_wise_stack_gradient(gradient: &Arr, lhs: &mut Arr, rhs: &mut Arr, op: &Ba #[derive(Debug)] pub struct ConcatenateNode { - axis: ndarray::Axis, + axis: Axis, value: RefCell, lhs_gradient: RefCell, rhs_gradient: RefCell, @@ -350,17 +350,14 @@ where LHS: Node, RHS: Node, { - pub fn new(lhs: Rc, rhs: Rc, axis: ndarray::Axis) -> Self { + pub fn new(lhs: Rc, rhs: Rc, axis: Axis) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = ndarray::stack( - axis, - &[lhs.value().deref().view(), rhs.value().deref().view()], - ) - .expect("Unable to concatenate arrays."); + let value = ndarray::stack(axis, &[lhs.value().view(), rhs.value().view()]) + .expect("Unable to concatenate arrays."); - let lhs_gradient = lhs.value().deref() * 0.0; - let rhs_gradient = rhs.value().deref() * 0.0; + let lhs_gradient = &*lhs.value() * 0.0; + let rhs_gradient = &*rhs.value() * 0.0; ConcatenateNode { axis, @@ -397,13 +394,9 @@ where match self.axis { // Vertically - ndarray::Axis(0) => { - row_wise_stack(self_value.deref_mut(), lhs_value.deref(), rhs_value.deref()) - } + Axis(0) => row_wise_stack(&mut *self_value, &*lhs_value, &*rhs_value), // Horizontally - ndarray::Axis(1) => { - column_wise_stack(self_value.deref_mut(), lhs_value.deref(), rhs_value.deref()) - } + Axis(1) => column_wise_stack(&mut *self_value, &*lhs_value, &*rhs_value), // Not allowed _ => panic!("Stacking tensors not allowed."), } @@ -414,16 +407,16 @@ where let mut rhs_grad = self.rhs_gradient.borrow_mut(); match self.axis { - ndarray::Axis(0) => row_wise_stack_gradient( + Axis(0) => row_wise_stack_gradient( gradient, - lhs_grad.deref_mut(), - rhs_grad.deref_mut(), + &mut *lhs_grad, + &mut *rhs_grad, &self.counter.backward(), ), - ndarray::Axis(1) => column_wise_stack_gradient( + Axis(1) => column_wise_stack_gradient( gradient, - lhs_grad.deref_mut(), - rhs_grad.deref_mut(), + &mut *lhs_grad, + &mut *rhs_grad, &self.counter.backward(), ), _ => panic!("Stacking tensors not allowed."), @@ -473,13 +466,13 @@ where let value = { let val = lhs.value(); let sliced = val.slice(slice); - let mut value = Arr::zeros((sliced.rows(), sliced.cols())); + let mut value = Arr::zeros((sliced.nrows(), sliced.ncols())); value.assign(&sliced); value }; - let lhs_gradient = lhs.value().deref() * 0.0; + let lhs_gradient = &*lhs.value() * 0.0; SliceNode { slice: *slice, @@ -515,13 +508,13 @@ where self.lhs_gradient .borrow_mut() .slice_mut(&self.slice) - .assign(gradient.deref()); + .assign(&*gradient); } BackwardAction::Increment => { self.lhs_gradient .borrow_mut() .slice_mut(&self.slice) - .add_assign(gradient.deref()); + .add_assign(&**gradient); } } @@ -612,12 +605,14 @@ impl GradientAccumulator { } pub fn add_sparse_row(&mut self, idx: usize, grad: &ndarray::ArrayView) { - if self.sparse_index.add(idx as u32) { + if self.sparse_index.add(idx.try_into().unwrap()) { self.gradient - .subview_mut(Axis(0), idx) + .index_axis_mut(Axis(0), idx) .slice_add_assign(grad); } else { - self.gradient.subview_mut(Axis(0), idx).slice_assign(grad); + self.gradient + .index_axis_mut(Axis(0), idx) + .slice_assign(grad); } self.sparse = true; @@ -631,7 +626,7 @@ impl GradientAccumulator { idx.into_iter().map(move |idx| { let idx = idx as usize; - (idx, grad.subview(Axis(0), idx)) + (idx, grad.index_axis(Axis(0), idx)) }) } @@ -657,7 +652,7 @@ impl GradientAccumulator { } else { let mut grad = &self.gradient * 0.0; for (idx, row) in self.sparse_iter() { - grad.subview_mut(Axis(0), idx).slice_assign(&row); + grad.index_axis_mut(Axis(0), idx).slice_assign(&row); } grad } @@ -677,7 +672,7 @@ impl GradientAccumulator { unimplemented!(); // for (idx, row) in self.sparse_iter() { // self.gradient - // .subview_mut(Axis(0), idx) + // .index_axis_mut(Axis(0), idx) // .fast_slice_mut() // .iter_mut() // .for_each(|x| *x = clamp(*x, min, max)); @@ -692,7 +687,7 @@ pub trait GradientSink { impl<'a, 'b> GradientSink<&'a Ref<'b, Arr>> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: &Ref) { - self.add_dense(gradient.deref()); + self.add_dense(gradient); } } @@ -704,7 +699,7 @@ impl<'a> GradientSink<&'a Arr> for GradientAccumulator { impl<'a> GradientSink<&'a mut Arr> for GradientAccumulator { fn accumulate_gradient(&mut self, gradient: &'a mut Arr) { - self.add_dense(gradient.deref()); + self.add_dense(gradient); } } @@ -757,7 +752,7 @@ impl HogwildParameter { pub fn new(value: Arr) -> Self { let squared_gradients = &value * 0.0; let moments = &value * 0.0; - let shape = (value.rows(), value.cols()); + let shape = (value.nrows(), value.ncols()); HogwildParameter { shape, @@ -805,8 +800,8 @@ impl ParameterNode { // This method can be called in multiple threads, so borrowing // (even immutably) will read to borrow failures. ( - (*value.value.as_ptr()).rows(), - (*value.value.as_ptr()).cols(), + (*value.value.as_ptr()).nrows(), + (*value.value.as_ptr()).ncols(), ) }; @@ -821,7 +816,7 @@ impl ParameterNode { /// Create a new parameter node. The parameters held by this node /// cannot be shared and optimized in parallel. pub fn new(value: Arr) -> Variable { - let shape = (value.rows(), value.cols()); + let shape = (value.nrows(), value.ncols()); let node = Rc::new(ParameterNode { value: Arc::new(HogwildParameter::new(value)), @@ -845,7 +840,7 @@ impl Node for ParameterNode { self.gradient.borrow_mut().accumulate_gradient(gradient); } fn value(&self) -> Bor { - Bor::Reference(unsafe { &*(self.value.value.as_ptr() as *const Arr) }) + Bor::Reference(unsafe { &*self.value.value.as_ptr() }) } fn needs_gradient(&self) -> bool { true @@ -875,10 +870,10 @@ where { pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = lhs.value().deref() - rhs.value().deref(); + let value = &*lhs.value() - &*rhs.value(); - let rhs_gradient = rhs.value().deref() * 0.0; - let lhs_gradient = lhs.value().deref() * 0.0; + let rhs_gradient = &*rhs.value() * 0.0; + let lhs_gradient = &*lhs.value() * 0.0; SubNode { value: RefCell::new(value), @@ -909,11 +904,7 @@ where let mut dest = self.value.borrow_mut(); - numerics::sub( - self.lhs.value().deref(), - self.rhs.value().deref(), - dest.deref_mut(), - ); + numerics::sub(&*self.lhs.value(), &*self.rhs.value(), &mut *dest); } fn backward(&self, gradient: &Ref) { @@ -937,10 +928,10 @@ where } BackwardAction::Increment => { let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - rhs_gradient.slice_sub_assign(gradient.deref()); + rhs_gradient.slice_sub_assign(&**gradient); let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - lhs_gradient.slice_add_assign(gradient.deref()); + lhs_gradient.slice_add_assign(&**gradient); } } @@ -982,7 +973,7 @@ where { pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = lhs.value().deref() * rhs.value().deref(); + let value = &*lhs.value() * &*rhs.value(); let lhs_gradient = &value * 0.0; let rhs_gradient = &value * 0.0; @@ -1016,45 +1007,25 @@ where let mut dest = self.value.borrow_mut(); - numerics::mul( - self.lhs.value().deref(), - self.rhs.value().deref(), - dest.deref_mut(), - ); + numerics::mul(&*self.lhs.value(), &*self.rhs.value(), &mut *dest); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { BackwardAction::Set => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); - numerics::mul( - self.rhs.value().deref(), - gradient.deref(), - lhs_gradient.deref_mut(), - ); + numerics::mul(&*self.rhs.value(), &*gradient, &mut *lhs_gradient); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::mul( - self.lhs.value().deref(), - gradient.deref(), - rhs_gradient.deref_mut(), - ); + numerics::mul(&*self.lhs.value(), &*gradient, &mut *rhs_gradient); } BackwardAction::Increment => { let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::increment_mul( - self.rhs.value().deref(), - gradient.deref(), - lhs_gradient.deref_mut(), - ); - numerics::increment_mul( - self.lhs.value().deref(), - gradient.deref(), - rhs_gradient.deref_mut(), - ); + numerics::increment_mul(&*self.rhs.value(), &*gradient, &mut *lhs_gradient); + numerics::increment_mul(&*self.lhs.value(), &*gradient, &mut *rhs_gradient); } } @@ -1096,7 +1067,7 @@ where { pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = lhs.value().deref() / rhs.value().deref(); + let value = &*lhs.value() / &*rhs.value(); let lhs_gradient = &value * 0.0; let rhs_gradient = &value * 0.0; @@ -1130,11 +1101,7 @@ where let mut dest = self.value.borrow_mut(); - numerics::div( - self.lhs.value().deref(), - self.rhs.value().deref(), - dest.deref_mut(), - ); + numerics::div(&*self.lhs.value(), &*self.rhs.value(), &mut *dest); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { @@ -1142,11 +1109,7 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); - numerics::div( - gradient.deref(), - rhs_value.deref(), - lhs_gradient.deref_mut(), - ); + numerics::div(&*gradient, &*rhs_value, &mut *lhs_gradient); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -1164,11 +1127,7 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let rhs_value = self.rhs.value(); - numerics::increment_div( - gradient.deref(), - rhs_value.deref(), - lhs_gradient.deref_mut(), - ); + numerics::increment_div(&*gradient, &*rhs_value, &mut *lhs_gradient); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); @@ -1226,11 +1185,11 @@ where { pub fn new(lhs: Rc, rhs: Rc) -> Self { let needs_gradient = lhs.needs_gradient() || rhs.needs_gradient(); - let value = lhs.value().dot(rhs.value().deref()); + let value = lhs.value().dot(&*rhs.value()); let gradient = &value * 0.0; - let lhs_gradient = lhs.value().deref() * 0.0; - let rhs_gradient = rhs.value().deref() * 0.0; + let lhs_gradient = &*lhs.value() * 0.0; + let rhs_gradient = &*rhs.value() * 0.0; DotNode { value: RefCell::new(value), @@ -1263,22 +1222,20 @@ where numerics::mat_mul( 1.0, - self.lhs.value().deref(), - self.rhs.value().deref(), + &*self.lhs.value(), + &*self.rhs.value(), 0.0, - self.value.borrow_mut().deref_mut(), + &mut *self.value.borrow_mut(), ); } fn backward(&self, gradient: &Ref) { match self.counter.backward() { BackwardAction::Set => { - self.gradient.borrow_mut().slice_assign(gradient.deref()); + self.gradient.borrow_mut().slice_assign(&**gradient); } BackwardAction::Increment => { - self.gradient - .borrow_mut() - .slice_add_assign(gradient.deref()); + self.gradient.borrow_mut().slice_add_assign(&**gradient); } } @@ -1292,20 +1249,8 @@ where let mut lhs_gradient = self.lhs_gradient.borrow_mut(); let mut rhs_gradient = self.rhs_gradient.borrow_mut(); - numerics::mat_mul( - 1.0, - gradient.deref(), - &rhs_value.t(), - 0.0, - &mut lhs_gradient, - ); - numerics::mat_mul( - 1.0, - &lhs_value.t(), - gradient.deref(), - 0.0, - &mut rhs_gradient, - ); + numerics::mat_mul(1.0, &*gradient, &rhs_value.t(), 0.0, &mut lhs_gradient); + numerics::mat_mul(1.0, &lhs_value.t(), &*gradient, 0.0, &mut rhs_gradient); } self.lhs.backward(&self.lhs_gradient.borrow()); @@ -1365,17 +1310,17 @@ where lhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()), + .map(|x| x.to_slice().unwrap()), rhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()) + .map(|x| x.to_slice().unwrap()) ) { *result = numerics::simd_dot(lhs, rhs); } - let lhs_gradient = lhs_value.deref() * 0.0; - let rhs_gradient = rhs_value.deref() * 0.0; + let lhs_gradient = &*lhs_value * 0.0; + let rhs_gradient = &*rhs_value * 0.0; (value, lhs_gradient, rhs_gradient, needs_gradient) }; @@ -1416,11 +1361,11 @@ where lhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()), + .map(|x| x.to_slice().unwrap()), rhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()) + .map(|x| x.to_slice().unwrap()) ) { *result = numerics::simd_dot(lhs, rhs); } @@ -1443,7 +1388,7 @@ where rhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()), + .map(|x| x.to_slice().unwrap()), gradient.as_slice().unwrap() ) { numerics::simd_scaled_assign(backward_row, rhs_row, gradient) @@ -1456,7 +1401,7 @@ where lhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()), + .map(|x| x.to_slice().unwrap()), gradient.as_slice().unwrap() ) { numerics::simd_scaled_assign(backward_row, lhs_row, gradient) @@ -1474,7 +1419,7 @@ where rhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()), + .map(|x| x.to_slice().unwrap()), gradient.as_slice().unwrap() ) { numerics::simd_scaled_add(backward_row, rhs_row, gradient) @@ -1487,7 +1432,7 @@ where lhs_value .genrows() .into_iter() - .map(|x| x.into_slice().unwrap()), + .map(|x| x.to_slice().unwrap()), gradient.as_slice().unwrap() ) { numerics::simd_scaled_add(backward_row, lhs_row, gradient) @@ -1560,7 +1505,7 @@ where let mut dest = self.value.borrow_mut(); - dest.assign(self.operand.value().deref()); + dest.assign(&*self.operand.value()); dest.map_inplace(|x| *x = x.powi(2)); } @@ -1650,7 +1595,7 @@ where let mut dest = self.value.borrow_mut(); - dest.assign(self.operand.value().deref()); + dest.assign(&*self.operand.value()); dest.map_inplace(|x| *x = numerics::ln(*x)); } @@ -1739,9 +1684,7 @@ where self.operand.forward(); let mut dest = self.value.borrow_mut(); - numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { - numerics::tanh(x) - }); + numerics::map_assign(&mut *dest, &*self.operand.value(), numerics::tanh); } fn backward(&self, gradient: &Ref) { @@ -1801,7 +1744,7 @@ where T: Node, { pub fn new(operand: Rc) -> Self { - let value = operand.value().deref().map(|&x| numerics::sigmoid(x)); + let value = operand.value().map(|&x| numerics::sigmoid(x)); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -1831,9 +1774,7 @@ where { let mut dest = self.value.borrow_mut(); - numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { - numerics::sigmoid(x) - }); + numerics::map_assign(&mut *dest, &*self.operand.value(), numerics::sigmoid); } } @@ -1844,7 +1785,7 @@ where numerics::map_assign_binary( &mut operand_gradient, - self.value.borrow().deref(), + &*self.value.borrow(), gradient, |sigmoid, grad| grad * sigmoid * (1.0 - sigmoid), ); @@ -1854,7 +1795,7 @@ where numerics::map_inplace_assign_binary( &mut operand_gradient, - self.value.borrow().deref(), + &*self.value.borrow(), gradient, |dest, sigmoid, grad| *dest += grad * sigmoid * (1.0 - sigmoid), ); @@ -1896,10 +1837,7 @@ where T: Node, { pub fn new(operand: Rc) -> Self { - let value = operand - .value() - .deref() - .map(|&x| if x < 0.0 { 0.0 } else { x }); + let value = operand.value().map(|&x| if x < 0.0 { 0.0 } else { x }); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -1928,7 +1866,7 @@ where let mut dest = self.value.borrow_mut(); - numerics::map_assign(dest.deref_mut(), self.operand.value().deref(), |x| { + numerics::map_assign(&mut *dest, &*self.operand.value(), |x| { if x < 0.0 { 0.0 } else { @@ -1944,7 +1882,7 @@ where numerics::map_assign_binary( &mut operand_gradient, - self.value.borrow().deref(), + &*self.value.borrow(), gradient, |x, grad| if x <= 0.0 { 0.0 } else { grad }, ); @@ -1954,7 +1892,7 @@ where numerics::map_inplace_assign_binary( &mut operand_gradient, - self.value.borrow().deref(), + &*self.value.borrow(), gradient, |dest, x, grad| *dest += if x <= 0.0 { 0.0 } else { grad }, ); @@ -1996,7 +1934,7 @@ where T: Node, { pub fn new(operand: Rc) -> Self { - let value = -operand.value().deref(); + let value = -&*operand.value(); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -2026,7 +1964,7 @@ where let mut dest = self.value.borrow_mut(); - dest.assign(self.operand.value().deref()); + dest.assign(&*self.operand.value()); dest.map_inplace(|x| *x = -*x); } @@ -2084,7 +2022,7 @@ where OP: Node, { pub fn new(operand: Rc) -> Self { - let value = operand.value().deref().map(|&x| numerics::exp(x)); + let value = operand.value().map(|&x| numerics::exp(x)); let gradient = &value * 0.0; let needs_gradient = operand.needs_gradient(); @@ -2112,7 +2050,7 @@ where self.operand.forward(); let mut dest = self.value.borrow_mut(); - dest.assign(self.operand.value().deref()); + dest.assign(&*self.operand.value()); dest.map_inplace(|x| *x = numerics::exp(*x)); } fn backward(&self, gradient: &Ref) { @@ -2170,10 +2108,10 @@ where { pub fn new(operand: Rc) -> Self { let needs_gradient = operand.needs_gradient(); - let mut value = Arr::zeros((operand.value().cols(), operand.value().rows())); + let mut value = Arr::zeros((operand.value().ncols(), operand.value().nrows())); value.assign(&operand.value().t()); let value = RefCell::new(value); - let gradient = RefCell::new(operand.value().deref() * 0.0); + let gradient = RefCell::new(&*operand.value() * 0.0); TransposeNode { value, @@ -2248,7 +2186,6 @@ where let value = { let max = operand .value() - .deref() .as_slice() .unwrap() .iter() @@ -2287,7 +2224,7 @@ where self.operand.forward(); let mut dest = self.value.borrow_mut(); - dest.slice_assign(self.operand.value().deref()); + dest.slice_assign(&*self.operand.value()); let max = self .operand @@ -2334,9 +2271,9 @@ where numerics::mat_mul( 1.0, gradient, - jacobian.deref_mut(), + &*jacobian, beta, - self.operand_gradient.borrow_mut().deref_mut(), + &mut *self.operand_gradient.borrow_mut(), ); } @@ -2374,7 +2311,7 @@ where pub fn new(operand: Rc) -> Self { let value = { let operand_value = operand.value(); - let operand_slice = operand_value.deref().as_slice().unwrap(); + let operand_slice = operand_value.as_slice().unwrap(); let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); let denominator = max @@ -2384,7 +2321,7 @@ where .sum::() .ln(); - operand_value.deref() - denominator + &*operand_value - denominator }; let gradient = &value * 0.0; @@ -2420,10 +2357,10 @@ where self.operand.forward(); let mut dest = self.value.borrow_mut(); - dest.assign(self.operand.value().deref()); + dest.assign(&*self.operand.value()); let operand_value = self.operand.value(); - let operand_slice = operand_value.deref().as_slice().unwrap(); + let operand_slice = operand_value.as_slice().unwrap(); let max = operand_slice.iter().fold(std::f32::MIN, |x, y| x.max(*y)); let denominator = max + numerics::softmax_exp_sum(operand_slice, max).ln(); @@ -2498,7 +2435,7 @@ where value }; - let gradient = operand.value().deref() * 0.0; + let gradient = &*operand.value() * 0.0; let needs_gradient = operand.needs_gradient(); SumNode { @@ -2649,19 +2586,19 @@ impl Node for IndexNode { ); for (&idx, mut row) in idx_value.iter().zip(arr_value.genrows_mut()) { - let new_val = operand_value.subview(Axis(0), idx); + let new_val = operand_value.index_axis(Axis(0), idx); row.slice_assign(&new_val); } } fn backward(&self, gradient: &Ref) { - self.counter.backward(); + let _ = self.counter.backward(); self.operand .gradient .borrow_mut() - .accumulate_gradient((&self.index_value.borrow()[..], gradient.deref())); - self.counter.recurse_backward(); + .accumulate_gradient((&self.index_value.borrow()[..], &**gradient)); + let _ = self.counter.recurse_backward(); } fn value(&self) -> Bor { @@ -2687,7 +2624,7 @@ mod tests { #[test] fn test_sub_counter() { let x = ParameterNode::new(nn::xavier_normal(1, 1)); - let y = x.clone() - x.clone(); + let y = x.clone() - x; let mut z = y.clone() + y.clone() + y.clone(); diff --git a/amadeus-ml/src/numerics.rs b/amadeus-ml/src/numerics.rs index 686708fa..1dcdab45 100644 --- a/amadeus-ml/src/numerics.rs +++ b/amadeus-ml/src/numerics.rs @@ -195,23 +195,23 @@ pub fn mat_mul( S2: Data, S3: DataMut, { - match (lhs.rows(), rhs.cols()) { + match (lhs.nrows(), rhs.ncols()) { (_, 1) => { general_mat_vec_mul( alpha, lhs, - &rhs.subview(Axis(1), 0), + &rhs.index_axis(Axis(1), 0), beta, - &mut out.subview_mut(Axis(1), 0), + &mut out.index_axis_mut(Axis(1), 0), ); } (1, _) => { general_mat_vec_mul( alpha, &rhs.t(), - &lhs.subview(Axis(0), 0), + &lhs.index_axis(Axis(0), 0), beta, - &mut out.subview_mut(Axis(0), 0), + &mut out.index_axis_mut(Axis(0), 0), ); } _ => { @@ -408,7 +408,8 @@ where #[cfg(test)] mod tests { - use rand::{self, Rng}; + use approx::AbsDiffEq; + use rand::Rng; use super::*; use crate::nn; @@ -509,14 +510,14 @@ mod tests { .map(|_| rand::thread_rng().gen()) .collect::>(); - let _dot = dot(&xs[..], &ys[..]); - let _unrolled_dot = unrolled_dot(&xs[..], &ys[..]); - let _simd_dot = simd_dot(&xs[..], &ys[..]); + let dot = dot(&xs[..], &ys[..]); + let unrolled_dot = unrolled_dot(&xs[..], &ys[..]); + let simd_dot = simd_dot(&xs[..], &ys[..]); let epsilon = 1e-5; - assert!((_dot - _unrolled_dot).abs() < epsilon); - assert!((_dot - _simd_dot).abs() < epsilon, "{} {}", _dot, _simd_dot); + assert!((dot - unrolled_dot).abs() < epsilon); + assert!((dot - simd_dot).abs() < epsilon, "{} {}", dot, simd_dot); } } @@ -539,7 +540,7 @@ mod tests { #[allow(dead_code)] fn assert_close(x: &Arr, y: &Arr, tol: f32) { assert!( - x.all_close(y, tol), + x.abs_diff_eq(y, tol), "{:#?} not within {} of {:#?}", x, tol, diff --git a/amadeus-ml/src/optim/adagrad.rs b/amadeus-ml/src/optim/adagrad.rs index 8a46e9cd..4f808f04 100644 --- a/amadeus-ml/src/optim/adagrad.rs +++ b/amadeus-ml/src/optim/adagrad.rs @@ -77,8 +77,8 @@ impl Adagrad { } } else { for (row_idx, grad) in sink.sparse_iter() { - let mut param_row = param_value.subview_mut(Axis(0), row_idx); - let mut squared_row = squared_gradient.subview_mut(Axis(0), row_idx); + let mut param_row = param_value.index_axis_mut(Axis(0), row_idx); + let mut squared_row = squared_gradient.index_axis_mut(Axis(0), row_idx); for (value, &gradient, squared_gradient) in izip!( param_row.fast_slice_mut(), diff --git a/amadeus-ml/src/optim/adam.rs b/amadeus-ml/src/optim/adam.rs index 0dff2e48..fea51a7c 100644 --- a/amadeus-ml/src/optim/adam.rs +++ b/amadeus-ml/src/optim/adam.rs @@ -62,6 +62,7 @@ impl Adam { } fn param_fields<'par>(&self, value: &'par Arc) -> AdamParameters<'par> { + let _ = self; AdamParameters { value: unsafe { value.value_mut() }, m: unsafe { value.squared_gradient_mut() }, @@ -70,6 +71,7 @@ impl Adam { } } + #[allow(clippy::trivially_copy_pass_by_ref)] #[inline(always)] fn update(&self, value: &mut f32, gradient: f32, m: &mut f32, v: &mut f32, t: &i32) { // Apply L2 to gradient. @@ -108,9 +110,9 @@ impl Adam { } } else { for (row_idx, ref grad) in sink.sparse_iter() { - let mut value_row = param.value.subview_mut(Axis(0), row_idx); - let mut m_row = param.m.subview_mut(Axis(0), row_idx); - let mut v_row = param.v.subview_mut(Axis(0), row_idx); + let mut value_row = param.value.index_axis_mut(Axis(0), row_idx); + let mut m_row = param.m.index_axis_mut(Axis(0), row_idx); + let mut v_row = param.v.index_axis_mut(Axis(0), row_idx); for (value, &gradient, m, v) in izip!( value_row.as_slice_mut().unwrap(), diff --git a/amadeus-ml/src/optim/barrier.rs b/amadeus-ml/src/optim/barrier.rs index 520c0f2b..78170597 100644 --- a/amadeus-ml/src/optim/barrier.rs +++ b/amadeus-ml/src/optim/barrier.rs @@ -146,7 +146,7 @@ impl SynchronizationBarrierCore { } fn register_thread(&self) -> usize { - self.start_barrier.increment_num_threads(); + let _ = self.start_barrier.increment_num_threads(); let thread_num = self.end_barrier.increment_num_threads(); self.thread_queue.lock().unwrap().push_back(thread_num); @@ -155,8 +155,8 @@ impl SynchronizationBarrierCore { } fn deregister_thread(&self, thread_id: usize) { - self.start_barrier.decrement_num_threads(); - self.end_barrier.decrement_num_threads(); + let _ = self.start_barrier.decrement_num_threads(); + let _ = self.end_barrier.decrement_num_threads(); self.thread_queue .lock() @@ -177,11 +177,11 @@ impl SynchronizationBarrierCore { } fn start_wait(&self) { - self.start_barrier.wait(); + let _ = self.start_barrier.wait(); } fn end_wait(&self) { - self.end_barrier.wait(); + let _ = self.end_barrier.wait(); } } diff --git a/amadeus-ml/src/optim/sgd.rs b/amadeus-ml/src/optim/sgd.rs index 825e8003..3d781d56 100644 --- a/amadeus-ml/src/optim/sgd.rs +++ b/amadeus-ml/src/optim/sgd.rs @@ -46,7 +46,7 @@ impl SGD { ) { let param_value = unsafe { param.value_mut() }; let learning_rate = self.learning_rate; - let sink = sink.deref_mut(); + let sink = &mut *sink; if let Some((min, max)) = self.clamp { sink.clamp(min, max); @@ -56,7 +56,7 @@ impl SGD { param_value.scaled_add(-self.learning_rate, sink.gradient()); } else { for (row_idx, grad) in sink.sparse_iter() { - let param_row = param_value.subview_mut(Axis(0), row_idx); + let param_row = param_value.index_axis_mut(Axis(0), row_idx); numerics::map_add_assign_slice( param_row.into_slice().unwrap(), From 28e3c2e6013ce84b8a13ed056ed4c83a2fc46c38 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sat, 8 Aug 2020 09:57:21 +0100 Subject: [PATCH 106/108] fix flaky tests --- amadeus-ml/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/amadeus-ml/src/lib.rs b/amadeus-ml/src/lib.rs index 12bb376b..537ca829 100644 --- a/amadeus-ml/src/lib.rs +++ b/amadeus-ml/src/lib.rs @@ -890,10 +890,10 @@ mod tests { let mut z = (x.clone() + x.clone()).log_softmax(); let v = (x.clone() + x.clone()).softmax().ln(); - assert_close(&*v.value(), &*z.value(), TOLERANCE); + assert_close(&*v.value(), &*z.value(), TOLERANCE * 4.0); let (finite_difference, gradient) = finite_difference(&mut x, &mut z); - assert_close(&finite_difference, &gradient, TOLERANCE); + assert_close(&finite_difference, &gradient, TOLERANCE * 4.0); } #[test] fn sparse_categorical_cross_entropy_finite_difference() { @@ -1200,7 +1200,7 @@ mod tests { let optimizer = SGD::new(); let losses: Vec = optimizer - .synchronized(rayon::current_num_threads()) + .synchronized(2.min(rayon::current_num_threads())) .into_par_iter() .map(|optimizer| { let u_embedding = ParameterNode::shared(u_parameters.clone()); @@ -1248,6 +1248,6 @@ mod tests { let sum_loss: f32 = losses.iter().sum(); - assert!(sum_loss / (losses.len() as f32) < 1e-3); + assert!(sum_loss / (losses.len() as f32) < 1e-2); } } From 2e626fde8568381ef7aac1705d88a89babda8d10 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sat, 8 Aug 2020 10:15:50 +0100 Subject: [PATCH 107/108] add ml to CI --- Cargo.toml | 8 ++++--- amadeus-ml/Cargo.toml | 12 +++++----- amadeus-ml/NOTICE.txt | 10 +++++++++ amadeus-ml/src/lib.rs | 46 +++++++++++++++++++++------------------ amadeus-ml/src/nn/lstm.rs | 11 ++-------- azure-pipelines.yml | 36 +++++++++++++++--------------- 6 files changed, 66 insertions(+), 57 deletions(-) create mode 100644 amadeus-ml/NOTICE.txt diff --git a/Cargo.toml b/Cargo.toml index 291fde01..cf053228 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,12 @@ parquet = ["amadeus-parquet", "amadeus-derive/parquet"] postgres = ["amadeus-postgres", "amadeus-derive/postgres"] csv = ["amadeus-serde", "amadeus-derive/serde"] json = ["amadeus-serde", "amadeus-derive/serde"] +ml = ["amadeus-ml"] +ml-openblas = ["amadeus-ml/openblas"] bench = ["serde-csv", "once_cell", "arrow-parquet", "rayon"] [package.metadata.docs.rs] -features = ["constellation", "aws", "commoncrawl", "parquet", "postgres", "csv", "json"] +features = ["constellation", "aws", "commoncrawl", "parquet", "postgres", "csv", "json", "ml"] [dependencies] amadeus-core = { version = "=0.4.1", path = "amadeus-core" } @@ -41,7 +43,7 @@ amadeus-derive = { version = "=0.4.1", path = "amadeus-derive" } amadeus-types = { version = "=0.4.1", path = "amadeus-types" } amadeus-aws = { version = "=0.4.1", path = "amadeus-aws", optional = true } amadeus-commoncrawl = { version = "=0.4.1", path = "amadeus-commoncrawl", optional = true } -amadeus-ml = { version = "=0.4.1", path = "amadeus-ml" } +amadeus-ml = { version = "=0.4.1", path = "amadeus-ml", optional = true } amadeus-parquet = { version = "=0.4.1", path = "amadeus-parquet", optional = true } amadeus-postgres = { version = "=0.4.1", path = "amadeus-postgres", optional = true } amadeus-serde = { version = "=0.4.1", path = "amadeus-serde", optional = true } @@ -50,7 +52,7 @@ async-channel = "1.1" bincode = { version = "1.3", optional = true } constellation-rs = { version = "0.2.0-alpha.2", default-features = false, optional = true } derive-new = "0.5" -event-listener = "=2.3.1" # https://github.com/stjepang/event-listener/issues/9 +event-listener = "2.3.3" futures = "0.3" num_cpus = "1.13" pin-project = "0.4" diff --git a/amadeus-ml/Cargo.toml b/amadeus-ml/Cargo.toml index e2f7f1ff..75d6789c 100644 --- a/amadeus-ml/Cargo.toml +++ b/amadeus-ml/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-ml" -version = "0.4.0" +version = "0.4.1" license = "Apache-2.0" authors = ["Alec Mocatta ", "Maciej Kula "] categories = ["data-structures", "algorithms", "science"] @@ -20,15 +20,17 @@ maintenance = { status = "actively-developed" } [features] fast-math = [] +openblas = ["ndarray/blas", "blas-src/openblas", "openblas-src/static"] [dependencies] approx = "0.3" +blas-src = { version = "0.2", optional = true } hibitset = "0.6" itertools = "0.9" ndarray = { version = "0.13", features = ["approx", "serde"] } +openblas-src = { version = "0.6", default-features = false, optional = true } rand = { version = "0.7", features = ["serde1"] } rand_distr = "0.2" -rayon = "1.3" serde = { version = "1", features = ["derive", "rc"] } smallvec = { version = "1.4", features = ["serde"] } @@ -36,10 +38,8 @@ smallvec = { version = "1.4", features = ["serde"] } rustversion = "1.0" [dev-dependencies] -# ndarray = { version = "0.13", features = ["blas"] } -# blas-src = { version = "0.2", default-features = false, features = ["openblas"] } -# openblas-src = { version = "0.6", default-features = false, features = ["static"] } -criterion = "0.2.3" +criterion = "0.3" +rayon = "1.3" [[bench]] name = "benchmark" diff --git a/amadeus-ml/NOTICE.txt b/amadeus-ml/NOTICE.txt new file mode 100644 index 00000000..0b7ec865 --- /dev/null +++ b/amadeus-ml/NOTICE.txt @@ -0,0 +1,10 @@ +This product includes software from the wyrm project (MIT) +https://github.com/maciejkula/wyrm + +Copyright 2017 Maciej Kula + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/amadeus-ml/src/lib.rs b/amadeus-ml/src/lib.rs index 537ca829..021be3e1 100644 --- a/amadeus-ml/src/lib.rs +++ b/amadeus-ml/src/lib.rs @@ -1,3 +1,13 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  â”‚  ðŸ“‘  GitHub  â”‚  ðŸ’¬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.3/amadeus/source/index.html). +//! +//! --- +//! //! A reverse mode, define-by-run, low-overhead autodifferentiation library. //! //! # Features @@ -17,12 +27,13 @@ //! The following defines a univariate linear regression model, then //! backpropagates through it. //! -//! ```rust +//! ``` //! # use amadeus_ml::*; +//! # //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } -//! # fn main() { +//! # //! let slope = ParameterNode::new(random_matrix(1, 1)); //! let intercept = ParameterNode::new(random_matrix(1, 1)); //! @@ -31,19 +42,18 @@ //! //! let y_hat = slope.clone() * x.clone() + intercept.clone(); //! let mut loss = (y.clone() - y_hat).square(); -//! # } //! ``` //! //! To optimize the parameters, create an optimizer object and //! go through several epochs of learning: //! -//! ```rust -//! # use amadeus_ml::*; -//! # use amadeus_ml::optim::*; +//! ``` +//! # use amadeus_ml::{*, optim::*}; +//! # //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } -//! # fn main() { +//! # //! # let slope = ParameterNode::new(random_matrix(1, 1)); //! # let intercept = ParameterNode::new(random_matrix(1, 1)); //! # let x = InputNode::new(random_matrix(1, 1)); @@ -67,21 +77,21 @@ //! //! optimizer.step(loss.parameters()); //! } -//! # } //! ``` //! //! You can use `rayon` to fit your model in parallel, by first creating a set of shared //! parameters, then building a per-thread copy of the model: //! -//! ```rust -//! # use std::sync::Arc; +//! ``` //! # use rayon::prelude::*; -//! # use amadeus_ml::*; -//! # use amadeus_ml::optim::*; +//! # use std::sync::Arc; +//! # +//! # use amadeus_ml::{*, optim::*}; +//! # //! # fn random_matrix(rows: usize, cols: usize) -> Arr { //! # Arr::zeros((rows, cols)).map(|_| rand::random::()) //! # } -//! # fn main() { +//! # //! let slope_param = Arc::new(HogwildParameter::new(random_matrix(1, 1))); //! let intercept_param = Arc::new(HogwildParameter::new(random_matrix(1, 1))); //! let num_epochs = 10; @@ -111,18 +121,12 @@ //! optimizer.step(loss.parameters()); //! } //! }); -//! # } //! ``` //! //! ## BLAS support -//! You should enable BLAS support to get (much) better performance out of matrix-multiplication-heavy -//! workloads. To do so, add the following to your `Cargo.toml`: //! -//! ```text -//! ndarray = { version = "0.11.0", features = ["blas", "serde-1"] } -//! blas-src = { version = "0.1.2", default-features = false, features = ["openblas"] } -//! openblas-src = { version = "0.5.6", default-features = false, features = ["cblas"] } -//! ``` +//! You should enable BLAS support to get (much) better performance out of matrix-multiplication-heavy +//! workloads. To do so, enable the `openblas` feature. //! //! ## Fast numerics //! diff --git a/amadeus-ml/src/nn/lstm.rs b/amadeus-ml/src/nn/lstm.rs index 391d55ce..c7f56c31 100644 --- a/amadeus-ml/src/nn/lstm.rs +++ b/amadeus-ml/src/nn/lstm.rs @@ -3,16 +3,9 @@ //! You can create an LSTM layer by first initializing its parameters, //! then applying it to your inputs: //! -//! ```rust -//! # use std::sync::Arc; -//! # use std::rc::Rc; -//! # -//! # use amadeus_ml::{HogwildParameter, InputNode, Node, ParameterNode}; -//! # -//! # use amadeus_ml::nn::xavier_normal; -//! # use amadeus_ml::nn::lstm; +//! ``` +//! # use amadeus_ml::{nn::{lstm, xavier_normal}, Arr, HogwildParameter, InputNode, Node, ParameterNode, Variable}; //! # -//! # use amadeus_ml::{Arr, Variable}; //! let input_dim = 10; //! let hidden_dim = 5; //! diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8fcf189a..386f36bb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,18 +22,18 @@ jobs: rust_toolchain: nightly rust_lint_toolchain: nightly-2020-07-26 rust_flags: '' - rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;constellation aws commoncrawl parquet postgres csv json bench' - rust_features: 'constellation aws commoncrawl parquet postgres csv json bench' - rust_doc_features: 'constellation aws commoncrawl parquet postgres csv json' + rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;ml;constellation aws commoncrawl parquet postgres csv json ml bench' + rust_features: 'constellation aws commoncrawl parquet postgres csv json ml bench' + rust_doc_features: 'constellation aws commoncrawl parquet postgres csv json ml' rust_target_check: '' rust_target_build: '' rust_target_run: '' matrix: windows: imageName: 'windows-latest' - rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;aws commoncrawl parquet postgres csv json bench' - rust_features: 'aws commoncrawl parquet postgres csv json bench' - rust_doc_features: 'aws commoncrawl parquet postgres csv json' + rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;ml;aws commoncrawl parquet postgres csv json ml bench' + rust_features: 'aws commoncrawl parquet postgres csv json ml bench' + rust_doc_features: 'aws commoncrawl parquet postgres csv json ml' rust_target_run: 'x86_64-pc-windows-msvc' mac: imageName: 'macos-latest' @@ -54,9 +54,9 @@ jobs: rust_toolchain: stable rust_lint_toolchain: nightly-2020-07-26 rust_flags: '' - rust_features_clippy: ';aws;commoncrawl;postgres;csv;json;aws commoncrawl postgres csv json' - rust_features: 'aws commoncrawl postgres csv json' - rust_doc_features: 'aws commoncrawl postgres csv json' + rust_features_clippy: ';aws;commoncrawl;postgres;csv;json;ml;aws commoncrawl postgres csv json ml' + rust_features: 'aws commoncrawl postgres csv json ml' + rust_doc_features: 'aws commoncrawl postgres csv json ml' rust_target_check: '' rust_target_build: '' rust_target_run: '' @@ -81,9 +81,9 @@ jobs: rust_lint_toolchain: nightly-2020-07-26 rust_flags: '' rust_packages: '-p amadeus-core -p amadeus-derive -p amadeus-parquet -p amadeus-serde -p amadeus-types -p amadeus' - rust_features_clippy: ';parquet;csv;json;parquet csv json' - rust_features: 'parquet csv json' - rust_doc_features: 'parquet csv json' + rust_features_clippy: ';parquet;csv;json;ml;parquet csv json ml' + rust_features: 'parquet csv json ml' + rust_doc_features: 'parquet csv json ml' rust_target_check: '' rust_target_build: '' rust_target_run: '' @@ -94,7 +94,7 @@ jobs: # rust_target_run: 'wasm32-unknown-unknown' linux: imageName: 'ubuntu-latest' - rust_target_run: 'wasm32-unknown-unknown' + rust_target_build: 'wasm32-unknown-unknown' # TODO: headless browser fails: driver status: exit code: 1 # windows: # imageName: 'windows-latest' @@ -109,19 +109,19 @@ jobs: rust_lint_toolchain: nightly-2020-07-26 rust_flags: '' rust_packages: '-p amadeus-core -p amadeus-derive -p amadeus-serde -p amadeus-types -p amadeus' - rust_features_clippy: ';csv;json;csv json' - rust_features: 'csv json' - rust_doc_features: 'csv json' + rust_features_clippy: ';csv;json;ml;csv json ml' + rust_features: 'csv json ml' + rust_doc_features: 'csv json ml' rust_target_check: '' rust_target_build: '' rust_target_run: '' matrix: mac: imageName: 'macos-latest' - rust_target_run: 'wasm32-unknown-unknown' + rust_target_build: 'wasm32-unknown-unknown' linux: imageName: 'ubuntu-latest' - rust_target_run: 'wasm32-unknown-unknown' + rust_target_build: 'wasm32-unknown-unknown' # TODO: headless browser fails: driver status: exit code: 1 # windows: # imageName: 'windows-latest' From 45e319c69794d3ca970821a316276482c7fe34e9 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sat, 8 Aug 2020 14:14:52 +0100 Subject: [PATCH 108/108] fix a stack overflow in CI on windows --- amadeus-ml/src/nn/lstm.rs | 10 +++++++--- amadeus-ml/src/nodes.rs | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/amadeus-ml/src/nn/lstm.rs b/amadeus-ml/src/nn/lstm.rs index c7f56c31..e736edb7 100644 --- a/amadeus-ml/src/nn/lstm.rs +++ b/amadeus-ml/src/nn/lstm.rs @@ -4,8 +4,11 @@ //! then applying it to your inputs: //! //! ``` -//! # use amadeus_ml::{nn::{lstm, xavier_normal}, Arr, HogwildParameter, InputNode, Node, ParameterNode, Variable}; +//! # use amadeus_ml::{nn::{lstm, xavier_normal}, InputNode}; //! # +//! # // Overflows with 1MiB stack (default on Windows), so we spawn a thread which uses Rust's +//! # // [2MiB](https://github.com/rust-lang/rust/blob/c989ac132a81503ceaaa1054da0a3a8c5c305ef0/library/std/src/sys/windows/thread.rs#L13) +//! # std::thread::spawn(|| { //! let input_dim = 10; //! let hidden_dim = 5; //! @@ -15,12 +18,12 @@ //! //! // Construct the input nodes. //! let input: Vec<_> = (0..200) -//! .map(|_| InputNode::new(xavier_normal(1, input_dim))).collect(); +//! .map(|_| InputNode::new(xavier_normal(1, input_dim))).collect(); //! //! // Construct an LSTM with 200 steps of recursion. //! let mut hidden = lstm.forward(&input); //! -//! let mut last_hidden = hidden.last_mut().unwrap(); +//! let last_hidden = hidden.last_mut().unwrap(); //! //! // Run as usual. //! last_hidden.forward(); @@ -29,6 +32,7 @@ //! //! // Reset the hidden state between sequences //! lstm.reset_state(); +//! # }).join().unwrap(); //! ``` use serde::{Deserialize, Serialize}; diff --git a/amadeus-ml/src/nodes.rs b/amadeus-ml/src/nodes.rs index b7a75476..fae5ab3f 100644 --- a/amadeus-ml/src/nodes.rs +++ b/amadeus-ml/src/nodes.rs @@ -131,18 +131,23 @@ pub trait Node: fmt::Debug + 'static { impl Node for Rc> { type Value = Arr; type InputGradient = Arr; + #[inline(always)] fn forward(&self) { (**self).forward() } + #[inline(always)] fn backward(&self, gradient: &Ref) { (**self).backward(gradient) } + #[inline(always)] fn value(&self) -> Bor { (**self).value() } + #[inline(always)] fn needs_gradient(&self) -> bool { (**self).needs_gradient() } + #[inline(always)] fn clear(&self) { (**self).clear() }