diff --git a/.gitignore b/.gitignore index 1cf4b4d7..92b3e5af 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.mod build data/*/*.dat +doc diff --git a/README.md b/README.md index accbcde4..41e55f67 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Read the paper [here](https://arxiv.org/abs/1902.06714). - [Training the network](https://github.com/modern-fortran/neural-fortran#training-the-network) - [Saving and loading from file](https://github.com/modern-fortran/neural-fortran#saving-and-loading-from-file) - [MNIST training example](https://github.com/modern-fortran/neural-fortran#mnist-training-example) +* [API documentation](https://github.com/modern-fortran/neural-fortran#api-documentation) * [Contributing](https://github.com/modern-fortran/neural-fortran#contributing) * [Contributors](https://github.com/modern-fortran/neural-fortran#contributors) * [Related projects](https://github.com/modern-fortran/neural-fortran#related-projects) @@ -369,6 +370,18 @@ for example on 16 cores using [OpenCoarrays](https://github.com/sourceryinstitut $ cafrun -n 16 ./example_mnist ``` +## API documentation + +API documentation can be generated with [FORD](https://github.com/Fortran-FOSS-Programmers/ford/). +Assuming you have FORD installed on your system, run + +``` +ford ford.md +``` + +from the neural-fortran top-level directory to generate the API documentation in doc/html. +Point your browser to doc/html/index.html to read it. + ## Contributing neural-fortran is currently a proof-of-concept with potential for diff --git a/ford.md b/ford.md new file mode 100644 index 00000000..919d201b --- /dev/null +++ b/ford.md @@ -0,0 +1,23 @@ +project: +summary: A parallel neural net microframework +src_dir: src +output_dir: doc/html +preprocess: true +display: public + protected + private +source: true +graph: true +md_extensions: markdown.extensions.toc +coloured_edges: true +sort: permission-alpha +extra_mods: iso_fortran_env:https://gcc.gnu.org/onlinedocs/gfortran/ISO_005fFORTRAN_005fENV.html + iso_c_binding:https://gcc.gnu.org/onlinedocs/gfortran/ISO_005fC_005fBINDING.html#ISO_005fC_005fBINDING +author: Milan Curcic +print_creation_date: true +creation_date: %Y-%m-%d %H:%M %z +project_github: https://github.com/modern-fortran/neural-fortran +project_download: https://github.com/modern-fortran/neural-fortran/releases +github: https://github.com/modern-fortran + +{!README.md!} diff --git a/src/mod_activation.f90 b/src/mod_activation.f90 index 083c6022..8997e581 100644 --- a/src/mod_activation.f90 +++ b/src/mod_activation.f90 @@ -1,6 +1,6 @@ module mod_activation - ! A collection of activation functions and their derivatives. + !! A collection of activation functions and their derivatives. use mod_kinds, only: ik, rk @@ -26,14 +26,14 @@ end function activation_function contains pure function gaussian(x) result(res) - ! Gaussian activation function. + !! Gaussian activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = exp(-x**2) end function gaussian pure function gaussian_prime(x) result(res) - ! First derivative of the Gaussian activation function. + !! First derivative of the Gaussian activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = -2 * x * gaussian(x) @@ -47,7 +47,7 @@ pure function relu(x) result(res) end function relu pure function relu_prime(x) result(res) - ! First derivative of the REctified Linear Unit (RELU) activation function. + !! First derivative of the REctified Linear Unit (RELU) activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) where (x > 0) @@ -58,21 +58,21 @@ pure function relu_prime(x) result(res) end function relu_prime pure function sigmoid(x) result(res) - ! Sigmoid activation function. + !! Sigmoid activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = 1 / (1 + exp(-x)) endfunction sigmoid pure function sigmoid_prime(x) result(res) - ! First derivative of the sigmoid activation function. + !! First derivative of the sigmoid activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = sigmoid(x) * (1 - sigmoid(x)) end function sigmoid_prime pure function step(x) result(res) - ! Step activation function. + !! Step activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) where (x > 0) @@ -83,24 +83,24 @@ pure function step(x) result(res) end function step pure function step_prime(x) result(res) - ! First derivative of the step activation function. + !! First derivative of the step activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = 0 end function step_prime pure function tanhf(x) result(res) - ! Tangent hyperbolic activation function. - ! Same as the intrinsic tanh, but must be - ! defined here so that we can use procedure - ! pointer with it. + !! Tangent hyperbolic activation function. + !! Same as the intrinsic tanh, but must be + !! defined here so that we can use procedure + !! pointer with it. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = tanh(x) end function tanhf pure function tanh_prime(x) result(res) - ! First derivative of the tanh activation function. + !! First derivative of the tanh activation function. real(rk), intent(in) :: x(:) real(rk) :: res(size(x)) res = 1 - tanh(x)**2 diff --git a/src/mod_layer.f90 b/src/mod_layer.f90 index 78b6a20a..4ac409cd 100644 --- a/src/mod_layer.f90 +++ b/src/mod_layer.f90 @@ -1,6 +1,6 @@ module mod_layer - ! Defines the layer type and its methods. + !! Defines the layer type and its methods. use mod_activation use mod_kinds, only: ik, rk @@ -12,13 +12,13 @@ module mod_layer public :: array1d, array2d, db_init, db_co_sum, dw_init, dw_co_sum, layer_type type :: layer_type - real(rk), allocatable :: a(:) ! activations - real(rk), allocatable :: b(:) ! biases - real(rk), allocatable :: w(:,:) ! weights - real(rk), allocatable :: z(:) ! arg. to activation function + real(rk), allocatable :: a(:) !! activations + real(rk), allocatable :: b(:) !! biases + real(rk), allocatable :: w(:,:) !! weights + real(rk), allocatable :: z(:) !! arg. to activation function procedure(activation_function), pointer, nopass :: activation => null() procedure(activation_function), pointer, nopass :: activation_prime => null() - character(len=:), allocatable :: activation_str ! activation character string + character(len=:), allocatable :: activation_str !! activation character string contains procedure, public, pass(self) :: set_activation end type layer_type @@ -46,9 +46,9 @@ module mod_layer contains type(layer_type) function constructor(this_size, next_size) result(layer) - ! Layer class constructor. this_size is the number of neurons in the layer. - ! next_size is the number of neurons in the next layer, used to allocate - ! the weights. + !! Layer class constructor. this_size is the number of neurons in the layer. + !! next_size is the number of neurons in the next layer, used to allocate + !! the weights. integer(ik), intent(in) :: this_size, next_size allocate(layer % a(this_size)) allocate(layer % z(this_size)) @@ -59,21 +59,21 @@ type(layer_type) function constructor(this_size, next_size) result(layer) end function constructor pure type(array1d) function array1d_constructor(length) result(a) - ! Overloads the default type constructor. + !! Overloads the default type constructor. integer(ik), intent(in) :: length allocate(a % array(length)) a % array = 0 end function array1d_constructor pure type(array2d) function array2d_constructor(dims) result(a) - ! Overloads the default type constructor. + !! Overloads the default type constructor. integer(ik), intent(in) :: dims(2) allocate(a % array(dims(1), dims(2))) a % array = 0 end function array2d_constructor pure subroutine db_init(db, dims) - ! Initialises biases structure. + !! Initialises biases structure. type(array1d), allocatable, intent(in out) :: db(:) integer(ik), intent(in) :: dims(:) integer(ik) :: n, nm @@ -86,7 +86,7 @@ pure subroutine db_init(db, dims) end subroutine db_init pure subroutine dw_init(dw, dims) - ! Initialises weights structure. + !! Initialises weights structure. type(array2d), allocatable, intent(in out) :: dw(:) integer(ik), intent(in) :: dims(:) integer(ik) :: n, nm @@ -99,7 +99,7 @@ pure subroutine dw_init(dw, dims) end subroutine dw_init subroutine db_co_sum(db) - ! Performs a collective sum of bias tendencies. + !! Performs a collective sum of bias tendencies. type(array1d), allocatable, intent(in out) :: db(:) integer(ik) :: n do n = 2, size(db) @@ -110,7 +110,7 @@ subroutine db_co_sum(db) end subroutine db_co_sum subroutine dw_co_sum(dw) - ! Performs a collective sum of weights tendencies. + !! Performs a collective sum of weights tendencies. type(array2d), allocatable, intent(in out) :: dw(:) integer(ik) :: n do n = 1, size(dw) - 1 @@ -121,9 +121,9 @@ subroutine dw_co_sum(dw) end subroutine dw_co_sum pure elemental subroutine set_activation(self, activation) - ! Sets the activation function. Input string must match one of - ! provided activation functions, otherwise it defaults to sigmoid. - ! If activation not present, defaults to sigmoid. + !! Sets the activation function. Input string must match one of + !! provided activation functions, otherwise it defaults to sigmoid. + !! If activation not present, defaults to sigmoid. class(layer_type), intent(in out) :: self character(len=*), intent(in) :: activation select case(trim(activation)) diff --git a/src/mod_mnist.f90 b/src/mod_mnist.f90 index 1f41b2e6..7c5f897b 100644 --- a/src/mod_mnist.f90 +++ b/src/mod_mnist.f90 @@ -1,9 +1,9 @@ module mod_mnist - ! Procedures to work with MNIST dataset, usable with data format - ! as provided in this repo and not the original data format (idx). + !! Procedures to work with MNIST dataset, usable with data format + !! as provided in this repo and not the original data format (idx). - use iso_fortran_env, only: real32 ! TODO make MNIST work with arbitrary precision + use iso_fortran_env, only: real32 !! TODO make MNIST work with arbitrary precision use mod_io, only: read_binary_file use mod_kinds, only: ik, rk @@ -16,11 +16,11 @@ module mod_mnist contains pure function digits(x) - ! Returns an array of 10 reals, with zeros everywhere - ! and a one corresponding to the input number, for example: - ! digits(0) = [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.] - ! digits(1) = [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.] - ! digits(6) = [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.] + !! Returns an array of 10 reals, with zeros everywhere + !! and a one corresponding to the input number, for example: + !! digits(0) = [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.] + !! digits(1) = [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.] + !! digits(6) = [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.] real(rk), intent(in) :: x real(rk) :: digits(10) digits = 0 @@ -28,8 +28,8 @@ pure function digits(x) end function digits pure function label_digits(labels) result(res) - ! Converts an array of MNIST labels into a form - ! that can be input to the network_type instance. + !! Converts an array of MNIST labels into a form + !! that can be input to the network_type instance. real(rk), intent(in) :: labels(:) real(rk) :: res(10, size(labels)) integer(ik) :: i @@ -40,7 +40,7 @@ end function label_digits subroutine load_mnist(tr_images, tr_labels, te_images,& te_labels, va_images, va_labels) - ! Loads the MNIST dataset into arrays. + !! Loads the MNIST dataset into arrays. real(rk), allocatable, intent(in out) :: tr_images(:,:), tr_labels(:) real(rk), allocatable, intent(in out) :: te_images(:,:), te_labels(:) real(rk), allocatable, intent(in out), optional :: va_images(:,:), va_labels(:) @@ -69,7 +69,7 @@ subroutine load_mnist(tr_images, tr_labels, te_images,& end subroutine load_mnist subroutine print_image(images, labels, n) - ! Prints a single image and label to screen. + !! Prints a single image and label to screen. real(rk), intent(in) :: images(:,:), labels(:) integer(ik), intent(in) :: n real(rk) :: image(28, 28) diff --git a/src/mod_network.f90 b/src/mod_network.f90 index 4c107be6..405099d8 100644 --- a/src/mod_network.f90 +++ b/src/mod_network.f90 @@ -47,9 +47,9 @@ module mod_network contains type(network_type) function net_constructor(dims, activation) result(net) - ! Network class constructor. Size of input array dims indicates the total - ! number of layers (input + hidden + output), and the value of its elements - ! corresponds the size of each layer. + !! Network class constructor. Size of input array dims indicates the total + !! number of layers (input + hidden + output), and the value of its elements + !! corresponds the size of each layer. integer(ik), intent(in) :: dims(:) character(len=*), intent(in), optional :: activation call net % init(dims) @@ -63,9 +63,9 @@ end function net_constructor pure real(rk) function accuracy(self, x, y) - ! Given input x and output y, evaluates the position of the - ! maximum value of the output and returns the number of matches - ! relative to the size of the dataset. + !! Given input x and output y, evaluates the position of the + !! maximum value of the output and returns the number of matches + !! relative to the size of the dataset. class(network_type), intent(in) :: self real(rk), intent(in) :: x(:,:), y(:,:) integer(ik) :: i, good @@ -80,8 +80,8 @@ end function accuracy pure subroutine backprop(self, y, dw, db) - ! Applies a backward propagation through the network - ! and returns the weight and bias gradients. + !! Applies a backward propagation through the network + !! and returns the weight and bias gradients. class(network_type), intent(in out) :: self real(rk), intent(in) :: y(:) type(array2d), allocatable, intent(out) :: dw(:) @@ -111,8 +111,8 @@ end subroutine backprop pure subroutine fwdprop(self, x) - ! Performs the forward propagation and stores arguments to activation - ! functions and activations themselves for use in backprop. + !! Performs the forward propagation and stores arguments to activation + !! functions and activations themselves for use in backprop. class(network_type), intent(in out) :: self real(rk), intent(in) :: x(:) integer(ik) :: n @@ -127,7 +127,7 @@ end subroutine fwdprop subroutine init(self, dims) - ! Allocates and initializes the layers with given dimensions dims. + !! Allocates and initializes the layers with given dimensions dims. class(network_type), intent(in out) :: self integer(ik), intent(in) :: dims(:) integer(ik) :: n @@ -143,12 +143,12 @@ end subroutine init subroutine load(self, filename) - ! Loads the network from file. + !! Loads the network from file. class(network_type), intent(in out) :: self character(len=*), intent(in) :: filename integer(ik) :: fileunit, n, num_layers, layer_idx integer(ik), allocatable :: dims(:) - character(len=100) :: buffer ! activation string + character(len=100) :: buffer !! activation string open(newunit=fileunit, file=filename, status='old', action='read') read(fileunit, *) num_layers allocate(dims(num_layers)) @@ -169,7 +169,7 @@ end subroutine load pure real(rk) function loss(self, x, y) - ! Given input x and expected output y, returns the loss of the network. + !! Given input x and expected output y, returns the loss of the network. class(network_type), intent(in) :: self real(rk), intent(in) :: x(:), y(:) loss = 0.5 * sum((y - self % output(x))**2) / size(x) @@ -177,8 +177,8 @@ end function loss pure function output_single(self, x) result(a) - ! Use forward propagation to compute the output of the network. - ! This specific procedure is for a single sample of 1-d input data. + !! Use forward propagation to compute the output of the network. + !! This specific procedure is for a single sample of 1-d input data. class(network_type), intent(in) :: self real(rk), intent(in) :: x(:) real(rk), allocatable :: a(:) @@ -193,8 +193,8 @@ end function output_single pure function output_batch(self, x) result(a) - ! Use forward propagation to compute the output of the network. - ! This specific procedure is for a batch of 1-d input data. + !! Use forward propagation to compute the output of the network. + !! This specific procedure is for a batch of 1-d input data. class(network_type), intent(in) :: self real(rk), intent(in) :: x(:,:) real(rk), allocatable :: a(:,:) @@ -207,7 +207,7 @@ end function output_batch subroutine save(self, filename) - ! Saves the network to a file. + !! Saves the network to a file. class(network_type), intent(in out) :: self character(len=*), intent(in) :: filename integer(ik) :: fileunit, n @@ -228,9 +228,9 @@ end subroutine save pure subroutine set_activation_equal(self, activation) - ! A thin wrapper around layer % set_activation(). - ! This method can be used to set an activation function - ! for all layers at once. + !! A thin wrapper around layer % set_activation(). + !! This method can be used to set an activation function + !! for all layers at once. class(network_type), intent(in out) :: self character(len=*), intent(in) :: activation call self % layers(:) % set_activation(activation) @@ -238,17 +238,17 @@ end subroutine set_activation_equal pure subroutine set_activation_layers(self, activation) - ! A thin wrapper around layer % set_activation(). - ! This method can be used to set different activation functions - ! for each layer separately. + !! A thin wrapper around layer % set_activation(). + !! This method can be used to set different activation functions + !! for each layer separately. class(network_type), intent(in out) :: self character(len=*), intent(in) :: activation(size(self % layers)) call self % layers(:) % set_activation(activation) end subroutine set_activation_layers subroutine sync(self, image) - ! Broadcasts network weights and biases from - ! specified image to all others. + !! Broadcasts network weights and biases from + !! specified image to all others. class(network_type), intent(in out) :: self integer(ik), intent(in) :: image integer(ik) :: n @@ -263,9 +263,9 @@ end subroutine sync subroutine train_batch(self, x, y, eta) - ! Trains a network using input data x and output data y, - ! and learning rate eta. The learning rate is normalized - ! with the size of the data batch. + !! Trains a network using input data x and output data y, + !! and learning rate eta. The learning rate is normalized + !! with the size of the data batch. class(network_type), intent(in out) :: self real(rk), intent(in) :: x(:,:), y(:,:), eta type(array1d), allocatable :: db(:), db_batch(:) @@ -273,8 +273,8 @@ subroutine train_batch(self, x, y, eta) integer(ik) :: i, im, n, nm integer(ik) :: is, ie, indices(2) - im = size(x, dim=2) ! mini-batch size - nm = size(self % dims) ! number of layers + im = size(x, dim=2) !! mini-batch size + nm = size(self % dims) !! number of layers ! get start and end index for mini-batch indices = tile_indices(im) @@ -304,7 +304,7 @@ end subroutine train_batch subroutine train_epochs(self, x, y, eta, num_epochs, batch_size) - ! Trains for num_epochs epochs with mini-bachtes of size equal to batch_size. + !! Trains for num_epochs epochs with mini-bachtes of size equal to batch_size. class(network_type), intent(in out) :: self integer(ik), intent(in) :: num_epochs, batch_size real(rk), intent(in) :: x(:,:), y(:,:), eta @@ -335,8 +335,8 @@ end subroutine train_epochs pure subroutine train_single(self, x, y, eta) - ! Trains a network using a single set of input data x and output data y, - ! and learning rate eta. + !! Trains a network using a single set of input data x and output data y, + !! and learning rate eta. class(network_type), intent(in out) :: self real(rk), intent(in) :: x(:), y(:), eta type(array2d), allocatable :: dw(:) @@ -348,8 +348,8 @@ end subroutine train_single pure subroutine update(self, dw, db, eta) - ! Updates network weights and biases with gradients dw and db, - ! scaled by learning rate eta. + !! Updates network weights and biases with gradients dw and db, + !! scaled by learning rate eta. class(network_type), intent(in out) :: self class(array2d), intent(in) :: dw(:) class(array1d), intent(in) :: db(:) @@ -357,11 +357,11 @@ pure subroutine update(self, dw, db, eta) integer(ik) :: n associate(layers => self % layers, nm => size(self % dims)) - ! update biases + !! update biases do concurrent(n = 2:nm) layers(n) % b = layers(n) % b - eta * db(n) % array end do - ! update weights + !! update weights do concurrent(n = 1:nm-1) layers(n) % w = layers(n) % w - eta * dw(n) % array end do diff --git a/src/mod_parallel.f90 b/src/mod_parallel.f90 index dda8cb19..0d7503da 100644 --- a/src/mod_parallel.f90 +++ b/src/mod_parallel.f90 @@ -9,19 +9,19 @@ module mod_parallel contains pure function tile_indices(dims) - ! Given input global array size, return start and end index - ! of a parallel 1-d tile that correspond to this image. + !! Given input global array size, return start and end index + !! of a parallel 1-d tile that correspond to this image. integer(ik), intent(in) :: dims integer(ik) :: tile_indices(2) integer(ik) :: offset, tile_size tile_size = dims / num_images() - ! start and end indices assuming equal tile sizes + !! start and end indices assuming equal tile sizes tile_indices(1) = (this_image() - 1) * tile_size + 1 tile_indices(2) = tile_indices(1) + tile_size - 1 - ! if we have any remainder, distribute it to the tiles at the end + !! if we have any remainder, distribute it to the tiles at the end offset = num_images() - mod(dims, num_images()) if (this_image() > offset) then tile_indices(1) = tile_indices(1) + this_image() - offset - 1 diff --git a/src/mod_random.f90 b/src/mod_random.f90 index d8c1b4af..78814c40 100644 --- a/src/mod_random.f90 +++ b/src/mod_random.f90 @@ -1,7 +1,7 @@ module mod_random - ! Provides a random number generator with - ! normal distribution, centered on zero. + !! Provides a random number generator with + !! normal distribution, centered on zero. use mod_kinds, only: ik, rk @@ -19,7 +19,7 @@ module mod_random contains function randn1d(n) result(r) - ! Generates n random numbers with a normal distribution. + !! Generates n random numbers with a normal distribution. integer(ik), intent(in) :: n real(rk) :: r(n), r2(n) call random_number(r) @@ -28,7 +28,7 @@ function randn1d(n) result(r) end function randn1d function randn2d(m, n) result(r) - ! Generates m x n random numbers with a normal distribution. + !! Generates m x n random numbers with a normal distribution. integer(ik), intent(in) :: m, n real(rk) :: r(m, n), r2(m, n) call random_number(r)