From e9af5b43c2ec86587d111656b63e193eb6fa80ab Mon Sep 17 00:00:00 2001 From: Milan Curcic Date: Wed, 28 Sep 2022 14:29:31 -0400 Subject: [PATCH] Implement the reshape layer (#97) * Interface and constructor for the reshape3d layer * Use reshape for the constructor function and reshape3d for the internal layer implementation * Add the submodule for the concrete reshape3d_layer * Forward and backward passes for the * Add type guards for reshape layer to forward and backward subroutines * Test that the resulting shape and values of a reshape layer are correct * Bump version to 0.8.0 (unreleased) * Add reshape layer to list of features * Update CMake build for the reshape layer * Ignore submodule files * Enable reading reshape layers from Keras h5 --- .gitignore | 5 +- CMakeLists.txt | 2 + README.md | 11 +-- fpm.toml | 2 +- src/nf.f90 | 3 +- src/nf/nf_datasets.f90 | 3 + src/nf/nf_keras.f90 | 3 + src/nf/nf_keras_submodule.f90 | 7 ++ src/nf/nf_layer.f90 | 30 +++++++-- src/nf/nf_layer_constructors.f90 | 13 +++- src/nf/nf_layer_constructors_submodule.f90 | 15 +++++ src/nf/nf_layer_submodule.f90 | 65 +++++++++++++++--- src/nf/nf_network_submodule.f90 | 10 ++- src/nf/nf_reshape_layer.f90 | 76 +++++++++++++++++++++ src/nf/nf_reshape_layer_submodule.f90 | 51 ++++++++++++++ test/CMakeLists.txt | 1 + test/test_reshape_layer.f90 | 78 ++++++++++++++++++++++ 17 files changed, 350 insertions(+), 25 deletions(-) create mode 100644 src/nf/nf_reshape_layer.f90 create mode 100644 src/nf/nf_reshape_layer_submodule.f90 create mode 100644 test/test_reshape_layer.f90 diff --git a/.gitignore b/.gitignore index 8a4f2abb..bc43a875 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ *.gz *.o *.mod +*.smod *.dat *.h5 -build -doc +/build +/doc diff --git a/CMakeLists.txt b/CMakeLists.txt index 564b3b0d..08538eea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,8 @@ add_library(neural src/nf/nf_parallel_submodule.f90 src/nf/nf_random.f90 src/nf/nf_random_submodule.f90 + src/nf/nf_reshape_layer.f90 + src/nf/nf_reshape_layer_submodule.f90 src/nf/io/nf_io_binary.f90 src/nf/io/nf_io_binary_submodule.f90 src/nf/io/nf_io_hdf5.f90 diff --git a/README.md b/README.md index b99c7a97..d4a241b4 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Read the paper [here](https://arxiv.org/abs/1902.06714). * Dense, fully connected neural layers * Convolutional and max-pooling layers (experimental, forward propagation only) -* Flatten layers (forward and backward pass) +* Flatten and reshape layers (forward and backward passes) * Loading dense and convolutional models from Keras h5 files * Stochastic and mini-batch gradient descent for back-propagation * Data-based parallelism @@ -29,10 +29,11 @@ Read the paper [here](https://arxiv.org/abs/1902.06714). | Layer type | Constructor name | Supported input layers | Rank of output array | Forward pass | Backward pass | |------------|------------------|------------------------|----------------------|--------------|---------------| | Input (1-d and 3-d) | `input` | n/a | 1, 3 | n/a | n/a | -| Dense (fully-connected) | `dense` | `input` (1-d) | 1 | ✅ | ✅ | -| Convolutional (2-d) | `conv2d` | `input` (3-d), `conv2d`, `maxpool2d` | 3 | ✅ | ❌ | -| Max-pooling (2-d) | `maxpool2d` | `input` (3-d), `conv2d`, `maxpool2d` | 3 | ✅ | ❌ | -| Flatten | `flatten` | `input` (3-d), `conv2d`, `maxpool2d` | 1 | ✅ | ✅ | +| Dense (fully-connected) | `dense` | `input1d` | 1 | ✅ | ✅ | +| Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d` | 3 | ✅ | ❌ | +| Max-pooling (2-d) | `maxpool2d` | `input3d`, `conv2d`, `maxpool2d` | 3 | ✅ | ❌ | +| Flatten | `flatten` | `input3d`, `conv2d`, `maxpool2d` | 1 | ✅ | ✅ | +| Reshape (1-d to 3-d) | `reshape` | `input1d`, `dense`, `flatten` | 3 | ✅ | ✅ | ## Getting started diff --git a/fpm.toml b/fpm.toml index e9225657..b3464dd8 100644 --- a/fpm.toml +++ b/fpm.toml @@ -1,5 +1,5 @@ name = "neural-fortran" -version = "0.7.0" +version = "0.8.0" license = "MIT" author = "Milan Curcic" maintainer = "milancurcic@hey.com" diff --git a/src/nf.f90 b/src/nf.f90 index 34de190b..c2c9ce63 100644 --- a/src/nf.f90 +++ b/src/nf.f90 @@ -2,7 +2,8 @@ module nf !! User API: everything an application needs to reference directly use nf_datasets_mnist, only: label_digits, load_mnist use nf_layer, only: layer - use nf_layer_constructors, only: conv2d, dense, flatten, input, maxpool2d + use nf_layer_constructors, only: & + conv2d, dense, flatten, input, maxpool2d, reshape use nf_network, only: network use nf_optimizers, only: sgd end module nf diff --git a/src/nf/nf_datasets.f90 b/src/nf/nf_datasets.f90 index 2df35cb9..24e9ac5c 100644 --- a/src/nf/nf_datasets.f90 +++ b/src/nf/nf_datasets.f90 @@ -12,6 +12,7 @@ module nf_datasets download_and_unpack, & keras_cnn_mnist_url, & keras_dense_mnist_url, & + keras_reshape_url, & mnist_url character(*), parameter :: keras_snippets_baseurl = & @@ -22,6 +23,8 @@ module nf_datasets keras_snippets_baseurl // '/8892585/keras_cnn_mnist.tar.gz' character(*), parameter :: keras_dense_mnist_url = & keras_snippets_baseurl // '/8788739/keras_dense_mnist.tar.gz' + character(*), parameter :: keras_reshape_url = & + keras_snippets_baseurl // '/9667603/keras_reshape.tar.gz' character(*), parameter :: mnist_url = & neural_fortran_baseurl // '/8498876/mnist.tar.gz' diff --git a/src/nf/nf_keras.f90 b/src/nf/nf_keras.f90 index 04c9385b..efe2905a 100644 --- a/src/nf/nf_keras.f90 +++ b/src/nf/nf_keras.f90 @@ -29,6 +29,9 @@ module nf_keras integer, allocatable :: pool_size(:) integer, allocatable :: strides(:) + ! Reshape + integer, allocatable :: target_shape(:) + end type keras_layer interface diff --git a/src/nf/nf_keras_submodule.f90 b/src/nf/nf_keras_submodule.f90 index 161b38be..b5c0d292 100644 --- a/src/nf/nf_keras_submodule.f90 +++ b/src/nf/nf_keras_submodule.f90 @@ -82,6 +82,13 @@ module function get_keras_h5_layers(filename) result(res) res(n) % pool_size = reverse(res(n) % pool_size) res(n) % strides = reverse(res(n) % strides) + case('Reshape') + ! Only read target shape + call json % get(layer_config_json, & + 'target_shape', res(n) % target_shape, found) + ! Reverse to account for C -> Fortran order + res(n) % target_shape = reverse(res(n) % target_shape) + case default error stop 'This Keras layer is not supported' diff --git a/src/nf/nf_layer.f90 b/src/nf/nf_layer.f90 index 7613d84f..ac177e63 100644 --- a/src/nf/nf_layer.f90 +++ b/src/nf/nf_layer.f90 @@ -24,24 +24,25 @@ module nf_layer contains - procedure :: backward procedure :: forward procedure :: init procedure :: print_info procedure :: update - ! Specific output subroutines for different array ranks, - ! available via generic `get_output`. + ! Specific subroutines for different array ranks + procedure, private :: backward_1d + procedure, private :: backward_3d procedure, private :: get_output_1d procedure, private :: get_output_3d + generic :: backward => backward_1d, backward_3d generic :: get_output => get_output_1d, get_output_3d end type layer - interface + interface backward - pure module subroutine backward(self, previous, gradient) + pure module subroutine backward_1d(self, previous, gradient) !! Apply a backward pass on the layer. !! This changes the internal state of the layer. !! This is normally called internally by the `network % backward` @@ -52,7 +53,24 @@ pure module subroutine backward(self, previous, gradient) !! Previous layer instance real, intent(in) :: gradient(:) !! Array of gradient values from the next layer - end subroutine backward + end subroutine backward_1d + + pure module subroutine backward_3d(self, previous, gradient) + !! Apply a backward pass on the layer. + !! This changes the internal state of the layer. + !! This is normally called internally by the `network % backward` + !! method. + class(layer), intent(in out) :: self + !! Layer instance + class(layer), intent(in) :: previous + !! Previous layer instance + real, intent(in) :: gradient(:,:,:) + !! Array of gradient values from the next layer + end subroutine backward_3d + + end interface backward + + interface pure module subroutine forward(self, input) !! Apply a forward pass on the layer. diff --git a/src/nf/nf_layer_constructors.f90 b/src/nf/nf_layer_constructors.f90 index 0222a8c5..b18234e4 100644 --- a/src/nf/nf_layer_constructors.f90 +++ b/src/nf/nf_layer_constructors.f90 @@ -7,7 +7,7 @@ module nf_layer_constructors implicit none private - public :: conv2d, dense, flatten, input, maxpool2d + public :: conv2d, dense, flatten, input, maxpool2d, reshape interface input @@ -154,6 +154,17 @@ pure module function maxpool2d(pool_size, stride) result(res) !! Resulting layer instance end function maxpool2d + pure module function reshape(output_shape) result(res) + !! Rank-1 to rank-any reshape layer constructor. + !! Currently implemented is only rank-3 for the output of the reshape. + !! + !! This layer is for connecting 1-d inputs to conv2d or similar layers. + integer, intent(in) :: output_shape(:) + !! Shape of the output + type(layer) :: res + !! Resulting layer instance + end function reshape + end interface end module nf_layer_constructors diff --git a/src/nf/nf_layer_constructors_submodule.f90 b/src/nf/nf_layer_constructors_submodule.f90 index 8e991901..00ea7bf4 100644 --- a/src/nf/nf_layer_constructors_submodule.f90 +++ b/src/nf/nf_layer_constructors_submodule.f90 @@ -7,6 +7,7 @@ use nf_input1d_layer, only: input1d_layer use nf_input3d_layer, only: input3d_layer use nf_maxpool2d_layer, only: maxpool2d_layer + use nf_reshape_layer, only: reshape3d_layer implicit none @@ -109,4 +110,18 @@ pure module function maxpool2d(pool_size, stride) result(res) end function maxpool2d + pure module function reshape(output_shape) result(res) + integer, intent(in) :: output_shape(:) + type(layer) :: res + + res % name = 'reshape' + + if (size(output_shape) == 3) then + allocate(res % p, source=reshape3d_layer(output_shape)) + else + error stop 'size(output_shape) of the reshape layer must == 3' + end if + + end function reshape + end submodule nf_layer_constructors_submodule diff --git a/src/nf/nf_layer_submodule.f90 b/src/nf/nf_layer_submodule.f90 index dbffb6b1..2613606a 100644 --- a/src/nf/nf_layer_submodule.f90 +++ b/src/nf/nf_layer_submodule.f90 @@ -6,16 +6,18 @@ use nf_input1d_layer, only: input1d_layer use nf_input3d_layer, only: input3d_layer use nf_maxpool2d_layer, only: maxpool2d_layer + use nf_reshape_layer, only: reshape3d_layer contains - pure module subroutine backward(self, previous, gradient) + pure module subroutine backward_1d(self, previous, gradient) implicit none class(layer), intent(in out) :: self class(layer), intent(in) :: previous real, intent(in) :: gradient(:) - ! Backward pass currently implemented only for dense and flatten layers + ! Backward pass from a 1-d layer downstream currently implemented + ! only for dense and flatten layers select type(this_layer => self % p) type is(dense_layer) @@ -32,7 +34,7 @@ pure module subroutine backward(self, previous, gradient) type is(flatten_layer) - ! Downstream layers permitted: input3d, conv2d, maxpool2d + ! Upstream layers permitted: input3d, conv2d, maxpool2d select type(prev_layer => previous % p) type is(input3d_layer) call this_layer % backward(prev_layer % output, gradient) @@ -44,7 +46,34 @@ pure module subroutine backward(self, previous, gradient) end select - end subroutine backward + end subroutine backward_1d + + + pure module subroutine backward_3d(self, previous, gradient) + implicit none + class(layer), intent(in out) :: self + class(layer), intent(in) :: previous + real, intent(in) :: gradient(:,:,:) + + ! Backward pass from a 3-d layer downstream currently implemented + ! only for reshape3d layer + select type(this_layer => self % p) + + type is(reshape3d_layer) + + ! Upstream layers permitted: input1d, dense, flatten + select type(prev_layer => previous % p) + type is(input1d_layer) + call this_layer % backward(prev_layer % output, gradient) + type is(dense_layer) + call this_layer % backward(prev_layer % output, gradient) + type is(flatten_layer) + call this_layer % backward(prev_layer % output, gradient) + end select + + end select + + end subroutine backward_3d pure module subroutine forward(self, input) @@ -68,7 +97,7 @@ pure module subroutine forward(self, input) type is(conv2d_layer) - ! Upstream layers permitted: input3d, conv2d, maxpool2d + ! Upstream layers permitted: input3d, conv2d, maxpool2d, reshape3d select type(prev_layer => input % p) type is(input3d_layer) call this_layer % forward(prev_layer % output) @@ -76,11 +105,13 @@ pure module subroutine forward(self, input) call this_layer % forward(prev_layer % output) type is(maxpool2d_layer) call this_layer % forward(prev_layer % output) + type is(reshape3d_layer) + call this_layer % forward(prev_layer % output) end select type is(maxpool2d_layer) - ! Upstream layers permitted: input3d, conv2d, maxpool2d + ! Upstream layers permitted: input3d, conv2d, maxpool2d, reshape3d select type(prev_layer => input % p) type is(input3d_layer) call this_layer % forward(prev_layer % output) @@ -88,11 +119,13 @@ pure module subroutine forward(self, input) call this_layer % forward(prev_layer % output) type is(maxpool2d_layer) call this_layer % forward(prev_layer % output) + type is(reshape3d_layer) + call this_layer % forward(prev_layer % output) end select type is(flatten_layer) - ! Upstream layers permitted: input3d, conv2d, maxpool2d + ! Upstream layers permitted: input3d, conv2d, maxpool2d, reshape3d select type(prev_layer => input % p) type is(input3d_layer) call this_layer % forward(prev_layer % output) @@ -100,6 +133,20 @@ pure module subroutine forward(self, input) call this_layer % forward(prev_layer % output) type is(maxpool2d_layer) call this_layer % forward(prev_layer % output) + type is(reshape3d_layer) + call this_layer % forward(prev_layer % output) + end select + + type is(reshape3d_layer) + + ! Upstream layers permitted: input1d, dense, flatten + select type(prev_layer => input % p) + type is(input1d_layer) + call this_layer % forward(prev_layer % output) + type is(dense_layer) + call this_layer % forward(prev_layer % output) + type is(flatten_layer) + call this_layer % forward(prev_layer % output) end select end select @@ -141,8 +188,10 @@ pure module subroutine get_output_3d(self, output) allocate(output, source=this_layer % output) type is(maxpool2d_layer) allocate(output, source=this_layer % output) + type is(reshape3d_layer) + allocate(output, source=this_layer % output) class default - error stop '3-d output can only be read from an input3d, conv2d, or maxpool2d layer.' + error stop '3-d output can only be read from a conv2d, input3d, maxpool2d, or reshape3d layer.' end select diff --git a/src/nf/nf_network_submodule.f90 b/src/nf/nf_network_submodule.f90 index f33f3f06..a95db208 100644 --- a/src/nf/nf_network_submodule.f90 +++ b/src/nf/nf_network_submodule.f90 @@ -6,10 +6,11 @@ use nf_input1d_layer, only: input1d_layer use nf_input3d_layer, only: input3d_layer use nf_maxpool2d_layer, only: maxpool2d_layer + use nf_reshape_layer, only: reshape3d_layer use nf_io_hdf5, only: get_hdf5_dataset use nf_keras, only: get_keras_h5_layers, keras_layer use nf_layer, only: layer - use nf_layer_constructors, only: conv2d, dense, flatten, input, maxpool2d + use nf_layer_constructors, only: conv2d, dense, flatten, input, maxpool2d, reshape use nf_loss, only: quadratic_derivative use nf_optimizers, only: sgd use nf_parallel, only: tile_indices @@ -117,6 +118,9 @@ module function network_from_keras(filename) result(res) keras_layers(n) % strides(1) & ) + case('Reshape') + layers(n) = reshape(keras_layers(n) % target_shape) + case default error stop 'This Keras layer is not supported' @@ -165,6 +169,10 @@ module function network_from_keras(filename) result(res) ! Nothing to do continue + type is(reshape3d_layer) + ! Nothing to do + continue + class default error stop 'Internal error in network_from_keras(); ' & // 'mismatch in layer types between the Keras and ' & diff --git a/src/nf/nf_reshape_layer.f90 b/src/nf/nf_reshape_layer.f90 new file mode 100644 index 00000000..f17620e6 --- /dev/null +++ b/src/nf/nf_reshape_layer.f90 @@ -0,0 +1,76 @@ +module nf_reshape_layer + + !! This module provides the concrete reshape layer type. + !! It is used internally by the layer type. + !! It is not intended to be used directly by the user. + + use nf_base_layer, only: base_layer + + implicit none + + private + public :: reshape3d_layer + + type, extends(base_layer) :: reshape3d_layer + + !! Concrete implementation of a reshape layer type + !! It implements only rank-1 to rank-3 reshaping. + + integer :: input_shape(1) + integer :: output_shape(3) + real, allocatable :: gradient(:) + real, allocatable :: output(:,:,:) + + contains + + procedure :: backward + procedure :: forward + procedure :: init + + end type reshape3d_layer + + interface reshape3d_layer + pure module function reshape3d_layer_cons(output_shape) result(res) + !! This function returns the `reshape_layer` instance. + integer, intent(in) :: output_shape(3) + !! The shape of the output + type(reshape3d_layer) :: res + !! reshape_layer instance + end function reshape3d_layer_cons + end interface reshape3d_layer + + interface + + pure module subroutine backward(self, input, gradient) + !! Apply the backward pass for the reshape3d layer. + !! This is just flattening to a rank-1 array. + class(reshape3d_layer), intent(in out) :: self + !! Dense layer instance + real, intent(in) :: input(:) + !! Input from the previous layer + real, intent(in) :: gradient(:,:,:) + !! Gradient from the next layer + end subroutine backward + + pure module subroutine forward(self, input) + !! Apply the forward pass for the reshape3d layer. + !! This is just a reshape from rank-1 to rank-3 array. + class(reshape3d_layer), intent(in out) :: self + !! Dense layer instance + real, intent(in) :: input(:) + !! Input from the previous layer + end subroutine forward + + module subroutine init(self, input_shape) + !! Initialize the layer data structures. + !! + !! This is a deferred procedure from the `base_layer` abstract type. + class(reshape3d_layer), intent(in out) :: self + !! Dense layer instance + integer, intent(in) :: input_shape(:) + !! Shape of the input layer + end subroutine init + + end interface + +end module nf_reshape_layer diff --git a/src/nf/nf_reshape_layer_submodule.f90 b/src/nf/nf_reshape_layer_submodule.f90 new file mode 100644 index 00000000..989497ba --- /dev/null +++ b/src/nf/nf_reshape_layer_submodule.f90 @@ -0,0 +1,51 @@ +submodule(nf_reshape_layer) nf_reshape_layer_submodule + + use nf_base_layer, only: base_layer + + implicit none + +contains + + pure module function reshape3d_layer_cons(output_shape) result(res) + integer, intent(in) :: output_shape(3) + type(reshape3d_layer) :: res + res % output_shape = output_shape + end function reshape3d_layer_cons + + + pure module subroutine backward(self, input, gradient) + class(reshape3d_layer), intent(in out) :: self + real, intent(in) :: input(:) + real, intent(in) :: gradient(:,:,:) + ! The `input` dummy argument is not used but nevertheless declared + ! because the abstract type requires it. + self % gradient = pack(gradient, .true.) + end subroutine backward + + + pure module subroutine forward(self, input) + class(reshape3d_layer), intent(in out) :: self + real, intent(in) :: input(:) + self % output = reshape(input, self % output_shape) + end subroutine forward + + + module subroutine init(self, input_shape) + class(reshape3d_layer), intent(in out) :: self + integer, intent(in) :: input_shape(:) + + self % input_shape = input_shape + + allocate(self % gradient(input_shape(1))) + self % gradient = 0 + + allocate(self % output( & + self % output_shape(1), & + self % output_shape(2), & + self % output_shape(3) & + )) + self % output = 0 + + end subroutine init + +end submodule nf_reshape_layer_submodule diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5538d198..b4a21fdd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -5,6 +5,7 @@ foreach(execid conv2d_layer maxpool2d_layer flatten_layer + reshape_layer dense_network io_hdf5 keras_read_model diff --git a/test/test_reshape_layer.f90 b/test/test_reshape_layer.f90 new file mode 100644 index 00000000..405afd98 --- /dev/null +++ b/test/test_reshape_layer.f90 @@ -0,0 +1,78 @@ +program test_reshape_layer + + use iso_fortran_env, only: stderr => error_unit + use nf, only: input, network, reshape_layer => reshape + use nf_datasets, only: download_and_unpack, keras_reshape_url + + implicit none + + type(network) :: net + real, allocatable :: sample_input(:), output(:,:,:) + integer, parameter :: output_shape(3) = [3, 32, 32] + integer, parameter :: input_size = product(output_shape) + character(*), parameter :: keras_reshape_path = 'keras_reshape.h5' + logical :: file_exists + logical :: ok = .true. + + ! Create the network + net = network([ & + input(input_size), & + reshape_layer(output_shape) & + ]) + + if (.not. size(net % layers) == 2) then + write(stderr, '(a)') 'the network should have 2 layers.. failed' + ok = .false. + end if + + ! Initialize test data + allocate(sample_input(input_size)) + call random_number(sample_input) + + ! Propagate forward and get the output + call net % forward(sample_input) + call net % layers(2) % get_output(output) + + if (.not. all(shape(output) == output_shape)) then + write(stderr, '(a)') 'the reshape layer produces expected output shape.. failed' + ok = .false. + end if + + if (.not. all(reshape(sample_input, output_shape) == output)) then + write(stderr, '(a)') 'the reshape layer produces expected output values.. failed' + ok = .false. + end if + + ! Now test reading the reshape layer from a Keras h5 model. + inquire(file=keras_reshape_path, exist=file_exists) + if (.not. file_exists) call download_and_unpack(keras_reshape_url) + + net = network(keras_reshape_path) + + if (.not. size(net % layers) == 2) then + write(stderr, '(a)') 'the reshape network from Keras has the correct size.. failed' + ok = .false. + end if + + if (.not. net % layers(2) % name == 'reshape') then + write(stderr, '(a)') 'the 2nd layer of the reshape network from Keras is a reshape layer.. failed' + ok = .false. + end if + + ! Test that the output shape checks out + call net % layers(1) % get_output(sample_input) + call net % layers(2) % get_output(output) + + if (.not. all(shape(output) == [1, 28, 28])) then + write(stderr, '(a)') 'the target shape of the reshape layer is correct.. failed' + ok = .false. + end if + + if (ok) then + print '(a)', 'test_reshape_layer: All tests passed.' + else + write(stderr, '(a)') 'test_reshape_layer: One or more tests failed.' + stop 1 + end if + +end program test_reshape_layer