Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial implementation of autograd #30

Merged
merged 20 commits into from
Jul 6, 2017
Merged

Initial implementation of autograd #30

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
5ed7a88
Reorganizing the CMake files
pavanky Jul 2, 2017
dfa8fda
Reorganizing the include files and namespace
pavanky Jul 2, 2017
562b860
First attempt at implementing autograd
pavanky Jul 2, 2017
3f832a0
Store gradients as autograd::Variable instead of af::array
pavanky Jul 2, 2017
c94ee3d
Refactor autograd::Variable, option to disable grad calculations
pavanky Jul 4, 2017
7bb0b6c
Changing autograd::backward function to Variable::backward method
pavanky Jul 5, 2017
a316af0
Moving autograd from header only lib to a compiled lib
pavanky Jul 5, 2017
d7edafc
Adding negate, reciprocal, subtract and divide
pavanky Jul 5, 2017
664cf7c
Add scalar support for operators
pavanky Jul 5, 2017
45b21da
Adding exp, sin, cos, tanh, and sigmoid functions
pavanky Jul 5, 2017
3b985b0
Adding expandAs, reduceAs, and transpose
pavanky Jul 5, 2017
9b05273
Adding matmul, matmulTN, and matmulNT functions
pavanky Jul 5, 2017
49b8917
Add option to explicitly request higher order gradients.
pavanky Jul 5, 2017
8bf7f1b
Convert Variable::build and Variable::buildSubGraph to static functions
pavanky Jul 6, 2017
5eda600
Overhaul of af::nn to use af::autograd
pavanky Jul 6, 2017
9aefea4
Fixing bugs in backward pass for activation functions
pavanky Jul 6, 2017
6d5751a
Fixing perceptron example to use smaller batch size
pavanky Jul 6, 2017
a01504b
Adding model.eval() and model.train()
pavanky Jul 6, 2017
2776aa2
Formatting changes
pavanky Jul 6, 2017
04cd450
Use references while iterating when possible
pavanky Jul 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 0 additions & 65 deletions AFBuildMacros.cmake
View file
Open in desktop

This file was deleted.

62 changes: 36 additions & 26 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,36 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(ARRAYFIRE_ML)

SET_PROPERTY(GLOBAL PROPERTY USE_FOLDERS ON)
ADD_DEFINITIONS(-Wall -std=c++11 -fvisibility=hidden)

OPTION(BUILD_TEST "Build Tests" ON)

# Header files
IF(NOT DEFINED AFML_INSTALL_INC_DIR)
SET(AFML_INSTALL_INC_DIR "include" CACHE PATH "Installation path for headers")
ENDIF()

IF (BUILD_TEST)
FILE(GLOB FILES "test/*.cpp")
INCLUDE("${CMAKE_CURRENT_SOURCE_DIR}/AFBuildMacros.cmake")
BUILD_ALL("${FILES}")
ENDIF()

INSTALL(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION "${AFML_INSTALL_INC_DIR}"
COMPONENT headers
FILES_MATCHING
PATTERN "*.h"
PATTERN "*.hpp"
PATTERN ".gitignore" EXCLUDE
)
cmake_minimum_required(VERSION 3.5.1)

project(ArrayFireML
VERSION 0.1.0
LANGUAGES C CXX)

find_package(ArrayFire REQUIRED)

add_library(afml SHARED "")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you don't add SHARED then you can control the type of library you make by BUILD_SHARED_LIBS variable


target_sources(afml
PRIVATE
src/autograd/Variable.cpp
src/autograd/Functions.cpp
)

target_include_directories(afml
PUBLIC
${ArrayFire_INCLUDE_DIRS}
${CMAKE_CURRENT_SOURCE_DIR}/include
)

target_link_libraries(afml
PUBLIC
af
)

set_target_properties(afml
PROPERTIES
VERSION "${ArrayFireML_VERSION}"
SOVERSION "${ArrayFireML_VERSION_MAJOR}"
CXX_STANDARD 11
)


add_subdirectory(examples)
8 changes: 4 additions & 4 deletions test/Activations.cpp → examples/Activations.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@
* http://arrayfire.com/licenses/BSD-3-Clause
********************************************************/

#include <afml/nn/Activations.hpp>
#include <af/nn/Activations.hpp>

using namespace afml::nn;
using namespace af::nn;

int main()
{
const int num = 5;

afml::ArrayVector in = {100 * af::randu(num, 1) - 50};
afml::ArrayVector grad = {100 * af::randu(num, 1)};
af::ArrayVector in = {100 * af::randu(num, 1) - 50};
af::ArrayVector grad = {100 * af::randu(num, 1)};

ReLU r = ReLU(num, 0);
Sigmoid s = Sigmoid(num);
Expand Down
18 changes: 18 additions & 0 deletions examples/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
function(build_example SRC)
get_filename_component(src_name ${SRC} NAME_WE)
set(target "${src_name}")
add_executable(${target} ${SRC})
target_link_libraries(${target}
PRIVATE
afml
)
target_compile_features(${target}
PRIVATE cxx_range_for)
endfunction(build_example)

build_example(Activations.cpp)
build_example(FFNet.cpp)
build_example(Node.cpp)
build_example(perceptron.cpp)
build_example(Weights.cpp)
build_example(autograd.cpp)
6 changes: 3 additions & 3 deletions test/FFNet.cpp → examples/FFNet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
* http://arrayfire.com/licenses/BSD-3-Clause
********************************************************/

#include <afml/nn.h>
#include <af/nn.h>

using namespace af;
using namespace afml;
using namespace afml::nn;
using namespace af::nn;

int main()
{
af::info();
const int inputSize = 2;
const int hiddenSize = 3;
const int outputSize = 1;
Expand Down
4 changes: 2 additions & 2 deletions test/Node.cpp → examples/Node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
* http://arrayfire.com/licenses/BSD-3-Clause
********************************************************/

#include <afml/nn/Nodes/Node.hpp>
#include <af/nn/Nodes/Node.hpp>

using namespace afml::nn;
using namespace af::nn;

int main()
{
Expand Down
4 changes: 2 additions & 2 deletions test/Weights.cpp → examples/Weights.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
* http://arrayfire.com/licenses/BSD-3-Clause
********************************************************/

#include <afml/nn/Weights.hpp>
#include <af/nn/Weights.hpp>

using namespace afml::nn;
using namespace af::nn;

int main()
{
Expand Down
172 changes: 172 additions & 0 deletions examples/autograd.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
/*******************************************************
* Copyright (c) 2017, ArrayFire
* All rights reserved.
*
* This file is distributed under 3-clause BSD license.
* The complete license agreement can be obtained at:
* http://arrayfire.com/licenses/BSD-3-Clause
********************************************************/

#include <af/autograd.h>

#define VERIFY(VAL) do { \
auto res = af::allTrue<bool>(af::abs(VAL) < 1E-5); \
printf("%s:%d %s\n", __FUNCTION__, __LINE__, \
res ? "PASS" : "FAIL"); \
} while(0)

using af::autograd::Variable;
void test_multiply()
{
auto x = Variable(af::randu(5), true);
auto y = x * x;
auto dy = Variable(af::constant(1.0, 5), false);
y.backward(dy);
auto dx = x.grad();
VERIFY(dx.array() - 2 * x.array());
}

void test_multipl_add()
{
auto x = Variable(af::randu(5), true);
auto y = Variable(af::randu(5), true);
auto z = x * x + x * y + y * y;
auto dz = Variable(af::constant(1.0, 5), false);
z.backward(dz);
auto dx = x.grad();
auto dy = y.grad();
VERIFY(dx.array() - 2 * x.array() - y.array());
VERIFY(dy.array() - 2 * y.array() - x.array());
}

void test_no_calc_grad()
{
auto x = Variable(af::randu(5), false);
auto y = Variable(af::randu(5), true);
auto z = x * x + x * y + y * y;
auto dz = Variable(af::constant(1.0, 5), false);
z.backward(dz);
auto dy = y.grad();
VERIFY(dy.array() - 2 * y.array() - x.array());
try {
auto dx = x.grad();
} catch(af::exception &ex) {
std::cout << ex.what() << std::endl;
return;
}
printf("%s:%d No Gradient check Failed\n");
}

void test_multiply_sub()
{
auto x = Variable(af::randu(5), true);
auto y = Variable(af::randu(5), true);
auto z = x * x - x * y;
auto dz = Variable(af::constant(1.0, 5), false);
z.backward(dz);
auto dx = x.grad();
auto dy = y.grad();
VERIFY(dx.array() - (2 * x.array() - y.array()));
VERIFY(dy.array() - (-x.array()));
}

void test_divide_add()
{
auto x = Variable(af::randu(5), true);
auto y = Variable(af::randu(5), true);
auto z = x + x / y + y;
auto dz = Variable(af::constant(1.0, 5), false);
z.backward(dz);
auto dx = x.grad();
auto dy = y.grad();
VERIFY(dx.array() - (1.0 + 1.0 / y.array()));
VERIFY(dy.array() - (1.0 - x.array() / (y.array() * y.array())));
}

void test_multiply_add_scalar()
{
auto x = Variable(af::randu(5), true);
auto y = Variable(af::randu(5), true);
auto z = 2 * x + x * y + y;
auto dz = Variable(af::constant(1.0, 5), false);
z.backward(dz);
auto dx = x.grad();
auto dy = y.grad();
VERIFY(dx.array() - (2.0 + y.array()));
VERIFY(dy.array() - (1.0 + x.array()));
}

void test_exp()
{
auto x = Variable(af::randu(5), true);
auto y = exp(x);
auto dy = Variable(af::constant(1.0, 5), false);
y.backward(dy);
auto dx = x.grad();
VERIFY(dx.array() - (af::exp(x.array())));
}

void test_sigmoid()
{
auto x = Variable(af::randu(5), true);
auto y = sigmoid(x);
auto dy = Variable(af::constant(1.0, 5), false);
y.backward(dy);
auto dx = x.grad();
VERIFY(dx.array() - (y.array() * (1 - y.array())));
VERIFY(dx.array() - (af::sigmoid(x.array()) * (1 - af::sigmoid(x.array()))));
}

void test_tanh()
{
auto x = Variable(af::randu(5), true);
auto y = tanh(x);
auto dy = Variable(af::constant(1.0, 5), false);
y.backward(dy);
auto dx = x.grad();
VERIFY(dx.array() - (1 - y.array() * y.array()));
VERIFY(dx.array() - (1 + af::tanh(x.array())) * (1 - af::tanh(x.array())));
}

void test_expand()
{
auto x = Variable(af::randu(5), true);
auto y = Variable(af::randu(5, 2), true);
auto z = y * expandAs(x, y);
auto dz = Variable(af::constant(1.0, 5, 2), false);
z.backward(dz);
auto dy = y.grad();
auto dx = x.grad();
VERIFY(dy.array() - af::tile(x.array(), 1, 2));
VERIFY(dx.array() - af::sum(y.array(), 1));
}

void test_reduce()
{
auto x = Variable(af::randu(5), true);
auto y = Variable(af::randu(5, 2), true);
auto z = x * reduceAs(y, x);
auto dz = Variable(af::constant(1.0, 5), false);
z.backward(dz);
auto dy = y.grad();
auto dx = x.grad();
VERIFY(dy.array() - af::tile(x.array(), 1, 2));
VERIFY(dx.array() - af::sum(y.array(), 1));
}

int main()
{
af::info();
test_multiply();
test_multipl_add();
test_no_calc_grad();
test_multiply_sub();
test_divide_add();
test_multiply_add_scalar();
test_exp();
test_sigmoid();
test_tanh();
test_expand();
test_reduce();
return 0;
}
6 changes: 3 additions & 3 deletions test/perceptron.cpp → examples/perceptron.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
* http://arrayfire.com/licenses/BSD-3-Clause
********************************************************/

#include <afml/nn.h>
#include <af/nn.h>

using namespace afml;
using namespace afml::nn;
using namespace af;
using namespace af::nn;

int main()
{
Expand Down
Loading