From 0edcfc190de0c107f6278813901e705615077484 Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 2 Apr 2018 14:46:44 +0200 Subject: [PATCH] Initial commit --- .gitignore | 3 + LICENSE | 21 +++++ README.md | 57 +++++++++++++ Setup.hs | 2 + package.yaml | 44 ++++++++++ src/AI/Nn.hs | 236 +++++++++++++++++++++++++++++++++++++++++++++++++++ stack.yaml | 66 ++++++++++++++ test/Spec.hs | 70 +++++++++++++++ 8 files changed, 499 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 Setup.hs create mode 100644 package.yaml create mode 100644 src/AI/Nn.hs create mode 100644 stack.yaml create mode 100644 test/Spec.hs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f5def0b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.stack-work/ +nn.cabal +*~ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1e3ef50 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Sascha Grunert + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1dc078e --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# nn +## A tiny neural network 🧠 + +This small neural network is based on the +[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) algorithm. + +## Usage + +A minimal usage example would look like this: + +```haskell +main :: IO () +main = do + {- Creates a new network with two inputs, two hidden layers and one output -} + network <- newIO [2, 2, 1] + + {- Train the network for a common logical AND, + until the maximum error of 0.01 is reached -} + let trainedNetwork = train 0.01 network [([0, 0], [0]) + ,([0, 1], [0]) + ,([1, 0], [0]) + ,([1, 1], [1])] + + {- Predict the learned values -} + let r00 = predict trainedNetwork [0, 0] + let r01 = predict trainedNetwork [0, 1] + let r10 = predict trainedNetwork [1, 0] + let r11 = predict trainedNetwork [1, 1] + + {- Print the results -} + putStrLn $ printf "0 0 -> %.2f" (head r00) + putStrLn $ printf "0 1 -> %.2f" (head r01) + putStrLn $ printf "1 0 -> %.2f" (head r10) + putStrLn $ printf "1 1 -> %.2f" (head r11) +``` + +The result should be something like: + +```console +0 0 -> -0.02 +0 1 -> -0.02 +1 0 -> -0.01 +1 1 -> 1.00 +``` + +## Hacking +To start hacking simply clone this repository and make sure that +[stack](https://docs.haskellstack.org/en/stable/README/) is installed. Then +simply hack around and build the project with: + +```console +> stack build --file-watch +``` + +## Contributing +You want to contribute to this project? Wow, thanks! So please just fork it and +send me a pull request. diff --git a/Setup.hs b/Setup.hs new file mode 100644 index 0000000..9a994af --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/package.yaml b/package.yaml new file mode 100644 index 0000000..7857cac --- /dev/null +++ b/package.yaml @@ -0,0 +1,44 @@ +--- +name: nn +version: 0.1.0 +github: "saschagrunert/nn" +license: MIT +author: "Sascha Grunert" +maintainer: "mail@saschagrunert.de" +copyright: "2018 Sascha Grunert" + +extra-source-files: + - README.md + +synopsis: A tiny neural network +category: AI +description: Please see the README on Github at + + +dependencies: + - base >= 4.7 && < 5 + +library: + source-dirs: src + ghc-options: + - -Wall + - -Wcompat + dependencies: + - random + - split + +tests: + nn-test: + main: Spec.hs + source-dirs: test + ghc-options: + - -Wall + - -Wcompat + - -threaded + - -rtsopts + - -with-rtsopts=-N + dependencies: + - nn + - tasty + - tasty-hspec + - tasty-quickcheck diff --git a/src/AI/Nn.hs b/src/AI/Nn.hs new file mode 100644 index 0000000..c7b9e6d --- /dev/null +++ b/src/AI/Nn.hs @@ -0,0 +1,236 @@ +-- | This module contains everything related to the main library interface +-- +-- @since 0.1.0 + +module AI.Nn + ( Network + , predict + , new + , newIO + , train + ) where + +import Data.List (find + ,transpose) +import Data.List.Split (chunksOf) +import Data.Maybe (fromJust) +import System.Random (StdGen + ,getStdGen + ,randomRs) + +-- | The network +-- +-- @since 0.1.0 +type Network = Network' () + +-- | The alias for a list of layers +-- +-- @since 0.1.0 +type Network' a = [Layer a] + +-- | The network layer +-- +-- @since 0.1.0 +type Layer a = [(Neuron,a)] + +-- | A network neuron +-- +-- @since 0.1.0 +data Neuron = Neuron { inputWeights :: [Double] -- ^ The input weights + , activate :: Double -> Double -- ^ The activation function + , activate' :: Double -> Double -- ^ The first derivation of the activation function + } + +-- | The forward layer type +-- +-- @since 0.1.0 +data Forward = Forward { output :: Double + , sumInputWeight :: Double + , inputs :: [Double] + } deriving Show + +-- | The alias for a list of input weights +-- +-- @since 0.1.0 +type Neuron' = [Double] + +-- | The sigmoid activation function +-- +-- @since 0.1.0 +sigmoid :: Double -> Double +sigmoid x = 1.0 / (1 + exp (-x)) + +-- | The first derivation of the sigmoid function +-- +-- @since 0.1.0 +sigmoid' :: Double -> Double +sigmoid' x = sigmoid x * (1 - sigmoid x) + +-- | Create a sigmoid neuron from given input weights +-- +-- @since 0.1.0 +sigmoidNeuron :: Neuron' -> Neuron +sigmoidNeuron ws = Neuron ws sigmoid sigmoid' + +-- | Create a output neuron from given weights +-- +-- @since 0.1.0 +outputNeuron :: Neuron' -> Neuron +outputNeuron ws = Neuron ws id (const 1) + +-- | Create a bias neuron from given number of inputs +-- +-- @since 0.1.0 +biasNeuron :: Int -> Neuron +biasNeuron i = Neuron (replicate i 1) (const 1) (const 0) + +-- | Create a new Layer from a list of Neuron' +-- +-- @since 0.1.0 +createLayer :: Functor f => f t -> (t -> a) -> f (a, ()) +createLayer n x = (\p -> (x p, ())) <$> n + +-- | Create a new sigmoid Layer from a list of Neuron' +-- +-- @since 0.1.0 +sigmoidLayer :: [Neuron'] -> Layer () +sigmoidLayer n = (biasNeuron x, ()) : createLayer n sigmoidNeuron + where x = length $ head n + +-- | Create a new output Layer from a list of Neuron' +-- +-- @since 0.1.0 +outputLayer :: [Neuron'] -> Layer () +outputLayer n = createLayer n outputNeuron + +-- | Create a new network for a StdGen and a number of layer and neurons +-- +-- @since 0.1.0 +new :: [Int] -> StdGen -> Network +new n g = (sigmoidLayer <$> init wss) ++ [outputLayer (last wss)] + where + rest = init n + hiddenIcsNcs = zip ((+ 1) <$> rest) (tail rest) + (outputIc, outputNc) = (snd (last hiddenIcsNcs) + 1, last n) + rs = randomRs (-1, 1) g + (hidden, rs') = foldl + ( \(wss', rr') (ic, nc) -> + let (sl, rs'') = pack ic nc rr' in (wss' ++ [sl], rs'') + ) + ([], rs) + hiddenIcsNcs + (outputWss, _) = pack outputIc outputNc rs' + wss = hidden ++ [outputWss] + pack ic nc ws = (take nc $ chunksOf ic ws, drop (ic * nc) ws) + +-- | Create a new standard network for a number of layer and neurons +-- +-- @since 0.1.0 +newIO :: [Int] -> IO Network +newIO n = new n <$> getStdGen + +-- | Do the complete back propagation +-- +-- @since 0.1.0 +backpropagate :: Network -> ([Double], [Double]) -> Network +backpropagate nw (xs, ys) = weightUpdate (forwardLayer nw xs) ys + +-- | The learning rate +-- +-- @since 0.1.0 +rate :: Double +rate = 0.5 + +-- | Generate forward pass info +-- +-- @since 0.1.0 +forwardLayer :: Network -> [Double] -> Network' Forward +forwardLayer nw xs = reverse . fst $ foldl pf ([], 1 : xs) nw + where + pf (nw', xs') l = (l' : nw', xs'') + where + l' = (\(n, _) -> (n, forwardNeuron n xs')) <$> l + xs'' = (output . snd) <$> l' + +-- | Generate forward pass info for a neuron +-- +-- @since 0.1.0 +forwardNeuron :: Neuron -> [Double] -> Forward +forwardNeuron n xs = Forward + { output = activate n net' + , sumInputWeight = net' + , inputs = xs + } + where net' = calcNet xs (inputWeights n) + +-- | Calculate the product sum +-- +-- @since 0.1.0 +calcNet :: [Double] -> [Double] -> Double +calcNet xs ws = sum $ zipWith (*) xs ws + +-- | Updates the weights for an entire network +-- +-- @since 0.1.0 +weightUpdate + :: Network' Forward + -> [Double] -- ^ desired output value + -> Network +weightUpdate fpnw ys = fst $ foldr updateLayer ([], ds) fpnw + where ds = zipWith (-) ys ((output . snd) <$> last fpnw) + +-- | Updates the weights for a layer +-- +-- @since 0.1.0 +updateLayer :: Layer Forward -> (Network, [Double]) -> (Network, [Double]) +updateLayer fpl (nw, ds) = (l' : nw, ds') + where + (l, es) = unzip $ zipWith updateNeuron fpl ds + ds' = + map sum . transpose $ map (\(n, e) -> (* e) <$> inputWeights n) (zip l es) + l' = (\n -> (n, ())) <$> l + +-- | Updates the weights for a neuron +-- +-- @since 0.1.0 +updateNeuron :: (Neuron, Forward) -> Double -> (Neuron, Double) +updateNeuron (n, fpi) d = (n { inputWeights = ws' }, e) + where + e = activate' n (sumInputWeight fpi) * d + ws' = zipWith (\x w -> w + (rate * e * x)) (inputs fpi) (inputWeights n) + + +-- | Trains a network with a set of vector pairs until the global error is +-- smaller than epsilon +-- +-- @since 0.1.0 +train :: Double -> Network -> [([Double], [Double])] -> Network +train epsilon nw samples = fromJust + $ find (\x -> globalQuadError x samples < epsilon) (trainUl nw samples) + +-- | Create an indefinite sequence of networks +-- +-- @since 0.1.0 +trainUl :: Network -> [([Double], [Double])] -> [Network] +trainUl nw samples = iterate (\x -> foldl backpropagate x samples) nw + +-- | Quadratic error for multiple pairs +-- +-- @since 0.1.0 +globalQuadError :: Network -> [([Double], [Double])] -> Double +globalQuadError nw samples = sum $ quadErrorNet nw <$> samples + +-- | Quadratic error for a single vector pair +-- +-- @since 0.1.0 +quadErrorNet :: Network -> ([Double], [Double]) -> Double +quadErrorNet nw (xs, ys) = + sum $ zipWith (\o y -> (y - o) ** 2) (predict nw xs) ys + +-- | Calculates the output of a network for a given input vector +-- +-- @since 0.1.0 +predict :: Network -> [Double] -> [Double] +predict nw xs = foldl calculateLayer (1 : xs) nw + where + calculateLayer s = map (\(n, _) -> activate n (calcNet s (inputWeights n))) diff --git a/stack.yaml b/stack.yaml new file mode 100644 index 0000000..5fd7976 --- /dev/null +++ b/stack.yaml @@ -0,0 +1,66 @@ +# This file was automatically generated by 'stack init' +# +# Some commonly used options have been documented as comments in this file. +# For advanced use and comprehensive documentation of the format, please see: +# https://docs.haskellstack.org/en/stable/yaml_configuration/ + +# Resolver to choose a 'specific' stackage snapshot or a compiler version. +# A snapshot resolver dictates the compiler version and the set of packages +# to be used for project dependencies. For example: +# +# resolver: lts-3.5 +# resolver: nightly-2015-09-21 +# resolver: ghc-7.10.2 +# resolver: ghcjs-0.1.0_ghc-7.10.2 +# resolver: +# name: custom-snapshot +# location: "./custom-snapshot.yaml" +resolver: lts-11.2 + +# User packages to be built. +# Various formats can be used as shown in the example below. +# +# packages: +# - some-directory +# - https://example.com/foo/bar/baz-0.0.2.tar.gz +# - location: +# git: https://github.com/commercialhaskell/stack.git +# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a +# - location: https://github.com/commercialhaskell/stack/commit/e7b331f14bcffb8367cd58fbfc8b40ec7642100a +# extra-dep: true +# subdirs: +# - auto-update +# - wai +# +# A package marked 'extra-dep: true' will only be built if demanded by a +# non-dependency (i.e. a user package), and its test suites and benchmarks +# will not be run. This is useful for tweaking upstream packages. +packages: +- . +# Dependency packages to be pulled from upstream that are not in the resolver +# (e.g., acme-missiles-0.3) +# extra-deps: [] + +# Override default flag values for local packages and extra-deps +# flags: {} + +# Extra package databases containing global packages +# extra-package-dbs: [] + +# Control whether we use the GHC we find on the path +# system-ghc: true +# +# Require a specific version of stack, using version ranges +# require-stack-version: -any # Default +# require-stack-version: ">=1.6" +# +# Override the architecture used by stack, especially useful on Windows +# arch: i386 +# arch: x86_64 +# +# Extra directories used by stack for building +# extra-include-dirs: [/path/to/dir] +# extra-lib-dirs: [/path/to/dir] +# +# Allow a newer minor version of GHC than the snapshot specifies +# compiler-check: newer-minor \ No newline at end of file diff --git a/test/Spec.hs b/test/Spec.hs new file mode 100644 index 0000000..e87ae85 --- /dev/null +++ b/test/Spec.hs @@ -0,0 +1,70 @@ +-- | The main test module +-- +-- @since 0.1.0 + +module Main + ( main + ) where + +import AI.Nn (newIO + ,predict + ,train) +import Test.Tasty (TestTree + ,defaultMain + ,localOption + ,testGroup) +import Test.Tasty.Hspec (Spec + ,it + ,parallel + ,shouldBe + ,testSpec) +import Test.Tasty.QuickCheck (QuickCheckTests (QuickCheckTests)) + +-- The main test routine +main :: IO () +main = do + uTests <- unitTests + defaultMain . opts $ testGroup "Tests" [uTests] + where opts = localOption $ QuickCheckTests 5000 + +-- Unit tests based on hspec +unitTests :: IO TestTree +unitTests = do + actionUnitTests <- testSpec "Nn" nnSpec + return $ testGroup "Unit Tests" [actionUnitTests] + +-- Nn.hs related tests +nnSpec :: Spec +nnSpec = parallel $ do + it "should succeed to train logical AND" $ do + n <- newIO [2, 2, 1] + let + nw = train 0.001 + n + [([0, 0], [0]), ([0, 1], [0]), ([1, 0], [0]), ([1, 1], [1])] + round (head $ predict nw [1, 1]) `shouldBe` (1 :: Int) + round (head $ predict nw [1, 0]) `shouldBe` (0 :: Int) + round (head $ predict nw [0, 1]) `shouldBe` (0 :: Int) + round (head $ predict nw [0, 0]) `shouldBe` (0 :: Int) + + it "should succeed to train logical OR" $ do + n <- newIO [2, 2, 1] + let + nw = train 0.001 + n + [([0, 0], [0]), ([0, 1], [1]), ([1, 0], [1]), ([1, 1], [1])] + round (head $ predict nw [1, 1]) `shouldBe` (1 :: Int) + round (head $ predict nw [1, 0]) `shouldBe` (1 :: Int) + round (head $ predict nw [0, 1]) `shouldBe` (1 :: Int) + round (head $ predict nw [0, 0]) `shouldBe` (0 :: Int) + + it "should succeed to train addition" $ do + n <- newIO [2, 2, 1] + let + nw = train 0.001 + n + [([0, 1], [1]), ([1, 1], [2]), ([1, 0], [1]), ([1, 2], [3])] + round (head $ predict nw [0, 1]) `shouldBe` (1 :: Int) + round (head $ predict nw [1, 0]) `shouldBe` (1 :: Int) + round (head $ predict nw [1, 1]) `shouldBe` (2 :: Int) + round (head $ predict nw [1, 2]) `shouldBe` (3 :: Int)