Merge pull request #80 from DrChainsaw/batchsizeutils

Add batchsize utils

.gitignore

@@ -7,3 +7,4 @@ Manifest.toml
 *.bson
 models
 .mempool
+.vscode

Project.toml

@@ -1,7 +1,7 @@
 name = "NaiveGAflux"
 uuid = "81ede08e-ab29-11e9-16d3-79edd30a1d76"
 authors = ["DrChainsaw"]
-version = "0.9.1"
+version = "0.10.0"
 
 [deps]
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
@@ -23,12 +23,12 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 CUDA = "3"
-Flux = "0.13"
+Flux = "0.13.4"
 Functors = "0.2"
 IterTools = "1"
 MemPool = "0.3"
 NaiveNASflux = "2"
-NaiveNASlib = "2"
+NaiveNASlib = "2.0.6"
 Reexport = "0.2.0, 1"
 Setfield = "0.3.4, 0.5, 0.6, 0.7, 0.8"
-julia = "1"
+julia = "1.7"

docs/Project.toml

@@ -1,4 +1,8 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
 NaiveGAflux = "81ede08e-ab29-11e9-16d3-79edd30a1d76"
+
+[compat]
+Documenter = "0.27"

docs/make.jl

@@ -1,5 +1,7 @@
 using Documenter, Literate, NaiveGAflux, NaiveGAflux.AutoFlux, NaiveGAflux.AutoFlux.ImageClassification
 
+import NaiveGAflux: AbstractIteratorMap, maptrain, mapvalidation, limit_maxbatchsize
+
 const nndir = joinpath(dirname(pathof(NaiveGAflux)), "..")
 
 function literate_example(sourcefile; rootdir=nndir, sourcedir = "test/examples", destdir="docs/src/examples")
@@ -16,6 +18,7 @@ crossover_ex = literate_example("crossover.jl")
 fitness_ex = literate_example("fitness.jl")
 candidate_ex = literate_example("candidate.jl")
 evolution_ex = literate_example("evolution.jl")
+iteratormaps_ex = literate_example("iteratormaps.jl")
 iterators_ex = literate_example("iterators.jl")
 
 makedocs(   sitename="NaiveGAflux",
@@ -37,6 +40,7 @@ makedocs(   sitename="NaiveGAflux",
                     fitness_ex,
                     candidate_ex,
                     evolution_ex,
+                    iteratormaps_ex,
                     iterators_ex
                 ],
                 "API Reference" => [
@@ -46,6 +50,8 @@ makedocs(   sitename="NaiveGAflux",
                     "reference/fitness.md",
                     "reference/candidate.md",
                     "reference/evolution.md",
+                    "reference/batchsize.md",
+                    "reference/iteratormaps.md",
                     "reference/iterators.md",
                     "reference/utils.md",
                 ]

docs/src/index.md

@@ -7,6 +7,10 @@ done in a train-validate-select-evolve loop where the validation metric serves a
 There is however absolutely no enforcement of this structure and the parts are designed to work standalone
 and in a composable manner to support a wide variety of search strategies.
 
+It is also not limited to model architecture related hyperparameters. Support for inclusion of optimizers,
+learningrates and batchsizes into the search space is built in and the framework supports adding any 
+hyperparameter (e.g data augmentation strategies and loss functions) through simple interfaces.
+
 ## Readers Guideline
 
 The [Quick Tutorial](@ref) serves as a starting point to get an idea of the syntax and type of capabilities of NaiveGAflux.
@@ -24,7 +28,8 @@ self-contained, allowing you to pick and choose the ones you like when building
 4. [Fitness Functions](@ref)
 5. [Candidate Utilities](@ref)
 6. [Evolution Strategies](@ref)
-7. [Iterators](@ref)
+7. [Iterator Maps](@ref) 
+8. [Iterators](@ref)
 
 
 

docs/src/reference/batchsize.md

@@ -0,0 +1,9 @@
+# [Batch Size Utilities](@id BatchSizeUtilsAPI)
+
+```@docs
+batchsizeselection
+BatchSizeSelectionWithDefaultInShape
+BatchSizeSelectionScaled
+BatchSizeSelectionFromAlternatives
+BatchSizeSelectionMaxSize
+```

docs/src/reference/candidate.md

@@ -3,8 +3,9 @@
 ```@docs
 CandidateModel 
 CandidateOptModel
+CandidateDataIterMap
 FittedCandidate
-evolvemodel
+MapCandidate
 Population
 model
 opt

docs/src/reference/crossover.md

@@ -14,6 +14,12 @@ CrossoverSwap
 OptimizerCrossover
 LearningRateCrossover
 ```
+
+## Core IteratorMap Crossover Operations
+```@docs
+IteratorMapCrossover
+```
+
 ## Functions
 ```@docs
 NaiveGAflux.crossover

docs/src/reference/iteratormaps.md

@@ -0,0 +1,16 @@
+# [Iterator Maps](@id IteratorMapsAPI)
+
+## Iterator map types
+```@docs
+AbstractIteratorMap
+ShieldedIteratorMap
+BatchSizeIteratorMap
+IteratorMaps
+```
+
+## [Interface functions (not exported)](@id IteratorMapInterfaceFunctionsAPI)
+```@docs
+maptrain
+mapvalidation
+limit_maxbatchsize
+```

docs/src/reference/mutation.md

@@ -18,6 +18,11 @@ LearningRateMutation
 AddOptimizerMutation
 ```
 
+## Core IteratorMap Mutation Operations
+```@docs
+TrainBatchSizeMutation
+```
+
 ## Wrapping Mutation Operations
 
 ```@docs

src/NaiveGAflux.jl

@@ -3,6 +3,7 @@ module NaiveGAflux
 using Base: release
 using Reexport
 @reexport using NaiveNASflux
+using NaiveNASlib: name
 using NaiveNASflux: FluxDense, FluxConv, FluxConvolutional, FluxNoParLayer, FluxParNorm, FluxRnn, FluxBatchNorm
 using NaiveNASflux: nograd, layertype
 using NaiveNASlib.Advanced, NaiveNASlib.Extend
@@ -29,40 +30,59 @@ const rng_default = MersenneTwister(abs(rand(Int)))
 const modeldir = "models"
 
 # Fitness
-export fitness, AbstractFitness, LogFitness, GpuFitness, AccuracyFitness, TrainThenFitness, TrainAccuracyFitness, MapFitness, EwmaFitness, TimeFitness, SizeFitness, AggFitness
+export fitness, AbstractFitness, LogFitness, GpuFitness, AccuracyFitness, TrainThenFitness, TrainAccuracyFitness, MapFitness
+export EwmaFitness, TimeFitness, SizeFitness, AggFitness
 
 # Candidate
-export evolvemodel, AbstractCandidate, CandidateModel, CandidateOptModel, FittedCandidate, model, opt, lossfun
+export AbstractCandidate, CandidateModel, CandidateOptModel, CandidateDataIterMap, FittedCandidate, MapCandidate, model, opt, lossfun
 
 # Evolution
-export evolve, AbstractEvolution, NoOpEvolution, AfterEvolution, EliteSelection, SusSelection, TournamentSelection, CombinedEvolution, EvolutionChain, PairCandidates, ShuffleCandidates, EvolveCandidates
+export evolve, AbstractEvolution, NoOpEvolution, AfterEvolution, EliteSelection, SusSelection, TournamentSelection, CombinedEvolution
+export EvolutionChain, PairCandidates, ShuffleCandidates, EvolveCandidates
 
 # Population
 export Population, generation
 
 # misc types
 export Probability, MutationShield, ApplyIf, RemoveIfSingleInput, PersistentArray, ShieldedOpt
 
+# Batch size selection
+export BatchSizeSelectionWithDefaultInShape, BatchSizeSelectionScaled, BatchSizeSelectionFromAlternatives, BatchSizeSelectionMaxSize, batchsizeselection
+
 # Iterators. These should preferably come from somewhere else, but I haven't found anything which fits the bill w.r.t repeatability over subsets
-export RepeatPartitionIterator, SeedIterator, MapIterator, GpuIterator, BatchIterator, ShuffleIterator, TimedIterator, TimedIteratorStop, StatefulGenerationIter
+export RepeatPartitionIterator, SeedIterator, MapIterator, GpuIterator, BatchIterator, ShuffleIterator, TimedIterator, TimedIteratorStop
+export StatefulGenerationIter
+
+# Iterator mapping types for evolving hyperparameters related to datasets, e.g. augmentation and batch size
+export BatchSizeIteratorMap, IteratorMaps, ShieldedIteratorMap
 
 # Persistence
 export persist
 
 # Vertex selection types
 export AbstractVertexSelection, AllVertices, FilterMutationAllowed
 
-# mutation types
-export AbstractMutation, MutationProbability, WeightedMutationProbability, HighUtilityMutationProbability, LowUtilityMutationProbability, MutationChain, RecordMutation, LogMutation, MutationFilter, PostMutation, VertexMutation, NoutMutation, AddVertexMutation, RemoveVertexMutation, AddEdgeMutation, RemoveEdgeMutation, KernelSizeMutation, KernelSizeMutation2D, ActivationFunctionMutation, PostMutation, OptimizerMutation, LearningRateMutation, AddOptimizerMutation
+# generic mutation types
+export AbstractMutation, MutationProbability, WeightedMutationProbability, HighUtilityMutationProbability, LowUtilityMutationProbability 
+export MutationChain, RecordMutation, LogMutation, MutationFilter
+# graph mutation types
+export VertexMutation, NoutMutation, AddVertexMutation, RemoveVertexMutation, AddEdgeMutation, RemoveEdgeMutation, KernelSizeMutation
+export KernelSizeMutation2D, ActivationFunctionMutation
+# optimizer mutation types
+export OptimizerMutation, LearningRateMutation, AddOptimizerMutation
+# Iterator wrapping mutation types
+export TrainBatchSizeMutation
 
 # Crossover types
-export AbstractCrossover, VertexCrossover, CrossoverSwap, OptimizerCrossover, LearningRateCrossover
+export AbstractCrossover, VertexCrossover, CrossoverSwap, OptimizerCrossover, LearningRateCrossover, IteratorMapCrossover
 
 # architecture spaces
-export AbstractArchSpace, LoggingArchSpace, VertexSpace, NoOpArchSpace, ArchSpace, ConditionalArchSpace, RepeatArchSpace, ArchSpaceChain, ForkArchSpace, ResidualArchSpace, FunctionSpace, GlobalPoolSpace
+export AbstractArchSpace, LoggingArchSpace, VertexSpace, NoOpArchSpace, ArchSpace, ConditionalArchSpace, RepeatArchSpace, ArchSpaceChain
+export ForkArchSpace, ResidualArchSpace, FunctionSpace, GlobalPoolSpace
 
 #  Other search space types
-export BaseLayerSpace, AbstractParSpace, SingletonParSpace, Singleton2DParSpace, ParSpace, ParSpace1D, ParSpace2D, CoupledParSpace, NamedLayerSpace, LoggingLayerSpace, DenseSpace, ConvSpace, BatchNormSpace, PoolSpace, LayerVertexConf, Shielded, ConcConf
+export BaseLayerSpace, AbstractParSpace, SingletonParSpace, Singleton2DParSpace, ParSpace, ParSpace1D, ParSpace2D, CoupledParSpace
+export NamedLayerSpace, LoggingLayerSpace, DenseSpace, ConvSpace, BatchNormSpace, PoolSpace, LayerVertexConf, Shielded, ConcConf
 
 #weight inits
 export AbstractWeightInit, DefaultWeightInit, IdentityWeightInit, PartialIdentityWeightInit, ZeroWeightInit
@@ -74,13 +94,21 @@ export nparams
 export AutoFlux
 
 # Visulization
-export PlotFitness, ScatterPop, ScatterOpt, MultiPlot, CbAll
+export PlotFitness, ScatterPop, ScatterOpt, ScatterBatchSize, MultiPlot, CbAll
 
 include("util.jl")
 include("shape.jl")
+include("batchsize.jl")
+include("iteratormaps.jl")
 include("archspace.jl")
-include("mutation.jl")
-include("crossover.jl")
+include("mutation/generic.jl")
+include("mutation/graph.jl")
+include("mutation/optimizer.jl")
+include("mutation/iteratormaps.jl")
+include("crossover/generic.jl")
+include("crossover/graph.jl")
+include("crossover/optimizer.jl")
+include("crossover/iteratormaps.jl")
 include("candidate.jl")
 include("fitness.jl")
 include("evolve.jl")

src/app/imageclassification/ImageClassification.jl

@@ -9,7 +9,7 @@ using NaiveNASlib.Advanced, NaiveNASlib.Extend
 import Flux
 using Flux: Dense, Conv, ConvTranspose, DepthwiseConv, CrossCor, LayerNorm, BatchNorm, InstanceNorm, GroupNorm, 
             MaxPool, MeanPool, Dropout, AlphaDropout, GlobalMaxPool, GlobalMeanPool, cpu, gpu
-using Flux: Descent, Momentum, Nesterov, ADAM, NADAM, ADAGrad, WeightDecay
+using Flux: Descent, Momentum, Nesterov, Adam, NAdam, AdaGrad, WeightDecay
 import Functors
 using Functors: fmap
 using Random
@@ -131,12 +131,15 @@ function generate_persistent(nr, newpop, mdir, insize, outsize, cwrap=identity,
     end
 
     iv(i) = conv2dinputvertex(join(["model", i, ".input"]), insize[3])
-    return Population(PersistentArray(mdir, nr, i -> create_model(join(["model", i]), archspace, iv(i), cwrap)))
+    return Population(PersistentArray(mdir, nr, i -> create_model(join(["model", i]), archspace, iv(i), cwrap, insize)))
 end
-function create_model(name, as, in, cwrap)
+function create_model(name, as, in, cwrap, insize)
     optselect = optmutation(1.0)
     opt = optselect(Descent(rand() * 0.099 + 0.01))
-    cwrap(CandidateOptModel(opt, CompGraph(in, as(name, in))))
+    bslimit = batchsizeselection(insize[1:end-1]; alternatives=ntuple(i->2^i, 10))
+    imstart = BatchSizeIteratorMap(64, 64, bslimit)
+    im = itermapmutation(1.0)(imstart)
+    cwrap(CandidateDataIterMap(im, CandidateOptModel(opt, CompGraph(in, as(name, in)))))
 end
 
 end

src/app/imageclassification/strategy.jl

@@ -61,7 +61,7 @@ function TrainSplitAccuracy(;split=0.1,
             accuracyconfig=BatchedIterConfig(),
             accuracyfitness=AccuracyVsSize,
             trainconfig=TrainIterConfig(),
-            trainfitness=(iter, accf) -> GpuFitness(TrainThenFitness(StatefulGenerationIter(iter), Flux.Losses.logitcrossentropy, ADAM(), accf, 0.0)))
+            trainfitness=(iter, accf) -> GpuFitness(TrainThenFitness(StatefulGenerationIter(iter), Flux.Losses.logitcrossentropy, Adam(), accf, 0.0)))
 
     return TrainSplitAccuracy(split, accuracyconfig, accuracyfitness, trainconfig, trainfitness)
 end
@@ -123,7 +123,7 @@ function TrainAccuracyVsSize(;
                         trainconfig=TrainIterConfig(),
                         trainfitness = dataiter -> sizevs(GpuFitness(TrainAccuracyFitness(
                                                                             dataiter=StatefulGenerationIter(dataiter), 
-                                                                            defaultloss=Flux.Losses.logitcrossentropy, defaultopt = ADAM())))) 
+                                                                            defaultloss=Flux.Losses.logitcrossentropy, defaultopt = Adam())))) 
         return TrainAccuracyVsSize(trainconfig, trainfitness)
 end
 function fitnessfun(s::TrainAccuracyVsSize, x, y) 
@@ -133,7 +133,7 @@ end
 
 """
     struct BatchedIterConfig{T, V}
-    BatchedIterConfig(;batchsize=32, dataaug=identity, iterwrap=GpuIterator) 
+    BatchedIterConfig(;batchsize=32, dataaug=identity, iterwrap=identity) 
 
 Configuration for creating batch iterators from array data.
 
@@ -146,12 +146,12 @@ struct BatchedIterConfig{T, V}
     dataaug::T
     iterwrap::V
 end
-BatchedIterConfig(;batchsize=32, dataaug=identity, iterwrap=GpuIterator) = BatchedIterConfig(batchsize, dataaug, iterwrap)
+BatchedIterConfig(;batchsize=1024, dataaug=identity, iterwrap=identity) = BatchedIterConfig(batchsize, dataaug, iterwrap)
 dataiter(s::BatchedIterConfig, x, y) = dataiter(x, y, s.batchsize, s.dataaug) |> s.iterwrap
 
 """
     struct ShuffleIterConfig{T, V}
-    ShuffleIterConfig(;batchsize=32, seed=123, dataaug=identity, iterwrap=GpuIterator) 
+    ShuffleIterConfig(;batchsize=32, seed=123, dataaug=identity, iterwrap=identity) 
 
 Configuration for creating shuffled batch iterators from array data. Data will be re-shuffled every time the iterator restarts.
 
@@ -168,7 +168,7 @@ struct ShuffleIterConfig{T, V}
     dataaug::T
     iterwrap::V
 end
-ShuffleIterConfig(;batchsize=32, seed=123, dataaug=identity, iterwrap=GpuIterator) = ShuffleIterConfig(batchsize, seed, dataaug, iterwrap)
+ShuffleIterConfig(;batchsize=1024, seed=123, dataaug=identity, iterwrap=identity) = ShuffleIterConfig(batchsize, seed, dataaug, iterwrap)
 dataiter(s::ShuffleIterConfig, x, y) = dataiter(x, y, s.batchsize, s.seed, s.dataaug) |> s.iterwrap
 
 
@@ -317,7 +317,7 @@ Crossover is done using [`CrossoverSwap`](@ref) for models and [`LearningRateCro
 
 Mutation is applied both to the model itself (change sizes, add/remove vertices/edges) as well as to the optimizer (change learning rate and optimizer algorithm).
 """
-crossovermutate(;pcrossover=0.3, pmutate=0.9) = function(inshape)
+crossovermutate(;pcrossover=0.3, pmutate=0.8) = function(inshape)
     cross = candidatecrossover(pcrossover)
     crossoverevo = AfterEvolution(PairCandidates(EvolveCandidates(cross)), align_vertex_names)
 
@@ -327,8 +327,8 @@ crossovermutate(;pcrossover=0.3, pmutate=0.9) = function(inshape)
     return EvolutionChain(crossoverevo, mutationevo)
 end
 
-candidatemutation(p, inshape) = evolvemodel(MutationProbability(graphmutation(inshape), p), optmutation())
-candidatecrossover(p) = evolvemodel(MutationProbability(graphcrossover(), p), optcrossover())
+candidatemutation(p, inshape) = MapCandidate(MutationProbability(graphmutation(inshape), p), optmutation(), itermapmutation())
+candidatecrossover(p) = MapCandidate(MutationProbability(graphcrossover(), p), optcrossover(), itermapcrossover())
 
 function clear_redundant_vertices(pop)
     foreach(cand -> NaiveGAflux.model(check_apply, cand), pop)
@@ -359,6 +359,13 @@ function rename_model(i, cand)
     end
 end
 
+itermapcrossover(p= 0.2) = MutationProbability(IteratorMapCrossover(), p) |> IteratorMapCrossover
+
+function itermapmutation(p=0.1)
+    m = TrainBatchSizeMutation(-0.2, 0.2, ntuple(i -> 2^(i+2), 8))
+    return MutationProbability(m, p)
+end
+
 function optcrossover(poptswap=0.3, plrswap=0.4)
     lrc = MutationProbability(LearningRateCrossover(), plrswap) |> OptimizerCrossover
     oc = MutationProbability(OptimizerCrossover(), poptswap) |> OptimizerCrossover
@@ -367,7 +374,7 @@ end
 
 function optmutation(p=0.1)
     lrm = LearningRateMutation()
-    om = MutationProbability(OptimizerMutation([Descent, Momentum, Nesterov, ADAM, NADAM, ADAGrad]), p)
+    om = MutationProbability(OptimizerMutation([Descent, Momentum, Nesterov, Adam, NAdam, AdaGrad]), p)
     return MutationChain(lrm, om)
 end