From aa88bd1f1d450045b5b9b7d6c162891d64927795 Mon Sep 17 00:00:00 2001 From: samanthawangdl Date: Fri, 17 Jan 2025 14:08:39 +0000 Subject: [PATCH] Add Float GEMM on PULP with Tiling 1. add pulpfpgemm binding, tiler, connstraint 2. add floatgemm test to CI Siracusa 3. change deeploytest for float input & output 4. change testMVP input default datatype from int64 to float64 --- .github/workflows/CI.yml | 9 ++ CHANGELOG.md | 16 +- Deeploy/Targets/PULPOpen/Bindings.py | 18 ++- Deeploy/Targets/PULPOpen/Platform.py | 29 ++-- .../Templates/UniformRequantShiftTemplate.py | 6 +- .../TileConstraints/GEMMTileConstraint.py | 144 ++++++++++++++++++ Deeploy/Targets/PULPOpen/Tiler.py | 17 ++- .../Platforms/Siracusa/src/deeploytest.c | 23 ++- DeeployTest/Tests/testFloatGEMM/inputs.npz | Bin 1264 -> 13036 bytes DeeployTest/Tests/testFloatGEMM/network.onnx | Bin 618 -> 8414 bytes DeeployTest/Tests/testFloatGEMM/outputs.npz | Bin 446 -> 4362 bytes DeeployTest/testMVP.py | 6 +- 12 files changed, 219 insertions(+), 49 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 420aa9d7..58a05335 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -200,6 +200,7 @@ jobs: RQHardswish testBacktracking FloatAdder + testFloatGEMM num-cores: 8 siracusa-models: @@ -251,6 +252,10 @@ jobs: { "name": "RQHardswish", "L1": [750] + }, + { + "name": "testFloatGEMM", + "L1": [8000] } ] num-cores: 8 @@ -291,6 +296,10 @@ jobs: { "name": "RQHardswish", "L1": [750] + }, + { + "name": "testFloatGEMM", + "L1": [8000] } ] num-cores: 8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df97d14..42f41c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,9 +69,15 @@ - Add the RTL library to the snitch_cluster build process in the Makefile, required for GVSOC simulation -## Add Float Support & Float GEMM for Generic +## Add Float Support & Float GEMM for Generic and PULP + +### Added - Float Support for Constbuffer -- Float GEMM on Generic -- Added FP GEMM to CI -- Fixed Float bug on Testslice, CMSIS TestUtil, DivInterger -- Fixed AbstractDatayType Float Bugs +- Simple Float GEMM on Generic and Pulp +- FP GEMM to CI +- FP GEMM Tiling on PULP + +### Fixed +- Float bug on Testslice, CMSIS TestUtil, DivInterger +- AbstractDatayType Float Bugs + diff --git a/Deeploy/Targets/PULPOpen/Bindings.py b/Deeploy/Targets/PULPOpen/Bindings.py index cb7515bc..ac61768f 100644 --- a/Deeploy/Targets/PULPOpen/Bindings.py +++ b/Deeploy/Targets/PULPOpen/Bindings.py @@ -32,14 +32,15 @@ from Deeploy.CommonExtensions.CodeTransformationPasses.Closure import ClosureGeneration, MemoryAwareClosureGeneration from Deeploy.CommonExtensions.CodeTransformationPasses.MemoryAllocation import ArgumentStructGeneration, \ MemoryManagementGeneration -from Deeploy.CommonExtensions.DataTypes import IntegerDataTypes, SignedIntegerDataTypes, int8_t, int32_t, uint8_t +from Deeploy.CommonExtensions.DataTypes import IntegerDataTypes, SignedIntegerDataTypes, float32_t, int8_t, int32_t, \ + uint8_t from Deeploy.DeeployTypes import CodeTransformation, NodeBinding, NodeTemplate from Deeploy.FutureExtension.Bindings.AutoFutureBinding import AutoFutureBinding from Deeploy.FutureExtension.CodeTransformationPasses.FutureCodeTransformation import FutureGeneration -from Deeploy.Targets.Generic.Templates import ConcatTemplate, RQSiGELUTemplate, iHardswishTemplate -from Deeploy.Targets.Generic.TypeCheckers import ConcatChecker, GELUChecker, HardswishChecker, MatMulChecker, \ - MulChecker, ReduceMeanChecker, RQAddChecker, RQHardswishChecker, SliceChecker, SoftmaxChecker, TransposeChecker, \ - iLayerNormChecker +from Deeploy.Targets.Generic.Templates import ConcatTemplate, FloatGemmTemplate, RQSiGELUTemplate, iHardswishTemplate +from Deeploy.Targets.Generic.TypeCheckers import ConcatChecker, GELUChecker, GEMMChecker, HardswishChecker, \ + MatMulChecker, MulChecker, ReduceMeanChecker, RQAddChecker, RQHardswishChecker, SliceChecker, SoftmaxChecker, \ + TransposeChecker, iLayerNormChecker from Deeploy.Targets.PULPOpen.CodeTransformationPasses.PULPClusterSynch import PULPSynchCoresPass from Deeploy.Targets.PULPOpen.CodeTransformationPasses.PULPClusterTiling import PULPClusterTiling from Deeploy.Targets.PULPOpen.CodeTransformationPasses.PULPL3Tiling import PULPL3Tiling @@ -196,6 +197,13 @@ ForkTransformer) for type1, type2 in zip([int8_t, uint8_t, int8_t, uint8_t], [int8_t, uint8_t, uint8_t, int8_t]) ] +PULPFloatGEMMBindings = [ + NodeBinding( + GEMMChecker([PointerClass(float32_t), PointerClass(float32_t), + PointerClass(float32_t)], [PointerClass(float32_t)]), FloatGemmTemplate.referenceTemplate, + ForkTransformer) +] + PULPRQSMatrixVecBindings = [ NodeBinding( PULPLinearChecker([PointerClass(type1), diff --git a/Deeploy/Targets/PULPOpen/Platform.py b/Deeploy/Targets/PULPOpen/Platform.py index bac2d823..4c7cda84 100644 --- a/Deeploy/Targets/PULPOpen/Platform.py +++ b/Deeploy/Targets/PULPOpen/Platform.py @@ -35,13 +35,13 @@ from Deeploy.Targets.CortexM.Parsers import CMSISMaxPool2DParser from Deeploy.Targets.Generic.Bindings import BasicGatherBindings, BasicPad1DBindings, BasicPad2DBindings, \ BasicReshapeBindings, BasicRQIntegerDivBinding -from Deeploy.Targets.Generic.Layers import AddLayer, ConcatLayer, GatherLayer, MatMulLayer, MaxPoolLayer, MulLayer, \ - PadLayer, ReduceMeanLayer, RequantShiftLayer, ReshapeLayer, RQIntegerDivLayer, RQSiGELULayer, RQSiHardswishLayer, \ - SliceLayer, TransposeLayer, iHardswishLayer, iRMSNormLayer, iSoftmaxLayer -from Deeploy.Targets.Generic.Parsers import AddParser, ConcatParser, FlattenParser, GatherParser, MatMulParser, \ - MulParser, Pad1DParser, Pad2DParser, ReduceMeanParser, RequantShiftParser, ReshapeParser, RQAddParser, \ - RQIntegerDivParser, RQSiGELUParser, RQSiHardswishParser, SliceParser, TransposeParser, UniformRequantShiftParser, \ - UnsqueezeParser, iHardswishParser, iRMSNormParser, iSoftmaxParser +from Deeploy.Targets.Generic.Layers import AddLayer, ConcatLayer, GatherLayer, GEMMLayer, MatMulLayer, MaxPoolLayer, \ + MulLayer, PadLayer, ReduceMeanLayer, RequantShiftLayer, ReshapeLayer, RQIntegerDivLayer, RQSiGELULayer, \ + RQSiHardswishLayer, SliceLayer, TransposeLayer, iHardswishLayer, iRMSNormLayer, iSoftmaxLayer +from Deeploy.Targets.Generic.Parsers import AddParser, ConcatParser, FlattenParser, GatherParser, GEMMParser, \ + MatMulParser, MulParser, Pad1DParser, Pad2DParser, ReduceMeanParser, RequantShiftParser, ReshapeParser, \ + RQAddParser, RQIntegerDivParser, RQSiGELUParser, RQSiHardswishParser, SliceParser, TransposeParser, \ + UniformRequantShiftParser, UnsqueezeParser, iHardswishParser, iRMSNormParser, iSoftmaxParser from Deeploy.Targets.Generic.Templates import AllocateTemplate as BasicAllocateTemplate from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import IntegerDivRequantMergePass, \ MergeConstAddAndRequantPass, MergeTrueIntegerDivRequantShiftPass, RQSSplitPass, SkipEmptyConcatPass, \ @@ -53,12 +53,13 @@ PULPDWConv2DParser, PULPGEMMParser, PULPMatrixVecParser, PULPTallGEMMParser from Deeploy.Targets.PULPOpen.Templates import AllocateTemplate, FreeTemplate from Deeploy.Targets.PULPOpen.Tiler import PULPAddTilingReadyBindings, PULPConcatTilingReadyBindings, \ - PULPFlattenTilingReadyBindings, PULPiHardswishTilingReadyBindings, PULPiRMSNormTilingReadyBindings, \ - PULPiRQSGELUTilingReadyBindings, PULPiSoftmaxTilingReadyBindings, PULPMatMulTilingReadyBindings, \ - PULPMaxPool2DTilingReadyBindings, PULPMulTilingReadyBindings, PULPRQAddTilingReadyBindings, \ - PULPRQSConv2DTilingReadyBindings, PULPRQSDWConv2DTilingReadyBindings, PULPRQSGEMMTilingReadyBindings, \ - PULPRQSiHardswishTilingReadyBindings, PULPRQSMatrixVecTilingReadyBindings, PULPRQSTallGEMMTilingReadyBindings, \ - PULPRQSTilingReadyBindings, PULPTransposeTilingReadyBindings, PULPUniformRQSTilingReadyBindings + PULPFlattenTilingReadyBindings, PULPFPGEMMTilingReadyBindings, PULPiHardswishTilingReadyBindings, \ + PULPiRMSNormTilingReadyBindings, PULPiRQSGELUTilingReadyBindings, PULPiSoftmaxTilingReadyBindings, \ + PULPMatMulTilingReadyBindings, PULPMaxPool2DTilingReadyBindings, PULPMulTilingReadyBindings, \ + PULPRQAddTilingReadyBindings, PULPRQSConv2DTilingReadyBindings, PULPRQSDWConv2DTilingReadyBindings, \ + PULPRQSGEMMTilingReadyBindings, PULPRQSiHardswishTilingReadyBindings, PULPRQSMatrixVecTilingReadyBindings, \ + PULPRQSTallGEMMTilingReadyBindings, PULPRQSTilingReadyBindings, PULPTransposeTilingReadyBindings, \ + PULPUniformRQSTilingReadyBindings from Deeploy.Targets.PULPOpen.TopologyOptimizationPasses.Passes import PULPAddRequantMergePass, \ PULPConvRequantMergePass, PULPGEMMRequantMergePass, PULPMatMulRequantMergePass @@ -87,6 +88,7 @@ Conv2DMapper = NodeMapper(PULPConv2DParser(), PULPRQSConv2DTilingReadyBindings) DWConv2DMapper = NodeMapper(PULPDWConv2DParser(), PULPRQSDWConv2DTilingReadyBindings) GEMMMapper = NodeMapper(PULPGEMMParser(), PULPRQSGEMMTilingReadyBindings) +FloatGEMMMapper = NodeMapper(GEMMParser(), PULPFPGEMMTilingReadyBindings) MatrixVecMapper = NodeMapper(PULPMatrixVecParser(), PULPRQSMatrixVecTilingReadyBindings) TallGEMMMapper = NodeMapper(PULPTallGEMMParser(), PULPRQSTallGEMMTilingReadyBindings) MaxPool2DMapper = NodeMapper(CMSISMaxPool2DParser(), PULPMaxPool2DTilingReadyBindings) @@ -104,6 +106,7 @@ PULPMapping = { 'RequantizedConv': PULPRQSConvLayer([Conv2DMapper, DWConv2DMapper, Conv1DMapper, DWConv1DMapper]), 'RequantizedGemm': PULPRQSGEMMLayer([MatrixVecMapper, TallGEMMMapper, GEMMMapper]), + 'Gemm': GEMMLayer([FloatGEMMMapper]), 'MaxPool': MaxPoolLayer([MaxPool2DMapper]), 'RequantizediGELU': RQSiGELULayer([RQGELU_int8_Mapper]), 'RQIntegerDiv': RQIntegerDivLayer([RQIntegerDivMapper]), diff --git a/Deeploy/Targets/PULPOpen/Templates/UniformRequantShiftTemplate.py b/Deeploy/Targets/PULPOpen/Templates/UniformRequantShiftTemplate.py index 9592b0d3..4cfd3d6f 100644 --- a/Deeploy/Targets/PULPOpen/Templates/UniformRequantShiftTemplate.py +++ b/Deeploy/Targets/PULPOpen/Templates/UniformRequantShiftTemplate.py @@ -77,10 +77,10 @@ def alignToContext(self, ctxt: NetworkContext, inSignage = "s" if signedI else "u" outSignage = "s" if signedO else "u" -mul_intimmediate = int(mul_immediate) -add_intimmediate = int(add_immediate) +mul_int_immediate = int(mul_immediate) +add_int_immediate = int(add_immediate) %> // UniformRequantShift (Name: ${nodeName}, Op: ${nodeOp}) -UniformRequantShift_${inSignage}${data_in_type.referencedType.typeWidth}_${outSignage}${data_out_type.referencedType.typeWidth}(${data_in}, ${size}, ${mul_intimmediate}, ${add_intimmediate}, ${data_out}, ${log2Dstring}, ${channel_width}, 0, 0 , ${output_min}, ${output_max}, 1); +UniformRequantShift_${inSignage}${data_in_type.referencedType.typeWidth}_${outSignage}${data_out_type.referencedType.typeWidth}(${data_in}, ${size}, ${mul_int_immediate}, ${add_int_immediate}, ${data_out}, ${log2Dstring}, ${channel_width}, 0, 0 , ${output_min}, ${output_max}, 1); """) diff --git a/Deeploy/Targets/PULPOpen/TileConstraints/GEMMTileConstraint.py b/Deeploy/Targets/PULPOpen/TileConstraints/GEMMTileConstraint.py index 437b6478..17d3c378 100644 --- a/Deeploy/Targets/PULPOpen/TileConstraints/GEMMTileConstraint.py +++ b/Deeploy/Targets/PULPOpen/TileConstraints/GEMMTileConstraint.py @@ -216,3 +216,147 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo tm = GEMMTileConstraint.addPolicyConstraint(tilerModel, parseDict, ctxt) return tm + + +class FloatGEMMTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + # Get to-be-tiled tensor's buffers + bufferA = ctxt.lookup(name = parseDict['A']) + bufferB = ctxt.lookup(name = parseDict['B']) + bufferC = ctxt.lookup(name = parseDict['C']) + outputBuffer = ctxt.lookup(name = parseDict['data_out']) + + # Add I/O dimensions to the model as variables + for bufferName in [bufferA.name, bufferB.name, bufferC.name, outputBuffer.name]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + dimOffsetA = len(bufferA.shape) - 2 + dimOffsetB = len(bufferB.shape) - 2 + dimOffsetOut = len(outputBuffer.shape) - 2 + + AFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, dimIdx = dimOffsetA + parseDict['transA']) + ASecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, + dimIdx = dimOffsetA + 1 - parseDict['transA']) + BFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, dimIdx = dimOffsetB + parseDict['transB']) + BSecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, + dimIdx = dimOffsetB + 1 - parseDict['transB']) + outputFirstDimVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = dimOffsetOut) + outputSecondDimVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = dimOffsetOut + 1) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputFirstDimVar == AFirstDimVar) + tilerModel.addConstraint(outputSecondDimVar == BSecondDimVar) + + # Add GEMM Geometrical constraints + tilerModel.addConstraint(ASecondDimVar == BFirstDimVar) + + addDimVar_1 = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = 0) + addDimVar_2 = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = 1) + tilerModel.addConstraint(outputFirstDimVar == addDimVar_1) + tilerModel.addConstraint(outputSecondDimVar == addDimVar_2) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + bufferA = ctxt.lookup(name = parseDict['A']) + bufferB = ctxt.lookup(name = parseDict['B']) + + dimOffsetA = len(bufferA.shape) - 2 + dimOffsetB = len(bufferB.shape) - 2 + + AFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, dimIdx = dimOffsetA + parseDict['transA']) + + ASecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, + dimIdx = dimOffsetA + 1 - parseDict['transA']) + BFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, dimIdx = dimOffsetB + parseDict['transB']) + BSecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, + dimIdx = dimOffsetB + 1 - parseDict['transB']) + + # VIC: We don't want to deal with intermediate results between kernel calls + tilerModel.addConstraint(ASecondDimVar == parseDict['N']) + tilerModel.addConstraint(BFirstDimVar == parseDict['N']) + + if (parseDict["O"] >= 16): + # modulus = tilerModel.addMinTileSizeConstraint(parseDict, 'O', BSecondDimVar, 8, prefix="8_") + modulus = tilerModel.addTileSizeDivisibleConstraint(parseDict, 'O', BSecondDimVar, 16, prefix = "16_") + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['A', 'B', 'C', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + varA = operatorRepresentation['A'] + + NSize = ctxt.lookup(varA).shape[-1] + NOffset = 0 + + inputACubes = [] + inputBCubes = [] + inputAddCubes = [] + + replacements = {"M": [], "O": [], "batch": []} + + # Every output is constructed by a pair of inputs. Reconstruct this pair. + for cube in outputCubes: + + BSize = 1 + BOffset = 0 + BatchSize = 1 + BatchOffset = 0 + + if len(cube.offset) == 2: + (MOffset, OOffset) = cube.offset + (MSize, OSize) = cube.dims + elif len(cube.offset) == 3: + (BatchOffset, MOffset, OOffset) = cube.offset + (BatchSize, MSize, OSize) = cube.dims + else: + (BatchOffset, BOffset, MOffset, OOffset) = cube.offset + (BatchSize, BSize, MSize, OSize) = cube.dims + + replacements["M"].append(MSize) + replacements["O"].append(OSize) + replacements["batch"].append(BSize) + + ACube = HyperRectangle((BatchOffset, BOffset, MOffset, NOffset), (BatchSize, BSize, MSize, NSize)) + BCube = HyperRectangle((BatchOffset, BOffset, NOffset, OOffset), (BatchSize, BSize, NSize, OSize)) + + CCube = HyperRectangle(cube.offset, cube.dims) + + inputACubes.append(ACube) + inputBCubes.append(BCube) + inputAddCubes.append(CCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + replacements["N"] = [NSize] * len(outputCubes) + + replacementTypes = { + "M": PointerClass(uint16_t), + "N": PointerClass(uint16_t), + "O": PointerClass(uint16_t), + "batch": PointerClass(uint8_t) + } + + for a, b, c in zip(inputACubes, inputBCubes, inputAddCubes): + inputLoadSchedule.append({"A": a, "B": b, "C": c}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + schedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + + return VariableReplacementScheme(replacements, replacementTypes), schedule diff --git a/Deeploy/Targets/PULPOpen/Tiler.py b/Deeploy/Targets/PULPOpen/Tiler.py index 26fd2518..20c671f9 100644 --- a/Deeploy/Targets/PULPOpen/Tiler.py +++ b/Deeploy/Targets/PULPOpen/Tiler.py @@ -40,15 +40,15 @@ from Deeploy.Targets.Generic.TileConstraints.TransposeTileConstraint import TransposeTileConstraint from Deeploy.Targets.Generic.TileConstraints.UnaryTileConstraint import UnaryTileConstraint from Deeploy.Targets.Generic.TileConstraints.UntiledTileConstraint import UntiledTileConstraint -from Deeploy.Targets.PULPOpen.Bindings import PULPConcatBindings, PULPiHardswishBindings, PULPiRMSNormBindings, \ - PULPiRQSGELUBindings, PULPMatMulBinding, PULPMaxPool2DBindings, PULPMulBindings, PULPRQAddBindings, \ - PULPRQSBindings, PULPRQSConv2DBindings, PULPRQSDWConv2DBindings, PULPRQSGEMMBindings, PULPRQSiHardswishBindings, \ - PULPRQSMatrixVecBindings, PULPRQSTallGEMMBindings, PULPSoftmaxBindings, PULPTransposeBindings, \ - PULPUniformRQSBindings, SimpleTransformer +from Deeploy.Targets.PULPOpen.Bindings import PULPConcatBindings, PULPFloatGEMMBindings, PULPiHardswishBindings, \ + PULPiRMSNormBindings, PULPiRQSGELUBindings, PULPMatMulBinding, PULPMaxPool2DBindings, PULPMulBindings, \ + PULPRQAddBindings, PULPRQSBindings, PULPRQSConv2DBindings, PULPRQSDWConv2DBindings, PULPRQSGEMMBindings, \ + PULPRQSiHardswishBindings, PULPRQSMatrixVecBindings, PULPRQSTallGEMMBindings, PULPSoftmaxBindings, \ + PULPTransposeBindings, PULPUniformRQSBindings, SimpleTransformer from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint from Deeploy.Targets.PULPOpen.TileConstraints.DWConvTileConstraint import DWConv2DTileConstraint -from Deeploy.Targets.PULPOpen.TileConstraints.GEMMTileConstraint import GEMMTileConstraint, MatrixVecTileConstraint, \ - TallGEMMTileConstraint +from Deeploy.Targets.PULPOpen.TileConstraints.GEMMTileConstraint import FloatGEMMTileConstraint, GEMMTileConstraint, \ + MatrixVecTileConstraint, TallGEMMTileConstraint from Deeploy.Targets.PULPOpen.TileConstraints.iSoftmaxTileConstraint import iSoftmaxTileConstraint from Deeploy.Targets.PULPOpen.TileConstraints.MatMulTileConstraint import MatMulTileConstraint from Deeploy.Targets.PULPOpen.TileConstraints.MaxPoolTileConstraint import MaxPoolTileConstraint @@ -64,6 +64,9 @@ PULPRQSGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = PULPRQSGEMMBindings, tileConstraint = GEMMTileConstraint()) +PULPFPGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = PULPFloatGEMMBindings, + tileConstraint = FloatGEMMTileConstraint()) + PULPRQSMatrixVecTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = PULPRQSMatrixVecBindings, tileConstraint = MatrixVecTileConstraint()) diff --git a/DeeployTest/Platforms/Siracusa/src/deeploytest.c b/DeeployTest/Platforms/Siracusa/src/deeploytest.c index c111055a..98b7f501 100644 --- a/DeeployTest/Platforms/Siracusa/src/deeploytest.c +++ b/DeeployTest/Platforms/Siracusa/src/deeploytest.c @@ -88,7 +88,8 @@ void main(void) { #ifndef CI printf("Output:\r\n"); #endif - int32_t diff, tot_err, tot_tested; + float32_t diff, expected_float, actual_float; + uint32_t tot_err, tot_tested; tot_err = 0; tot_tested = 0; char *compbuf; @@ -102,22 +103,18 @@ void main(void) { compbuf = DeeployNetwork_outputs[buf]; } - for (int i = 0; i < DeeployNetwork_outputs_bytes[buf]; i++) { - diff = ((char *)testOutputVector[buf])[i] - ((char *)compbuf)[i]; + for (int i = 0; i < DeeployNetwork_outputs_bytes[buf] / sizeof(float32_t); i++) { tot_tested++; - if (diff) { + expected_float = ((float32_t *)testOutputVector[buf])[i]; + actual_float = ((float32_t *)compbuf)[i]; + diff = expected_float - actual_float; + if (diff < -1e-5 || diff > 1e-5) { tot_err += 1; #ifndef CI - printf("Expected: %i\t\t", ((int8_t *)testOutputVector[buf])[i]); - printf("Actual: %i \t\t", ((int8_t *)compbuf)[i]); + printf("Expected: %f\t\t", expected_float); + printf("Actual: %f \t\t", actual_float); + printf("Diff: %f at Index %u \r\n", diff, i); #endif -#ifndef CI - printf("Diff: %i at Index %u \r\n", diff, i); -#endif - } else { - /* #ifndef CI */ - /* printf("\r\n"); */ - /* #endif */ } } if (DeeployNetwork_outputs[buf] < 0x1000000) { diff --git a/DeeployTest/Tests/testFloatGEMM/inputs.npz b/DeeployTest/Tests/testFloatGEMM/inputs.npz index fae7083cab62cfa40eb86d0fd3d2a0796aef0ff6..47c86bd226986d076412c5c49ab43ede6b060bb9 100644 GIT binary patch literal 13036 zcmd73c~nl(|2Ev9lqp1^R2rm7iUxJ>>nf!*ib_Z-nrG6afl`K)%#krfg$kL;JkRqO zk(rDcGCueB{jJ~guHUoP`_KE=v+s4Ed!2pGS?BEgeD-I!u50^v$hYeu^Ixr>j8}uN z<;<=!GXME1%k+?mPDo0bHZUP6Lsq8i|I_)O6aRDYKW&M(zn9NAS(#LsOvAA7Gxeha4ja~?`CV@2#G z#8!M1Ka8sEquX1u-#2RnuT@}InG3??sVf~S75U<00R5EhD2%>{q$etL-8K=CJ)NjG zvJ^Y!Iy0vBBp&pi%*tLi_b~r*rhno+xqmps%PC<8Er9_oHw&rVhr6 zMa%Jh`fsryt`D6RqhaJ<1@&#&Tx8IOH+o-TcdCyFUu1!!vwul?rv8TIv9BU$&jjwg zISm^x&V#Uu<(Kqej?rvFNoE%3ZO+342X&6H>cF`DDy-VD9tlR)yx;DPnEIw3rJvuR z{QVSmEeL_q7&pq@KLO)&?tE$x%&sO6;aR;0JMKRdEANkHyFu^K;nXK|3>ir$A73u= zoQ}1U?>OHjmO1Thd1YY$Z?`70?`d^9%<>Z4ScE?a~iEopzd3Xi-$0Tyyui@-fU5Rub zZ?+xuq1j9s#)tdTzJnX9wx2+@z9#zT>_yzG-LQRZfx60W{2uWHhrb7MZf#Gvq?`~b zH~jH9*N-L59#jaA;%C1_=v@;|{SV7*`pJH*@2a<5igTRhyTR4DrtX9%>`)#mUkJ6nVx`TnZG%1J~ltqaMP@D3Wi> z>P1Eu12)#^Q!{aeFnyT9vu_MoUhczLW#f4Hiz!E1b>fnY(>VI7L^NE|qKm<2;dCK~ zT@rl7pQ&Au;Va9a8MXGOvOZu~*HzFFHxSVrMhl&3lw1rDi9d93dBY;H_)jN<9omM? zjza{852I;owPbGnSseV^gcY;;pxcup*tw5P58r`({wBO*=+3#zWf^Db#mLi#5juGQ zSJ!!R@jp*C&i;iQgWs4Z+ls%Fwqd8d_hzukK*HtCD6L$%u6d(aICKl1HawEsfG1uYHP>Q8AY&j(ctwlzG6a_6CIa& z(5`iaSex$6$FV=KwSFSIMa0qlfgX=N%Hn_tg4>j*;8EIA3|Mjxy}zaLafBSz_U3SG zOm`N_ofoMQYtWXN$^E;Yz<+u+*B?_pSYQH-2+7Xb^Bgu(cJ+_Zlt zercS6S;7pKt4+W}k8BRO_a4R%9wT7JJLt^n%8QkzTywi88=f!3OBr*{@O>&mT*G*0 zjNscjPei(nF;jZ$Qu}HUT8jE%ag`?*i9nn@y-w_FmSe9DpRrG4hA0>?9(#L~NoKv% z;AhD5zP6X}`?3aBzMDi$q#W-T{y@J`!*I{ei?jEdakX9+?p&C`BibJ@Yw$ZHn?Dfq z-rmJaHVRiGBc{f7W5*Ad@GS9#{n`GXal=xZQ+vgVvzvpdYEXn(tDYl2b1F~REx_Ey z7vgqq19p8rfa#V!`K5X>BE8N@W>&W6saj`_nrqJQhZH&A$&r;YEjTx^7o#ITqF?Wa z5F=i|>clBgwLXg@%ZIbA*92-Pcj38PHh8QyfsQNOnR@#tHlJF9-DCSPUoDzvU-sv! ztWkWGs>p?>$8wfjFFwCH81o0WXKd_a6gXtExH^!Jz9}MUgg-lLlp%C*FRo1t=i}@8 zEbRZpK5MQslwGH?EMJk=F2`{0YHeEdox-Ny(q}x>f%e~r;91RZUe@Z&B>il9e%Gdg zycw1DUB==HEs^oC!;0rG>7Z8KoL>{upB-?a8CG<@#Kr)$db z>C1d5436b~muJ{>LWg1X9a(e7j=@S%^mkSg*LGKn>0<-!+pg8)S8Al#tf|Z8w?05; z=~*Os$Wi_72%3LB4UdFLG#H@Dysqa^xaB5Pr2ay+MukVTlrbeqmPb^Sm}aTQs||lJ zHuDYCU$Z z?#hZnzfQ4yzSoXdH*Ld(vpJk=*Ok4ai*aD=R35i=rROte25h~6}{^g?rVqhjCFUuxco%CDqMy&ZQa?|tS=`F z8bYTNb)wYb60G;QiODN9XuobGH9G8-WM3Ri!$)c2eEUx9H@yd4JEm~SkpLW1QQ`8` z6^Po@gt!@AnBCcm%vlT1aDV{f^7`A5~F64M}-=1Q8INr`a z&uSSK##F<+*KtwqGZja5;@Nvu7B6b((7b0M7GCO(gOA=}&!t?%#aJ+Cqu}xtqgnj9 z33sLpWMXqBeW%HA!{)x!bKQrLFBJH0hYA11j6`LRW|3s9$D4O%qq%hqKkrVW`Qepl zJD$Qr{bfb>-wJ#oA%m1>!eVAHwX%nBWveYOANzz=MX6koc^O6Tvl+K1OX_n!qEOM0 zKmJbP5w%K8FYpEGzevuVn2YOUXW^>G12`WuqRP&9@M>(9XaqU&#V-k!d)&g>t%`i- z(1S(BkFhkv1+h-E#g0T9*xS4E>!yy}^ezq$Uk^oFY3OJNq!W`uraJ0Vx zotc)ryDXT6(swm<#x>MFl;$k&UyD8qW6)`JDnI6JMSA#n&eR>nn)pOIHC)2oyaa}f z&*ZlU)wr!xkEpTTd245*=$m;MsS9>udjat^a3GXx6gZ%D8#b++NPVeYoa<@B>!z)6 z)<1z0>wVC}s}toLdeN$5DYn>_VcDNFK7ZbW`6c^sV(Br_Z^&PvZrV=RN7%9I)>Vv5 zXixPy0etLag_-RIYh<6H@|h9Smri5fFZE*6p9GG}G^K5M5MRm2@RYMk6hhpy$P5cuRbdcE~!gV7zymb!A3KXGDJ(Mx?YGoGH9Ovp{-l&O!zoQwo|#98yyt(|yuL67nBIyA0$ zhok%cijzHc81gxU3R!2c=}A50d^@1&*DO$F1jE)X5{VUkIICg+ASp#ZApI}6-InG6e@cOeAIP*4= zLo=74_*M^QC~XoSQd@9gV+v0i4`tAbojBgT5ksde!mBTFoHbv<;_W7U$6Cu1D78gMZpEi%+;Tt`6qM z2jkcdW#-la{_{FOtJ07O`EuxU(w15O9C<$16SsoxF+IqQ>YJsx>JAlDPMyYy_x8iW z-JzhV52;p^(&<&W0k)7Ld>4hVrrL%_csb4F)pIL|YY14RTg&vbW zwTL2(VW@p+#x0#j^Rz-LeP$OTzU~temxt3PCyB%NJ925#7br#g@yQcqyy&n4F^=^T z`7OR;XRaOd8YO)GBt$xwau~Bmk45@5n5{XELGQcMYfCAjdd^1GNp*hlx{c2Tg+iwP zH5`?7;JbE~T&a`^-B}s1xP1r_=gvtC@)OZm-5IK%>}i*D8?O~(xvy#=oU^1cZP;Hd zysN<03RxJ4@zf8Q0{wJXzH-*$?_np<@0%ZeG`sQB5oN0D$*|_D9J}m2jvd;$Le|la zJ8k-L<&wbC~~I$7`Dkw#IsIUu%@V7J=k}-8&Z!r(_iveWChJb<%|F%Huq(t{6F-2Jxh3cs`4hzU|dx` z=H3~_J8Knb_Ai_ofh$FTq)E7HM&s}8sc>o3V&cQaI63MM^m?Bb&bv-wzNG`Ne(gZL zF`Y&J_Q71G3dvD@MR9en49jCJnEG9V^Y!*)Ve=%O&I#qWnkXi0l41}Zfhdxn!Nd?N zRx3Dje{&4w>-%s`jtlSBn6P>JX1tr*nde4DaM#Rmnk9qfS5M*A9vRA=K950X7o#e` zjuAS3?3MBwpOeP(&C%~TU?xj>8uRPNHBj5OUSub|6@3*9B@ey!BgoGRMLTYxU+o03 zG|Nmn{x8BR>IJsWK8oeOCy_n+83wn_#A%z~2#y)b9y-!`Vc8UxDW}l+pAjca_$v0s zufV!Mckwb|2IkG}OTXsRV$Xs&c6as0v!n#3TX*9z(TJHpl=&+37y>4B!owvwJaRIH zy1I8zR(Jp!U5E2^$ay463HwqV_Vd-l+D$rqpkWTp>~>W3O2Dn3dB~sZ!Yau;47s4nzzRPO@6m^Dzk=ZV zQI<{T_M_8sHQrYKineV}V9|D7OewRI);$#viz4a0!kp1b2Gra?gCFC}MMZ=XTYX)* zM>!N(_oa558pg?zVR$&G0~R@1@v6a4s`O|KCnInX6j zbnZKi)nkV7R?ppV&IT{eGT>ZmT^yP>l|wDtprK{O-B$)cY0pn=+VlralQmJXu0SlQ zbD&RFAD-WG7khu}@pr91RdclX$fX0n>@()1v~DaKWy;Rgk!&g~!ZDvL26ndOr`Kbd z+pPk=mi8P!tuJ@4h~b^BacqiPhU0TI_;al>Ust_D>W=Y@&+|iQlQkv%Vz_d5C3KWW z^5-^lKK-i2&YdRHZ%s0FXMRVIAX5%o9bUs;s=*?`!FFe;8x#)mK8V0+AxnyX&H_o)lZ4G+P0XeK&EnzPv~ zi!)!W68XiIlF=a(sBbocZFh2UVR|TU#CGK3#anRc&Un%FaVJE^1&hH4x7sgE?Tdg6 zec|VJ4C}8MaMQAM>R64WQjNLj-R?SGdUmH|wJ*C~%jAOejl#lqB+HA>W4UU7?mcG6 zO)K2EZ2mZUhH5Znfg$_bH5&gkK$~S{&Mi)7CPQxqED=TkeUFr?;@A@jX^QSt54Is_;=x2xq;&1{2j{cz2w^ zzxDFmaAz0JNfPNcdOc1Z6V%?JLI2xkT<&}T&$iD&mq~dzKd=Ed2J+&wn>tTh2U6a8 z0uPVKU}2FdgB2z)>tQLL-HK&k|9a&A2E)pn;k#LCQ`Jwf|JXDR6`u6_){|;Q-jdah z_n?;?h;Jv7ai-=ry4NMsX?!9>6pHX-_HyKpX@S|wXW00-9!H1u;_^wI(Po=1s!saQ z>)ZlZyl6t%hgz&$m;wclGzQEYNT2JPJmqf05e2#!Y_ktrUB}YYu^Fu$VuVcGJ+XVm zNEQs4BUW|k!vv}>>@dWYyS|DaeGrNHE~?U`kzL&f&8eBv6y!r>P9 zI?#^u7R|?}pnqr+?{KL$fX(XhtP8gk-QTJ4{?+kZF;kHtg-&c()ScQ_T&N^>3f>iy zP*7!qGVQxiZwaQ~2p?9hoy;{KOnFb$jz$W{psSdOqs!C8LH`nAI^U2dUW9SlRXysd zZAY~0IlMKW$}rEDk`1Fq@a1}Eb~(CEs~mJ zttvb}ZWN~H$Z`GsuDlD6@0rkJ(DR({(KB z{eGkK`|&K?ZO(jmJI0)eMP#@&olAV_DfOdcA7{ZcH%y9EpF_yGSEw-})un#CRz8xK zs`}AnZL*l9TP$&qyNYAUmb~_I42SD}fxhlnR2+GM4soY&`}rAIA3u-${#8g=a{{F@ z>i9TMaP6f(*f?k+E?OsHkK;fZ-|fVBOLw;0G+UgMHHW%V9d57N3|;$kSgg>38F{m@ zw!`1tH4+BcXXrEWV?xBkWx>oa)Rx>D$z@5#E04~0Rmbf*3-k(^6a<(6D+ z)OyHaqoyA`Y?YuI7DgrOQS4=(!0y%ReEip%-712ZFmVP~{#k^^BbH1a5Jhk0eYl~Y z#J6v}adE6Pjz4+>C*K#iFgb$xa+dTgydXXpZ^QM+o$2EvN0*8BQSj~~PUI?x-6g&0 zKe8_^wfv|atjR0!?U|jFO7-pQv22%Ma$|QU?7M~*4JjX#wp#K_@&e0y`AD(8CVbHg zfW-`V8hwg~yA;QIEe$YNUKOL;BI3= zW)4l{HP})+mESgY;{?N@IB#yoOO|8kHFvRS=z0%w;mSO&sK;4Rq*b>m?B28vx8DZf zPt9TMZ?1ss&<|)^AO|z2lo)QnG}fs_OJ zE#=bXA5P*c-zc~){;S>i@3tM+_I`%L3cv8->@BD? zCs3q2aLdp(9*%8<+P+NA)AkYRWJdl- zzT7yO;}=ea_7Zn$Y|ItQ+67SY$URBioOAyNcH!aVar?iq3neLbF)2DU`Tthcg>_y> zes0y{E#FBD3Y~|oD~-8CEta=a0^3|80cs zbmW5*ws76iiDrKaaIh>)SXw`a4GkeFd5q|}X(HvlF~_yjrbBuXXO`MhzqbPaI)zio zv=VN21~LV$2$kJ~#sSX4cjye3s;MI^!-mh+NFz}ZAMfW5yR0 zeGKEyF}tv{=VyG&mPVi%w&)R`%9P~O!f;eI7WlQp?LU`sv}Z6U<=OI8QY&uSDYKws zM{a1iD=A<23-c6ek=P(*t#7#VbNyRv^O1u=Y@VnvzYn_U{fa! zOUe{qm%kDF7xd&AzK72de^!{BMbwe?aCIHU9Q0(5_G!^H16EuQ!Dpr>DvxSIVHdb$$( zwE07!&}_oZC2Dv!=Pm|Gk%m>H4&Yje3N$VbWaLvh4!-gdVS|o2%&HYTUkqpE@tI=jizIQej|MMA_kqE% z1vpr`K`4)PXRx6gM<49VmQ8zb&@>W9y0&8AWd~8~t^rdm=`8u5;mt0ruBb!mFdx3|GLe{;+_{r+&l-3!=;!MeGi9E7;)ph0sQ@YrSLrS z9^ro1@Zt6_`iD=Yy0@IjUyv?pGn_ejjSU;OJ%*-+3l|LbVdnSeC<$_>;~I0$Tk!?5 zizm_Oqa7mlu0Vp5A`UO|r`m!L>KzW}8mU@m;hHx4gv;8j`Z`^tho`ghTnSR+f>H2$ z1mhP^V~Nf|B$i!A$Dy@&rm2s1X*=QC(Vi2fw%^!E$}H9{g36{87JmJQ8_rEwntKje zAyXN7AP3b4YEU+_1+n?T{O2={7!F#|7pdx_^F(HIgUSe?nR56^j>#V@!&(ADET*)vHhbNB(07H z9hk_ZrT`A8w&l?y?p$)AMiRHkg9FF8vNo-Y`1aQyO*&)v!LAYWJO(0RkQR-<%V5n` zTR6<$C?XY{m}UAybh_{d%la3hX3kS|TR==$mP|=YBew5O#pxm5G{0uZTmSB3@LEqs z&-X-MuLW>B6iSQAyIAlyiK~>`iNXy%DDRoT&@Mv}KY1YEeC*0G4JiyaIf3e_{TQ{% z6w^Jg!4J}?5@EsO_CwKqx)OWrGe(Dbdtt09wUb$@^j6d4o@L>TY-+&5s)0OPI0riS z_d>Zqj*4?L#nRh0d~^O6=6kfq*`w28dqa)4wj|QA?v-eaxg=r-gz(3Vjd&8(A~bdW zi3K8+qaRcY`NjM3=Sc>vOnUQj&sYkFx2WEoCtfdB;Fy2*bllR6LmMt3ICnJ9rkFG1 z@E@s)sy{0?>tNS{E3h4(3-3<@*|4!5f4ww0(mRbDxkr+GD4m18D)RJ_S>n@U6R1}g zOYX0Wq{Gk4!q4gl4#^M2qog8qK6n88&rIa7XP8#e}XG z#H2`FHbvM#t>^?K=WEeivI~|edr&W};`fJ=O}h*^c9T8xR8pzqlt~ZkPtZ|UpvvS8 zLQSz9uWrwhqL4{E*bvCq`GXMMyb$_-Zen%eNo<+BAMQQRK*`^TanS>arcg=OxN}G^ zw3k*<<+zs-#_xakp*TcFydAVr6c6&}t{@XuWy}}pN_y@=_Wphr{PLl5jGh2pkbyIF&VH2zdNrNxrZbo$1#%`tFNKr=>iN~ zvI|>vo~!6wp|tM;4_UpEHen#(Yrzi-~a5%sJS&p>HrDA~hTzKgZ zr|LE}hS<2_qHd|sH@z#Kdit{|Z@bVB`;NGc12{r`Je7w& zWPIo!^k0-i$(0JE#g69vPW4E9GMWWag<9*&5-9Z_#F>w6Ic7#FQs(!<*W^DaJ+=a; z%V*%g${kR#=*X9<#B<0~7vB0eY%lmj!^+zA}o_hyT-&!E+8-}P%Yu>2S<|Nm85u5oBiATrq zjCv^!ywm0L?Vi#(xn0Oam~(31Dr}8@D|$2>#sl4CM#U7vcjs~}(ufiL&%DKtT(Y*^ zPORLMhXu9$Ik7<%5v@}wb1Z;Nz6yg=A-u0$iCx#_CE1f(B@HjlIr-&mgiU!W5?#mh zLW%*s?|IYas4NR_pT>+w^WgN|5p_MA#aj(~4As=*&xO|!T;7HH8o>y1=!ui@-RNEP z0aw0{6>po3(9^yHB=rYH+Sz`>SBn1B>x_fCtuY4-vtgB`6@70<@ugKewrS~$&GuJu z&m^0ju21IoDlguWog|HQIeeIIP6er2tg+gdeGiS|8TS}ro6v!mo#c6|xJsPzjOU-A zU}&wsDaKq3WXI@Ekdx+dAEZouQvN8uk)nGJ*+Tr=ds}4bSK!9lv9y@}2Ve4+!2N<7 zS6xrx*dHU{a-MUiR zlPvi6M!l2^cm~^;>0DLhMMEubksY1R*P+*u!cCZ-)tzFW4+jqTjiG(Fp?6y!n&yq+ zR?`WLKR502$83tvf8&l&6`MQ%2n>dv2ufjF{X9p=A0(CM}%yQ*p-M|BQzpKce& z_71QM`&}(s4W!7O6!qR*JPSwKyxCQ*8}|2h6&43p;8AUF`W)ITRF-_ggwc+CIs)vN zI0+YvAHsdlSniqsM9e#=#EMi?8cSKY`OnQ5JxYZux9`W4S@F=?6egWd`8b(2g_`>6 zoRn(EGL1s9(J_Y~=M`YV`02dn=EBhE(fo8W4JLbTV9J#e%y9KVyZTAIawnNupNB&4 z>tgi%w^tm#REsN(T`~QeI$pesWwYH{F~xU^7_~)RyzkN?t*nOea;^+VFVAAPDGq$v zPM_vYgK2kW7y2xTBYSoeZ-Dz5@6#yy7mO3nrPcm3i%Nuzk?`pG8&5q#W-D)&2a>AxPQM@wK1EEi3c)sIRJaLy`fRybq-gO(f zy{d7*Hj>@FtZ+HGTnryNj6F=J2tTJVYR^|;?&fr6Hyy&1uAk9PCXj=?Gx@H049kml zqWS!A)J^;i|6Y1rp>Z5jekMZah5^4`TZBuh+GMmHY@`aw4H;5KYmPcv>Ky4gCXnZT z88O^{IAu2d79-sPS@`4u&j0DeMT4Da{P4V_L*+Wm-50@oK2PzpG?-CgBYDKWQqm=H zvt;m)k<8PaCT03A;+uzs6oqU>+MJhSy^bpr9Q!e_M=PGmErzO70)MSnVf2lTh?wt! zg>^=fipr0O3k%0gGkccI>A?$gG>MyjIGUZz8m+!8e_AEtD-*b=H5uMoZ;>~|m$yE- zi9GiJwydebwrSt-e8U$Ud^QStM?-nR`?x4>dxS4v%u}}Bz#+=7eoBP;;Z##R5>LzO(>^c~o zWglQ=VUtArxw0f(s>o~|DkHM~Ito?Glj5SI8>d>CF!4<+o%?OW{lS~C+oD!9=;<)< zY9{|JvBaR?kKyn=p6XX@=x|t;jvYUV*2osg^^pa*vFswAmj=SAVwyzL$Ohn&&6rL{X%@AW}zQ}rj=C4T%*!>Pqv_es8TqOJ%LGvE=d^}2rH}1yrO3hPT{}+vh zcL|i2vf6isA42M3GoCRnM+c=5JTb_KTZj2`()dr}a?M~a-nk4bj_g8F(r!uqsc^AV z+lyVY{tE4i=NLG4J_foBhE75amaXZ@lEK@NV0Q&d1+N7u@K}jh&>5-77y|!SsVvRhM&260FyU?|M{V@1`^+Z;?Tlx@zobkiiZY z70wu&#g_%cxNx>D?)(nKunGDS4T&uW9~@23Ob_W?>%o(gE=rLLS-Bqa|6fZ+{%d8( zf3`Da|Fiwy3rPOY%l@Za<3FeVSDPt&P`bS2{}pfipV$3QsN}z|`p@ms`pNun;Kav6 TLGeGw92+5fA;~3#fP$^<+turP18pr;Dj&+R9rYB~v$@j5#+gEmNaqg(*_j zphO`|?T-=NP(!V-i%hU9Sai1wT{T5A6FW!IQs_@Uhxfd^&-Xolp6B~MRk+w*#*D&) ziEJE-`@s)FIWRItXE0`EE-@H$M9l0FQrIXg3OF7a6QSBJVsN2;0bv;6w>@2!cr-Q94 zNa=q!8$D(1`Os|v6kF|eDGRcNTEG*pzf`p(DGVkvSgU*y@HY6wE+zag1|g+LP1^_n%9}p zVwtJ3!PUCd0K`l@wroriheZBwPMZD%zTbd*v=rd2?9e(nqNsTD+I0VfFJA zdZkzoy@k*4r4$m~AT8Wsr3Ei)NOh9KrG{AO+@#38*+HulmP7Y3g<{jpP!bFGug9qI z*V{>%RMjzQD|qLHlJc1(>eS}3Q!H=-#`=^J;ZN~Y6RAr7t_*GsR$MR7g`p)6XWEkB z`yc~nwjCz9(<(V#H9;XrPHtj4LtIq?xXQ|xi$yUR}ay(o%OuiB0HJ*U(d#J6JOY=bP=ic?Z>#lXzS?7=UoOiAF?6dZM_I{qdm6Us`gonn)2E@jPgvQv^+N%Vs zC`863B+XJ$ObLw)kC>$*9~>E!*i$h+X_oYP%1P=WQAJKwqlgnUsb{NhaQf5{e z;Il*p+Ix){mm`P1=dGC0ZqK?bciauI#q6oB)Yz@VnTNVz@61`8_Mj5xJ|0Y7G+$y6 zHJsdo=ZpBJetgoQ9sS?V7w+z=yn{=aye|if8^*I?t0ITA zhf>pjqwtk93Ky+N{Jl35&QG)%|9BP7k8gv1pNqoj$OSC5u;YzyD%79YMdTbD#!ae_ zoHkGtH;&10d$c)|e`s>4ekE2k1@q$EAbu~2VBD^8^c~}mybkjiA85&91t(TEMX^J9 zUv8P}%=;xKY?{3r9~O6E?f5Vr$qeO?L~#3!3%FY$L%EA}7IVwNoP_tHpMsI(@z_dC^|nObp}Xi`Iz_C>7$UX*i?ED% zh5ZXpW1ZJ|%$e{4!+vGr;>cD6L|Lh@_rsb>JDyf7OW@1vbo)ErM7XPLB;{j7%`3R^>c?iZK&KVM_e!<-!W(AY?q0`fYLu zr%N30c&G|iI$H9Ep*6epYDRW;GMj`8-xZh$+0uDbsr1IYY7Lq?r;9HAX0dpp1MjLG zh0`4H>U={k9-)VmOJ>sA;ukcvEqSzlAe1V8W7n=WG)~t-(Y6I*b(tMKGy1Y_&wU(g z)n{v|4^`)C^B-pwemQQ;;FKQB8$Xy`io@BMn};)=8T9XB!Irm^nAM{QUKY0Wo7Im; zH$<^ve+(OA*5d3UP5$0$%(weKAo-9VW3#;x)Hs5Y{!!d`bT4$3$MN?8Gd};O%`Tm% z(|b!I^)i2;*VMrr>N1#Z-^X&=oPl)z_(Y<+{kB;4`K)csI8BkJwFVnMm~g0nUvY5( zDF0U$%iDG!x;li4I;97&9=>D`b!eau&uoL@jS0wj?Z<>Mdc0Gf_n}*!A zHkG=TaJpsM4CXWc^3a(zAGKc7SQJE7d5 zJcRu|=fG%REa!~af;CC4IOf`w%RL>qV^Rw0BuxvN}GdGa)|Gfzl)n)MLG>?DFJ8(zC5!6cJ z={lhh7tRRk9MYuEy&+uZRD~A@7olr#HtI~PG19Pu`0T2|izEEmVZ;=k8lA@6yul1m zn8J+5>+#}lH2nvZBc~M%DR6?PWaA6$d|Hmv4!yZ9xC?$+%@O;~dva`TKFnV=qTpjG zHm*p5!k85LE-|I&EiGPf8^qBI^e}AXaqM@Q#KHDW_^J{mWMUqOqZ`I?!SF?5Q`f$H z9a4zCH`FO4&G@2If{MsPqSw+BVq1g)6EAmUhNUhQJIeBzOCWPcnd6(O4VSE3ik7ME z_$5BzTB$FaG-6p6Y9V@lQ0K!Ne%z3$$iQ4jR1`s*<20%Ur;GNxvltp2!;=q(^S)y|7X%BM{EFcUs}N>aPT-p8 zk*qrZ5RO|l*zOv`RcW{J#OX3NJz9%05zBtN6shuwhzuTtW~F0Tw5Lw8Elg^2TPzd3 z)bRJ$DM)N*FxAYThbxtM>P9yhY#fP4pL=ossAupSoeABIm$2=X9r?8z>n4xK?74C* zeAt~2!_r`vI0RuUGT7Q{7Ui~_5uQ`7VRDlNOW*Y1p&)yHX(|%3#=o#2xHoeo>oMl0 zxtKR23zk(LJpCXR?`QXB!1p`2?ykj+j@^0ak|PRiT=;XL5*xZtLb-PAW}n$ih0ry^B!ickV$QiH?aa+z|y^IDvEfL&$tqnVe zPQ%p^38=6)rSbjFjJ0s1{H}%KysQ~Cl*({#+ivLD)?$@HGv;M4#8#CC;lAIP$v0O) zAQGmg&&?U&prX~w4) zuhFr32$ISUB6)u+o{hMK$6q%J-8wavU41MJd#5t_?{Z0PvMTpv>7aCs9Cm7XV~mv& zR6|0jG-5n^+s3hHu?C<19l;(&QyDjH9yhkF#FH8eCJv0Chw^dUHb~(6cOJMpNt(z1 zc?UU%_o2_Yezef`rbd7k z>tj1|PC_y@4i;kV5y8YKJsEfWCYm)(*pRYW@<;Lt>w0_g`Ibg}(f5Vr=luJUjR-0G12XoRjS7Z0O>6CsP-=xo_Ne4|fm(Jw( zojo|k$QpHKLwL<%BF8RXC91nWfLy3D&noJ3egtXRV+MORZo|FzzGy2sh03NP$Xb8I zuMM)S`r}QtqmS@J_B&oJT8oF#n(S{ih!Y(K(aF=1>)H(Ppf+BtA3p-K6DM*-jUu=9 z-UY|_QC#HW!5+)DpnTbEnoW{{+^h}Qe6$Cg#@6DKaaZ`MInuH{h<*2$VA!IeR2Xty zbX_!n_oOwaQKuOqN$Vkknl3i2Dt)(olK=5`EuD~F5d zbJG}|9md_=%MnsJ1B0tg_%U1-xlg@$G=B)Rw+-j&#?R>JAIDwW8W5T@0M)OwS$@ic zl@(v`u0&4KthXD@dD-weAI{?eefavNoy1=$gb_<_qr@+PhKtSE8u<$y9tZP{R|LLV z7>NUI%b?bM46|H+;O47rC;wVz>xX1bza+bt! z^vr*dRcy&S+FHV{W+&zu`eKf5KKdLWt0I)y8WF`Ivmarmi4wguE6~z^3k-8C_^Cr6 zXT6@uwf|)BRP+<5A5Z5J9Z!+E-Ibqizr!+RQ{GjP*YT@u7$-#K}|7ijvvODqfSAE{~3g*ij5$?`h1OHE+rdMg_AQQ%+4P$~`H3)coyCgCfBWGnDwpp)>jp2zj-8VE@n z$roFs7~jK_iXEbG?E4}VmW-pugaQ1N_62#LLU?%M5gb<8(UB_v)00w7U@lC>4+_6#Sf=->dqx!yN`-(qUqEL$XY9Btl?Z(gL?{UCW z4u;X$qR8waY?9R3B*n^i_fJa>z8gU`S6h0nt-%w`R4J#Lk8RiN__DST>WX&k+P+Gh zIpf5|>TVLZJ=57b!5Lkmqv(0%2?p-!OoxPY@on8ZQJJsCOZ*p}H9jmdxr~UKLb$k$ z=Unt*+zU?*S?9)%A+IsWtrXjDS#yW!RIKT5$l3?;%rgY;{aOS~#Vc5H%NcJT2XOC# z`@n$d%$TzHWbxcZ{UNTKZ6#O;QjapJZL$IvJw~8 z=vcDL)llv|n<>`3N)T83YVulSUl=;%TW}Wb|x1f!l1z$+=@XgGT-L?+G zg|!D@l;e!7J-x}7Pm$H(0|wiF5!1HM=Aj$m=$E<_3$MFzw6!VUG_De-c004Us0_&t zo_yQc8$TnwInOE&Rm-BOb<>{RQsWs`7RI@cd(kpkh_Nt0-iM1gyfA>fKF=4g9*pJa zJGIa}wI0Ed4{++-AntrHkgcs7h5My{5$b&tAMZKPCv+w?Jmf@9eyS)+ZBl+aO zQ)p>AGk=&T(|^3g@~LjL-(tok8@@nxRWJvAvO(Cf4Ty78#Hp1&RL>8j{;5!I2^_~2 zTYlNbUDx5hZ?i>eXe#&CE=O`q02Z{4X6&k2T&{Zp@ddZg$+{FTvwz?EEXaUtN=Mvr_TjpQJ_v35iUYATIsJMJe;z)D zW>@K14XSwbF$ovgmK=kzs?<;B ztI|VVixq1_8Qxfp75hwiId>6sA0C790y!!!P8VzLjpVz!KUg}ZBQBqw4XfMgyt^l! z_GNFxlc;MVdSD=b&fAG+5zRtNw_W6mWKMWgEIO>JMBB48Seo?Vb+u>;yZ0zQnl0Y0 zQsBgPTiWkw!pR-i5Rf&2my^txcB&0?3dFLRhD@D`tBd|!SK)IA3R|b)dM~paW zmo2lqB~#ZionuC{KvzS7-KOsl>WcEbaWF%w?xW4q@e;{qjaejF|Y!y zT?$3kNr{+ipU$++H&OIFAEv91V88A&=q$9OjjIDjhI?^S#X0e9r!Q_gJFxCb4IJD? zvNd-Uf3&Sb%JlVOpvPj2H5f(J1L_PM>58j*>xIGK`{KE~4;!-&3WJazh}k)iqc!}f z>=3~P+ch|C!EMR=KCv8i%M?b%i_tu$8cwAncwA0aGCeaLfyYWvB;@@=FoOF|Fg?~PiNjxfU6Q3Xgp`3{D4 zi_j@|7&FWlOY8m!lq*~CLxxmywr<0Km2)YnFG5Q61U~Fsj`(L2xM0Uh@%8m`C=D3O z%%@hIIBz|YmiETC#5SxyvjG>k&qLM5L+EDSiLX_Id01T&53^IS!mSFgny!h#eWLk& zR2OP~eu$?pKH+z}0)KZ3r0hU&`V9q)|9D=K^!S~~cP+xYz8+Nl*_VA5Hz4ABGi1F& z5RpEDx65=G>{2eG)7ue$dLl1rtVh)cJ-$5XE{(~9LMF_NGyCns{>b;DSM@18(o1AS z)G~M-UWb*MQDVTQ_xPDbmdYQ-#)@p@mk!{xYE^`Nok5v1zGUJJ7+whEL!G@ia;t-6 z&h)R6>eptR{(2!oW;_@1E`Gd{WJr$(9vpdEmbv#XV%|SX;P}HHWok|0y{0X!weVhQ{i>T4!pZ;pSa*2%eJWj&@Q|qCSLVtr^wEblh$z`Z6+`wXFT6Ybu+s;LbM;d zCo&9*aC_?{n$K>-mz>pbyW+}Cw~{#N=V&@H>eTzw zZ>tsy<)%_A;XR7g*7 zKV#7Oo&~$BYGbbIB4j;3D9#)kXcN*}EWR2_H8H84ynESv)co>bcex&@?BgQLt2W@D z(mwP&c}#R$-GV6-?D>2&*eO03SC>78Tg4<+EPW=HoKRv>@?aWE`&&z24q@c@ZrpgV z5;Nwkz%C*3joy9Qo;h-Q<`Rx!hCh8Vv`L;Tyd zS;|=)cs)yo6V_$0#|%3@mp7nU<1pG>I)c8dV@S0g;{BRc2)}*SCb2ph!`lsrtn(r_ zDut3;iTrwX7Hc}akbEnW$6Y%UZfLNFNlQ5@(>qgBQI~c2jgCP+tmrU-Q>&+PlkQo_ zDaNw#@C^FSi@~W_V}@?BW#-CpEY(p$q5c7Qw~VLHg302glt;fX-;0om5}vNREjgXP z9{aOGS#&HD>n3%h#qwcXD6?PEDdjS@PHw=jdTU;OH=3)Cs?%_gBX(6pus(APf}Y8+ zuG0-XbCY18w2xwZut64z99$VeNEjPm4; zG-!X<*;=!tJ+!O%KE_VRmH4z@#?~y&+i+5XGMYfwSo44%4fmuKBa>o~(crhOOr-OLKmPi1AI0}B zInZ}}hw@R4Xp1*u$A5g`Z8)6${R*XZLWpgs#z5L6x8e5lQ9S?c5N?&$+opQ-;M}KD zpZnB|?}vMf;>Hm;(R~=Y$Uefx+(wDcOJzywK6ie#mJu0$?S-nvd2!X=l`}0(82>Js zPW=zy;jmpeYF;X;^>yigBc1K5Eikn8DeQj4Qlow(?M}(kzEg|%8s03qHEscJuf2+w z>;2(aG)tm2$P&)h@{+&n4cO2>i5K(wFwoeEuQDE?>01JQkAA>2?I5f-&J#WiqggLo zJ{_;i+xMecU-BHc+9Of@A&woS{o;mECy{(=2rn6LhlmnidWr+yT?a2FMR*I@s_Uu#; z$yAqcXs9Z4u8cSFD__zcuSUJRTTotFhK6icb}sj2{qHvz{IM3fbI(Wu^!xHduOb}V zmBPe5GU!@Xj6>BjION=o^Co5R^#TX3SZIZYR)085F_36VtT^n%1iGh>k;Ymto)5kX rhyNKr@YDFuR6@_LDxH;Ndn&2ODe3xKa&an|BZuOch&y_7JrzL delta 479 zcmV<50U-X~LFxn%2niAjXJu_|Uu|z>Wo$at1d$POf9U}T1_%id0V3D|*^3uHW&;mD zi|-&m0+$ItOu!sJzAJ@3RGWl7m83R5#>_@OyYy#2x^SpI9XlpJ;yQ*tNAVIrVe_6o zN1UiVWCi~|xgZvC`5s=b8d&(3)yvCnC zHJ9Bye@;5HKFojmK8sRyKhc~UKTstnKQ0^eK4{QpKj#Q)KS17RKH)q?KWu~*KPTV_ zKLu&~J}6z6J{aGpK2Vk?KeW15KWQBZKYYW`K0seHKWXY`K2~aTKRrGmKOgRrKD+$o zK0*d}Woc(}mQd_Qi$em|nhV?Qa=aX+G1h(0cGCtKi}mRgvXrc4D_6v=T;aVmZZ(@F`;Yh2o1dP3vXlvv#z#$KvsSUzy3UJT zJ~!Sq)aV*EKhV|F=sJH{{EGOPrL&gB&x`%!9~rZFc`V~!zAz>(mbrHe@b@$Z_`7== z|6%-pn$Z1{2SSn@pncEB-tj6F%1Gvs{XP+YEYb!-1nNkB+;tabl?)jbpm0@5@$tcgYsIB^AVO zRw6M}i1XQ2c)Qse{B<0Bo7_Un{`iw-MmJObt14>UTu1Mg+#u`8eRSB}0~`|<0e^6i z#Ve+@OO^PZ)kiZ&u<+B|b{g@rha`4J%qkQ>KjA&C*L6_QxIDV8u!YvBgQ1Iumv?or zDAMAyFFL9Bp&Er>v9QP09%Gk@kv&QP>0k@&d|F1k%IhfT92--j>L^vh#*=L#=={xC zJ*I*t91|gJ<4f{Br^fUW3k+}lNF$%WBFTe3YTf36&piervO$IZ3@*65NYoO+%_OltH874?R4^@9H9@CxOGvD&M)gI zSmcgB^u|-$m0p!mDiBJ+<*fz zE1U?Zq>v!>zS53Yq7?`!lhO_t$R{$SMak|JQbJ$8rl@KNQ93wH)! z)_Fa)o^nLy^-`MZYl#gVd|b}Z;%IIYy?yQkw}uexsj?oLa@de+j+$!ugsmecud3MkJxAkdSAv5UFbzCeT3ONuDP zQVgTB1ZA6SVP2$1YKIk^Wc`%na-WhmRMPDM)g<%CrNq&#R6U7>Ih2ZJRBPgG&5 znuR>M1d)#!E=iYScY+k>qAfATw|jZWo{e+P8Fm zUOP?Vig5j5K3UEYpDf4)JD{(vn-nRPN5h zTx&Vzo_;|~$pUMRmeH#eF?wHc&>}CUtLtCVxq?i}?e3)L2p(4NYo@nhZ)o?E9BS$B zAg9t+8Z|zb0wS~NHG`qeC!f$TCQe3UIa#>$(}@ig6n?sdJhNZZt)di)ACgJpl@F*e zt(vmx?vu2pffig-VN~s9nr_iZTk~#`$K?;{QZFSXM$dksR*)Y1}jC&e84K(|lSQBtKI zYsZD*^<5F_FF0T~hYiU$u6Xu9hd(l9==wzm<$fPrwUS`QOeeIL+2MG*JpvSdh|$=h z=Q9ney^5)tddYjB0y|VZB+eAzV4W6YCi>xIv;>aNG#IZ^L7eV~#G@WqV$eYD)k&F^ z0*ucKM&gzrwEM|0C%^+O8!hlt_yCkP`@(0a3&eRQ*smXr$Vd;s6o^eXJ+Zha0ORfi zWB;G7sOAJ>?hm!JB|?t!GdhGaYqoJGAH}8*erYX_n&`3r;52HKl>FHrHga@TCY!%}E^Dg?jUJm)WM#^$l;=&ml zWJf!qbb=Q8i&{*N=RyCbnyM3;$opdv70l`)y9q38HC55jQ+hPb?4zn28KSFtX9ywH8Z4n=1T5FQUeX9O4V zBV4gNO^5g2dSSlN3yDmw<0WV@<-2?ejg#VHssv{W`FQ-ajr^LOu}L284; z3;E!mazSsQBYv4E#K31=)c#(AsSHmB<(TpFcOA5%!5JSu2rzrE89NVn;9QIbrZ8JL zF7m;%LsHBrwM8}C83L{y7X0Oo;3WpgTs5d0*-C?_^igKJ0TUQ*h?``EZ<%k`g$~Es z32bakV&?ISMyg-$iGfN_yx(d@Wt$bgyDCIyfdy_&Q9{|%MwfJ5RQOvn#W}FBUn)XK zY#IIdgpc)yln5zmrOUo*h}R0Sq0k!Ue@W5Dhi4gVq}EbXbrU&FEGJf%83SwF;dWI5$w_zA{mMo3 zVOQ8hxS=P@5!J0yob2&|?vfVO9rg&?AVbPiU(8e!mM0q#Tj7FbUv?7bnk8nW3m_`e zA#<0x_OpbFxm}4c)`KKHMPFX@Y(i&G+_EXKa zE^-)1E2iyH>9A-3E zTq_?Z(;w2pHSefl9Se&cggBd8Nk!*mSeoXHvBw1X?MFTiYCqC9!<>=2j1MJAFzZS! zO*vLcnlLl^)_Gy-V~Ks}vr!qP!uvccESuwu zd_O-#Ue=&p?f}mL^^}oi1#_|p+eR`xb(Vv_-s&;rs2_%Uau6*yVt^BqlS`bLoa~Py zKQY>%PmIQnUQ(`5Lf4={%-T9?X%J&7^FMojGHhdy^J{CY!&GlxAO-^fGiX(M)> zk>clMA*OD!Lz&bXZpY*pk@JGKL|LME3=b8ICJPJULS!jJ;nzN}5w=kHLJfSUbFn$x z5nX0AHfE?Wz_XOfo^uhz@adgQJLI=EQ+ksOYmH5`t=I{^p&C3-kt1!874E*%B17L! zF*h|xX1L1KIuPbQGlEqzjP%eTq~HxD+~}q!8GV%N`H@oMOUc*68jl{kqa=b4hj?2Y ziL9nc0TQUEDd5^-jhxsjy5iPFEVTi<$4d}?y@?zj2$7i1!ool0u+=M3dB++H|Diw= zqa%J$v{6w(56#-7#DfYwUKW{Q$g_t?Xo&+SUQ!8z-_%ep>K1VDOM4sbyw1YCFe65< zZKrkZ4tSB)Ogl>*Fg{d;?t?bCr4?YvjBbkFW=7rr*x|-SG4|*bXqa9?m3$GtuU2F8 zTOPv3@^RJQ4N375+*xD5+A=ZR$GG8~*8ohqB7 zAxb1K@~d#P>@4-=OkY1?9!`pUv5YSAf?Ru=5Ek&7UXC!FUt?l24NQH!EFPv=@;;R;4_^)?{gS7$Ytk~#dII3x`1-g#6 zQGUovvQO(G&#`IbbDf8oCwchb#7Ebc)il|LhwyoAH0AgQTH}>T=~V)h?&Tu1y^*+K zTuj-XO50Yzc27+3 z8LmK;jRoF?+8`=|%~5jyuj<8rYZRZ_4eU?t|JE|1B6$2y9yW7?FxTJzQ_uJ}_k4y! delta 292 zcmeBD+Q)1b;LXe;!TA{}0MI{GYDjFe#DO!Fb&h`xpap z2Qv?LhX;kX?b(jAJJkJKYX4&|kHd?eV26VfH`^~i?&#pEYvT~q+H395x=Gk!#gSL` zUvmH3pLrYO;A<-Gutw#Ny?@;!`%~4o?9-W89C+t5IWR3?bFhitYrm