From 2ac098719082a189b5e76c7d8fa502f3c23212bc Mon Sep 17 00:00:00 2001 From: leondavi Date: Tue, 7 May 2024 21:01:53 +0300 Subject: [PATCH] [SOURCE_CSV] Init new implementation of source csv file read implemented with NIF --- .github/workflows/pr.yml | 6 + CMakeLists.txt | 3 + src_cpp/CMakeLists.txt | 1 + src_cpp/opennnBridge/CMakeLists.txt | 1 - src_cpp/opennnBridge/ModelParams.h | 112 ------------------ src_cpp/opennnBridge/get_set_weights.h | 1 - .../opennnBridge/openNNExtensionFunction.h | 1 - src_cpp/opennnBridge/openNNmodelNif.h | 1 - src_cpp/opennnBridge/openNNnif.h | 1 - src_cpp/source/CMakeLists.txt | 36 ++++++ src_cpp/source/Source.h | 1 + src_cpp/source/SourceCSV.cpp | 1 + src_cpp/source/SourceCSV.h | 4 + src_cpp/source/SourceNIF.h | 62 ++++++++++ src_erl/NerlnetApp/src/Source/sourceNIF.erl | 30 +++++ .../NerlnetApp/src/Source/sourceNIFdefs.hrl | 11 ++ .../NerlnetApp/src/Source/testSourceNIF.erl | 10 ++ tests/NerlnetNifTest.sh | 4 - 18 files changed, 165 insertions(+), 121 deletions(-) delete mode 100644 src_cpp/opennnBridge/ModelParams.h create mode 100644 src_cpp/source/CMakeLists.txt create mode 100644 src_cpp/source/Source.h create mode 100644 src_cpp/source/SourceCSV.cpp create mode 100644 src_cpp/source/SourceCSV.h create mode 100644 src_cpp/source/SourceNIF.h create mode 100644 src_erl/NerlnetApp/src/Source/sourceNIF.erl create mode 100644 src_erl/NerlnetApp/src/Source/sourceNIFdefs.hrl create mode 100644 src_erl/NerlnetApp/src/Source/testSourceNIF.erl diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 85dd2bb9a..15f25e897 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,6 +28,12 @@ jobs: run: | ./tests/NerlnetNifTest.sh timeout-minutes: 15 + - name: Source NIF unit tests + id: sourcenif + if: steps.build.outcome == 'success' + run: | + ./tests/SourceNifTest.sh + timeout-minutes: 15 - name: Run FullFlow test id: fullflow if: steps.nif.outcome == 'success' diff --git a/CMakeLists.txt b/CMakeLists.txt index f6ff73b91..72f9dbf28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,3 +39,6 @@ if (NERLWOLF) add_library(nerlnet_wolf SHARED $) target_link_libraries(nerlnet_wolf PUBLIC wolframBridge) endif() + +add_library(source_nif SHARED $) +target_link_libraries(source_nif PUBLIC SourceNIF) \ No newline at end of file diff --git a/src_cpp/CMakeLists.txt b/src_cpp/CMakeLists.txt index db3835232..26aea536d 100644 --- a/src_cpp/CMakeLists.txt +++ b/src_cpp/CMakeLists.txt @@ -9,6 +9,7 @@ project(src_cpp) add_subdirectory(common) add_subdirectory(opennnBridge) +add_subdirectory(source) if(NERLWOLF) message("[NERLNET] Wolfram Engine nif extension is enabled") diff --git a/src_cpp/opennnBridge/CMakeLists.txt b/src_cpp/opennnBridge/CMakeLists.txt index 8908d629f..f43a5bf7b 100644 --- a/src_cpp/opennnBridge/CMakeLists.txt +++ b/src_cpp/opennnBridge/CMakeLists.txt @@ -31,7 +31,6 @@ set(SRC_CODE "definitionsNN.h" "openNNnif.h" "openNNnif.cpp" - "ModelParams.h" "nerlWorkerOpenNN.h" "nerlWorkerOpenNN.cpp" "nerlWorkerNIF.h" diff --git a/src_cpp/opennnBridge/ModelParams.h b/src_cpp/opennnBridge/ModelParams.h deleted file mode 100644 index e2e545ce7..000000000 --- a/src_cpp/opennnBridge/ModelParams.h +++ /dev/null @@ -1,112 +0,0 @@ -/***************************************************** - * Authors: Evgeni Andrachnic - * 29/10/2021 - * - * @copyright Copyright (c) 2021 Nerlnet - *****************************************************/ - -#pragma once - -#include -#include -#include -#include "../opennn/opennn/opennn.h" - -/* - model parameters class: - modelId - " " in erlang . represents the model Id in singelton. - modeltype - " " in erlang . represents the model type . - 1 - Approximation , 2 - Classification , 3 - Forecasting. - - layers_sizes - list in erlang. represents the layers sizes of the neural network. - layers_sizes - list in erlang. represents the layers types of the neural network. - 1 - Scaling , 2 - Convolutional , 3 - Perceptron , 4 - Pooling , 5 - Probabilistic , 6 - LongShortTermMemory , - 7 - Recurrent , 8 - Unscaling , 9 - Bounding. - - activation functions - - 1 - Threshold , 2 - SymmetricThreshold , 3 - Logistic , 4 - HyperbolicTangent , - 5 - Linear , 6 - RectifiedLinear , 7 - ExponentialLinear , 8 - ScaledExponentialLinear , - 9 - SoftPlus , 10 - SoftSign , 11 - HardSigmoid. - - layers_sizes - activation functions: - - - Learning_rate - number (0-1). Usually 1/number_of_samples - Train_set_size - percentage number. Usually 70%-80%. Represents the portion of the data that we train - Activation_list (optional) - list -*/ -class ModelParams -{ -public: - ModelParams() {}; - ModelParams(unsigned long modelId, int modelType, int scalingMethod, - std::shared_ptr> activationList, std::shared_ptr> layersSizes, - std::shared_ptr> layersTypes) : - - //_optimizer(optimizer), - //_learningRate(learningRate), - - _modelId(modelId), - _modelType(modelType), - _scalingMethod(scalingMethod), - _activationList(activationList), - _layersSizes(layersSizes), - _layersTypes(layersTypes) - - - { - - }; - - - - unsigned long GetModelId() { return _modelId; }; - int GetModelType() { return _modelType; }; - int GetScalingMethod() { return _scalingMethod; }; - std::shared_ptr> GetAcvtivationList() { return _activationList; }; - std::shared_ptr> GetLayersSizes() { return _layersSizes; }; - std::shared_ptr> GetLayersTypes() { return _layersTypes; }; - - //int GetOptimizer() { return _optimizer; }; - //double GetLearningRate() { return _learningRate; }; - - - - void setModelId(unsigned long x){this->_modelId = x; }; - void setModelType(int x){this->_modelType = x; }; - void setScalingMethod(int x){this->_scalingMethod = x; }; - void setAcvtivationList(std::shared_ptr> x){this->_activationList = x; }; - void setLayersSizes(std::shared_ptr> x){this->_layersSizes = x; }; - void setLayersTypes(std::shared_ptr> x){this->_layersTypes = x; }; - - //void setOptimizer(int x){this->_optimizer = x; }; - //void setLearningRate(double x){this->_learningRate = x; }; - - - - unsigned long _modelId; - int _modelType; - int _scalingMethod; - std::shared_ptr> _activationList; //shared pointers std::shared_ptr _activationListPtr; - std::shared_ptr> _layersSizes; //TODO try to use shared pointers - std::shared_ptr> _layersTypes; - - //double _learningRate; - //int _optimizer; -private: - -}; - -// Train mode parameters struct -// rows, col - Represents the rows and columns of the data matrix -// labels - Represents the label portion of the label matrix from the data_label matrix -// mid - model id number -// data_Label_mat - list in erlang. Represents tha data and the label matrix (last columns) together -// tid - unique thread identification -// pid - unique erlang process identification - - - - - - diff --git a/src_cpp/opennnBridge/get_set_weights.h b/src_cpp/opennnBridge/get_set_weights.h index 090b755d9..15a616d92 100644 --- a/src_cpp/opennnBridge/get_set_weights.h +++ b/src_cpp/opennnBridge/get_set_weights.h @@ -3,7 +3,6 @@ #include #include "../simple-cpp-logger/include/Logger.h" -#include "ModelParams.h" #include "nifppNerltensorEigen.h" #include "bridgeController.h" #include "nerlWorkerOpenNN.h" diff --git a/src_cpp/opennnBridge/openNNExtensionFunction.h b/src_cpp/opennnBridge/openNNExtensionFunction.h index ff5b77f87..e39cbc433 100644 --- a/src_cpp/opennnBridge/openNNExtensionFunction.h +++ b/src_cpp/opennnBridge/openNNExtensionFunction.h @@ -9,7 +9,6 @@ #include #include -#include "ModelParams.h" #include "definitionsNN.h" #include diff --git a/src_cpp/opennnBridge/openNNmodelNif.h b/src_cpp/opennnBridge/openNNmodelNif.h index 9a34f6e59..ea4d55b93 100644 --- a/src_cpp/opennnBridge/openNNmodelNif.h +++ b/src_cpp/opennnBridge/openNNmodelNif.h @@ -14,7 +14,6 @@ * */ -#include "ModelParams.h" #include "openNNExtensionFunction.h" #include "CustumNN.h" #include "../simple-cpp-logger/include/Logger.h" diff --git a/src_cpp/opennnBridge/openNNnif.h b/src_cpp/opennnBridge/openNNnif.h index b21446857..9cbac1968 100644 --- a/src_cpp/opennnBridge/openNNnif.h +++ b/src_cpp/opennnBridge/openNNnif.h @@ -19,7 +19,6 @@ #include "../opennn/opennn/opennn.h" #include "bridgeController.h" #include "get_set_weights.h" -#include "ModelParams.h" #include "nifppNerltensorEigen.h" #include "nerlWorkerNIF.h" diff --git a/src_cpp/source/CMakeLists.txt b/src_cpp/source/CMakeLists.txt new file mode 100644 index 000000000..2992cf3a9 --- /dev/null +++ b/src_cpp/source/CMakeLists.txt @@ -0,0 +1,36 @@ + +project(SourceNIF) + +set(CMAKE_CXX_STANDARD 17) +set(NIFPP_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../nifpp/") +set(COMMON_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../common") +set(SIMPLE_LOGGER_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../simple-cpp-logger/include") + +set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_CXX_FLAGS "-fpic") +set(ERL_NIF_DEFAULT_LOCATION "/usr/local/lib/erlang/usr/include") + +# cpp Simple logger options +add_definitions( -D LOGGER_MAX_LOG_LEVEL_PRINTED=6 ) +add_definitions( -D LOGGER_PREFIX_LEVEL=2 ) +add_definitions( -D LOGGER_ENABLE_COLORS=1 ) +add_definitions( -D LOGGER_ENABLE_COLORS_ON_USER_HEADER=0 ) + +set(SRC_CODE + "Source.h" + "SourceNIF.h" + "SourceCSV.h" + "SourceCSV.cpp" +) + + +add_library(${PROJECT_NAME} SHARED ${SRC_CODE}) + +target_link_libraries(${PROJECT_NAME} PUBLIC common) + +# Include NIF, OpenNN and Simple Cpp Logger +target_include_directories(${PROJECT_NAME} PUBLIC + ${COMMON_PATH} + ${NIFPP_PATH} + ${SIMPLE_LOGGER_PATH} + ${ERL_NIF_DEFAULT_LOCATION}) \ No newline at end of file diff --git a/src_cpp/source/Source.h b/src_cpp/source/Source.h new file mode 100644 index 000000000..7b9637ef9 --- /dev/null +++ b/src_cpp/source/Source.h @@ -0,0 +1 @@ +#pragma once \ No newline at end of file diff --git a/src_cpp/source/SourceCSV.cpp b/src_cpp/source/SourceCSV.cpp new file mode 100644 index 000000000..c5662fdca --- /dev/null +++ b/src_cpp/source/SourceCSV.cpp @@ -0,0 +1 @@ +#include "SourceCSV.h" \ No newline at end of file diff --git a/src_cpp/source/SourceCSV.h b/src_cpp/source/SourceCSV.h new file mode 100644 index 000000000..8838f8ff0 --- /dev/null +++ b/src_cpp/source/SourceCSV.h @@ -0,0 +1,4 @@ +#pragma once + +#include "Source.h" + diff --git a/src_cpp/source/SourceNIF.h b/src_cpp/source/SourceNIF.h new file mode 100644 index 000000000..a085260df --- /dev/null +++ b/src_cpp/source/SourceNIF.h @@ -0,0 +1,62 @@ +#pragma once + +#include "nifpp.h" +#include "SourceCSV.h" + + +set_source(SourceType, DataType, BatchSize, CustomParamsStr) when is_list(CustomParams)-> + exit(nif_library_not_loaded). + +source_get_batches() -> + exit(nif_library_not_loaded). + +source_more_batches() -> + exit(nif_library_not_loaded). + +static ERL_NIF_TERM set_source_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + +} + +static ERL_NIF_TERM source_get_batches_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + +} + +static ERL_NIF_TERM source_more_batches_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + +} + +static ErlNifFunc nif_funcs[] = +{ + {"set_source_nif",0, set_source_nif}, + {"source_get_batches_nif", 0 , source_get_batches_nif}, + {"source_more_batches_nif", 0 , source_more_batches_nif}, +}; + + +// load_info is the second argument to erlang:load_nif/2. +// *priv_data can be set to point to some private data if the library needs to keep a state between NIF calls. +// enif_priv_data returns this pointer. *priv_data is initialized to NULL when load is called. +// The library fails to load if load returns anything other than 0. load can be NULL if initialization is not needed. +static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info) +{ + //nifpp::register_resource(env, nullptr, "GetcppBridgeController"); + //nifpp::register_resource(env, nullptr, "cppBridgeController"); + // nifpp::register_resource(env, nullptr, "cppBridgeController"); + return 0; +} + +// This is the magic macro to initialize a NIF library. It is to be evaluated in global file scope. +// ERL_NIF_INIT(MODULE, ErlNifFunc funcs[], load, NULL, upgrade, unload) +// MODULE - The first argument must be the name of the Erlang module as a C-identifier. It will be stringified by the macro. +// ErlNifFunc - The second argument is the array of ErlNifFunc structures containing name, arity, and function pointer of each NIF. +// load - is called when the NIF library is loaded and no previously loaded library exists for this module. +// NULL - The fourth argument NULL is ignored. It was earlier used for the deprecated reload callback which is no longer supported since OTP 20. +// The remaining arguments are pointers to callback functions that can be used to initialize the library. +// They are not used in this simple example, hence they are all set to NULL. +ERL_NIF_INIT(SourceNIF, nif_funcs, load, NULL, NULL, NULL) + +//ERL_NIF_INIT(nerlNIF,nif_funcs,NULL,NULL,NULL,NULL) + diff --git a/src_erl/NerlnetApp/src/Source/sourceNIF.erl b/src_erl/NerlnetApp/src/Source/sourceNIF.erl new file mode 100644 index 000000000..507e62f4e --- /dev/null +++ b/src_erl/NerlnetApp/src/Source/sourceNIF.erl @@ -0,0 +1,30 @@ +-module(sourceNIF). +-author("David Leon"). + +-include("sourceNIFdefs.hrl"). + +-export([nif_init/0]). +-export([set_source_nif/4, source_get_batches_nif/0, source_more_batches_nif/0]). + +-on_load(nif_init/0). + +nif_init() -> + SOURCE_NIF_LIB_PATH = ?NERLNET_PATH++?BUILD_TYPE_RELEASE++"/"++?SOURCE_NIF_LIB, + RES = erlang:load_nif(SOURCE_NIF_LIB_PATH, 0), + RES. + + +%% Reads a csv file and returns a binary nerltensor of the given data type +%% This is user responsibility to validate that given file has the correct data type +%% generally - float can read int, but int data cannot read by float +%% data type size (# of bytes) must be equal or larger than data type size +%% E.g. int32 can read int8, int16, int32, but int8 cannot read int16, int32 +set_source_nif(_SourceType, _DataType, _BatchSize, _CustomParamsStr) -> + exit(nif_library_not_loaded). + +source_get_batches_nif() -> + exit(nif_library_not_loaded). + +source_more_batches_nif() -> + exit(nif_library_not_loaded). + diff --git a/src_erl/NerlnetApp/src/Source/sourceNIFdefs.hrl b/src_erl/NerlnetApp/src/Source/sourceNIFdefs.hrl new file mode 100644 index 000000000..2d928f350 --- /dev/null +++ b/src_erl/NerlnetApp/src/Source/sourceNIFdefs.hrl @@ -0,0 +1,11 @@ + +%% nerlNIF defines +-define(SOURCE_NIF_LIB,"libsource_nif"). +-define(NERLNET_PATH,"/usr/local/lib/nerlnet-lib/NErlNet"). +-define(BUILD_TYPE_DEBUG,"debug"). +-define(BUILD_TYPE_RELEASE,"/build/release"). + + +%% Source supported types and data types +-define(SOURCE_TYPES,[csv]). +-define(SOURCE_DATA_TYPES,[int8,int16,int32,int64,float16,float,double]). \ No newline at end of file diff --git a/src_erl/NerlnetApp/src/Source/testSourceNIF.erl b/src_erl/NerlnetApp/src/Source/testSourceNIF.erl new file mode 100644 index 000000000..0ed0e63f7 --- /dev/null +++ b/src_erl/NerlnetApp/src/Source/testSourceNIF.erl @@ -0,0 +1,10 @@ +-module(testSourceNIF). +-author("David Leon"). + +-compile(sourceNIF). +-export([run_tests/0]). + +run_tests() -> + io:format("cwd is: ~s~n", [filename:absname(".")]), + io:format("Running tests...~n"), + ok. \ No newline at end of file diff --git a/tests/NerlnetNifTest.sh b/tests/NerlnetNifTest.sh index 62963ee1d..e23a1cce2 100755 --- a/tests/NerlnetNifTest.sh +++ b/tests/NerlnetNifTest.sh @@ -44,10 +44,6 @@ is_rasp="$(grep -c raspbian /etc/os-release)" if [ $is_rasp -gt "0" ]; then export LD_PRELOAD=/usr/lib/arm-linux-gnueabihf/libatomic.so.1.2.0 fi -cmake -S . -B build/release -DCMAKE_BUILD_TYPE=RELEASE -cd build/release -make . -cd - print "Change directory to $NERLNET_TEST_DIR:" cd $NERLNET_TEST_DIR