[SOURCE_CSV] Init new implementation of source csv file read implemen…

…ted with NIF

.github/workflows/pr.yml

@@ -28,6 +28,12 @@ jobs:
         run: |
           ./tests/NerlnetNifTest.sh
         timeout-minutes: 15
+      - name: Source NIF unit tests
+        id: sourcenif
+        if: steps.build.outcome == 'success'
+        run: |
+          ./tests/SourceNifTest.sh
+        timeout-minutes: 15
       - name: Run FullFlow test
         id: fullflow
         if: steps.nif.outcome == 'success'

CMakeLists.txt

@@ -39,3 +39,6 @@ if (NERLWOLF)
     add_library(nerlnet_wolf SHARED $<TARGET_OBJECTS:wolframBridge>)
     target_link_libraries(nerlnet_wolf PUBLIC wolframBridge)
 endif()
+
+add_library(source_nif SHARED $<TARGET_OBJECTS:SourceNIF>)
+target_link_libraries(source_nif PUBLIC SourceNIF)

src_cpp/CMakeLists.txt

@@ -9,6 +9,7 @@ project(src_cpp)
 
 add_subdirectory(common)
 add_subdirectory(opennnBridge)
+add_subdirectory(source)
 
 if(NERLWOLF)
     message("[NERLNET] Wolfram Engine nif extension is enabled")

src_cpp/opennnBridge/CMakeLists.txt

@@ -31,7 +31,6 @@ set(SRC_CODE
 	"definitionsNN.h"
 	"openNNnif.h"
 	"openNNnif.cpp"
-    "ModelParams.h"
 	"nerlWorkerOpenNN.h"
 	"nerlWorkerOpenNN.cpp"
 	"nerlWorkerNIF.h"

src_cpp/opennnBridge/get_set_weights.h

@@ -3,7 +3,6 @@
 
 #include <string>
 #include "../simple-cpp-logger/include/Logger.h"
-#include "ModelParams.h"
 #include "nifppNerltensorEigen.h"
 #include "bridgeController.h"
 #include "nerlWorkerOpenNN.h"

src_cpp/opennnBridge/openNNExtensionFunction.h

@@ -9,7 +9,6 @@
 
 #include <vector>
 #include <string>
-#include "ModelParams.h"
 #include "definitionsNN.h"
 #include <map>
 

src_cpp/opennnBridge/openNNmodelNif.h

@@ -14,7 +14,6 @@
  * 
  */
 
-#include "ModelParams.h"
 #include "openNNExtensionFunction.h"
 #include "CustumNN.h"
 #include "../simple-cpp-logger/include/Logger.h"

src_cpp/opennnBridge/openNNnif.h

@@ -19,7 +19,6 @@
 #include "../opennn/opennn/opennn.h"
 #include "bridgeController.h"
 #include "get_set_weights.h"
-#include "ModelParams.h"
 
 #include "nifppNerltensorEigen.h"
 #include "nerlWorkerNIF.h"

src_cpp/source/CMakeLists.txt

@@ -0,0 +1,36 @@
+
+project(SourceNIF)
+
+set(CMAKE_CXX_STANDARD 17)
+set(NIFPP_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../nifpp/")
+set(COMMON_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../common")
+set(SIMPLE_LOGGER_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../simple-cpp-logger/include")
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+set(CMAKE_CXX_FLAGS "-fpic")
+set(ERL_NIF_DEFAULT_LOCATION "/usr/local/lib/erlang/usr/include")
+
+# cpp Simple logger options
+add_definitions( -D LOGGER_MAX_LOG_LEVEL_PRINTED=6 )
+add_definitions( -D LOGGER_PREFIX_LEVEL=2 )
+add_definitions( -D LOGGER_ENABLE_COLORS=1 )
+add_definitions( -D LOGGER_ENABLE_COLORS_ON_USER_HEADER=0 )
+
+set(SRC_CODE
+    "Source.h"
+    "SourceNIF.h"
+    "SourceCSV.h"
+    "SourceCSV.cpp"
+)
+
+
+add_library(${PROJECT_NAME} SHARED ${SRC_CODE})
+
+target_link_libraries(${PROJECT_NAME} PUBLIC common)
+
+# Include NIF, OpenNN and Simple Cpp Logger
+target_include_directories(${PROJECT_NAME} PUBLIC
+							${COMMON_PATH}
+							${NIFPP_PATH}
+							${SIMPLE_LOGGER_PATH}
+							${ERL_NIF_DEFAULT_LOCATION})

src_cpp/source/Source.h

@@ -0,0 +1 @@
+#pragma once

src_cpp/source/SourceCSV.cpp

@@ -0,0 +1 @@
+#include "SourceCSV.h"

src_cpp/source/SourceCSV.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#include "Source.h"
+

src_cpp/source/SourceNIF.h

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "nifpp.h"
+#include "SourceCSV.h"
+
+
+set_source(SourceType, DataType, BatchSize, CustomParamsStr) when is_list(CustomParams)->
+    exit(nif_library_not_loaded).
+
+source_get_batches() ->
+    exit(nif_library_not_loaded).
+
+source_more_batches() ->
+    exit(nif_library_not_loaded).
+
+static ERL_NIF_TERM set_source_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+
+}
+
+static ERL_NIF_TERM source_get_batches_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+
+}
+
+static ERL_NIF_TERM source_more_batches_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+
+}
+
+static ErlNifFunc nif_funcs[] =
+{
+    {"set_source_nif",0, set_source_nif},
+    {"source_get_batches_nif", 0 , source_get_batches_nif},
+    {"source_more_batches_nif", 0 , source_more_batches_nif},
+};
+
+
+// load_info is the second argument to erlang:load_nif/2.
+// *priv_data can be set to point to some private data if the library needs to keep a state between NIF calls.
+// enif_priv_data returns this pointer. *priv_data is initialized to NULL when load is called.
+// The library fails to load if load returns anything other than 0. load can be NULL if initialization is not needed.
+static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info)
+{
+    //nifpp::register_resource<GetcppBridgeController>(env, nullptr, "GetcppBridgeController");
+    //nifpp::register_resource<cppBridgeController>(env, nullptr, "cppBridgeController");
+   // nifpp::register_resource<SANN::Model>(env, nullptr, "cppBridgeController");
+    return 0;
+}
+
+// This is the magic macro to initialize a NIF library. It is to be evaluated in global file scope.
+// ERL_NIF_INIT(MODULE, ErlNifFunc funcs[], load, NULL, upgrade, unload)
+// MODULE -  The first argument must be the name of the Erlang module as a C-identifier. It will be stringified by the macro.
+// ErlNifFunc - The second argument is the array of ErlNifFunc structures containing name, arity, and function pointer of each NIF.
+// load -  is called when the NIF library is loaded and no previously loaded library exists for this module.
+// NULL - The fourth argument NULL is ignored. It was earlier used for the deprecated reload callback which is no longer supported since OTP 20.
+// The remaining arguments are pointers to callback functions that can be used to initialize the library.
+// They are not used in this simple example, hence they are all set to NULL.
+ERL_NIF_INIT(SourceNIF, nif_funcs, load, NULL, NULL, NULL)
+
+//ERL_NIF_INIT(nerlNIF,nif_funcs,NULL,NULL,NULL,NULL)
+

src_erl/NerlnetApp/src/Source/sourceNIF.erl

@@ -0,0 +1,30 @@
+-module(sourceNIF).
+-author("David Leon").
+
+-include("sourceNIFdefs.hrl").
+
+-export([nif_init/0]).
+-export([set_source_nif/4, source_get_batches_nif/0, source_more_batches_nif/0]).
+
+-on_load(nif_init/0).
+
+nif_init() ->
+    SOURCE_NIF_LIB_PATH = ?NERLNET_PATH++?BUILD_TYPE_RELEASE++"/"++?SOURCE_NIF_LIB,
+    RES = erlang:load_nif(SOURCE_NIF_LIB_PATH, 0),
+    RES.
+
+
+%% Reads a csv file and returns a binary nerltensor of the given data type
+%% This is user responsibility to validate that given file has the correct data type
+%% generally - float can read int, but int data cannot read by float
+%% data type size (# of bytes) must be equal or larger than data type size
+%% E.g. int32 can read int8, int16, int32, but int8 cannot read int16, int32
+set_source_nif(_SourceType, _DataType, _BatchSize, _CustomParamsStr) ->
+    exit(nif_library_not_loaded).
+
+source_get_batches_nif() ->
+    exit(nif_library_not_loaded).
+
+source_more_batches_nif() ->
+    exit(nif_library_not_loaded).
+

src_erl/NerlnetApp/src/Source/sourceNIFdefs.hrl

@@ -0,0 +1,11 @@
+
+%% nerlNIF defines
+-define(SOURCE_NIF_LIB,"libsource_nif").
+-define(NERLNET_PATH,"/usr/local/lib/nerlnet-lib/NErlNet").
+-define(BUILD_TYPE_DEBUG,"debug").
+-define(BUILD_TYPE_RELEASE,"/build/release").
+
+
+%% Source supported types and data types
+-define(SOURCE_TYPES,[csv]).
+-define(SOURCE_DATA_TYPES,[int8,int16,int32,int64,float16,float,double]).

src_erl/NerlnetApp/src/Source/testSourceNIF.erl

@@ -0,0 +1,10 @@
+-module(testSourceNIF).
+-author("David Leon").
+
+-compile(sourceNIF).
+-export([run_tests/0]).
+
+run_tests() ->
+    io:format("cwd is: ~s~n", [filename:absname(".")]),
+    io:format("Running tests...~n"),
+    ok.

tests/NerlnetNifTest.sh

@@ -44,10 +44,6 @@ is_rasp="$(grep -c raspbian /etc/os-release)"
 if [ $is_rasp -gt "0" ]; then 
     export LD_PRELOAD=/usr/lib/arm-linux-gnueabihf/libatomic.so.1.2.0 
 fi
-cmake -S . -B build/release -DCMAKE_BUILD_TYPE=RELEASE
-cd build/release 
-make . 
-cd -
 
 print "Change directory to $NERLNET_TEST_DIR:"
 cd $NERLNET_TEST_DIR