[ML] Generate windows resource files using CMake functionality

cmake/functions.cmake

@@ -28,23 +28,55 @@ endif()
 # that is linked into the Windows executables
 #
 function(ml_generate_resources _target)
-  if(NOT WIN32)
-    return()
+
+  set(ML_VERSION_STR "${ML_VERSION_NUM}")
+
+  if(ENV{VERSION_QUALIFIER})
+    set(ML_VERSION_STR "${ML_VERSION_STR}-$ENV{VERSION_QUALIFIER}")
+  endif()
+
+  if(NOT ENV{SNAPSHOT} STREQUAL no)
+    set(ML_VERSION_STR "${ML_VERSION_STR}-SNAPSHOT")
+  endif()
+
+  if(${ML_VERSION_STR} MATCHES "([0-9.]+)")
+    set(ML_VERSION ${CMAKE_MATCH_1})
+  endif()
+  string(REPLACE "." "," ML_VERSION "${ML_VERSION}")
+
+  set(ML_PATCH "0")
+
+  execute_process(COMMAND git -c core.fileMode=false update-index -q --refresh ERROR_FILE /dev/null OUTPUT_FILE /dev/null)
+  execute_process(COMMAND git -c core.fileMode=false diff-index --quiet HEAD --  RESULT_VARIABLE UNCOMMITTED_CHANGES)
+
+  if(UNCOMMITED_CHANGES EQUAL 0)
+    set(ML_FILEFLAGS "0")
+  else()
+    set(ML_FILEFLAGS "VS_FF_PRIVATEBUILD")
   endif()
-  set( ${_target}_LINKFLAGS ${CMAKE_CURRENT_BINARY_DIR}/${_target}.res )
-  set_target_properties( ${_target} PROPERTIES LINK_FLAGS ${${_target}_LINKFLAGS} )
-  execute_process(COMMAND bash -c "${CMAKE_SOURCE_DIR}/mk/make_rc_defines.sh ${_target}.exe" OUTPUT_VARIABLE
-    RC_DEFINES OUTPUT_STRIP_TRAILING_WHITESPACE)
-  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp.sh "rc -nologo ${CPPFLAGS} ${RC_DEFINES} -Fo${_target}.res ${CMAKE_SOURCE_DIR}/mk/ml.rc")
-  add_custom_target(
-    ${_target}.res
-    DEPENDS ${CMAKE_SOURCE_DIR}/mk/ml.rc ${CMAKE_SOURCE_DIR}/gradle.properties ${CMAKE_SOURCE_DIR}/mk/ml.ico ${CMAKE_SOURCE_DIR}/mk/make_rc_defines.sh ${CMAKE_CURRENT_BINARY_DIR}/tmp.sh
-    COMMAND bash -c ${CMAKE_CURRENT_BINARY_DIR}/tmp.sh
-    )
-  add_dependencies(${_target} ${_target}.res)
 
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_CURRENT_BINARY_DIR}/tmp.sh)
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_CURRENT_BINARY_DIR}/${_target}.res)
+  if("${_target}" MATCHES ".dll$")
+    set(ML_FILETYPE "VFT_DLL")
+  else()
+    set(ML_FILETYPE "VFT_APP")
+  endif()
+
+  set(ML_FILENAME ${_target})
+
+  if(${_target} MATCHES "([^.]+).")
+    set(ML_NAME ${CMAKE_MATCH_1})
+  endif()
+  set(ML_YEAR ${BUILD_YEAR})
+  set(ML_ICON ${CMAKE_SOURCE_DIR}/mk/ml.ico)
+
+  configure_file(
+	"${CMAKE_SOURCE_DIR}/mk/ml.rc.in"
+	"${CMAKE_CURRENT_BINARY_DIR}/${ML_NAME}.rc"
+        @ONLY
+        )
+
+  set(ML_FILEFLAGS ${ML_FILEFLAGS} PARENT_SCOPE)
+
 endfunction()
 
 #
@@ -95,6 +127,12 @@ function(ml_add_library _target _type)
 
   add_compile_definitions(BUILDING_lib${_target})
 
+  if (WIN32 AND _type STREQUAL "SHARED")
+    ml_generate_resources(lib${_target}.dll)
+    list(APPEND PLATFORM_SRCS ${CMAKE_CURRENT_BINARY_DIR}/lib${_target}.rc)
+    add_compile_definitions(ML_FILEFLAGS=${ML_FILEFLAGS})
+  endif()
+
   add_library(${_target} ${_type} ${PLATFORM_SRCS})
 
   if(ML_LINK_LIBRARIES)
@@ -140,6 +178,12 @@ function(ml_add_executable _target)
     add_library(Ml${_target} OBJECT ${PLATFORM_SRCS})
   endif()
 
+  if (WIN32)
+    ml_generate_resources(${_target}.exe)
+    list(APPEND PLATFORM_SRCS ${CMAKE_CURRENT_BINARY_DIR}/${_target}.rc)
+    add_compile_definitions(ML_FILEFLAGS=${ML_FILEFLAGS})
+  endif()
+
   add_executable(${_target} Main.cc ${PLATFORM_SRCS})
 
   if (ML_EXE_LINKER_FLAGS)
@@ -150,8 +194,6 @@ function(ml_add_executable _target)
 
   ml_install(${_target})
 
-  ml_generate_resources(${_target})
-
   if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
     target_link_libraries(${_target} PRIVATE "${COVERAGE}")
   endif()

cmake/variables.cmake

@@ -195,3 +195,14 @@ if(MSVC)
   set(CMAKE_SHARED_LIBRARY_PREFIX "lib")
   set(CMAKE_IMPORT_LIBRARY_PREFIX "lib")
 endif()
+
+string(TIMESTAMP BUILD_YEAR "%Y")
+
+if(WIN32)
+  set(ML_USER $ENV{USERNAME})
+else()
+  execute_process(COMMAND id COMMAND awk -F ")" "{ print $1 }" COMMAND awk -F "(" "{ print $2 }" OUTPUT_VARIABLE ML_USER
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
+
+execute_process(COMMAND git rev-parse --short=14 HEAD OUTPUT_VARIABLE ML_BUILD_STR OUTPUT_STRIP_TRAILING_WHITESPACE)

lib/api/CMakeLists.txt

@@ -44,6 +44,8 @@ ml_add_library(MlApi SHARED
   CDataFrameTrainBoostedTreeRegressionRunner.cc
   CDataFrameTrainBoostedTreeRunner.cc
   CDataProcessor.cc
+  CDataSummarizationJsonTags.cc
+  CDataSummarizationJsonWriter.cc
   CDetectionRulesJsonParser.cc
   CFieldDataCategorizer.cc
   CForecastRunner.cc
@@ -66,6 +68,8 @@ ml_add_library(MlApi SHARED
   CPerPartitionCategoryIdMapper.cc
   CPersistenceManager.cc
   CResultNormalizer.cc
+  CRetrainableModelJsonReader.cc
+  CSerializableToJson.cc
   CSimpleOutputWriter.cc
   CSingleFieldDataCategorizer.cc
   CSingleStreamDataAdder.cc

lib/api/unittest/CMakeLists.txt

@@ -13,6 +13,7 @@ project("ML Api unit tests")
 
 set (SRCS
   Main.cc
+  CAnnotationJsonWriterTest.cc
   CAnomalyJobConfigTest.cc
   CAnomalyJobLimitTest.cc
   CAnomalyJobTest.cc
@@ -26,9 +27,10 @@ set (SRCS
   CDataFrameAnalyzerFeatureImportanceTest.cc
   CDataFrameAnalyzerOutlierTest.cc
   CDataFrameAnalyzerTrainingTest.cc
+  CDataFrameMockAnalysisRunner.cc
   CDataFrameTrainBoostedTreeClassifierRunnerTest.cc
   CDataFrameTrainBoostedTreeRegressionRunnerTest.cc
-  CDataFrameMockAnalysisRunner.cc
+  CDataSummarizationJsonSerializerTest.cc
   CDetectionRulesJsonParserTest.cc
   CFieldDataCategorizerTest.cc
   CForecastRunnerTest.cc
@@ -40,7 +42,6 @@ set (SRCS
   CMemoryUsageEstimationResultJsonWriterTest.cc
   CMockDataAdder.cc
   CMockSearcher.cc
-  CAnnotationJsonWriterTest.cc
   CModelPlotDataJsonWriterTest.cc
   CModelSnapshotJsonWriterTest.cc
   CMultiFileDataAdderTest.cc
@@ -51,6 +52,7 @@ set (SRCS
   CPersistenceManagerTest.cc
   CRestorePreviousStateTest.cc
   CResultNormalizerTest.cc
+  CSerializableToJsonTest.cc
   CSingleFieldDataCategorizerTest.cc
   CSingleStreamDataAdderTest.cc
   CStateRestoreStreamFilterTest.cc

lib/maths/analytics/CMakeLists.txt

@@ -22,6 +22,8 @@ ml_add_library(MlMathsAnalytics SHARED
   CBoostedTreeHyperparameters.cc
   CBoostedTreeImpl.cc
   CBoostedTreeLeafNodeStatistics.cc
+  CBoostedTreeLeafNodeStatisticsIncremental.cc
+  CBoostedTreeLeafNodeStatisticsScratch.cc
   CBoostedTreeLeafNodeStatisticsThreading.cc
   CBoostedTreeLoss.cc
   CBoostedTreeUtils.cc

lib/maths/analytics/unittest/CMakeLists.txt

@@ -13,9 +13,11 @@ project("ML Maths Analytics unit tests")
 
 set (SRCS
   Main.cc
+  BoostedTreeTestData.cc
   CBoostedTreeLeafNodeStatisticsTest.cc
   CBoostedTreeLossTest.cc
   CBoostedTreeTest.cc
+  CBoostedTreeUtilsTest.cc
   CDataFrameCategoryEncoderTest.cc
   CDataFrameUtilsTest.cc
   CMicTest.cc

lib/ver/CMakeLists.txt

@@ -12,18 +12,10 @@
 project("ML Ver")
 
 # Prepare values to substitute into the template
-string(TIMESTAMP BUILD_YEAR "%Y")
-
-if(WIN32)
-  set(USER_NAME $ENV{USERNAME})
-else()
-  execute_process(COMMAND id COMMAND awk -F ")" "{ print $1 }" COMMAND awk -F "(" "{ print $2 }" OUTPUT_VARIABLE USER_NAME
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-endif()
 set(PRODUCT_VERSION "${ML_VERSION_NUM}")
 
 message(STATUS "PRODUCT_VERSION ${PRODUCT_VERSION}")
-message(STATUS "USER_NAME ${USER_NAME}")
+message(STATUS "ML_USER_NAME ${ML_USER_NAME}")
 message(STATUS "BUILD_YEAR ${BUILD_YEAR}")
 
 if(DEFINED ENV{VERSION_QUALIFIER})
@@ -33,7 +25,6 @@ endif()
 if("$ENV{SNAPSHOT}" STREQUAL "yes")
   set(PRODUCT_VERSION "${PRODUCT_VERSION}-SNAPSHOT")
 endif()
-execute_process(COMMAND git rev-parse --short=14 HEAD OUTPUT_VARIABLE ML_BUILD_NUM OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
   execute_process(COMMAND git -c core.fileMode=false update-index -q --refresh ERROR_FILE /dev/null OUTPUT_FILE /dev/null  COMMAND git -c core.fileMode=false diff-index --quiet HEAD --  RESULT_VARIABLE UNCOMMITTED_CHANGES)
@@ -50,9 +41,9 @@ set(TEMPLATE "template")
 
 set(warning.comment "DO NOT EDIT THIS FILE - instead edit the template")
 set(build.year "${BUILD_YEAR}")
-set(user.name "${USER_NAME}")
+set(user.name "${ML_USER}")
 set(version.number "${PRODUCT_VERSION}")
-set(build.number "${ML_BUILD_NUM}")
+set(build.number "${ML_BUILD_STR}")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CBuildInfo.cc.${TEMPLATE}
   ${CMAKE_CURRENT_SOURCE_DIR}/CBuildInfo.cc
   )

mk/ml.rc.in

@@ -0,0 +1,43 @@
+#include <Windows.h>
+
+1 VERSIONINFO
+    FILEVERSION    @ML_VERSION@,@ML_PATCH@
+    PRODUCTVERSION @ML_VERSION@,@ML_PATCH@
+    FILEFLAGSMASK  VS_FFI_FILEFLAGSMASK
+    FILEFLAGS      (0 | @ML_FILEFLAGS@)
+    FILEOS         VOS_NT_WINDOWS32
+    FILETYPE       @ML_FILETYPE@
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "080904E4"
+        BEGIN
+#if ML_FILEFLAGS
+            VALUE "Comments", "Build @ML_BUILD_STR@  by  @ML_USER@"
+#else
+            VALUE "Comments", "Build @ML_BUILD_STR@"
+#endif
+            VALUE "CompanyName", "Elasticsearch BV"
+            VALUE "FileDescription", "@ML_FILENAME@ (part of Elasticsearch machine learning)"
+            VALUE "FileVersion", "@ML_VERSION_STR@"
+            VALUE "InternalName", "@ML_NAME@"
+            VALUE "LegalCopyright", "Copyright (c) @ML_YEAR@ Elasticsearch BV"
+#if 0
+            VALUE "LegalTrademarks", "TODO"
+#endif
+            VALUE "OriginalFilename", "@ML_FILENAME@"
+#if ML_FILEFLAGS
+            VALUE "PrivateBuild", "Built by @ML_USER@"
+#endif
+            VALUE "ProductName", "Elasticsearch machine learning"
+            VALUE "ProductVersion", "@ML_VERSION_STR@"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x0809, 1252
+    END
+END
+
+2 ICON "@ML_ICON@"
+