distributed server

.gitignore

@@ -46,3 +46,6 @@ build/*
 .Rhistory
 *.tar.gz
 xlearn/*
+
+# Python-package
+*.pyc

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ps-lite"]
+	path = ps-lite
+	url = https://github.com/dmlc/ps-lite

CMakeLists.txt

@@ -30,8 +30,9 @@ cmake_minimum_required(VERSION 2.8)
 # Using c++11;
 # Optimization on architecture
 #-------------------------------------------------------------------------------
-add_definitions("-Wall -Wno-sign-compare -O3 -std=c++11 
--march=native -Wno-strict-aliasing -Wno-comment")
+#add_definitions("-Wall -Wno-sign-compare -O3 -std=c++11 -march=native 
+add_definitions("-Wall -Wno-sign-compare -std=c++11 -march=native 
+-Wno-strict-aliasing -Wno-comment")
 
 #-------------------------------------------------------------------------------
 # Declare where our project will be installed.
@@ -50,6 +51,7 @@ include_directories(
   "${PROJECT_SOURCE_DIR}"
   "${PROJECT_BINARY_DIR}"
   "${PROJECT_SOURCE_DIR}/gtest/include"
+  "${PROJECT_SOURCE_DIR}/ps-lite/include"
 )
 
 link_directories(
@@ -59,21 +61,24 @@ link_directories(
   "${PROJECT_BINARY_DIR}/src/score"
   "${PROJECT_BINARY_DIR}/src/loss"
   "${PROJECT_BINARY_DIR}/src/solver"
+  "${PROJECT_BINARY_DIR}/src/distributed"
   "${PROJECT_BINARY_DIR}/src/c_api"
+  "${PROJECT_BINARY_DIR}/ps-lite"
 )
 
 #-------------------------------------------------------------------------------
 # Declare packages in xLearn project.
 #-------------------------------------------------------------------------------
+add_subdirectory(ps-lite)
 add_subdirectory(gtest)
 add_subdirectory(demo)
-add_subdirectory(scripts)
 add_subdirectory(src/base)
 add_subdirectory(src/data)
 add_subdirectory(src/reader)
 add_subdirectory(src/score)
 add_subdirectory(src/loss)
 add_subdirectory(src/solver)
+add_subdirectory(src/distributed)
 add_subdirectory(src/c_api)
 add_subdirectory(python-package)
-#add_subdirectory(R-package)
+add_subdirectory(scripts)

cmake/FindZMQ.cmake

@@ -0,0 +1,17 @@
+# - Try to find ZMQ
+# Once done this will define
+# ZMQ_FOUND - System has ZMQ
+# ZMQ_INCLUDE_DIRS - The ZMQ include directories
+# ZMQ_LIBRARIES - The libraries needed to use ZMQ
+# ZMQ_DEFINITIONS - Compiler switches required for using ZMQ
+
+find_path ( ZMQ_INCLUDE_DIR zmq.h )
+find_library ( ZMQ_LIBRARY NAMES zmq )
+
+set ( ZMQ_LIBRARIES ${ZMQ_LIBRARY} )
+set ( ZMQ_INCLUDE_DIRS ${ZMQ_INCLUDE_DIR} )
+
+include ( FindPackageHandleStandardArgs )
+# handle the QUIETLY and REQUIRED arguments and set ZMQ_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args ( ZMQ DEFAULT_MSG ZMQ_LIBRARY ZMQ_INCLUDE_DIR )

cmake/ps-lite.cmake

@@ -0,0 +1,13 @@
+include(ExternalProject)
+
+set(pslite_URL https://github.com/dmlc/ps-lite.git)
+set(pslite_TAG 2ce8b9a256207947acfa2cb9b09ab74b8de74547)
+
+ExternalProject_Add(ps-lite
+  PREFIX ps-lite
+  GIT_REPOSITORY ${pslite_URL}
+  GIT_TAG ${pslite_TAG}
+  UPDATE_COMMAND ""
+  BUILD_COMMAND make
+)
+

cmake/zmq.cmake

@@ -0,0 +1,63 @@
+if (NOT __ZMQ_INCLUDED) # guard against multiple includes
+  set(__ZMQ_INCLUDED TRUE)
+  include(ExternalProject)
+
+  # use the system-wide ZMQ if present
+  find_package(ZMQ)
+  if (ZMQ_FOUND)
+    set(ZMQ_EXTERNAL FALSE)
+  else()
+    # ZMQ will use pthreads if it's available in the system, so we must link with it
+    find_package(Threads)
+
+    # build directory
+    set(ZMQ_PREFIX ${CMAKE_BINARY_DIR}/external/ZMQ-prefix)
+    # install directory
+    set(ZMQ_INSTALL ${CMAKE_BINARY_DIR}/external/ZMQ-install)
+
+    # we build ZMQ statically, but want to link it into the caffe shared library
+    # this requires position-independent code
+    if (UNIX)
+        set(ZMQ_EXTRA_COMPILER_FLAGS "-fPIC")
+    endif()
+
+    set(ZMQ_CXX_FLAGS ${CMAKE_CXX_FLAGS} ${ZMQ_EXTRA_COMPILER_FLAGS})
+    set(ZMQ_C_FLAGS ${CMAKE_C_FLAGS} ${ZMQ_EXTRA_COMPILER_FLAGS})
+
+    ExternalProject_Add(ZMQ
+      PREFIX ${ZMQ_PREFIX}
+      GIT_REPOSITORY "https://github.com/zeromq/libZMQ.git"
+      UPDATE_COMMAND ""
+      INSTALL_DIR ${ZMQ_INSTALL}
+      CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                 -DCMAKE_INSTALL_PREFIX=${ZMQ_INSTALL}
+                 -DBUILD_SHARED_LIBS=OFF
+                 -DBUILD_STATIC_LIBS=ON
+                 -DBUILD_PACKAGING=OFF
+                 -DBUILD_TESTING=OFF
+                 -DBUILD_NC_TESTS=OFF
+                 -BUILD_CONFIG_TESTS=OFF
+                 -DINSTALL_HEADERS=ON
+                 -DCMAKE_C_FLAGS=${ZMQ_C_FLAGS}
+                 -DCMAKE_CXX_FLAGS=${ZMQ_CXX_FLAGS}
+      LOG_DOWNLOAD 1
+      LOG_INSTALL 1
+      )
+
+    set(ZMQ_FOUND TRUE)
+    set(ZMQ_INCLUDE_DIRS ${ZMQ_INSTALL}/include)
+
+	if(MSVC)
+		FILE(GLOB_RECURSE ZMQ_LIBRARIES "${ZMQ_INSTALL}/lib/libzmq-${CMAKE_VS_PLATFORM_TOOLSET}*.lib")
+		#set(ZMQ_LIBRARIES ${ZMQ_INSTALL}/lib/ZMQ.lib ${CMAKE_THREAD_LIBS_INIT})
+	else()
+		FILE(GLOB_RECURSE ZMQ_LIBRARIES "${ZMQ_INSTALL}/lib/libzmq-*.a")
+		#set(ZMQ_LIBRARIES ${ZMQ_INSTALL}/lib/libZMQ.a ${CMAKE_THREAD_LIBS_INIT})
+	endif()
+    set(ZMQ_LIBRARY_DIRS ${ZMQ_INSTALL}/lib)
+    set(ZMQ_EXTERNAL TRUE)
+
+    list(APPEND external_project_dependencies ZMQ)
+  endif()
+
+endif()

python-package/setup.py

@@ -1,31 +1,100 @@
 # coding: utf-8
 """Setup xlearn package."""
 from __future__ import absolute_import
-import sys
+
 import os
-from setuptools import setup, find_packages
+import subprocess
+import shutil
+import sys
+
+from setuptools import find_packages
+from setuptools import setup
+from setuptools.command.install import install
+from setuptools.command.sdist import sdist
+
 sys.path.insert(0, '.')
 
 CURRENT_DIR = os.path.dirname(__file__)
 
-libpath_py = os.path.join(CURRENT_DIR, 'xlearn/libpath.py')
-libpath = {'__file__': libpath_py}
-exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)
-
-LIB_PATH = [os.path.relpath(libfile, CURRENT_DIR) for libfile in libpath['find_lib_path']()]
-print("Install libxlearn_api from: %s" % LIB_PATH)
-
-setup(name='xlearn',
-      version=open(os.path.join(CURRENT_DIR, 'xlearn/VERSION')).read().strip(),
-      description="xLearn Python Package",
-      maintainer='Chao Ma',
-      maintainer_email='[email protected]',
-      zip_safe=False,
-      packages=find_packages(),
-      # this will use MANIFEST.in during install where we specify additional files,
-      # this is the golden line
-      include_package_data=True,
-      data_files=[('xlearn', LIB_PATH)],
-      license='Apache-2.0',
-      classifiers=['License :: OSI Approved :: Apache Software License'],
-      url='https://github.com/aksnzhy/xlearn')
+def silent_call(cmd, raise_error=False, error_msg=''):
+    try:
+        with open(os.devnull, 'w') as shut_up:
+            subprocess.check_output(cmd, stderr=shut_up)
+            return 0
+    except Exception:
+        if raise_error:
+            raise Exception(error_msg);
+        return 1
+
+def copy_files():
+    if os.path.isdir('compile'):
+        shutil.rmtree('compile')
+    src_list = ['demo', 'gtest', 'scripts', 'src']
+    for src in src_list:
+        dst = 'compile/{}'.format(src)
+        if os.path.isdir(dst):
+            shutil.rmtree(dst)
+        shutil.copytree('../{}'.format(src), dst)
+    shutil.copy('../CMakeLists.txt', 'compile')
+    # create empty python-package for cmake
+    os.makedirs('compile/python-package')
+    open('compile/python-package/CMakeLists.txt', 'w')
+
+def compile_cpp():
+    build_path = os.path.join(CURRENT_DIR, 'build_cpp')
+    if os.path.isdir(build_path):
+        shutil.rmtree(build_path)
+    os.makedirs(build_path)
+    old_working_dir = os.getcwd()
+    os.chdir(build_path)
+    src_path = '../compile'
+    cmake_cmd = ['cmake', src_path]
+    if not os.path.isdir(src_path):
+        print('current path: {}'.format(os.getcwd()))
+        raise Exception('{} not exists'.format(src_path))
+    if os.name == "nt":
+        # Windows
+        print('Windows users please use github installation.')
+    else:
+        # Linux, Darwin (OS X), etc.
+        silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake first')
+        silent_call(["make", "-j4"], raise_error=True, 
+                error_msg='An error has occurred while building xlearn library file')
+        suffix_list = ['dylib', 'so']
+        for suffix in suffix_list:
+            if os.path.isfile('lib/libxlearn_api.{}'.format(suffix)):
+                shutil.copy('lib/libxlearn_api.{}'.format(suffix), '../xlearn/')
+
+    os.chdir(old_working_dir)
+
+class CustomInstall(install):
+
+    def run(self):
+        compile_cpp();
+        install.run(self)
+
+class CustomSdist(sdist):
+
+    def run(self):
+        copy_files()
+        sdist.run(self)
+
+if __name__ == "__main__":
+    setup(name='xlearn',
+          version="0.20.a1",
+          description="xLearn Python Package",
+          maintainer='Chao Ma',
+          maintainer_email='[email protected]',
+          zip_safe=False,
+          cmdclass={
+              'install': CustomInstall,
+              'sdist': CustomSdist,
+          },
+          packages=find_packages(),
+          # this will use MANIFEST.in during install where we specify additional files,
+          # this is the golden line
+          include_package_data=True,
+          # move data to MANIFEST.in
+          license='Apache-2.0',
+          classifiers=['License :: OSI Approved :: Apache Software License'],
+          url='https://github.com/aksnzhy/xlearn')

scripts/local.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# set -x
+if [ $# -lt 3 ]; then
+    echo "usage: $0 num_servers num_workers bin [args..]"
+    exit -1;
+fi
+
+export DMLC_NUM_SERVER=$1
+shift
+export DMLC_NUM_WORKER=$1
+shift
+bin=$1
+shift
+arg="$@"
+
+# start the scheduler
+export DMLC_PS_ROOT_URI='127.0.0.1'
+export DMLC_PS_ROOT_PORT=8000
+export DMLC_ROLE='scheduler'
+${bin} ${arg} &
+
+
+# start servers
+export DMLC_ROLE='server'
+for ((i=0; i<${DMLC_NUM_SERVER}; ++i)); do
+    export HEAPPROFILE=./S${i}
+    ${bin} ${arg} &
+done
+
+# start workers
+export DMLC_ROLE='worker'
+for ((i=0; i<${DMLC_NUM_WORKER}; ++i)); do
+    export HEAPPROFILE=./W${i}
+    ${bin} ${arg} &
+done
+
+wait

scripts/run_xlearn_dist_train.sh

@@ -0,0 +1,10 @@
+#! /bin/sh
+
+echo "=====================================BigDataLR================================="
+sh local.sh 1 3 /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/test/distributed/dist_xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 0 -x auc --dis-es -p sgd -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm
+#sh local.sh 1 3 /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/test/distributed/dist_xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 0 -x auc --dis-es -p adagrad -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm
+#sh local.sh 1 3 /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/test/distributed/dist_xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 0 -x auc --dis-es -p ftrl -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm
+#echo "=====================================BigDataFM================================="
+#sh local.sh 1 1 /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 1 -x auc --dis-es -p ftrl -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm
+#echo "=====================================BigDataFFM==============================="
+#sh local.sh 1 1 /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 2 -x auc --dis-es -p ftrl -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm

scripts/run_xlearn_local_train.sh

@@ -0,0 +1,13 @@
+#echo "====================================SmallData LR==================================="
+#/Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/demo/classification/criteo_ctr/small_train.txt -v /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/demo/classification/criteo_ctr/small_test.txt -s 0 -x auc --dis-es
+#echo "====================================SmallData FM==================================="
+#/Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/demo/classification/criteo_ctr/small_train.txt -v /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/demo/classification/criteo_ctr/small_test.txt -s 1 -x auc --dis-es
+#echo "====================================SmallData FFM=================================="
+#/Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/demo/classification/criteo_ctr/small_train.txt -v /Users/xiaoshuwang/documents/oneflow/opensource/xlearn/demo/classification/criteo_ctr/small_test.txt -s 2 -x auc --dis-es
+
+echo "=====================================BigDataLR================================="
+/Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 0 -x auc --dis-es -p ftrl -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm
+#echo "=====================================BigDataFM================================="
+/Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 1 -x auc --dis-es -p ftrl -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm
+#echo "=====================================BigDataFFM==============================="
+/Users/xiaoshuwang/documents/oneflow/opensource/xlearn/xlearn_build/xlearn_train ~/documents/data/libffm_toy/criteo.tr.r100.gbdt0.ffm -v ~/documents/data/libffm_toy/criteo.va.r100.gbdt0.ffm -s 2 -x auc --dis-es -p ftrl -alpha 5e-2 -beta 1.0 -lambda_1 5e-5 -lambda_2 15.0 --no-norm

scripts/stop_xlearn_dist_train.sh

@@ -0,0 +1,10 @@
+#! /bin/sh
+#
+# stop_xlearn.sh
+# Copyright (C) 2017 wangxiaoshu <[email protected]>
+#
+# Distributed under terms of the MIT license.
+#
+
+
+ps -ef | grep xlearn_dist_train | awk '{ print $2 }' | sudo xargs kill -9

scripts/stop_xlearn_local_train.sh

@@ -0,0 +1,10 @@
+#! /bin/sh
+#
+# stop_xlearn.sh
+# Copyright (C) 2017 wangxiaoshu <[email protected]>
+#
+# Distributed under terms of the MIT license.
+#
+
+
+ps -ef | grep xlearn_train | awk '{ print $2 }' | sudo xargs kill -9

src/c_api/CMakeLists.txt

@@ -21,13 +21,15 @@ add_library(xlearn_api_shared SHARED c_api.cc c_api_error.cc
 ../solver/checker.cc ../solver/trainer.cc 
 ../solver/inference.cc ../solver/solver.cc)
 
+target_link_libraries(xlearn_api_shared)
+
 # Set properties
 set_target_properties(xlearn_api_shared PROPERTIES OUTPUT_NAME "xlearn_api")
 set_target_properties(xlearn_api_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 set_target_properties(xlearn_api PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 
 # Build unittests.
-set(LIBS xlearn_api solver reader loss score data base pthread gtest)
+set(LIBS xlearn_api solver reader loss score data base)
 
 add_executable(c_api_test c_api_test.cc)
 target_link_libraries(c_api_test gtest_main ${LIBS})

src/distributed/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
+
+set(STA_DEPS data base pslite protobuf zmq)
+add_library(distributed STATIC dist_score_function.cc dist_linear_score.cc dist_fm_score.cc dist_ffm_score.cc dist_cross_entropy_loss.cc dist_checker.cc dist_loss.cc)
+target_link_libraries(distributed ${STA_DEPS})
+
+add_executable(dist_xlearn_train dist_train_main.cc dist_solver.cc dist_trainer.cc dist_inference.cc)
+target_link_libraries(dist_xlearn_train solver distributed)
+
+install(TARGETS distributed DESTINATION lib/distributed)
+FILE(GLOB HEADER_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.h")
+message(${HEADER_FILES})
+install(FILES ${HEADER_FILES} DESTINATION include/distributed)