[WIP] New build system.

Rearchitecture of the the build system.  A vast simplification.

Changes
- ./configure can produce an XCode project of the Turi source code.
- deps/ can be compiled and installed out of the regular build process.
  Helpful for the XCode project.
- All packages in src/external/ and src/visualization compile into static
  libraries.
- The rest of the code base compiles into a single shared library.  This
  means that most everything occurs in one single CMakeLists.txt file
  using standard cmake commands.
- All headers are installed into targets/include when `make install` is
  called.  The main library is installed into targets/lib.
- The code definitions that reflect compiler oddities (e.g.
  is std::hash<int128_t> defined) are dumped into a single header file,
  src/turi_common.h.  All source files and headers include this file
  first.  This allows other programs to link against this library /
  headers reliably.
- The python part of TuriCreate now builds through the standard
  setup.py method. Cython / pybind11 extensions are compiled and
  packaged using the standard extensions mechanisms and linking against
  the installed Turi headers.

WIP TODO: the python installation, deployment.

Author: Hoyt Koepke

.gitignore

@@ -11,6 +11,7 @@ syntax: glob
 debug/*
 release/*
 profile/*
+src/turi_common.h
 dist/turicreateapi*
 configure.deps
 config.log

build_python_wheel.sh

@@ -0,0 +1,112 @@
+#!/bin/bash -e
+
+
+function print_help {
+  echo "Configures and builds a specified target."
+  echo
+  echo "Usage: ./build.sh [build options] [configure options]"
+  echo
+  echo "Common Options:"
+  echo "  --target-dir, -t                 The target directory to install artifact to."
+  echo "                                   default: `pwd`/targets."
+  echo
+  echo "  --release                        Build in release mode."
+  echo "  --debug                          Build in debug mode (default)."
+  echo
+  echo "  --jobs,-j                        The number of parallel jobs to run."
+  echo
+  echo "  --cleanup,-c                     Clean up everything before building."
+  echo
+  echo "  --skip-configure,-s              Skip running ./configure."
+  echo
+  echo "  --build-number                   Set build number.  "
+  echo "                                   Defaults to part of git commit hash. "
+  echo
+  echo "  All additional options passed through to ./configure."
+  echo
+  exit 1
+} # end of print help
+
+function unknown_option {
+  echo "Unrecognized option: $1"
+  echo "To get help, run ./configure --help"
+  exit 1
+} # end of unknown option
+
+if [[ ${OSTYPE} == darwin* ]] ; then
+  apple=1
+else
+  apple=0
+fi
+
+
+# command flag options
+cleanup=0
+skip_configure=0
+jobs=4
+configure_options=""
+build_mode="release"
+target_dir=`pwd`/targets
+install_sysroot=""
+no_sudo=0
+copy_links=0
+build_number=`git rev-parse --short HEAD || echo "NULL"`
+
+###############################################################################
+#
+# Parse command line configure flags ------------------------------------------
+#
+while [ $# -gt 0 ]
+  do case $1 in
+
+    --cleanup|-c)           cleanup=1;;
+
+    --skip-configure|-s)    skip_configure=1;;
+
+    --copy-links)           copy_links=1;;
+
+    --build-number=*)       build_number=${1##--build-number=} ;;
+    --build-number)         build_number="$2"; shift;;
+
+    --target-dir=*)         target_dir="${1##--target-dir=}" ;;
+    --target-dir|-t)        target_dir="$2"; shift ;;
+
+    --jobs=*)               jobs=${1##--jobs=} ;;
+    --jobs|-j)              jobs=$2; shift ;;
+
+    --help)                 print_help; exit 0;;
+
+    -D)                     configure_options="${configure_options} -D $2"; shift ;;
+
+    *)                      configure_options="${configure_options} $1";;
+  esac
+  shift
+done
+
+build_dir=`pwd`/${build_mode}
+src_dir=`pwd`
+
+
+echo "Setting up build:"
+echo "build_mode = ${build_mode}"
+echo "target_dir = ${target_dir}"
+echo "target     = ${target}"
+echo "build_dir  = ${build_dir}"
+
+
+if [[ ${cleanup} -eq 1 ]]; then
+  ./configure --cleanup --yes || exit 1
+fi
+
+if [[ ${skip_configure} -eq 0 ]] ; then
+  ./configure ${configure_options} --with-python || exit 1
+else
+  echo "skipping configure script as requested."
+fi
+
+
+install_dir=${target_dir}/python
+rm -rf ${target_dir}/python
+mkdir -p ${target_dir}/python
+
+bash scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number="$build_number" --num_procs=${jobs} --${build_mode} --target-dir="${install_dir}"

cmake/CompilerFlags.cmake

@@ -2,6 +2,16 @@ include(CheckCXXCompilerFlag)
 include(CheckCCompilerFlag)
 include(CMakeParseArguments)
 
+# Set a define to be dumped into turi_common.h at the end.
+#
+# A compiler source define is one that determines or switches some part of the
+# compilation, but isn't related to the particular build being asked for.  These are dumped into
+# turi_common.h at the end.
+#
+macro(add_compiler_source_define FLAG)
+  add_definitions(-D${FLAG})
+  set(TC_COMPILER_SOURCE_DEFINES "${TC_COMPILER_SOURCE_DEFINES} ${FLAG}")
+endmacro()
 
 # check_and_set_compiler_flag
 #

cmake/CompilerOddities.cmake

@@ -1,4 +1,6 @@
 include(CheckCXXSourceCompiles)
+include(CompilerFlags)
+
 
 macro(Set_Compiler_Specific_Flags)
 
@@ -16,7 +18,7 @@ macro(Set_Compiler_Specific_Flags)
 
   if(COMPILER_HAS_IOS_BASE_FAILURE_WITH_ERROR_CODE)
     message(STATUS "Compiler supports ios_base::failure(str, error_code)")
-    add_definitions(-DCOMPILER_HAS_IOS_BASE_FAILURE_WITH_ERROR_CODE)
+    add_compiler_source_define(COMPILER_HAS_IOS_BASE_FAILURE_WITH_ERROR_CODE)
   else()
     message(STATUS "Compiler does not support ios_base::failure(str, error_code)")
   endif()
@@ -60,11 +62,11 @@ macro(Set_Compiler_Specific_Flags)
 
     message(FATAL_ERROR "Cannot determine noexcept fladg on std::ios_base::failure.  See log.")
   elseif(COMPILER_HAS_NOEXCEPT_WHAT_ON_EXCEPTIONS_V1) 
-    add_definitions(-DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT=noexcept)
+    add_compiler_source_define(COMPILER_MODIFIER_ON_EXCEPTION_WHAT=noexcept)
   elseif(COMPILER_HAS_NOEXCEPT_WHAT_ON_EXCEPTIONS_V2)
-    add_definitions(-DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT=_NOEXCEPT)
+    add_compiler_source_define(DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT=_NOEXCEPT)
   else()
-    add_definitions(-DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT="")
+    add_compiler_source_define(COMPILER_MODIFIER_ON_EXCEPTION_WHAT=)
   endif()
 
 endmacro()

configure

@@ -78,6 +78,13 @@ function print_help {
   echo
   echo "  -D var=value                      Specify FLAGS definitions to be passed on to cmake."
   echo
+  echo "  --create-xcode-project            Create an xcode project file as xcode/Turi.xcodeproj."
+  echo
+  echo "  --build-dependencies              Build the external dependencies as part of configure and install them into deps/local."
+  echo "                                    Needed for the xcode project build."
+  echo
+  echo "  --skip-dependencies               Skip building the dependencies."
+  echo
   echo "Relevant environment variables: "
   echo
   echo "  CMAKE                             Path to CMake executable."
@@ -153,12 +160,14 @@ with_release=1
 with_debug=1
 release_opt_for_size=0
 debug_opt_for_size=0
+create_xcode_project=0
 arch_list=""
 target=""
 builder="make"
 min_ios_version=${IOS_DEFAULT_MIN_VERSION}
 min_macos_version=${MACOS_DEFAULT_MIN_VERSION}
 enable_codesign=0
+build_dependencies=0
 list_source_files=0
 
 DEBUG_DIR="${TURI_HOME}/debug"
@@ -233,6 +242,11 @@ while [ $# -gt 0 ]
     --min-macos-version=*)  min_macos_version=${1##--min-macos-version=};;
     --min-macos-version)    min_macos_version=${2}; shift ;;
 
+    --create-xcode-project) create_xcode_project=1; build_dependencies=1; builder=xcode;;
+
+    --build-dependencies)   build_dependencies=1;;
+    --skip-dependencies)    skip_dependencies=1;;
+
     --list-source-files)    list_source_files=1 ;;
 
     *) unknown_option $1 ;;
@@ -338,6 +352,8 @@ mkdir -p ${PWD}/deps/local/bin
 CCCMD=`./scripts/find_compiler.sh cc --ccache=$with_ccache --script-dir=${PWD}/deps/local/bin/`
 CXXCMD=`./scripts/find_compiler.sh cxx --ccache=$with_ccache --script-dir=${PWD}/deps/local/bin/`
 
+CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_C_COMPILER=$CCCMD -DCMAKE_CXX_COMPILER=$CXXCMD"
+
 echo "Setting C compiler to $CCCMD."
 echo "Setting C++ compiler to $CXXCMD."
 
@@ -349,16 +365,21 @@ CMAKE=`./scripts/find_cmake.sh`
 ########################################################
 # Prepare to build
 
+
+
+# Configuration flags that are specific to the xcode builder
+CMAKE_XCODE_CONFIG_FLAGS=""
+
 set -e
 set -o pipefail
 
 case $builder in
- xcode)
-   CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -GXcode"
-   CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_CXX_STANDARD=11 -DCMAKE_CXX_STANDARD_REQUIRED=1"
+ xcode) 
+   CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -GXcode"
+   CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_CXX_STANDARD=11 -DCMAKE_CXX_STANDARD_REQUIRED=1"
 
-   if [[ ! -z ${arch_list} ]] ; then
-     CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_OSX_ARCHITECTURES='${arch_list}'"
+   if [[ ! -z ${arch_list} ]] ; then 
+     CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_OSX_ARCHITECTURES='${arch_list}'"
    fi
 
    ;;
@@ -377,11 +398,11 @@ if [[ ${enable_codesign} == 1 ]] ; then
 
 elif [[ ${builder} == xcode ]] ; then
 
-  # Deal with the code signing issues.
-  CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY= -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=0 -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM= "
+  # Deal with the code signing issues. 
+  CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY= -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=0 -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM= "
 
   # This also requires us to skip the code signing when trying to compile targets to check definitions.
-  CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY"
+  CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY"
 
   echo "Skipping code signing."
 fi
@@ -429,22 +450,41 @@ if [[ ${build_platform} == macosx ]] ; then
   fi
 fi
 
-build_cmd="$CMAKE \
-    $GENERATOR \
-    -D CMAKE_C_COMPILER=$CCCMD \
-    -D CMAKE_CXX_COMPILER=$CXXCMD \
-    ${CMAKE_CONFIG_FLAGS}"
-
-if [[ $with_debug == 1 ]] ; then
-  set -x
-  mkdir -p ${DEBUG_DIR}
-  cd ${DEBUG_DIR} && $build_cmd -DCMAKE_BUILD_TYPE=Debug --config Debug -D CMAKE_CONFIGURATION_TYPES='Debug;Release' ${TURI_HOME}
-  set +x
+if [[ $build_dependencies == 1 ]] ; then
+
+  # Build the dependencies through the make build scripts so they don't have to
+  # be part of the XCode project, which doesn't work.
+
+  deps_build_dir=${TURI_HOME}/deps/build
+  mkdir -p $deps_build_dir
+  cd ${deps_build_dir}
+  $CMAKE ${CMAKE_CONFIG_FLAGS} -DCMAKE_BUILD_TYPE=Release --config Release ${TURI_HOME}
+  make external_dependencies
+  CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DTC_EXTERNAL_DEPS_PREBUILT=1"
+else
+  CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DTC_EXTERNAL_DEPS_PREBUILT=0"
 fi
 
-if [[ $with_release == 1 ]] ; then
-  set -x
-  mkdir -p ${RELEASE_DIR}
-  cd ${RELEASE_DIR} && $build_cmd -DCMAKE_BUILD_TYPE=Release -D CMAKE_CONFIGURATION_TYPES='Release;Debug' --config Release ${TURI_HOME}
-  set +x
+
+if [[ $create_xcode_project == 1 ]] ; then
+  xcode_proj_dir=${TURI_HOME}/xcode/
+  mkdir -p $xcode_proj_dir
+  cd ${xcode_proj_dir}
+  $CMAKE ${CMAKE_CONFIG_FLAGS} ${CMAKE_XCODE_CONFIG_FLAGS} -DTC_EXTERNAL_DEPS_PREBUILT=1 -DCMAKE_BUILD_TYPE=Debug --config Debug -D CMAKE_CONFIGURATION_TYPES='Debug;Release' ${TURI_HOME}
+
+else
+  if [[ $with_debug == 1 ]] ; then
+    set -x
+    mkdir -p ${DEBUG_DIR}
+    cd ${DEBUG_DIR} && $CMAKE ${CMAKE_CONFIG_FLAGS} ${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_BUILD_TYPE=Debug --config Debug -D CMAKE_CONFIGURATION_TYPES='Debug;Release' ${TURI_HOME}
+    set +x
+  fi
+
+  if [[ $with_release == 1 ]] ; then
+    set -x
+    mkdir -p ${RELEASE_DIR}
+    cd ${RELEASE_DIR} && $CMAKE ${CMAKE_CONFIG_FLAGS} ${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_BUILD_TYPE=Release -D CMAKE_CONFIGURATION_TYPES='Release;Debug' --config Release ${TURI_HOME}
+    set +x
+  fi
 fi
+