[native] Add support for DEPENDENCY_DIR, INSTALL_PREFIX, PYTHON_VENV

.gitignore

@@ -40,6 +40,8 @@ cmake-build-*/
 *.swp
 *~
 a.out
+presto-native-execution/deps-download
+presto-native-execution/deps-install
 
 # Compiled Object files
 *.slo

presto-native-execution/CMakeLists.txt

@@ -121,6 +121,11 @@ set(VELOX_BUILD_TEST_UTILS
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+if(DEFINED ENV{INSTALL_PREFIX})
+  message(STATUS "Dependency install directory set to: $ENV{INSTALL_PREFIX}")
+  list(APPEND CMAKE_PREFIX_PATH "$ENV{INSTALL_PREFIX}")
+endif()
+
 set(Boost_USE_MULTITHREADED TRUE)
 find_package(
   Boost
@@ -228,4 +233,11 @@ if("${TREAT_WARNINGS_AS_ERRORS}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
 endif()
 
+if(DEFINED ENV{INSTALL_PREFIX})
+  # Allow installed package headers to be picked up before brew/system package
+  # headers. We set this after Velox since Velox handles INSTALL_PREFIX its own
+  # way.
+  include_directories(BEFORE "$ENV{INSTALL_PREFIX}/include")
+endif()
+
 add_subdirectory(presto_cpp)

presto-native-execution/Makefile

@@ -20,6 +20,7 @@ NUM_THREADS ?= $(shell getconf _NPROCESSORS_CONF 2>/dev/null || echo 1)
 CPU_TARGET ?= "avx"
 CMAKE_PREFIX_PATH ?= "/usr/local"
 PRESTOCPP_ROOT_DIR="$(shell pwd)"
+PYTHON_VENV ?= .venv
 
 EXTRA_CMAKE_FLAGS ?= ""
 
@@ -102,17 +103,33 @@ TypeSignature:		#: Build the Presto TypeSignature parser
 	cd presto_cpp/main/types; $(MAKE) TypeSignature
 
 format-fix: 			#: Fix formatting issues in the presto-native-execution directory
+ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","")
+	source ${PYTHON_VENV}/bin/activate; scripts/check.py format master --fix
+else
 	scripts/check.py format master --fix
+endif
 
 format-check: 			#: Check for formatting issues in the presto-native-execution directory
 	clang-format --version
+ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","")
+	source ${PYTHON_VENV}/bin/activate; scripts/check.py format master
+else
 	scripts/check.py format master
+endif
 
 header-fix:			#: Fix license header issues in the presto-native-execution directory
+ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","")
+	source ${PYTHON_VENV}/bin/activate; scripts/check.py header master --fix
+else
 	scripts/check.py header master --fix
+endif
 
 header-check:			#: Check for license header issues in the presto-native-execution directory
+ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","")
+	source ${PYTHON_VENV}/bin/activate; scripts/check.py header master
+else
 	scripts/check.py header master
+endif
 
 help:					#: Show the help messages
 	@cat $(firstword $(MAKEFILE_LIST)) | \

presto-native-execution/README.md

@@ -26,10 +26,19 @@ available inside `presto/presto-native-execution/scripts`.
 * CentOS Stream 9: `setup-centos.sh`
 * Ubuntu: `setup-ubuntu.sh`
 
-Create a directory say `dependencies` and invoke one of these scripts from
-this folder. All the dependencies are installed in the system default location eg: `/usr/local`.
-To change the installation location specify a path using the `INSTALL_PREFIX` environment variable.
-For example, change the location if the default location cannot be written to by the user running the setup script.
+The above setup scripts use the `DEPENDENCY_DIR` environment variable to set the
+location to download and build packages. This defaults to `deps-download` in the current
+working directory.
+
+Use `INSTALL_PREFIX` to set the install directory of the packages. This defaults to
+`deps-install` in the current working directory on macOS and to the default install
+location (for example, `/usr/local`) on linux.
+Using the default install location `/usr/local` on macOS is discouraged because this
+location is used by certain Homebrew versions.
+
+Manually add the `INSTALL_PREFIX` value in the IDE or bash environment, so subsequent
+Prestissimo builds can use the installed packages. Say
+`export INSTALL_PREFIX=/Users/$USERNAME/presto/presto-native-execution/deps-install` to `~/.zshrc`.
 
 The following libraries are installed by the above setup scripts.
 The Velox library installs other
@@ -70,7 +79,7 @@ Compilers (and versions) not mentioned are known to not work or have not been tr
 | MacOS | `clang14` |
 
 ### Build Prestissimo
-#### Parquet and S3 Supprt
+#### Parquet and S3 Support
 To enable Parquet and S3 support, set `PRESTO_ENABLE_PARQUET = "ON"`,
 `PRESTO_ENABLE_S3 = "ON"` in the environment.
 

presto-native-execution/scripts/setup-macos.sh

@@ -13,7 +13,14 @@
 
 set -eufx -o pipefail
 
-# Run the velox setup script first.
+SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
+PYTHON_VENV=${PYTHON_VENV:-"${SCRIPTDIR}/../.venv"}
+# Prestissimo fails to build DuckDB with error
+# "math cannot parse the expression" when this
+# script is invoked under the Presto git project.
+# Set DEPENDENCY_DIR to a directory outside of Presto
+# to build DuckDB.
+BUILD_DUCKDB="${BUILD_DUCKDB:-false}"
 source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-macos.sh"
 
 function install_proxygen {
@@ -41,4 +48,6 @@ else
   install_velox_deps
   install_presto_deps
   echo "All dependencies for Prestissimo installed!"
+  echo "To reuse the installed dependencies for subsequent builds, consider adding this to your ~/.zshrc"
+  echo "export INSTALL_PREFIX=$INSTALL_PREFIX"
 fi