GH-362: Build Arrow from scratch for dev cookbooks

.github/workflows/deploy_development_cookbooks.yml

@@ -66,6 +66,8 @@ jobs:
         run:
           echo ${CONDA_PREFIX}
       - name: Build cookbook
+        env:
+          - ARROW_NIGHTLY: 1
         run:
           make cpp
       - name: Upload cpp book

cpp/CONTRIBUTING.md

@@ -95,7 +95,7 @@ output block when the recipe is rendered into the cookbook.
 ## Referencing Arrow C++ Documentation
 
 The Arrow project has its own documentation for the C++ implementation that
-is hosted at https://arrow.apache.org/docs/cpp/index.html. Fortunately,
+is hosted at <https://arrow.apache.org/docs/cpp/index.html>. Fortunately,
 this documentation is also built with Sphinx and so we can use the extension
 `intersphinx` to reference sections of this documentation. To do so simply
 write a standard Sphinx reference like so:
@@ -121,6 +121,7 @@ cmake build. For example:
 ```
 mkdir cpp/code/build
 cd cpp/code/build
+# Optional: Run `export ARROW_NIGHTLY=1` to build Arrow from git.
 cmake ../code -DCMAKE_BUILD_TYPE=Debug
 cmake --build .
 ctest

cpp/code/CMakeLists.txt

@@ -24,14 +24,50 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
 endif()
 
 # Add Arrow and other required packages
-find_package(Arrow REQUIRED)
-if(NOT ${ARROW_VERSION} VERSION_GREATER "9.0.0")
-  get_filename_component(ARROW_CMAKE_BASE_DIR ${Arrow_CONFIG} DIRECTORY)
-  list(INSERT CMAKE_MODULE_PATH 0 ${ARROW_CMAKE_BASE_DIR})
+if(DEFINED ENV{ARROW_NIGHTLY})
+    set(CMAKE_BUILD_TYPE Debug)
+    set(ARROW_BUILD_SHARED True)
+    set(ARROW_DEPENDENCY_SOURCE "AUTO")
+    set(ARROW_ENABLE_THREADING ON)
+    set(ARROW_SIMD_LEVEL NONE) # macOS-specific workaround
+
+    set(ARROW_ACERO ON)
+    set(ARROW_COMPUTE ON)
+    set(ARROW_DATASET ON)
+    set(ARROW_FILESYSTEM ON)
+    set(ARROW_FLIGHT ON)
+    set(ARROW_IPC ON)
+    set(ARROW_PARQUET ON)
+
+    include(FetchContent)
+
+    FetchContent_Declare(Arrow
+        GIT_REPOSITORY https://github.com/apache/arrow.git
+        GIT_TAG main
+        GIT_SHALLOW TRUE SOURCE_SUBDIR cpp
+        OVERRIDE_FIND_PACKAGE
+    )
+
+    FetchContent_MakeAvailable(Arrow)
+
+    # These are some Linux-only things the FetchContent build needs in order
+    # to compile
+    file(INSTALL "${arrow_BINARY_DIR}/src/arrow/util/config.h"
+        DESTINATION "${arrow_SOURCE_DIR}/cpp/src/arrow/util")
+    file(INSTALL "${arrow_BINARY_DIR}/src/parquet/parquet_version.h"
+        DESTINATION "${arrow_SOURCE_DIR}/cpp/src/parquet")
+    target_include_directories(
+        arrow_shared
+        INTERFACE "$<BUILD_INTERFACE:${arrow_SOURCE_DIR}/cpp/src>"
+    )
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+
+else()
+    find_package(Arrow REQUIRED)
+    find_package(ArrowDataset REQUIRED)
+    find_package(ArrowFlight REQUIRED)
+    find_package(Parquet REQUIRED)
 endif()
-find_package(ArrowDataset REQUIRED)
-find_package(ArrowFlight REQUIRED)
-find_package(Parquet REQUIRED)
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     set(CMAKE_CXX_CLANG_TIDY "clang-tidy")
@@ -76,10 +112,9 @@ recipe(creating_arrow_objects)
 recipe(datasets)
 recipe(flight)
 
-
 # Add protobuf to flight
-find_package(gRPC CONFIG REQUIRED)
 find_package(Threads)
+find_package(gRPC CONFIG REQUIRED)
 
 set(PROTO_FILES
     protos/helloworld.proto

cpp/dev.yml

@@ -16,15 +16,13 @@
 
 name: cookbook-cpp-dev
 channels:
-  - arrow-nightlies
   - conda-forge
 dependencies:
   - python=3.9
   - compilers
-  - arrow-nightlies::libarrow
+  - cmake
   - sphinx
   - gtest
   - gmock
-  - arrow-nightlies::pyarrow
   - clang-tools
   - zlib