adding api changes

cylondata · Sep 10, 2024 · f6655d9 · f6655d9
1 parent 03e0338
commit f6655d9
Show file tree

Hide file tree

Showing 18 changed files with 310 additions and 202 deletions.
diff --git a/.gitignore b/.gitignore
@@ -38,6 +38,7 @@
 
 # ignore these folders
 build/
+debug/
 **/bin/
 
 

diff --git a/conda/environments/cylon.yml b/conda/environments/cylon.yml
@@ -3,19 +3,22 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  - python>=3.9,<3.12
+  - cmake
+  - pyarrow=16.1.0
+  - libarrow-acero==16.1.0.*
+  - libarrow-dataset==16.1.0.*
+  - libarrow==16.1.0.*
   - glog
-  - openmpi=4.1.3=ha1ae619_105
-  - ucx>=1.12.1
-  - cython>=0.29.31,<3
-  - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
+  - openmpi
+  - ucx
+  - cython>=0.29.31
+  - numpy>=1.23,<2.0a0
+  - pandas>=2.0,<2.2.3dev
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it
   - pytest
   - pytest-mpi
   - mpi4py
+
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -24,17 +24,18 @@
 cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-project(CYLON VERSION 0.6.0)
+set(CYLON_VERSION 0.7.0)
+
+project(CYLON VERSION ${CYLON_VERSION})
 
-set(CYLON_VERSION 0.6.0)
 
 ## defaults to release build
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif ()
 
 # cmake modules directories
-set(CYLON_ARROW_VERSION 9.0.0)
+set(CYLON_ARROW_VERSION 16.1.0)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake/Modules/" ${CMAKE_MODULE_PATH})
 list(APPEND CMAKE_MODULE_PATH ${CYLON_SOURCE_DIR}/CMake)
 
@@ -74,7 +75,7 @@ else ()
 endif ()
 
 # C++ standard
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(GCC_ABI_COMPILE_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")

diff --git a/cpp/src/cylon/arrow/arrow_comparator.cpp b/cpp/src/cylon/arrow/arrow_comparator.cpp
@@ -150,7 +150,7 @@ struct CompareFunc<ArrowT, Asc, arrow::enable_if_floating_point<ArrowT>> {
 template<typename ArrowT, bool Asc>
 struct CompareFunc<ArrowT, Asc, arrow::enable_if_has_string_view<ArrowT>> {
 
-  static int compare(const arrow::util::string_view &v1, const arrow::util::string_view &v2) {
+  static int compare(const std::string_view &v1, const std::string_view &v2) {
     if (Asc) {
       return v1.compare(v2);
     } else {

diff --git a/cpp/src/cylon/arrow/arrow_partition_kernels.cpp b/cpp/src/cylon/arrow/arrow_partition_kernels.cpp
@@ -260,7 +260,7 @@ class FixedSizeBinaryHashPartitionKernel : public HashPartitionKernel {
       return
           visit_chunked_array<arrow::FixedSizeBinaryType>(
               idx_col,
-              [&](uint64_t global_idx, arrow::util::string_view val) {
+              [&](uint64_t global_idx, std::string_view val) {
                 uint32_t hash = 0;
                 util::MurmurHash3_x86_32(&val, len, 0, &hash);
                 hash += 31 * target_partitions[global_idx];
@@ -276,7 +276,7 @@ class FixedSizeBinaryHashPartitionKernel : public HashPartitionKernel {
     } else {
       return visit_chunked_array<arrow::FixedSizeBinaryType>(
           idx_col,
-          [&](uint64_t global_idx, arrow::util::string_view val) {
+          [&](uint64_t global_idx, std::string_view val) {
             uint32_t hash = 0;
             util::MurmurHash3_x86_32(&val, len, 0, &hash);
             hash += 31 * target_partitions[global_idx];
@@ -302,7 +302,7 @@ class FixedSizeBinaryHashPartitionKernel : public HashPartitionKernel {
 
     return visit_chunked_array<arrow::FixedSizeBinaryType>
         (idx_col,
-         [&](uint64_t global_idx, arrow::util::string_view val) {
+         [&](uint64_t global_idx, std::string_view val) {
            uint32_t hash = 0;
            util::MurmurHash3_x86_32(&val, byte_width, 0, &hash);
            hash += 31 * partial_hashes[global_idx];
@@ -348,7 +348,7 @@ class BinaryHashPartitionKernel : public HashPartitionKernel {
       return
           visit_chunked_array<ArrowT>(
               idx_col,
-              [&](uint64_t global_idx, arrow::util::string_view val) {
+              [&](uint64_t global_idx, std::string_view val) {
                 uint32_t hash = 0;
                 util::MurmurHash3_x86_32(&val, static_cast<int>(val.size()), 0, &hash);
                 hash += 31 * target_partitions[global_idx];
@@ -365,7 +365,7 @@ class BinaryHashPartitionKernel : public HashPartitionKernel {
       return
           visit_chunked_array<ArrowT>(
               idx_col,
-              [&](uint64_t global_idx, arrow::util::string_view val) {
+              [&](uint64_t global_idx, std::string_view val) {
                 uint32_t hash = 0;
                 util::MurmurHash3_x86_32(&val, static_cast<int>(val.size()), 0, &hash);
                 hash += 31 * target_partitions[global_idx];
@@ -390,7 +390,7 @@ class BinaryHashPartitionKernel : public HashPartitionKernel {
     return
         visit_chunked_array<arrow::FixedSizeBinaryType>(
             idx_col,
-            [&](uint64_t global_idx, arrow::util::string_view val) {
+            [&](uint64_t global_idx, std::string_view val) {
               uint32_t hash = 0;
               util::MurmurHash3_x86_32(&val, static_cast<int>(val.size()), 0, &hash);
               hash += 31 * partial_hashes[global_idx];

diff --git a/cpp/src/cylon/arrow/arrow_type_traits.hpp b/cpp/src/cylon/arrow/arrow_type_traits.hpp
@@ -15,6 +15,8 @@
 #ifndef CYLON_CPP_SRC_CYLON_ARROW_ARROW_TYPE_TRAITS_HPP_
 #define CYLON_CPP_SRC_CYLON_ARROW_ARROW_TYPE_TRAITS_HPP_
 
+#include <string_view>
+
 namespace cylon {
 
 template<typename ArrowT, typename Enable = void>
@@ -35,7 +37,7 @@ template<typename ArrowT>
 struct ArrowTypeTraits<ArrowT, arrow::enable_if_has_string_view<ArrowT>> {
   using ScalarT = typename arrow::TypeTraits<ArrowT>::ScalarType;
   using ArrayT = typename arrow::TypeTraits<ArrowT>::ArrayType;
-  using ValueT = arrow::util::string_view;
+  using ValueT = std::string_view;
 
   static ValueT ExtractFromScalar(const std::shared_ptr<arrow::Scalar> &scalar) {
     return ValueT(*(std::static_pointer_cast<ScalarT>(scalar))->value);
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,6 +38,7 @@ @@
     # ignore these folders
     build/
+    debug/
     **/bin/
@@ Expand Down @@