[Cylon] Add arrow 16.1.0 support

Signed-off-by: Arup Sarker <arupcsedu@gmail.com>
cylondata · Jun 8, 2024 · 5778f14 · 5778f14
1 parent 758a0dd
commit 5778f14
Show file tree

Hide file tree

Showing 10 changed files with 25 additions and 21 deletions.
diff --git a/build.sh b/build.sh
@@ -262,7 +262,7 @@ build_cpp_with_custom_arrow() {
   print_line
   source "${PYTHON_ENV_PATH}"/bin/activate || exit 1
   read_python_requirements
-  pip install pyarrow==14.0.2 || exit 1
+  pip install pyarrow==16.1.0 || exit 1
   pip install parquet || exit 1
   ARROW_LIB=$(python3 -c 'import pyarrow as pa; import os; print(os.path.dirname(pa.__file__))') || exit 1
   ARROW_INC=$(python3 -c 'import pyarrow as pa; import os; print(os.path.join(os.path.dirname(pa.__file__), "include"))') || exit 1
@@ -356,7 +356,7 @@ build_gcylon() {
 
   # set install path to conda directory if not already set
   INSTALL_PATH=${INSTALL_PATH:=${PREFIX:=${CONDA_PREFIX}}}
-  export CUDA_HOME=$CONDA_PREFIX/bin
+  export CUDA_HOME=$CONDA_PREFIX
 
   echo "SOURCE_DIR: ${SOURCE_DIR}"
   BUILD_PATH=$(pwd)/build

diff --git a/conda/environments/cylon.yml b/conda/environments/cylon.yml
@@ -6,8 +6,8 @@ dependencies:
   #- python>=3.8
   - python>=3.8,<3.10
   - cmake>=3.23.1
-  - arrow-cpp=14.0.2
-  - pyarrow=14.0.2
+  - arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - openmpi=4.1.4
   - ucx>=1.12.1

diff --git a/conda/environments/cylon_MacOS.yml b/conda/environments/cylon_MacOS.yml
@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python>=3.9,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=14.0.2
-  - pyarrow=14.0.2
+  - arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - openmpi>=4.1.2
   - cython>=0.29.31,<3

diff --git a/conda/environments/cylon_NoUCX.yml b/conda/environments/cylon_NoUCX.yml
@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=14.0.2
-  - pyarrow=14.0.2
+  - arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - openmpi=4.1.4
   - cython>=0.29.31,<3

diff --git a/conda/environments/cylon_rivanna_1.yml b/conda/environments/cylon_rivanna_1.yml
@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=14.0.2
-  - pyarrow=14.0.2
+  - arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - openmpi=4.1.4
   - ucx>=1.12.1

diff --git a/conda/environments/cylon_rivanna_2.yml b/conda/environments/cylon_rivanna_2.yml
@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=14.0.2
-  - pyarrow=14.0.2
+  - arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - openmpi=4.1.4
   - ucx>=1.12.1

diff --git a/conda/environments/gcylon.yml b/conda/environments/gcylon.yml
@@ -5,8 +5,9 @@ channels:
   - nvidia
   - defaults
 dependencies:
-  - cudf=24.04
+  #- cudf=24.04
   - python=3.11
+  - cuda
   #- cuda-version=12.2
   #- pytorch
   - cmake>=3.23.1
@@ -17,7 +18,7 @@ dependencies:
   - numpy
   - pandas
   - fsspec
-  #- setuptools
+  - setuptools
   # these are for running tests only,
   # they are not needed for using pygcylon or compiling it
   - pytest
@@ -26,5 +27,6 @@ dependencies:
   - gcc=11.4
   - gxx=11.4
   - gxx_linux-64=11.4
-  - cuda-runtime
-  - cudatoolkit
+  #- cuda-runtime
+  #- cudatoolkit
+  - cudf
diff --git a/conda/environments/windows.yml b/conda/environments/windows.yml
@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python>=3.8,<3.10
   - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=14.0.2
-  - pyarrow=14.0.2
+  - arrow-cpp=16.1.0
+  - pyarrow=16.1.0
   - glog
   - msmpi
   - cython>=0.29.31,<3

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -34,7 +34,7 @@ if (NOT CMAKE_BUILD_TYPE)
 endif ()
 
 # cmake modules directories
-set(CYLON_ARROW_VERSION 14.0.2)
+set(CYLON_ARROW_VERSION 16.1.0)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake/Modules/" ${CMAKE_MODULE_PATH})
 list(APPEND CMAKE_MODULE_PATH ${CYLON_SOURCE_DIR}/CMake)
 

diff --git a/cpp/src/cylon/compute/aggregate_utils.hpp b/cpp/src/cylon/compute/aggregate_utils.hpp
@@ -99,8 +99,10 @@ cylon::Status AllReduce(const std::shared_ptr<CylonContext> &ctx,
         auto rcv_scalar = std::make_shared<ScalarT>(*send_scalar);
         std::memset(&rcv_scalar->value, 0, sizeof(CType));
 
-        RETURN_CYLON_STATUS_IF_FAILED(cylon::mpi::AllReduce(ctx, send_scalar->mutable_data(),
-                                                            rcv_scalar->mutable_data(),
+        auto rcv_data = rcv_scalar->data();
+
+        RETURN_CYLON_STATUS_IF_FAILED(cylon::mpi::AllReduce(ctx, send_scalar->data(),
+                                                            (void*)rcv_data,
                                                             1, data_type, reduce_ops[i]));
         rcv_scalar_vector.push_back(rcv_scalar);
       }