Merge pull request #408 from unum-cloud/407-feature-improve-build-and…

…-test-python-wheels-of-github-action

Refactor: Reducing 'Python Wheels Build' time

CMakeLists.txt

@@ -26,6 +26,7 @@ option(USTORE_BUILD_API_FLIGHT_CLIENT "Building Apache Arrow Flight RPC client f
 option(USTORE_BUILD_API_FLIGHT_SERVER "Building Apache Arrow Flight RPC server for all backends")
 option(USTORE_BUILD_API_REST_SERVER "Building REST API server for all backends")
 
+option(USTORE_USE_PYARROW "Ready-made arrow libraries from the installed pip package pyarrow")
 option(USTORE_USE_JEMALLOC "Faster allocator, that requires autoconf to be installed")
 option(USTORE_USE_ONEAPI "Faster concurrency primitives from Intel")
 option(USTORE_USE_UUID "Replaces default 64-bit keys with 128-bit UUID compatible integers")
@@ -36,6 +37,10 @@ if(${USTORE_BUILD_CLI})
   set(USTORE_BUILD_API_FLIGHT_CLIENT ON)
 endif()
 
+if(USTORE_BUILD_SDK_PYTHON)
+  set(USTORE_USE_PYARROW ON)
+endif()
+
 if(USTORE_BUILD_API_FLIGHT)
   set(USTORE_BUILD_API_FLIGHT_SERVER ON)
   set(USTORE_BUILD_API_FLIGHT_CLIENT ON)
@@ -225,15 +230,23 @@ if(${USTORE_BUILD_API_FLIGHT_CLIENT} OR ${USTORE_BUILD_API_FLIGHT_SERVER})
 endif()
 
 if(${USTORE_BUILD_SDK_PYTHON} OR ${USTORE_BUILD_API_FLIGHT_CLIENT} OR ${USTORE_BUILD_API_FLIGHT_SERVER} OR ${USTORE_BUILD_ENGINE_UCSET} OR ${USTORE_BUILD_TESTS})
-  # if(NOT ${USTORE_BUILD_SDK_PYTHON})
+  if(NOT ${USTORE_USE_PYARROW})
     if(${USE_CONAN})
       include_directories(${CONAN_INCLUDE_DIRS_ARROW})
     else()
       include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/arrow.cmake")
     endif()
-  # else()
-    # include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/arrow.cmake")
-  # endif()
+  else()
+    include_directories(ustore_embedded_ucset PRIVATE ${PYARROW_DIR}/include)
+
+    set(LIB_ARROW ${PYARROW_DIR}/libarrow.so.1000)
+    set(LIB_ARROW_PARQUET ${PYARROW_DIR}/libparquet.so.1000)
+    set(LIB_ARROW_DATASET ${PYARROW_DIR}/libarrow_dataset.so.1000)
+    set(LIB_ARROW_FLIGHT ${PYARROW_DIR}/libarrow_flight.so.1000)
+    set(LIB_ARROW_PYTHON ${PYARROW_DIR}/libarrow_python.so)
+    set(LIB_ARROW_PYTHON_FLIGHT ${PYARROW_DIR}/libarrow_python_flight.so)
+    set(LIB_ARROW_BUNDLED)
+  endif()
 endif()
 
 if(${USTORE_BUILD_CLI})
@@ -277,7 +290,7 @@ if(${USTORE_BUILD_BENCHMARKS})
 endif()
 
 if(${USE_CONAN})
-  # if (NOT ${USTORE_BUILD_SDK_PYTHON})
+  if (NOT ${USTORE_USE_PYARROW})
     add_library(arrow::arrow STATIC IMPORTED)
     set_property(TARGET arrow::arrow PROPERTY IMPORTED_LOCATION ${CONAN_LIB_DIRS_ARROW}/libarrow.a)
 
@@ -293,13 +306,14 @@ if(${USE_CONAN})
     # add_library(arrow::bundled STATIC IMPORTED)
     # set_property(TARGET arrow::bundled ${CONAN_LIBS_RE2} ${CONAN_LIBS_UTF8PROC} ${CONAN_LIBS_THRIFT} ${CONAN_LIBS_ABSEIL} ${CONAN_LIBS_SNAPPY} ${CONAN_LIBS_PROTOBUF} ${CONAN_LIBS_ZLIB} ${CONAN_LIBS_C-ARES} ${CONAN_LIBS_GRPC})
     set(LIB_ARROW_BUNDLED ${CONAN_LIBS_RE2} ${CONAN_LIBS_UTF8PROC} ${CONAN_LIBS_THRIFT} ${CONAN_LIBS_ABSEIL} ${CONAN_LIBS_SNAPPY} ${CONAN_LIBS_PROTOBUF} ${CONAN_LIBS_ZLIB} ${CONAN_LIBS_C-ARES} ${CONAN_LIBS_GRPC})
-  # endif()
 
-  set(LIB_ARROW arrow::arrow)
-  set(LIB_ARROW_FLIGHT arrow::flight)
-  set(LIB_ARROW_DATASET arrow::dataset)
-  set(LIB_ARROW_PARQUET arrow::parquet)
-  # set(LIB_ARROW_BUNDLED arrow::bundled)
+    set(LIB_ARROW arrow::arrow)
+    set(LIB_ARROW_FLIGHT arrow::flight)
+    set(LIB_ARROW_DATASET arrow::dataset)
+    set(LIB_ARROW_PARQUET arrow::parquet)
+    # set(LIB_ARROW_BUNDLED arrow::bundled)
+  endif()
+
 
   set(LIB_SSL ssl)
   set(LIB_CRYPTO crypto)
@@ -308,11 +322,13 @@ if(${USE_CONAN})
   set(LIB_BSON bson-static-1.0) 
   set(LIB_PCRE2 pcre2-8)
 else()
-  set(LIB_ARROW arrow::arrow) 
-  set(LIB_ARROW_FLIGHT arrow::flight)
-  set(LIB_ARROW_DATASET arrow::dataset)
-  set(LIB_ARROW_PARQUET arrow::parquet)
-  set(LIB_ARROW_BUNDLED arrow::bundled)
+  if(NOT ${USTORE_USE_PYARROW})
+    set(LIB_ARROW arrow::arrow) 
+    set(LIB_ARROW_FLIGHT arrow::flight)
+    set(LIB_ARROW_DATASET arrow::dataset)
+    set(LIB_ARROW_PARQUET arrow::parquet)
+    set(LIB_ARROW_BUNDLED arrow::bundled)
+  endif()
 
   set(LIB_SSL openssl::ssl)
   set(LIB_CRYPTO openssl::crypto)
@@ -339,7 +355,7 @@ endif()
 
 if(${USTORE_BUILD_ENGINE_ROCKSDB})
   add_library(ustore_embedded_rocksdb src/engine_rocksdb.cpp src/modality_docs.cpp src/modality_paths.cpp src/modality_graph.cpp src/modality_vectors.cpp)
-  target_link_libraries(ustore_embedded_rocksdb ${LIB_ROCKSDB} pthread yyjson simdjson ${LIB_BSON} ${LIB_PCRE2} ${LIB_ARROW_BUNDLED} ${JEMALLOC_LIBRARIES})
+  target_link_libraries(ustore_embedded_rocksdb ${LIB_ROCKSDB} pthread yyjson simdjson ${LIB_BSON} ${LIB_PCRE2} ${JEMALLOC_LIBRARIES})
   target_compile_definitions(ustore_embedded_rocksdb INTERFACE USTORE_VERSION="${USTORE_VERSION}")
   target_compile_definitions(ustore_embedded_rocksdb INTERFACE USTORE_ENGINE_IS_ROCKSDB=1)
 
@@ -477,20 +493,11 @@ if(${USTORE_BUILD_SDK_PYTHON})
 
     string(CONCAT wrap_lib "py_" ${client_lib_name})
     get_target_property(client_dependencies ${client_lib} LINK_LIBRARIES)
-    if(${USE_PYARROW_LIBS})
-      list(FILTER client_dependencies EXCLUDE REGEX "arrow")
-
-      set(LIB_ARROW ${PYARROW_DIR}/arrow)
-      set(LIB_ARROW_FLIGHT ${PYARROW_DIR}/arrow_flight)
-      set(LIB_ARROW_DATASET ${PYARROW_DIR}/arrow_dataset)
-      set(LIB_ARROW_PARQUET ${PYARROW_DIR}/arrow_parquet)
-      set(LIB_ARROW_BUNDLED)
-    endif()
 
     pybind11_add_module(${wrap_lib} MODULE python/pybind.cpp python/database.cpp python/networkx.cpp python/documents.cpp python/pandas.cpp)
     target_include_directories(${wrap_lib} PUBLIC python/ ${PYARROW_DIR}/include)
     target_link_directories(${wrap_lib} PUBLIC ${PYARROW_DIR})
-    target_link_libraries(${wrap_lib} PRIVATE pthread ${LIB_FMT} ${LIB_ARROW_DATASET} arrow_python ${LIB_ARROW_FLIGHT} ${LIB_ARROW_BUNDLED} ${LIB_ARROW} ${client_lib} ${client_dependencies})
+    target_link_libraries(${wrap_lib} PRIVATE pthread ${LIB_FMT} ${LIB_ARROW_FLIGHT} ${LIB_ARROW_PARQUET} ${LIB_ARROW_PYTHON_FLIGHT} ${LIB_ARROW} ${LIB_ARROW_PYTHON} ${client_lib} ${client_dependencies})
     target_compile_definitions(${wrap_lib} PRIVATE USTORE_VERSION="${USTORE_VERSION}" USTORE_PYTHON_MODULE_NAME=${client_lib_name})
     set_target_properties(${wrap_lib} PROPERTIES OUTPUT_NAME ${client_lib_name})
   endforeach()

pyproject.toml

@@ -10,28 +10,23 @@ filterwarnings = ["error"]
 testpaths = ["python/tests"]
 
 [tool.cibuildwheel]
+build-verbosity = 3
 #manylinux-x86_64-image = "manylinux_2_28"
-build-verbosity = 0
 environment = "CMAKE_ARGS_F=/tmp/cmake_args"
 
 before-all = [
     "yum install -y wget python3-devel bzip2-devel libffi-devel zlib-devel perl-IPC-Cmd perl-Test-Simple",
-    "cd /usr/src && wget https://www.openssl.org/source/openssl-3.1.0.tar.gz",
-    "tar -zxf openssl-3.1.0.tar.gz && rm openssl-3.1.0.tar.gz && cd /usr/src/openssl-3.1.0",
-    "./config && make -j && make install",
-    "ln -s /usr/local/lib64/libssl.so.3 /usr/lib64/libssl.so.3",
-    "ln -s /usr/local/lib64/libcrypto.so.3 /usr/lib64/libcrypto.so.3"
 ]
 
 before-build = [
     # Removing the contents of previous builds from project if they exist
     "cd {project}",
     "rm -rf {project}/CMakeCache.txt {project}/build {project}/CMakeFiles.txt {project}/_deps",
 
-    "pip install cmake pyarrow==10.0.1 auditwheel",
+    "pip install pyarrow==10.0.1 auditwheel",
     "export PYST=$(python -c 'import site; print(site.getsitepackages()[0])')",
     # Install dependency using conan
-    # "pip install conan==1.60.1",
+    # "pip install cmake conan==1.60.1",
     # "conan profile new --detect default",
     # "conan profile update settings.compiler=gcc default",
     # "conan profile update settings.compiler.version=10 default",
@@ -41,7 +36,7 @@ before-build = [
     # "tar -xzvf {project}/ustore_deps_x86_linux.tar.gz -C ~/.conan",
     # "conan install ustore_deps/0.12.1@unum/x86_linux -g cmake",
     # "rm -rf {project}/ustore_deps_x86_linux.tar.gz",
-
+    
     "echo \"-DCMAKE_BUILD_TYPE=Release -DPYARROW_DIR=${PYST}/pyarrow/\" > ${CMAKE_ARGS_F}",
     "mkdir -p build/ustore",
 ]

setup.py

@@ -51,7 +51,8 @@ def build_extension(self, ext):
             '-DUSTORE_BUILD_ENGINE_ROCKSDB=1',
             '-DUSTORE_BUILD_API_FLIGHT_CLIENT=1',
             '-DUSTORE_BUILD_SDK_PYTHON=1',
-            '-DUSE_PYARROW_LIBS=0',
+            '-DUSTORE_BUILD_TESTS=0',
+            '-DUSE_PYARROW_LIBS=1',
             '-DUSE_CONAN=0'
         ]