Merge branch 'add-pyarrow-stubs' into unsupported-operator

Author: rok

.pre-commit-config.yaml

@@ -331,6 +331,7 @@ repos:
           ?^ci/scripts/integration_arrow_build\.sh$|
           ?^ci/scripts/integration_arrow\.sh$|
           ?^ci/scripts/integration_dask\.sh$|
+          ?^ci/scripts/integration_hdfs\.sh$|
           ?^ci/scripts/integration_spark\.sh$|
           ?^ci/scripts/matlab_build\.sh$|
           ?^ci/scripts/msys2_system_clean\.sh$|

c_glib/meson.build

@@ -34,7 +34,7 @@ project(
     #   * 22.04: 0.61.2
     #   * 24.04: 1.3.2
     meson_version: '>=0.61.2',
-    version: '21.0.0-SNAPSHOT',
+    version: '22.0.0-SNAPSHOT',
 )
 
 version = meson.project_version()

c_glib/tool/generate-version-header.py

@@ -140,6 +140,7 @@ def generate_availability_macros(library: str) -> str:
 
 
 ALL_VERSIONS = [
+    (22, 0),
     (21, 0),
     (20, 0),
     (19, 0),

c_glib/vcpkg.json

@@ -1,6 +1,6 @@
 {
   "name": "arrow-glib",
-  "version-string": "21.0.0-SNAPSHOT",
+  "version-string": "22.0.0-SNAPSHOT",
   "$comment:dependencies": "We can enable gobject-introspection again once it's updated",
   "dependencies": [
     "glib",

ci/scripts/PKGBUILD

@@ -18,7 +18,7 @@
 _realname=arrow
 pkgbase=mingw-w64-${_realname}
 pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}"
-pkgver=20.0.0.9000
+pkgver=21.0.0.9000
 pkgrel=8000
 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)"
 arch=("any")

ci/scripts/integration_hdfs.sh

@@ -19,10 +19,12 @@
 
 set -e
 
+# shellcheck disable=SC2034
 source_dir=${1}/cpp
 build_dir=${2}/cpp
 
-export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath --glob)
+HADOOP_CLASSPATH=$("$HADOOP_HOME/bin/hadoop" classpath --glob)
+export CLASSPATH="${HADOOP_CLASSPATH}"
 export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
 export LIBHDFS3_CONF=$HADOOP_CONF_DIR/hdfs-site.xml
 export ARROW_LIBHDFS3_DIR=$CONDA_PREFIX/lib
@@ -42,7 +44,7 @@ function use_libhdfs_dir() {
 
 # execute cpp tests
 export ARROW_HDFS_TEST_LIBHDFS_REQUIRE=ON
-pushd ${build_dir}
+pushd "${build_dir}"
 
 debug/arrow-io-hdfs-test
 debug/arrow-hdfs-test

cpp/CMakeLists.txt

@@ -96,7 +96,7 @@ if(POLICY CMP0170)
   cmake_policy(SET CMP0170 NEW)
 endif()
 
-set(ARROW_VERSION "21.0.0-SNAPSHOT")
+set(ARROW_VERSION "22.0.0-SNAPSHOT")
 
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
 

cpp/meson.build

@@ -19,7 +19,7 @@ project(
     'arrow',
     'cpp',
     'c',
-    version: '21.0.0-SNAPSHOT',
+    version: '22.0.0-SNAPSHOT',
     license: 'Apache-2.0',
     meson_version: '>=1.3.0',
     default_options: ['c_std=gnu11,c11', 'warning_level=2', 'cpp_std=c++17'],

cpp/src/arrow/record_batch.cc

@@ -310,19 +310,58 @@ const std::string& RecordBatch::column_name(int i) const {
   return schema_->field(i)->name();
 }
 
-bool RecordBatch::Equals(const RecordBatch& other, bool check_metadata,
-                         const EqualOptions& opts) const {
-  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
-    return false;
+namespace {
+
+bool ContainFloatType(const std::shared_ptr<DataType>& type) {
+  if (is_floating(type->id())) {
+    return true;
   }
 
-  if (!schema_->Equals(*other.schema(), check_metadata)) {
-    return false;
+  for (const auto& field : type->fields()) {
+    if (ContainFloatType(field->type())) {
+      return true;
+    }
   }
 
-  if (device_type() != other.device_type()) {
+  return false;
+}
+
+bool ContainFloatType(const Schema& schema) {
+  for (auto& field : schema.fields()) {
+    if (ContainFloatType(field->type())) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool CanIgnoreNaNInEquality(const RecordBatch& batch, const EqualOptions& opts) {
+  if (opts.nans_equal()) {
+    return true;
+  } else if (!ContainFloatType(*batch.schema())) {
+    return true;
+  } else {
     return false;
   }
+}
+
+}  // namespace
+
+bool RecordBatch::Equals(const RecordBatch& other, bool check_metadata,
+                         const EqualOptions& opts) const {
+  if (this == &other) {
+    if (CanIgnoreNaNInEquality(*this, opts)) {
+      return true;
+    }
+  } else {
+    if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
+      return false;
+    } else if (!schema_->Equals(*other.schema(), check_metadata)) {
+      return false;
+    } else if (device_type() != other.device_type()) {
+      return false;
+    }
+  }
 
   for (int i = 0; i < num_columns(); ++i) {
     if (!column(i)->Equals(other.column(i), opts)) {
@@ -334,12 +373,16 @@ bool RecordBatch::Equals(const RecordBatch& other, bool check_metadata,
 }
 
 bool RecordBatch::ApproxEquals(const RecordBatch& other, const EqualOptions& opts) const {
-  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
-    return false;
-  }
-
-  if (device_type() != other.device_type()) {
-    return false;
+  if (this == &other) {
+    if (CanIgnoreNaNInEquality(*this, opts)) {
+      return true;
+    }
+  } else {
+    if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
+      return false;
+    } else if (device_type() != other.device_type()) {
+      return false;
+    }
   }
 
   for (int i = 0; i < num_columns(); ++i) {

cpp/src/arrow/record_batch_test.cc

@@ -142,6 +142,94 @@ TEST_F(TestRecordBatch, ApproxEqualOptions) {
   EXPECT_TRUE(b1->ApproxEquals(*b2, options));
 }
 
+class TestRecordBatchEqualsSameAddress : public TestRecordBatch {};
+
+TEST_F(TestRecordBatchEqualsSameAddress, NonFloatType) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", int64());
+
+  auto schema = ::arrow::schema({f0, f1});
+
+  auto a0 = ArrayFromJSON(f0->type(), "[0, 1, 2]");
+  auto a1 = ArrayFromJSON(f1->type(), "[0, 1, 2]");
+
+  auto b0 = RecordBatch::Make(schema, 3, {a0, a1});
+  auto b1 = b0;
+
+  auto options = EqualOptions::Defaults();
+
+  ASSERT_TRUE(b0->Equals(*b1, true, options));
+  ASSERT_TRUE(b0->Equals(*b1, true, options.nans_equal(true)));
+
+  ASSERT_TRUE(b0->ApproxEquals(*b1, options));
+  ASSERT_TRUE(b0->ApproxEquals(*b1, options.nans_equal(true)));
+}
+
+TEST_F(TestRecordBatchEqualsSameAddress, NestedTypesWithoutFloatType) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", struct_({{"f2", int64()}, {"f3", int8()}}));
+
+  auto schema = ::arrow::schema({f0, f1});
+
+  auto a0 = ArrayFromJSON(f0->type(), "[0, 1, 2]");
+  auto a1 = ArrayFromJSON(
+      f1->type(), R"([{"f2": 1, "f3": 4}, {"f2": 2, "f3": 5}, {"f2":3, "f3": 6}])");
+
+  auto b0 = RecordBatch::Make(schema, 3, {a0, a1});
+  auto b1 = b0;
+
+  auto options = EqualOptions::Defaults();
+
+  ASSERT_TRUE(b0->Equals(*b1, true, options));
+  ASSERT_TRUE(b0->Equals(*b1, true, options.nans_equal(true)));
+
+  ASSERT_TRUE(b0->ApproxEquals(*b1, options));
+  ASSERT_TRUE(b0->ApproxEquals(*b1, options.nans_equal(true)));
+}
+
+TEST_F(TestRecordBatchEqualsSameAddress, FloatType) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", float64());
+
+  auto schema = ::arrow::schema({f0, f1});
+
+  auto a0 = ArrayFromJSON(f0->type(), "[0, 1, 2]");
+  auto a1 = ArrayFromJSON(f1->type(), "[0.0, 1.0, 2.0, NaN]");
+
+  auto b0 = RecordBatch::Make(schema, 3, {a0, a1});
+  auto b1 = b0;
+
+  auto options = EqualOptions::Defaults();
+
+  ASSERT_FALSE(b0->Equals(*b1, true, options));
+  ASSERT_TRUE(b0->Equals(*b1, true, options.nans_equal(true)));
+
+  ASSERT_FALSE(b0->ApproxEquals(*b1, options));
+  ASSERT_TRUE(b0->ApproxEquals(*b1, options.nans_equal(true)));
+}
+
+TEST_F(TestRecordBatchEqualsSameAddress, NestedTypesWithFloatType) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", struct_({{"f2", int64()}, {"f3", float32()}}));
+
+  auto schema = ::arrow::schema({f0, f1});
+
+  auto a0 = ArrayFromJSON(f0->type(), "[0, 1, 2]");
+  auto a1 = ArrayFromJSON(
+      f1->type(), R"([{"f2": 1, "f3": 4.0}, {"f2": 2, "f3": 4.0}, {"f2":3, "f3": NaN}])");
+
+  auto b0 = RecordBatch::Make(schema, 3, {a0, a1});
+  auto b1 = b0;
+
+  auto options = EqualOptions::Defaults();
+
+  ASSERT_FALSE(b0->Equals(*b1, true, options));
+  ASSERT_TRUE(b0->Equals(*b1, true, options.nans_equal(true)));
+
+  ASSERT_FALSE(b0->ApproxEquals(*b1, options));
+  ASSERT_TRUE(b0->ApproxEquals(*b1, options.nans_equal(true)));
+}
+
 TEST_F(TestRecordBatch, Validate) {
   const int length = 10;