fix ray local init

Author: kevinjqliu

pyproject.toml

@@ -163,6 +163,9 @@ filterwarnings = [
   "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core",
   # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections
   "ignore:unclosed database in <sqlite3.Connection object*:ResourceWarning",
+  # Ignore Ray subprocess cleanup warnings
+  "ignore:unclosed file:ResourceWarning",
+  "ignore:subprocess.*is still running:ResourceWarning",
 ]
 
 [tool.black]

tests/conftest.py

@@ -2947,3 +2947,16 @@ def pyarrow_table_with_promoted_types(pyarrow_schema_with_promoted_types: "pa.Sc
         },
         schema=pyarrow_schema_with_promoted_types,
     )
+
+
+@pytest.fixture(scope="function")
+def ray_session() -> Generator[Any, None, None]:
+    """Fixture to manage Ray initialization and shutdown for tests."""
+    import ray
+
+    ray.init(
+        ignore_reinit_error=True,
+        runtime_env={"working_dir": None},  # Prevent Ray from serializing the working directory to workers
+    )
+    yield ray
+    ray.shutdown()

tests/integration/test_reads.py

@@ -21,6 +21,7 @@
 import uuid
 from datetime import datetime, timedelta
 from pathlib import PosixPath
+from typing import Any
 from urllib.parse import urlparse
 
 import pyarrow as pa
@@ -357,7 +358,7 @@ def test_bodo_nan(catalog: Catalog, monkeypatch: pytest.MonkeyPatch) -> None:
 @pytest.mark.integration
 @pytest.mark.filterwarnings("ignore")
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
-def test_ray_nan(catalog: Catalog) -> None:
+def test_ray_nan(catalog: Catalog, ray_session: Any) -> None:
     table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten")
     ray_dataset = table_test_null_nan_rewritten.scan().to_ray()
     assert ray_dataset.count() == 3
@@ -366,7 +367,7 @@ def test_ray_nan(catalog: Catalog) -> None:
 
 @pytest.mark.integration
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
-def test_ray_nan_rewritten(catalog: Catalog) -> None:
+def test_ray_nan_rewritten(catalog: Catalog, ray_session: Any) -> None:
     table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten")
     ray_dataset = table_test_null_nan_rewritten.scan(
         row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric")
@@ -379,15 +380,15 @@ def test_ray_nan_rewritten(catalog: Catalog) -> None:
 @pytest.mark.integration
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
 @pytest.mark.skip(reason="Fixing issues with NaN's: https://github.com/apache/arrow/issues/34162")
-def test_ray_not_nan_count(catalog: Catalog) -> None:
+def test_ray_not_nan_count(catalog: Catalog, ray_session: Any) -> None:
     table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten")
     ray_dataset = table_test_null_nan_rewritten.scan(row_filter=NotNaN("col_numeric"), selected_fields=("idx",)).to_ray()
     assert ray_dataset.count() == 2
 
 
 @pytest.mark.integration
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
-def test_ray_all_types(catalog: Catalog) -> None:
+def test_ray_all_types(catalog: Catalog, ray_session: Any) -> None:
     table_test_all_types = catalog.load_table("default.test_all_types")
     ray_dataset = table_test_all_types.scan().to_ray()
     pandas_dataframe = table_test_all_types.scan().to_pandas()