GH-40410: [C++] Skip only s3fs-tests and s3fs-module-tests that require MinIO if MinIO is not available (#50215)

### Rationale for this change

There are lots of S3 tests that doesn't require MinIO which we are not being exercised on CI because we skip the full test suite. We should skip only the tests that use MinIO if MinIO is not found but not the ones that doesn't require MinIO.

### What changes are included in this PR?

- Remove all occurrences of `exclude_tests="arrow-s3fs-test"`
- Add MinIO to path for macOS cpp.yml builds so MinIO tests are exercised
- Add new `MinioTestEnvironment::IsAvailable` to validate whether MinIO is available or not as part of the test suite
- `GTEST_SKIP` tests that require MinIO

### Are these changes tested?

Yes via CI and existing tests.
I've validated this locally both ways: without MinIO the non-MinIO tests run and the MinIO dependant ones are skipped, with MinIO on PATH the full suite runs and passes

### Are there any user-facing changes?

No

* GitHub Issue: #40410

Authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Signed-off-by: Antoine Pitrou <antoine@python.org>

Author: raulcd

.github/workflows/cpp.yml

@@ -293,6 +293,8 @@ jobs:
       - name: Test
         shell: bash
         run: |
+          # MinIO is installed on ${ARROW_HOME}/bin
+          export PATH="${ARROW_HOME}/bin:${PATH}"
           sudo sysctl -w kern.coredump=1
           sudo sysctl -w kern.corefile=/tmp/core.%N.%P
           ulimit -c unlimited  # must enable within the same shell

.github/workflows/cpp_extra.yml

@@ -340,10 +340,8 @@ jobs:
           ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
           PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
         run: |
-          # MinIO is required
-          exclude_tests="arrow-s3fs-test"
           # unstable
-          exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
+          exclude_tests="arrow-acero-asof-join-node-test"
           exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
           ctest \
             --exclude-regex "${exclude_tests}" \

ci/scripts/cpp_test.sh

@@ -50,17 +50,12 @@ fi
 if ! type storage-testbench >/dev/null 2>&1; then
   exclude_tests+=("arrow-gcsfs-test")
 fi
-if ! type minio >/dev/null 2>&1; then
-  exclude_tests+=("arrow-s3fs-test")
-fi
 case "$(uname)" in
   Linux)
     n_jobs=$(nproc)
     ;;
   Darwin)
     n_jobs=$(sysctl -n hw.ncpu)
-    # TODO: https://github.com/apache/arrow/issues/40410
-    exclude_tests+=("arrow-s3fs-test")
     ;;
   MINGW*)
     n_jobs=${NUMBER_OF_PROCESSORS:-1}

cpp/src/arrow/filesystem/s3_test_util.cc

@@ -192,6 +192,19 @@ void MinioTestEnvironment::SetUp() {
       MakeReadaheadGenerator(std::move(impl_->server_generator_), pool->GetCapacity());
 }
 
+bool MinioTestEnvironment::IsAvailable() {
+  if (!available_.has_value()) {
+    // If external S3-compatible service is configured there's no need to check for Minio
+    available_ = std::getenv(kEnvConnectString) && std::getenv(kEnvAccessKey) &&
+                 std::getenv(kEnvSecretKey);
+    if (!available_.value()) {
+      ::arrow::util::Process process;
+      available_ = process.SetExecutable(kMinioExecutableName).ok();
+    }
+  }
+  return available_.value();
+}
+
 Result<std::shared_ptr<MinioTestServer>> MinioTestEnvironment::GetOneServer() {
   return impl_->server_generator_().result();
 }

cpp/src/arrow/filesystem/s3_test_util.h

@@ -18,6 +18,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
 
@@ -74,6 +75,11 @@ class MinioTestEnvironment : public ::testing::Environment {
 
   Result<std::shared_ptr<MinioTestServer>> GetOneServer();
 
+  bool IsAvailable();
+
+ private:
+  std::optional<bool> available_;
+
  protected:
   struct Impl;
   std::unique_ptr<Impl> impl_;

cpp/src/arrow/filesystem/s3fs_module_test.cc

@@ -55,6 +55,17 @@ MinioTestEnvironment* GetMinioEnv() {
   return ::arrow::internal::checked_cast<MinioTestEnvironment*>(minio_env);
 }
 
+class S3ModuleTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    if (!GetMinioEnv()->IsAvailable()) {
+      GTEST_SKIP() << "Minio executable not found, skipping tests";
+    }
+    ASSERT_OK_AND_ASSIGN(minio_, GetMinioEnv()->GetOneServer());
+  }
+  std::shared_ptr<MinioTestServer> minio_;
+};
+
 class RegistrationTestEnvironment : public ::testing::Environment {
  public:
   void SetUp() override {
@@ -68,12 +79,10 @@ class RegistrationTestEnvironment : public ::testing::Environment {
 
 auto* lib_env = ::testing::AddGlobalTestEnvironment(new RegistrationTestEnvironment);
 
-TEST(S3Test, FromUri) {
-  ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
-
+TEST_F(S3ModuleTest, FromUri) {
   std::string path;
-  ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://" + minio->access_key() + ":" +
-                                                      minio->secret_key() +
+  ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://" + minio_->access_key() + ":" +
+                                                      minio_->secret_key() +
                                                       "@bucket/somedir/subdir/subfile",
                                                   &path));
 
@@ -83,12 +92,11 @@ TEST(S3Test, FromUri) {
             "&allow_bucket_creation=0&allow_bucket_deletion=0");
 }
 
-TEST(S3Test, FromUriAndOptionsCredentials) {
-  ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
+TEST_F(S3ModuleTest, FromUriAndOptionsCredentials) {
   std::string path;
   FileSystemFactoryOptions options{
-      {"access_key", std::string(minio->access_key())},
-      {"secret_key", std::string(minio->secret_key())},
+      {"access_key", std::string(minio_->access_key())},
+      {"secret_key", std::string(minio_->secret_key())},
   };
   // Credentials supplied via options, NOT in the URI.
   ASSERT_OK_AND_ASSIGN(
@@ -111,11 +119,10 @@ class NoopRetryStrategy : public S3RetryStrategy {
 };
 }  // namespace
 
-TEST(S3Test, FromUriAndOptionsRetryStrategy) {
-  ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
+TEST_F(S3ModuleTest, FromUriAndOptionsRetryStrategy) {
   FileSystemFactoryOptions options{
-      {"access_key", std::string(minio->access_key())},
-      {"secret_key", std::string(minio->secret_key())},
+      {"access_key", std::string(minio_->access_key())},
+      {"secret_key", std::string(minio_->secret_key())},
       {"retry_strategy",
        std::shared_ptr<S3RetryStrategy>(std::make_shared<NoopRetryStrategy>())},
   };

cpp/src/arrow/filesystem/s3fs_test.cc

@@ -192,6 +192,9 @@ class S3TestMixin : public AwsTestMixin {
  public:
   void SetUp() override {
     AwsTestMixin::SetUp();
+    if (!GetMinioEnv(enable_tls_)->IsAvailable()) {
+      GTEST_SKIP() << "Minio executable not found, skipping tests";
+    }
 
     // Starting the server may fail, for example if the generated port number
     // was "stolen" by another process. Run a dummy S3 operation to make sure it
@@ -624,6 +627,7 @@ class TestS3FS : public S3TestMixin {
  public:
   void SetUp() override {
     S3TestMixin::SetUp();
+    if (IsSkipped()) return;
     // Most tests will create buckets
     options_.allow_bucket_creation = true;
     options_.allow_bucket_deletion = true;
@@ -1777,6 +1781,7 @@ class TestS3FSGeneric : public S3TestMixin, public GenericFileSystemTest {
  public:
   void SetUp() override {
     S3TestMixin::SetUp();
+    if (IsSkipped()) return;
     // Set up test bucket
     {
       Aws::S3::Model::CreateBucketRequest req;