Skip only arrow-s3fs-tests and arrow-s3fs-module-tests that require minio if minIO is not available but run the rest of the tests that don't require minio

Author: raulcd

.github/workflows/cpp_extra.yml

@@ -340,8 +340,6 @@ jobs:
           ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
           PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
         run: |
-          # MinIO is required
-          exclude_tests="arrow-s3fs-test"
           # unstable
           exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
           exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"

ci/scripts/cpp_test.sh

@@ -50,9 +50,6 @@ fi
 if ! type storage-testbench >/dev/null 2>&1; then
   exclude_tests+=("arrow-gcsfs-test")
 fi
-if ! type minio >/dev/null 2>&1; then
-  exclude_tests+=("arrow-s3fs-test")
-fi
 case "$(uname)" in
   Linux)
     n_jobs=$(nproc)

cpp/src/arrow/filesystem/s3_test_util.cc

@@ -192,6 +192,15 @@ void MinioTestEnvironment::SetUp() {
       MakeReadaheadGenerator(std::move(impl_->server_generator_), pool->GetCapacity());
 }
 
+bool MinioTestEnvironment::IsAvailable() {
+  if (!checked_) {
+    ::arrow::util::Process process;
+    available_ = process.SetExecutable(kMinioExecutableName).ok();
+    checked_ = true;
+  }
+  return available_;
+}
+
 Result<std::shared_ptr<MinioTestServer>> MinioTestEnvironment::GetOneServer() {
   return impl_->server_generator_().result();
 }

cpp/src/arrow/filesystem/s3_test_util.h

@@ -74,6 +74,12 @@ class MinioTestEnvironment : public ::testing::Environment {
 
   Result<std::shared_ptr<MinioTestServer>> GetOneServer();
 
+  bool IsAvailable();
+
+ private:
+  bool checked_ = false;
+  bool available_ = false;
+
  protected:
   struct Impl;
   std::unique_ptr<Impl> impl_;

cpp/src/arrow/filesystem/s3fs_module_test.cc

@@ -55,6 +55,17 @@ MinioTestEnvironment* GetMinioEnv() {
   return ::arrow::internal::checked_cast<MinioTestEnvironment*>(minio_env);
 }
 
+class S3ModuleTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    if (!GetMinioEnv()->IsAvailable()) {
+      GTEST_SKIP() << "Minio executable not found, skipping tests";
+    }
+    ASSERT_OK_AND_ASSIGN(minio_, GetMinioEnv()->GetOneServer());
+  }
+  std::shared_ptr<MinioTestServer> minio_;
+};
+
 class RegistrationTestEnvironment : public ::testing::Environment {
  public:
   void SetUp() override {
@@ -68,12 +79,10 @@ class RegistrationTestEnvironment : public ::testing::Environment {
 
 auto* lib_env = ::testing::AddGlobalTestEnvironment(new RegistrationTestEnvironment);
 
-TEST(S3Test, FromUri) {
-  ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
-
+TEST_F(S3ModuleTest, FromUri) {
   std::string path;
-  ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://" + minio->access_key() + ":" +
-                                                      minio->secret_key() +
+  ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://" + minio_->access_key() + ":" +
+                                                      minio_->secret_key() +
                                                       "@bucket/somedir/subdir/subfile",
                                                   &path));
 
@@ -83,12 +92,11 @@ TEST(S3Test, FromUri) {
             "&allow_bucket_creation=0&allow_bucket_deletion=0");
 }
 
-TEST(S3Test, FromUriAndOptionsCredentials) {
-  ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
+TEST_F(S3ModuleTest, FromUriAndOptionsCredentials) {
   std::string path;
   FileSystemFactoryOptions options{
-      {"access_key", std::string(minio->access_key())},
-      {"secret_key", std::string(minio->secret_key())},
+      {"access_key", std::string(minio_->access_key())},
+      {"secret_key", std::string(minio_->secret_key())},
   };
   // Credentials supplied via options, NOT in the URI.
   ASSERT_OK_AND_ASSIGN(
@@ -111,11 +119,10 @@ class NoopRetryStrategy : public S3RetryStrategy {
 };
 }  // namespace
 
-TEST(S3Test, FromUriAndOptionsRetryStrategy) {
-  ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
+TEST_F(S3ModuleTest, FromUriAndOptionsRetryStrategy) {
   FileSystemFactoryOptions options{
-      {"access_key", std::string(minio->access_key())},
-      {"secret_key", std::string(minio->secret_key())},
+      {"access_key", std::string(minio_->access_key())},
+      {"secret_key", std::string(minio_->secret_key())},
       {"retry_strategy",
        std::shared_ptr<S3RetryStrategy>(std::make_shared<NoopRetryStrategy>())},
   };

cpp/src/arrow/filesystem/s3fs_test.cc

@@ -192,6 +192,9 @@ class S3TestMixin : public AwsTestMixin {
  public:
   void SetUp() override {
     AwsTestMixin::SetUp();
+    if (!GetMinioEnv(enable_tls_)->IsAvailable()) {
+      GTEST_SKIP() << "Minio executable not found, skipping tests";
+    }
 
     // Starting the server may fail, for example if the generated port number
     // was "stolen" by another process. Run a dummy S3 operation to make sure it
@@ -624,6 +627,7 @@ class TestS3FS : public S3TestMixin {
  public:
   void SetUp() override {
     S3TestMixin::SetUp();
+    if (IsSkipped()) return;
     // Most tests will create buckets
     options_.allow_bucket_creation = true;
     options_.allow_bucket_deletion = true;
@@ -1777,6 +1781,7 @@ class TestS3FSGeneric : public S3TestMixin, public GenericFileSystemTest {
  public:
   void SetUp() override {
     S3TestMixin::SetUp();
+    if (IsSkipped()) return;
     // Set up test bucket
     {
       Aws::S3::Model::CreateBucketRequest req;