HADOOP-19256: Add tests

apache · Mar 5, 2025 · 4739e17 · 4739e17
1 parent 7c4db3c
commit 4739e17
Show file tree

Hide file tree

Showing 2 changed files with 203 additions and 5 deletions.
diff --git a/...op-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestS3AConditionalCreateBehavior.java b/...op-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestS3AConditionalCreateBehavior.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.Statistic;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_CONDITIONAL_OVERWRITE;
+import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_CONDITIONAL_OVERWRITE_ETAG;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CONDITIONAL_CREATE_ENABLED;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
+import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD;
+import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assumptions.assumeThat;
+
+@RunWith(Parameterized.class)
+public class ITestS3AConditionalCreateBehavior extends AbstractS3ATestBase {
+
+  private static final byte[] SMALL_FILE_BYTES = dataset(TEST_FILE_LEN, 0, 255);
+
+  private final boolean conditionalCreateEnabled;
+
+  public ITestS3AConditionalCreateBehavior(boolean conditionalCreateEnabled) {
+    this.conditionalCreateEnabled = conditionalCreateEnabled;
+  }
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][]{
+            {true},
+            {false}
+    });
+  }
+
+  @Override
+  public Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    removeBaseAndBucketOverrides(
+            conf,
+            FS_S3A_CREATE_PERFORMANCE,
+            FS_S3A_PERFORMANCE_FLAGS,
+            MULTIPART_SIZE,
+            MIN_MULTIPART_THRESHOLD,
+            UPLOAD_PART_COUNT_LIMIT
+    );
+    if (!conditionalCreateEnabled) {
+      conf.setBoolean(FS_S3A_CONDITIONAL_CREATE_ENABLED, false);
+    }
+    S3ATestUtils.disableFilesystemCaching(conf);
+    return conf;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    super.setup();
+  }
+
+  @Test
+  public void testConditionalWrite() throws Throwable {
+    FileSystem fs = getFileSystem();
+    Path testFile = methodPath();
+    fs.mkdirs(testFile.getParent());
+
+    // create a file over an empty path
+    try (FSDataOutputStream stream = fs.create(testFile)) {
+      stream.write(SMALL_FILE_BYTES);
+    }
+
+    // attempted conditional overwrite fails
+    intercept(PathIOException.class, () -> {
+      FSDataOutputStreamBuilder cf = fs.createFile(testFile);
+      cf.opt(FS_OPTION_CREATE_CONDITIONAL_OVERWRITE, true);
+      try (FSDataOutputStream stream = cf.build()) {
+        stream.write(SMALL_FILE_BYTES);
+      }
+    });
+  }
+
+  @Test
+  public void testWriteWithEtag() throws Throwable {
+    assumeThat(conditionalCreateEnabled)
+            .as("Skipping as conditional create is enabled")
+            .isFalse();
+
+    FileSystem fs = getFileSystem();
+    Path testFile = methodPath();
+    fs.mkdirs(testFile.getParent());
+
+    // create a file over an empty path
+    try (FSDataOutputStream stream = fs.create(testFile)) {
+      stream.write(SMALL_FILE_BYTES);
+    }
+
+    String etag = ((S3AFileStatus) fs.getFileStatus(testFile)).getEtag();
+    Assertions.assertThat(etag)
+            .as("ETag should not be null after file creation")
+            .isNotNull();
+
+    // attempted write with etag. should fail
+    intercept(PathIOException.class, () -> {
+      FSDataOutputStreamBuilder cf = fs.createFile(testFile);
+      cf.must(FS_OPTION_CREATE_CONDITIONAL_OVERWRITE_ETAG, etag);
+      try (FSDataOutputStream stream = cf.build()) {
+        stream.write(SMALL_FILE_BYTES);
+      }
+    });
+  }
+
+  @Test
+  public void testWriteWithPerformanceFlagAndOverwriteFalse() throws Throwable {
+    assumeThat(conditionalCreateEnabled)
+            .as("Skipping as conditional create is enabled")
+            .isFalse();
+
+    FileSystem fs = getFileSystem();
+    Path testFile = methodPath();
+    fs.mkdirs(testFile.getParent());
+
+    // create a file over an empty path
+    try (FSDataOutputStream stream = fs.create(testFile)) {
+      stream.write(SMALL_FILE_BYTES);
+    }
+
+    // overwrite with performance flag
+    FSDataOutputStreamBuilder cf = fs.createFile(testFile);
+    cf.overwrite(false);
+    cf.must(FS_S3A_CREATE_PERFORMANCE, true);
+    IOStatistics ioStatistics;
+    try (FSDataOutputStream stream = cf.build()) {
+      stream.write(SMALL_FILE_BYTES);
+      ioStatistics = S3ATestUtils.getOutputStreamStatistics(stream).getIOStatistics();
+    }
+    verifyStatisticCounterValue(ioStatistics, Statistic.CONDITIONAL_CREATE.getSymbol(), 0);
+    verifyStatisticCounterValue(ioStatistics, Statistic.CONDITIONAL_CREATE_FAILED.getSymbol(), 0);
+  }
+}
diff --git a/...adoop/fs/s3a/impl/ITestS3APutIfMatch.java → ...mpl/ITestS3APutIfMatchAndIfNoneMatch.java b/...adoop/fs/s3a/impl/ITestS3APutIfMatch.java → ...mpl/ITestS3APutIfMatchAndIfNoneMatch.java
@@ -35,11 +35,11 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
 import org.apache.hadoop.fs.s3a.RemoteFileChangedException;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.Statistic;
-import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+import org.apache.hadoop.fs.s3a.Statistic;;
 import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics;
 
 import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_CONDITIONAL_OVERWRITE;
@@ -67,7 +67,7 @@
  * This test class verifies the behavior of "If-Match" and "If-None-Match"
  * conditions while writing files.
  */
-public class ITestS3APutIfMatch extends AbstractS3ACostTest {
+public class ITestS3APutIfMatchAndIfNoneMatch extends AbstractS3ATestBase {
 
   private static final int UPDATED_MULTIPART_THRESHOLD = 100 * _1KB;
 
@@ -564,8 +564,6 @@ public void testIfMatchCreateFileWithoutOverwriteWithPerformanceFlag() throws Th
     Path testFile = methodPath();
     fs.mkdirs(testFile.getParent());
 
-    getConfiguration().set(FS_S3A_PERFORMANCE_FLAGS, "create");
-
     // Create a file and retrieve the etag
     createFileWithFlags(fs, testFile, SMALL_FILE_BYTES, false, null);
     String etag = getEtag(fs, testFile);
@@ -576,6 +574,7 @@ public void testIfMatchCreateFileWithoutOverwriteWithPerformanceFlag() throws Th
     // Attempt to create a file at the same path without overwrite, using If-Match with the etag
     FileAlreadyExistsException exception = intercept(FileAlreadyExistsException.class, () -> {
       fs.createFile(testFile).overwrite(false)
+              .must(FS_S3A_CREATE_PERFORMANCE, true)
               .opt(FS_OPTION_CREATE_CONDITIONAL_OVERWRITE_ETAG, etag)
               .build();
     });
@@ -653,4 +652,27 @@ public void testConditionalWriteStatisticsWithIfNoneMatch() throws Throwable {
     verifyStatisticCounterValue(statistics.getIOStatistics(), Statistic.CONDITIONAL_CREATE.getSymbol(), 1);
     verifyStatisticCounterValue(statistics.getIOStatistics(), Statistic.CONDITIONAL_CREATE_FAILED.getSymbol(), 1);
   }
+
+  /**
+   * Tests that a conditional create operation is triggered when the performance flag is enabled
+   * and the overwrite option is set to false.
+   */
+  @Test
+  public void testConditionalCreateWhenPerformanceFlagEnabledAndOverwriteDisabled() throws Throwable {
+    FileSystem fs = getFileSystem();
+    Path testFile = methodPath();
+    fs.mkdirs(testFile.getParent());
+
+    // Create a file
+    createFileWithFlags(fs, testFile, SMALL_FILE_BYTES, false, null);
+
+    // Attempt to override the file without overwrite and performance flag.
+    // Should throw RemoteFileChangedException (due to conditional write operation)
+    intercept(RemoteFileChangedException.class, () -> {
+      FSDataOutputStreamBuilder cf = fs.createFile(testFile);
+      cf.overwrite(false);
+      cf.must(FS_S3A_CREATE_PERFORMANCE, true);
+      cf.build().close();
+    });
+  }
 }