HADOOP-13336 S3A to support per-bucket configuration (S3Guard specific changes). Contributed by Steve Loughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -765,7 +765,7 @@ public class S3AFileSystem extends FileSystem {
     // If we have a MetadataStore, track deletions/creations.
     List<Path> srcPaths = null;
     List<PathMetadata> dstMetas = null;
-    if (!S3Guard.isNullMetadataStore(metadataStore)) {
+    if (hasMetadataStore()) {
       srcPaths = new ArrayList<>();
       dstMetas = new ArrayList<>();
     }
@@ -834,7 +834,7 @@ public class S3AFileSystem extends FileSystem {
               dstKey + summary.getKey().substring(srcKey.length());
           copyFile(summary.getKey(), newDstKey, length);
 
-          if (!S3Guard.isNullMetadataStore(metadataStore)) {
+          if (hasMetadataStore()) {
             Path srcPath = keyToQualifiedPath(summary.getKey());
             Path dstPath = keyToQualifiedPath(newDstKey);
             if (objectRepresentsDirectory(summary.getKey(), length)) {
@@ -885,8 +885,11 @@ public class S3AFileSystem extends FileSystem {
     return getObjectMetadata(pathToKey(path));
   }
 
-  @VisibleForTesting
-  public boolean isMetadataStoreConfigured() {
+  /**
+   * Does this Filesystem have a metadata store?
+   * @return true if the FS has been instantiated with a metadata store
+   */
+  public boolean hasMetadataStore() {
     return !S3Guard.isNullMetadataStore(metadataStore);
   }
 
@@ -1549,7 +1552,7 @@ public class S3AFileSystem extends FileSystem {
     incrementStatistic(INVOCATION_MKDIRS);
     FileStatus fileStatus;
     List<Path> metadataStoreDirs = null;
-    if (!S3Guard.isNullMetadataStore(metadataStore)) {
+    if (hasMetadataStore()) {
       metadataStoreDirs = new ArrayList<>();
     }
 
@@ -1896,7 +1899,7 @@ public class S3AFileSystem extends FileSystem {
 
     // See note about failure semantics in s3guard.md doc.
     try {
-      if (!S3Guard.isNullMetadataStore(metadataStore)) {
+      if (hasMetadataStore()) {
         S3AFileStatus status = createUploadFileStatus(p,
             S3AUtils.objectRepresentsDirectory(key, length), length,
             getDefaultBlockSize(p), username);

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java

@@ -105,17 +105,6 @@ public final class S3Guard {
     }
   }
 
-  /**
-   * Predicate to check whether or not the metadata store is the null one.
-   * @param conf Configuration
-   * @return true if NullMetadataStore is configured for s3a, or if the
-   * configuration is missing.
-   */
-  public static boolean isNullMetadataStoreConfigured(Configuration conf) {
-    Class<? extends MetadataStore> msClass = getMetadataStoreClass(conf);
-    return msClass.equals(NullMetadataStore.class);
-  }
-
   private static Class<? extends MetadataStore> getMetadataStoreClass(
       Configuration conf) {
     if (conf == null) {

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java

@@ -21,7 +21,6 @@ package org.apache.hadoop.fs.s3a;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,7 +64,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase {
     resetMetricDiffs();
     S3AFileStatus status = fs.getFileStatus(simpleFile);
     assertTrue("not a file: " + status, status.isFile());
-    if (S3Guard.isNullMetadataStoreConfigured(fs.getConf())) {
+    if (!fs.hasMetadataStore()) {
       metadataRequests.assertDiffEquals(1);
     }
     listRequests.assertDiffEquals(0);
@@ -85,7 +84,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase {
     S3AFileStatus status = fs.getFileStatus(dir);
     assertTrue("not empty: " + status, status.isEmptyDirectory());
 
-    if (S3Guard.isNullMetadataStoreConfigured(fs.getConf())) {
+    if (!fs.hasMetadataStore()) {
       metadataRequests.assertDiffEquals(2);
     }
     listRequests.assertDiffEquals(0);
@@ -140,7 +139,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase {
           + "\n" + ContractTestUtils.ls(fs, dir)
           + "\n" + fsState);
     }
-    if (S3Guard.isNullMetadataStoreConfigured(fs.getConf())) {
+    if (!fs.hasMetadataStore()) {
       metadataRequests.assertDiffEquals(2);
       listRequests.assertDiffEquals(1);
     }
@@ -200,7 +199,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase {
     // operations, it depends on side effects happening internally. With
     // metadata store enabled, it is brittle to change. We disable this test
     // before the internal behavior w/ or w/o metadata store.
-    assumeFalse(fs.isMetadataStoreConfigured());
+    assumeFalse(fs.hasMetadataStore());
 
     Path srcBaseDir = path("src");
     mkdirs(srcBaseDir);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java

@@ -51,7 +51,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
 
     // This test will fail if NullMetadataStore (the default) is configured:
     // skip it.
-    Assume.assumeTrue(fs.isMetadataStoreConfigured());
+    Assume.assumeTrue(fs.hasMetadataStore());
 
     // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed
     // in listObjects() results via InconsistentS3Client

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java

@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.internal.AssumptionViolatedException;

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java

@@ -16,23 +16,29 @@ package org.apache.hadoop.fs.s3a.fileContext;
 import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContextURIBase;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestFileSystem;
+
 /**
  * S3a implementation of FileContextURIBase.
  */
 public class ITestS3AFileContextURI extends FileContextURIBase {
 
   private Configuration conf;
+  private boolean hasMetadataStore;
 
   @Before
   public void setUp() throws IOException, Exception {
     conf = new Configuration();
+    try(S3AFileSystem s3aFS = createTestFileSystem(conf)) {
+      hasMetadataStore = s3aFS.hasMetadataStore();
+    }
     fc1 = S3ATestUtils.createTestFileContext(conf);
     fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
     super.setUp();
@@ -48,7 +54,8 @@ public class ITestS3AFileContextURI extends FileContextURIBase {
   @Test
   @Override
   public void testModificationTime() throws IOException {
-    Assume.assumeTrue(S3Guard.isNullMetadataStoreConfigured(conf));
+    // skip modtime tests as there may be some inconsistency during creation
+    assume("modification time tests are skipped", !hasMetadataStore);
     super.testModificationTime();
   }
 }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java

@@ -113,7 +113,7 @@ public class ITestS3ADirectoryPerformance extends S3AScaleTestBase {
           listContinueRequests,
           listStatusCalls,
           getFileStatusCalls);
-      if (!fs.isMetadataStoreConfigured()) {
+      if (!fs.hasMetadataStore()) {
         assertEquals(listRequests.toString(), 2, listRequests.diff());
       }
       reset(metadataRequests,