HADOOP-15645. ITestS3GuardToolLocal.testDiffCommand fails if bucket has per-bucket binding to DDB. Contributed by Steve Loughran.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java

@@ -56,6 +56,7 @@ import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
 import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
+import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -142,12 +143,14 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
   public void setup() throws Exception {
     super.setup();
     S3ATestUtils.assumeS3GuardState(true, getConfiguration());
-    ms = getFileSystem().getMetadataStore();
+    S3AFileSystem fs = getFileSystem();
+    ms = fs.getMetadataStore();
 
     // Also create a "raw" fs without any MetadataStore configured
     Configuration conf = new Configuration(getConfiguration());
-    URI fsUri = getFileSystem().getUri();
+    clearBucketOption(conf, fs.getBucket(), S3_METADATA_STORE_IMPL);
     conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
+    URI fsUri = fs.getUri();
     S3AUtils.setBucketOption(conf,fsUri.getHost(),
         METADATASTORE_AUTHORITATIVE,
         S3GUARD_METASTORE_NULL);
@@ -394,13 +397,17 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
   }
 
   @Test
-  public void testDiffCommand() throws Exception {
+  public void
+  testDiffCommand() throws Exception {
     S3AFileSystem fs = getFileSystem();
     ms = getMetadataStore();
     Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
     Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
 
     Path testPath = path("test-diff");
+    // clean up through the store and behind it.
+    fs.delete(testPath, true);
+    rawFs.delete(testPath, true);
     mkdirs(testPath, true, true);
 
     Path msOnlyPath = new Path(testPath, "ms_only");

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java

@@ -50,9 +50,10 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
   @Override
   public void setup() throws Exception {
     super.setup();
-    Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
-        getConfiguration().get(Constants.S3_METADATA_STORE_IMPL).equals(
-            Constants.S3GUARD_METASTORE_DYNAMO));
+    MetadataStore ms = getMetadataStore();
+    Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard;"
+        + "Store is " + (ms == null ? "none" : ms.toString()),
+        ms instanceof DynamoDBMetadataStore);
   }
 
   // Check the existence of a given DynamoDB table.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java

@@ -31,6 +31,8 @@ import java.util.concurrent.Callable;
 
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.StringUtils;
+
+import org.junit.Assume;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -52,6 +54,15 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
   private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
       "-force", "-verbose"};
 
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    MetadataStore ms = getMetadataStore();
+    Assume.assumeTrue("Test only applies when a local store is used for S3Guard;"
+            + "Store is " + (ms == null ? "none" : ms.toString()),
+        ms instanceof LocalMetadataStore);
+  }
+
   @Test
   public void testImportCommand() throws Exception {
     S3AFileSystem fs = getFileSystem();