HADOOP-14027. S3Guard: Implicitly creating DynamoDB table ignores endpoint config. Contributed by Sean Mackrory

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java

@@ -216,7 +216,15 @@ public class DynamoDBMetadataStore implements MetadataStore {
         "DynamoDBMetadataStore only supports S3A filesystem.");
     final S3AFileSystem s3afs = (S3AFileSystem) fs;
     final String bucket = s3afs.getBucket();
-    region = s3afs.getBucketLocation();
+    String confRegion = s3afs.getConf().getTrimmed(S3GUARD_DDB_REGION_KEY);
+    if (!StringUtils.isEmpty(confRegion)) {
+      region = confRegion;
+      LOG.debug("Overriding S3 region with configured DynamoDB region: {}",
+          region);
+    } else {
+      region = s3afs.getBucketLocation();
+      LOG.debug("Inferring DynamoDB region from S3 bucket: {}", region);
+    }
     username = s3afs.getUsername();
     conf = s3afs.getConf();
     dynamoDB = createDynamoDB(conf, region);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
@@ -102,6 +103,10 @@ public class ITestS3AAWSCredentialsProvider {
       createFailingFS(conf);
     } catch (AccessDeniedException e) {
       // expected
+    } catch (AWSServiceIOException e) {
+      GenericTestUtils.assertExceptionContains(
+          "UnrecognizedClientException", e);
+      // expected
     }
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java

@@ -126,7 +126,7 @@ public class ITestS3ACredentialsInURL extends Assert {
     Configuration conf = new Configuration();
     String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
     Assume.assumeNotNull(fsname);
-    Assume.assumeFalse(S3ATestUtils.isMetadataStoreAuthoritative(conf));
+    assumeS3GuardNotEnabled(conf);
     URI original = new URI(fsname);
     URI testURI = createUriWithEmbeddedSecrets(original, "user", "//");
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java

@@ -244,8 +244,11 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
       verifyTableInitialized(tableName);
       assertNotNull(ddbms.getTable());
       assertEquals(tableName, ddbms.getTable().getTableName());
-      assertEquals("DynamoDB table should be in the same region as S3 bucket",
-          s3afs.getBucketLocation(tableName),
+      String expectedRegion = conf.get(Constants.S3GUARD_DDB_REGION_KEY,
+          s3afs.getBucketLocation(tableName));
+      assertEquals("DynamoDB table should be in configured region or the same" +
+              " region as S3 bucket",
+          expectedRegion,
           ddbms.getRegion());
     }
   }
@@ -259,6 +262,8 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
     final String tableName = "testInitializeWithConfiguration";
     final Configuration conf = getFileSystem().getConf();
     conf.unset(Constants.S3GUARD_DDB_TABLE_NAME_KEY);
+    String savedRegion = conf.get(Constants.S3GUARD_DDB_REGION_KEY,
+        getFileSystem().getBucketLocation());
     conf.unset(Constants.S3GUARD_DDB_REGION_KEY);
     try {
       DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
@@ -275,8 +280,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
     } catch (IllegalArgumentException ignored) {
     }
     // config region
-    conf.set(Constants.S3GUARD_DDB_REGION_KEY,
-        getFileSystem().getBucketLocation());
+    conf.set(Constants.S3GUARD_DDB_REGION_KEY, savedRegion);
     try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
       ddbms.initialize(conf);
       verifyTableInitialized(tableName);
@@ -390,7 +394,9 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @Test
   public void testTableVersionRequired() throws Exception {
     final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
-    Table table = verifyTableInitialized(BUCKET);
+    String tableName = getFileSystem().getConf().get(Constants
+        .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
+    Table table = verifyTableInitialized(tableName);
     table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
 
     // create existing table
@@ -405,7 +411,9 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @Test
   public void testTableVersionMismatch() throws Exception {
     final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
-    Table table = verifyTableInitialized(BUCKET);
+    String tableName = getFileSystem().getConf().get(Constants
+        .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
+    Table table = verifyTableInitialized(tableName);
     table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
     Item v200 = createVersionMarker(VERSION_MARKER, 200, 0);
     table.putItem(v200);