Browse Source

HADOOP-15987. ITestDynamoDBMetadataStore should check if table configured properly. Contributed by Gabor Bota.

Sean Mackrory 6 years ago
parent
commit
c35de95a22

+ 5 - 5
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -996,11 +996,7 @@ When the `s3guard` profile is enabled, following profiles can be specified:
 
 * `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
   not exist. You will have to pay the bills for DynamoDB web service.
-* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS
-  DynamoDB web service; launch the server and creating the table.
-  You won't be charged bills for using DynamoDB in test. As it runs in-JVM,
-  the table isn't shared across other tests running in parallel.
-* `non-auth`: treat the S3Guard metadata as authoritative.
+* `auth`: treat the S3Guard metadata as authoritative.
 
 ```bash
 mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
@@ -1022,6 +1018,10 @@ If the `s3guard` profile *is* set,
 1. The S3Guard options from maven (the dynamo and authoritative flags)
   overwrite any previously set in the configuration files.
 1. DynamoDB will be configured to create any missing tables.
+1. When using DynamoDB and running ITestDynamoDBMetadataStore, the fs.s3a.s3guard.ddb.test.table
+property should be configured, and the name of that table should be different
+ than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
+ and modified multiple times during the test.
 
 
 ### Scale Testing MetadataStore Directly

+ 18 - 4
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java

@@ -116,7 +116,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @Override
   public void setUp() throws Exception {
     Configuration conf = prepareTestConfiguration(new Configuration());
-    assertThatDynamoMetadataStoreImpl(conf);
+    assumeThatDynamoMetadataStoreImpl(conf);
     Assume.assumeTrue("Test DynamoDB table name should be set to run "
             + "integration tests.", testDynamoDBTableName != null);
     conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
@@ -144,10 +144,24 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @BeforeClass
   public static void beforeClassSetup() throws IOException {
     Configuration conf = prepareTestConfiguration(new Configuration());
-    assertThatDynamoMetadataStoreImpl(conf);
+    assumeThatDynamoMetadataStoreImpl(conf);
     testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
-    Assume.assumeTrue("Test DynamoDB table name should be set to run "
+
+    // We should assert that the table name is configured, so the test should
+    // fail if it's not configured.
+    assertTrue("Test DynamoDB table name '"
+        + S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' should be set to run "
         + "integration tests.", testDynamoDBTableName != null);
+
+    // We should assert that the test table is not the same as the production
+    // table, as the test table could be modified and destroyed multiple
+    // times during the test.
+    assertTrue("Test DynamoDB table name: '"
+        + S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' and production table name: '"
+        + S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.",
+        !conf.get(S3GUARD_DDB_TABLE_NAME_KEY).equals(testDynamoDBTableName));
+
+    // We can use that table in the test if these assertions are valid
     conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
 
     LOG.debug("Creating static ddbms which will be shared between tests.");
@@ -169,7 +183,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
     }
   }
 
-  private static void assertThatDynamoMetadataStoreImpl(Configuration conf){
+  private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){
     Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
         conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
             Constants.S3GUARD_METASTORE_DYNAMO));