浏览代码

HADOOP-15987. ITestDynamoDBMetadataStore should check if table configured properly. Contributed by Gabor Bota.

Sean Mackrory 6 年之前
父节点
当前提交
c35de95a22

+ 5 - 5
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -996,11 +996,7 @@ When the `s3guard` profile is enabled, following profiles can be specified:
 
 
 * `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
 * `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
   not exist. You will have to pay the bills for DynamoDB web service.
   not exist. You will have to pay the bills for DynamoDB web service.
-* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS
-  DynamoDB web service; launch the server and creating the table.
-  You won't be charged bills for using DynamoDB in test. As it runs in-JVM,
-  the table isn't shared across other tests running in parallel.
-* `non-auth`: treat the S3Guard metadata as authoritative.
+* `auth`: treat the S3Guard metadata as authoritative.
 
 
 ```bash
 ```bash
 mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
 mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
@@ -1022,6 +1018,10 @@ If the `s3guard` profile *is* set,
 1. The S3Guard options from maven (the dynamo and authoritative flags)
 1. The S3Guard options from maven (the dynamo and authoritative flags)
   overwrite any previously set in the configuration files.
   overwrite any previously set in the configuration files.
 1. DynamoDB will be configured to create any missing tables.
 1. DynamoDB will be configured to create any missing tables.
+1. When using DynamoDB and running ITestDynamoDBMetadataStore, the fs.s3a.s3guard.ddb.test.table
+property should be configured, and the name of that table should be different
+ than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
+ and modified multiple times during the test.
 
 
 
 
 ### Scale Testing MetadataStore Directly
 ### Scale Testing MetadataStore Directly

+ 18 - 4
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java

@@ -116,7 +116,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @Override
   @Override
   public void setUp() throws Exception {
   public void setUp() throws Exception {
     Configuration conf = prepareTestConfiguration(new Configuration());
     Configuration conf = prepareTestConfiguration(new Configuration());
-    assertThatDynamoMetadataStoreImpl(conf);
+    assumeThatDynamoMetadataStoreImpl(conf);
     Assume.assumeTrue("Test DynamoDB table name should be set to run "
     Assume.assumeTrue("Test DynamoDB table name should be set to run "
             + "integration tests.", testDynamoDBTableName != null);
             + "integration tests.", testDynamoDBTableName != null);
     conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
     conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
@@ -144,10 +144,24 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @BeforeClass
   @BeforeClass
   public static void beforeClassSetup() throws IOException {
   public static void beforeClassSetup() throws IOException {
     Configuration conf = prepareTestConfiguration(new Configuration());
     Configuration conf = prepareTestConfiguration(new Configuration());
-    assertThatDynamoMetadataStoreImpl(conf);
+    assumeThatDynamoMetadataStoreImpl(conf);
     testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
     testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
-    Assume.assumeTrue("Test DynamoDB table name should be set to run "
+
+    // We should assert that the table name is configured, so the test should
+    // fail if it's not configured.
+    assertTrue("Test DynamoDB table name '"
+        + S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' should be set to run "
         + "integration tests.", testDynamoDBTableName != null);
         + "integration tests.", testDynamoDBTableName != null);
+
+    // We should assert that the test table is not the same as the production
+    // table, as the test table could be modified and destroyed multiple
+    // times during the test.
+    assertTrue("Test DynamoDB table name: '"
+        + S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' and production table name: '"
+        + S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.",
+        !conf.get(S3GUARD_DDB_TABLE_NAME_KEY).equals(testDynamoDBTableName));
+
+    // We can use that table in the test if these assertions are valid
     conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
     conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
 
 
     LOG.debug("Creating static ddbms which will be shared between tests.");
     LOG.debug("Creating static ddbms which will be shared between tests.");
@@ -169,7 +183,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
     }
     }
   }
   }
 
 
-  private static void assertThatDynamoMetadataStoreImpl(Configuration conf){
+  private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){
     Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
     Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
         conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
         conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
             Constants.S3GUARD_METASTORE_DYNAMO));
             Constants.S3GUARD_METASTORE_DYNAMO));