Browse Source

HADOOP-15847. S3Guard testConcurrentTableCreations to set R/W capacity == 0

Contributed by lqjaclee

Change-Id: I4a4d5b29f2677c188799479e4db38f07fa0591d1
lqjaclee 5 years ago
parent
commit
cd967c75a7

+ 1 - 5
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -854,13 +854,9 @@ and, for test runs in or near the S3/DDB stores, throttling events.
 
 If you want to manage capacity, use `s3guard set-capacity` to increase it
 (performance) or decrease it (costs).
-For remote `hadoop-aws` test runs, the read/write capacities of "10" each should suffice;
+For remote `hadoop-aws` test runs, the read/write capacities of "0" each should suffice;
 increase it if parallel test run logs warn of throttling.
 
-Tip: for agility, use DynamoDB autoscaling, setting the minimum to something very low (e.g 5 units), the maximum to the largest amount you are willing to pay.
-This will automatically reduce capacity when you are not running tests against
-the bucket, slowly increase it over multiple test runs, if the load justifies it.
-
 ## <a name="tips"></a> Tips
 
 ### How to keep your credentials really safe

+ 10 - 0
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java

@@ -46,6 +46,8 @@ import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_READ_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY;
 
 /**
  * Tests concurrent operations on S3Guard.
@@ -55,6 +57,14 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
   @Rule
   public final Timeout timeout = new Timeout(5 * 60 * 1000);
 
+  protected Configuration createConfiguration() {
+    Configuration conf =  super.createConfiguration();
+    //patch the read/write capacity
+    conf.setInt(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, 0);
+    conf.setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, 0);
+    return conf;
+  }
+
   private void failIfTableExists(DynamoDB db, String tableName) {
     boolean tableExists = true;
     try {

+ 2 - 2
hadoop-tools/hadoop-aws/src/test/resources/core-site.xml

@@ -153,11 +153,11 @@
   <!-- Reduce DDB capacity on auto-created tables, to keep bills down. -->
   <property>
     <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
-    <value>10</value>
+    <value>0</value>
   </property>
   <property>
     <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
-    <value>10</value>
+    <value>0</value>
   </property>
 
   <!--