HADOOP-14013 S3Guard: fix multi-bucket integration tests. Contributed by Aaron Fabbri

with input from Mingliang Liu and Steve Loughran.

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -219,6 +219,18 @@ Attempting to perform more IO than the capacity requested simply throttles the
 IO; small capacity numbers are recommended when initially experimenting
 with S3Guard.
 
+## Credentials
+
+The DynamoDB metadata store takes advantage of the fact that the DynamoDB
+service uses uses the same authentication mechanisms as S3. With S3Guard,
+DynamoDB doesn't have any dedicated authentication configuration; it gets its
+credentials from the S3A client that is using it.
+
+The existing S3 authentication mechanisms can be used, except for one
+exception. Credentials placed in URIs are not supported for S3Guard.  The
+reason is that providing login details in filesystem URIs is considered
+unsafe and thus deprecated.
+
 ## S3Guard Command Line Interface (CLI)
 
 Note that in some cases an endpoint or a s3a:// URI can be provided.

hadoop-tools/hadoop-aws/src/test/resources/core-site.xml

@@ -30,19 +30,45 @@
     <final>true</final>
   </property>
 
-  <!-- Uncomment these two properties to run integration tests with
-       local metadata store (S3Guard).
+  <property>
+    <name>fs.s3a.bucket.landsat-pds.endpoint</name>
+    <value>${central.endpoint}</value>
+    <description>The endpoint for s3a://landsat-pds URLs</description>
+  </property>
 
+  <!-- Make sure S3Guard is disabled for read-only bucket tests. -->
   <property>
-    <name>fs.s3a.metadatastore.authoritative</name>
-    <value>true</value>
+    <name>fs.s3a.bucket.landsat-pds.metadatastore.impl</name>
+    <value>${s3guard.null}</value>
+    <description>The read-only landsat-pds repository isn't
+        managed by s3guard</description>
   </property>
+
+  <!-- Convenience definitions. -->
   <property>
-    <name>fs.s3a.metadatastore.impl</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore</value>
+    <name>s3guard.null</name>
+    <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
   </property>
 
-  -->
+  <property>
+    <name>s3guard.dynamo</name>
+    <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value>
+  </property>
+
+  <property>
+    <name>frankfurt.endpoint</name>
+    <value>s3.eu-central-1.amazonaws.com</value>
+  </property>
+
+  <property>
+    <name>ireland.endpoint</name>
+    <value>s3-eu-west-1.amazonaws.com</value>
+  </property>
+
+  <property>
+    <name>central.endpoint</name>
+    <value>s3.amazonaws.com</value>
+  </property>
 
   <!-- Scale integration tests may time out on slower connections
        you can reduce the operation count like so to mitigate this.