HADOOP-16055. Upgrade AWS SDK to 1.11.271 in branch-2.

Contains HADOOP-12705 Upgrade Jackson 2.2.3 to 2.7.8.

This change was required to address license compatibility issues with the JSON parser in the older AWS SDKs.

A consequence of this is that the version of Jackson 2 shipped is now 2.7.8.

Author:    Akira Ajisaka <aajisaka@apache.org>

NOTICE.txt

@@ -17,13 +17,11 @@ which has the following notices:
 * This product includes software developed by IBM Corporation and others.
 
 The binary distribution of this product bundles binaries of
-AWS Java SDK 1.10.6,
+AWS Java SDK 1.11.271,
 which has the following notices:
- * This software includes third party software subject to the following
- copyrights: - XML parsing and utility functions from JetS3t - Copyright
- 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org -
- Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility
- functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+ * This software includes third party software subject to the following copyrights:
+   - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+   - PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
 
 The binary distribution of this product bundles binaries of
 Gson 2.2.4,

hadoop-project/pom.xml

@@ -68,7 +68,7 @@
 
     <!-- jackson versions -->
     <jackson.version>1.9.13</jackson.version>
-    <jackson2.version>2.2.3</jackson2.version>
+    <jackson2.version>2.7.8</jackson2.version>
 
     <!-- ProtocolBuffer version, used to verify the protoc version and -->
     <!-- define the protobuf JAR version                               -->
@@ -117,7 +117,7 @@
     <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
     <native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
     <surefire.fork.timeout>900</surefire.fork.timeout>
-    <aws-java-sdk.version>1.10.6</aws-java-sdk.version>
+    <aws-java-sdk.version>1.11.271</aws-java-sdk.version>
   </properties>
 
   <dependencyManagement>
@@ -684,13 +684,14 @@
       </dependency>
       <dependency>
         <groupId>com.amazonaws</groupId>
-        <artifactId>aws-java-sdk-s3</artifactId>
-        <version>${aws-java-sdk.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>com.amazonaws</groupId>
-        <artifactId>aws-java-sdk-sts</artifactId>
+        <artifactId>aws-java-sdk-bundle</artifactId>
         <version>${aws-java-sdk.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>io.netty</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>org.apache.mina</groupId>

hadoop-tools/hadoop-aws/pom.xml

@@ -316,7 +316,7 @@
     </dependency>
     <dependency>
       <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk-s3</artifactId>
+      <artifactId>aws-java-sdk-bundle</artifactId>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -335,11 +335,6 @@
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk-sts</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java

@@ -184,7 +184,7 @@ interface S3ClientFactory {
         userAgent = userAgentPrefix + ", " + userAgent;
       }
       LOG.debug("Using User-Agent: {}", userAgent);
-      awsConf.setUserAgent(userAgent);
+      awsConf.setUserAgentPrefix(userAgent);
     }
 
     /**
@@ -229,7 +229,9 @@ interface S3ClientFactory {
       final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
       if (pathStyleAccess) {
         LOG.debug("Enabling path style access!");
-        s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+        s3.setS3ClientOptions(S3ClientOptions.builder()
+          .setPathStyleAccess(true)
+          .build());
       }
     }
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java

@@ -148,7 +148,8 @@ public class ITestS3AAWSCredentialsProvider {
     @SuppressWarnings("unused")
     public GoodCredentialsProvider(Configuration conf) {
       super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
-          conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
+          conf.get(SECRET_KEY)),
+          InstanceProfileCredentialsProvider.getInstance());
     }
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java

@@ -436,7 +436,8 @@ public class ITestS3AConfiguration {
     assertNotNull(s3);
     ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
         "clientConfiguration");
-    assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
+    assertEquals("Hadoop " + VersionInfo.getVersion(),
+        awsConf.getUserAgentPrefix());
   }
 
   @Test
@@ -450,7 +451,7 @@ public class ITestS3AConfiguration {
     ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
         "clientConfiguration");
     assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
-        awsConf.getUserAgent());
+        awsConf.getUserAgentPrefix());
   }
 
   @Test