HADOOP-18890. Remove use of okhttp in runtime code (#6057)

Contributed by PJ Fanning

LICENSE-binary

@@ -243,8 +243,6 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.microsoft.azure:azure-storage:7.0.1
 com.nimbusds:nimbus-jose-jwt:9.8.1
-com.squareup.okhttp3:okhttp:4.10.0
-com.squareup.okio:okio:3.4.0
 com.yammer.metrics:metrics-core:2.2.0
 com.zaxxer:HikariCP-java7:2.4.12
 commons-beanutils:commons-beanutils:1.9.4
@@ -361,8 +359,6 @@ org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217
 org.apache.zookeeper:zookeeper:3.6.3
 org.ehcache:ehcache:3.3.1
 org.ini4j:ini4j:0.5.4
-org.jetbrains.kotlin:kotlin-stdlib:1.4.10
-org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.1

NOTICE-binary

@@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
------------------------------------------------------------------------
-
-This product contains a modified portion of 'OkHttp', an open source
-HTTP & SPDY client for Android and Java applications, which can be obtained
-at:
-
-  * LICENSE:
-    * okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/square/okhttp
-  * LOCATION_IN_GRPC:
-    * okhttp/third_party/okhttp
-
 This product contains a modified portion of 'Netty', an open source
 networking library, which can be obtained at:
 

hadoop-client-modules/hadoop-client/pom.xml

@@ -114,18 +114,6 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.jetbrains.kotlin</groupId>
-          <artifactId>kotlin-stdlib</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.jetbrains.kotlin</groupId>
-          <artifactId>kotlin-stdlib-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.squareup.okhttp3</groupId>
-          <artifactId>okhttp</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>

hadoop-common-project/hadoop-common/pom.xml

@@ -374,8 +374,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.squareup.okio</groupId>
-      <artifactId>okio-jvm</artifactId>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib-jdk8</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml

@@ -93,17 +93,4 @@
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
 
-  <!--okhttp classes from Kotlin are not analysed for NP check. -->
-  <Match>
-    <Class name="org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider" />
-    <Method name="refresh" />
-    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
-  </Match>
-
-  <Match>
-    <Class name="org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider" />
-    <Method name="refresh" />
-    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
-  </Match>
-
 </FindBugsFilter>

hadoop-hdfs-project/hadoop-hdfs-client/pom.xml

@@ -34,28 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>com.squareup.okhttp3</groupId>
-      <artifactId>okhttp</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>com.squareup.okio</groupId>
-          <artifactId>okio-jvm</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>com.squareup.okio</groupId>
-      <artifactId>okio-jvm</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>kotlin-stdlib</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>kotlin-stdlib-common</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
@@ -71,6 +49,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java

@@ -19,13 +19,10 @@
 package org.apache.hadoop.hdfs.web.oauth2;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -33,7 +30,17 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.Timer;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@@ -103,34 +110,37 @@ public class ConfRefreshTokenBasedAccessTokenProvider
   }
 
   void refresh() throws IOException {
-    OkHttpClient client =
-        new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
-                TimeUnit.MILLISECONDS)
-            .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
-            .build();
-
-    String bodyString =
-        Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId);
-
-    RequestBody body = RequestBody.create(bodyString, URLENCODED);
-
-    Request request = new Request.Builder().url(refreshURL).post(body).build();
-    try (Response response = client.newCall(request).execute()) {
-      if (!response.isSuccessful()) {
-        throw new IOException("Unexpected code " + response);
-      }
-      if (response.code() != HttpStatus.SC_OK) {
-        throw new IllegalArgumentException(
-            "Received invalid http response: " + response.code() + ", text = "
-                + response.toString());
+    final List<NameValuePair> pairs = new ArrayList<>();
+    pairs.add(new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN));
+    pairs.add(new BasicNameValuePair(REFRESH_TOKEN, refreshToken));
+    pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
+    final RequestConfig config = RequestConfig.custom()
+        .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .build();
+    try (CloseableHttpClient client =
+             HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
+      final HttpPost httpPost = new HttpPost(refreshURL);
+      httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
+      httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
+      try (CloseableHttpResponse response = client.execute(httpPost)) {
+        final int statusCode = response.getStatusLine().getStatusCode();
+        if (statusCode != HttpStatus.SC_OK) {
+          throw new IllegalArgumentException(
+              "Received invalid http response: " + statusCode + ", text = " +
+                  EntityUtils.toString(response.getEntity()));
+        }
+        Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
+            EntityUtils.toString(response.getEntity()));
+
+        String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
+        accessTokenTimer.setExpiresIn(newExpiresIn);
+
+        accessToken = responseBody.get(ACCESS_TOKEN).toString();
       }
-
-      Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(response.body().string());
-
-      String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
-      accessTokenTimer.setExpiresIn(newExpiresIn);
-
-      accessToken = responseBody.get(ACCESS_TOKEN).toString();
+    } catch (RuntimeException e) {
+      throw new IOException("Exception while refreshing access token", e);
     } catch (Exception e) {
       throw new IOException("Exception while refreshing access token", e);
     }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java

@@ -19,13 +19,10 @@
 package org.apache.hadoop.hdfs.web.oauth2;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -33,7 +30,17 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.Timer;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@@ -97,38 +104,37 @@ public abstract class CredentialBasedAccessTokenProvider
   }
 
   void refresh() throws IOException {
-    OkHttpClient client = new OkHttpClient.Builder()
-            .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
-            .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
-            .build();
-
-    String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
-        GRANT_TYPE, CLIENT_CREDENTIALS,
-        CLIENT_ID, clientId);
-
-    RequestBody body = RequestBody.create(bodyString, URLENCODED);
-
-    Request request = new Request.Builder()
-        .url(refreshURL)
-        .post(body)
+    final List<NameValuePair> pairs = new ArrayList<>();
+    pairs.add(new BasicNameValuePair(CLIENT_SECRET, getCredential()));
+    pairs.add(new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS));
+    pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
+    final RequestConfig config = RequestConfig.custom()
+        .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
         .build();
-    try (Response response = client.newCall(request).execute()) {
-      if (!response.isSuccessful()) {
-        throw new IOException("Unexpected code " + response);
-      }
-
-      if (response.code() != HttpStatus.SC_OK) {
-        throw new IllegalArgumentException("Received invalid http response: "
-            + response.code() + ", text = " + response.toString());
+    try (CloseableHttpClient client =
+             HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
+      final HttpPost httpPost = new HttpPost(refreshURL);
+      httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
+      httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
+      try (CloseableHttpResponse response = client.execute(httpPost)) {
+        final int statusCode = response.getStatusLine().getStatusCode();
+        if (statusCode != HttpStatus.SC_OK) {
+          throw new IllegalArgumentException(
+              "Received invalid http response: " + statusCode + ", text = " +
+                  EntityUtils.toString(response.getEntity()));
+        }
+        Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
+            EntityUtils.toString(response.getEntity()));
+
+        String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
+        timer.setExpiresIn(newExpiresIn);
+
+        accessToken = responseBody.get(ACCESS_TOKEN).toString();
       }
-
-      Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
-          response.body().string());
-
-      String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
-      timer.setExpiresIn(newExpiresIn);
-
-      accessToken = responseBody.get(ACCESS_TOKEN).toString();
+    } catch (RuntimeException e) {
+      throw new IOException("Unable to obtain access token from credential", e);
     } catch (Exception e) {
       throw new IOException("Unable to obtain access token from credential", e);
     }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java

@@ -18,7 +18,6 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import okhttp3.MediaType;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -30,8 +29,8 @@ import org.apache.hadoop.classification.InterfaceStability;
 public final class OAuth2Constants {
   private OAuth2Constants() { /** Private constructor. **/ }
 
-  public static final MediaType URLENCODED
-      = MediaType.parse("application/x-www-form-urlencoded; charset=utf-8");
+  public static final String URLENCODED
+      = "application/x-www-form-urlencoded; charset=utf-8";
 
   /* Constants for OAuth protocol */
   public static final String ACCESS_TOKEN = "access_token";

hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml

@@ -199,6 +199,16 @@
       <artifactId>bcprov-jdk15on</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>mockwebserver</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib-jdk8</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-project/pom.xml

@@ -136,9 +136,7 @@
     <derby.version>10.14.2.0</derby.version>
     <mssql.version>6.2.1.jre7</mssql.version>
     <okhttp3.version>4.10.0</okhttp3.version>
-    <okio.version>3.4.0</okio.version>
-    <kotlin-stdlib.verion>1.6.20</kotlin-stdlib.verion>
-    <kotlin-stdlib-common.version>1.6.20</kotlin-stdlib-common.version>
+    <kotlin-stdlib.version>1.6.20</kotlin-stdlib.version>
     <jdom.version>1.1</jdom.version>
     <jna.version>5.2.0</jna.version>
     <gson.version>2.9.0</gson.version>
@@ -224,59 +222,31 @@
     <dependencies>
       <dependency>
         <groupId>com.squareup.okhttp3</groupId>
-        <artifactId>okhttp</artifactId>
+        <artifactId>mockwebserver</artifactId>
         <version>${okhttp3.version}</version>
+        <scope>test</scope>
         <exclusions>
-          <exclusion>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-stdlib</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib-common</artifactId>
           </exclusion>
           <exclusion>
-            <groupId>com.squareup.okio</groupId>
-            <artifactId>okio-jvm</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-        <groupId>com.squareup.okio</groupId>
-        <artifactId>okio-jvm</artifactId>
-        <version>${okio.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.jetbrains.kotlin</groupId>
-        <artifactId>kotlin-stdlib</artifactId>
-        <version>${kotlin-stdlib.verion}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.jetbrains</groupId>
-            <artifactId>annotations</artifactId>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-stdlib-jdk8</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
       <dependency>
         <groupId>org.jetbrains.kotlin</groupId>
         <artifactId>kotlin-stdlib-common</artifactId>
-        <version>${kotlin-stdlib-common.version}</version>
+        <version>${kotlin-stdlib.version}</version>
+        <scope>test</scope>
       </dependency>
       <dependency>
-        <groupId>com.squareup.okhttp3</groupId>
-        <artifactId>mockwebserver</artifactId>
-        <version>${okhttp3.version}</version>
+        <groupId>org.jetbrains.kotlin</groupId>
+        <artifactId>kotlin-stdlib-jdk8</artifactId>
+        <version>${kotlin-stdlib.version}</version>
         <scope>test</scope>
-        <exclusions>
-          <exclusion>
-            <groupId>com.squareup.okio</groupId>
-            <artifactId>okio-jvm</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-stdlib-jdk8</artifactId>
-          </exclusion>
-        </exclusions>
       </dependency>
       <dependency>
         <groupId>jdiff</groupId>