HADOOP-17651. Backport to branch-3.1 HADOOP-17371, HADOOP-17621, HADOOP-17625 to update Jetty to 9.4.39. (#2879) (#2935)

* HADOOP-17371. Bump Jetty to the latest version 9.4.34. Contributed by Wei-Chiu Chuang. (#2453)

(cherry picked from commit 66ee0a6df0dc0dd8242018153fd652a3206e73b5)
(cherry picked from commit 6340ac857b7ff3f73bbcf95b59b98aac134f33af)

 Conflicts:
	hadoop-client-modules/hadoop-client-minicluster/pom.xml

Change-Id: I673ac136922740cb1d426cb9593aa1bd3e9acd32

* HADOOP-17621. hadoop-auth to remove jetty-server dependency. (#2865)

Reviewed-by: Akira Ajisaka <aajisaka@apache.org>
(cherry picked from commit dac60b8282013d7776667415a429e7ca35efba66)
(cherry picked from commit 1110b03752b45bc4695baaa6d9655e18de67303a)

* HADOOP-17625. Update to Jetty 9.4.39. (#2870)

Reviewed-by: cxorm <lianp964@gmail.com>
(cherry picked from commit 6040e86e99aae5e29c17b03fddb0a805da8fcae8)
(cherry picked from commit 7f7535534d5541e10b6b14dee3aa38a00058f201)
(cherry picked from commit 8ff61f9b076a7022a1dfc067b9f94434756b64a7)

 Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java

Change-Id: I465f6003b6f4c5df9c41c83eac3738bac56403e1

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -782,6 +782,18 @@
                         <exclude>*/**</exclude>
                       </excludes>
                     </filter>
+                    <filter>
+                      <artifact>org.eclipse.jetty:jetty-util-ajax</artifact>
+                      <excludes>
+                        <exclude>*/**</exclude>
+                      </excludes>
+                    </filter>
+                    <filter>
+                      <artifact>org.eclipse.jetty:jetty-server</artifact>
+                      <excludes>
+                        <exclude>jetty-dir.css</exclude>
+                      </excludes>
+                    </filter>
                   </filters>
 
                   <!-- relocate classes from mssql-jdbc -->

hadoop-common-project/hadoop-auth/pom.xml

@@ -188,7 +188,7 @@
       <artifactId>guava</artifactId>
       <scope>compile</scope>
     </dependency>
-  </dependencies>
+ </dependencies>
 
   <build>
     <plugins>

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java

@@ -619,11 +619,17 @@ public class AuthenticationFilter implements Filter {
                 KerberosAuthenticator.WWW_AUTHENTICATE))) {
           errCode = HttpServletResponse.SC_FORBIDDEN;
         }
+        // After Jetty 9.4.21, sendError() no longer allows a custom message.
+        // use setStatus() to set a custom message.
+        String reason;
         if (authenticationEx == null) {
-          httpResponse.sendError(errCode, "Authentication required");
+          reason = "Authentication required";
         } else {
-          httpResponse.sendError(errCode, authenticationEx.getMessage());
+          reason = authenticationEx.getMessage();
         }
+
+        httpResponse.setStatus(errCode, reason);
+        httpResponse.sendError(errCode, reason);
       }
     }
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java

@@ -37,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 
+import org.eclipse.jetty.server.Response;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -271,6 +272,10 @@ public class RestCsrfPreventionFilter implements Filter {
 
     @Override
     public void sendError(int code, String message) throws IOException {
+      if (httpResponse instanceof Response) {
+        ((Response)httpResponse).setStatusWithReason(code, message);
+      }
+
       httpResponse.sendError(code, message);
     }
   }

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthentica
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
 import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
 import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
+import org.eclipse.jetty.server.Response;
 
 import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
@@ -113,6 +114,18 @@ public class KMSAuthenticationFilter
     public void sendError(int sc, String msg) throws IOException {
       statusCode = sc;
       this.msg = msg;
+
+      ServletResponse response = getResponse();
+
+      // After Jetty 9.4.21, sendError() no longer allows a custom message.
+      // use setStatusWithReason() to set a custom message.
+      if (response instanceof Response) {
+        ((Response) response).setStatusWithReason(sc, msg);
+      } else {
+        KMS.LOG.warn("The wrapped response object is instance of {}" +
+            ", not org.eclipse.jetty.server.Response. Can't set custom error " +
+            "message", response.getClass());
+      }
       super.sendError(sc, HtmlQuoting.quoteHtmlChars(msg));
     }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java

@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.eclipse.jetty.server.Response;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSUtil;
@@ -118,7 +119,7 @@ public class ImageServlet extends HttpServlet {
     if (nnImage == null) {
       String errorMsg = "NameNode initialization not yet complete. "
           + "FSImage has not been set in the NameNode.";
-      response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg);
+      sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg);
       throw new IOException(errorMsg);
     }
     return nnImage;
@@ -207,7 +208,7 @@ public class ImageServlet extends HttpServlet {
       
     } catch (Throwable t) {
       String errMsg = "GetImage failed. " + StringUtils.stringifyException(t);
-      response.sendError(HttpServletResponse.SC_GONE, errMsg);
+      sendError(response, HttpServletResponse.SC_GONE, errMsg);
       throw new IOException(errMsg);
     } finally {
       response.getOutputStream().close();
@@ -223,7 +224,7 @@ public class ImageServlet extends HttpServlet {
             conf)) {
       String errorMsg = "Only Namenode, Secondary Namenode, and administrators may access "
           + "this servlet";
-      response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg);
+      sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg);
       LOG.warn("Received non-NN/SNN/administrator request for image or edits from "
           + request.getUserPrincipal().getName()
           + " at "
@@ -236,7 +237,7 @@ public class ImageServlet extends HttpServlet {
         && !myStorageInfoString.equals(theirStorageInfoString)) {
       String errorMsg = "This namenode has storage info " + myStorageInfoString
           + " but the secondary expected " + theirStorageInfoString;
-      response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg);
+      sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg);
       LOG.warn("Received an invalid request file transfer request "
           + "from a secondary with storage info " + theirStorageInfoString);
       throw new IOException(errorMsg);
@@ -552,7 +553,7 @@ public class ImageServlet extends HttpServlet {
                 // we need a different response type here so the client can differentiate this
                 // from the failure to upload due to (1) security, or (2) other checkpoints already
                 // present
-                response.sendError(HttpServletResponse.SC_EXPECTATION_FAILED,
+                sendError(response, HttpServletResponse.SC_EXPECTATION_FAILED,
                     "Nameode "+request.getLocalAddr()+" is currently not in a state which can "
                         + "accept uploads of new fsimages. State: "+state);
                 return null;
@@ -567,7 +568,7 @@ public class ImageServlet extends HttpServlet {
               // if the node is attempting to upload an older transaction, we ignore it
               SortedSet<ImageUploadRequest> larger = currentlyDownloadingCheckpoints.tailSet(imageRequest);
               if (larger.size() > 0) {
-                response.sendError(HttpServletResponse.SC_CONFLICT,
+                sendError(response, HttpServletResponse.SC_CONFLICT,
                     "Another checkpointer is already in the process of uploading a" +
                         " checkpoint made up to transaction ID " + larger.last());
                 return null;
@@ -575,7 +576,7 @@ public class ImageServlet extends HttpServlet {
 
               //make sure no one else has started uploading one
               if (!currentlyDownloadingCheckpoints.add(imageRequest)) {
-                response.sendError(HttpServletResponse.SC_CONFLICT,
+                sendError(response, HttpServletResponse.SC_CONFLICT,
                     "Either current namenode is checkpointing or another"
                         + " checkpointer is already in the process of "
                         + "uploading a checkpoint made at transaction ID "
@@ -622,7 +623,7 @@ public class ImageServlet extends HttpServlet {
                     (txid - lastCheckpointTxid) + " expecting at least "
                     + checkpointTxnCount;
                 LOG.info(message);
-                response.sendError(HttpServletResponse.SC_CONFLICT, message);
+                sendError(response, HttpServletResponse.SC_CONFLICT, message);
                 return null;
               }
 
@@ -632,7 +633,7 @@ public class ImageServlet extends HttpServlet {
                       + "another checkpointer already uploaded an "
                       + "checkpoint for txid " + txid;
                   LOG.info(message);
-                  response.sendError(HttpServletResponse.SC_CONFLICT, message);
+                  sendError(response, HttpServletResponse.SC_CONFLICT, message);
                   return null;
                 }
 
@@ -669,11 +670,20 @@ public class ImageServlet extends HttpServlet {
           });
     } catch (Throwable t) {
       String errMsg = "PutImage failed. " + StringUtils.stringifyException(t);
-      response.sendError(HttpServletResponse.SC_GONE, errMsg);
+      sendError(response, HttpServletResponse.SC_GONE, errMsg);
       throw new IOException(errMsg);
     }
   }
 
+  private void sendError(HttpServletResponse response, int code, String message)
+      throws IOException {
+    if (response instanceof Response) {
+      ((Response)response).setStatusWithReason(code, message);
+    }
+
+    response.sendError(code, message);
+  }
+
   /*
    * Params required to handle put image request
    */

hadoop-project/pom.xml

@@ -35,7 +35,7 @@
 
     <failIfNoTests>false</failIfNoTests>
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
-    <jetty.version>9.4.20.v20190813</jetty.version>
+    <jetty.version>9.4.39.v20210325</jetty.version>
     <test.exclude>_</test.exclude>
     <test.exclude.pattern>_</test.exclude.pattern>