HDFS-1178. The NameNode servlets should not use RPC to connect to the NameNode. Contributed by Kan Zhang.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/trunk@979489 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -91,6 +91,9 @@ Trunk (unreleased changes)
     HDFS-1315. Add fsck event to audit log and remove other audit log events 
     corresponding to FSCK listStatus and open calls. (suresh)
 
+    HDFS-1178. The NameNode servlets should not use RPC to connect to the 
+    NameNode. (Kan Zhang via jghoman)
+
   OPTIMIZATIONS
 
     HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)

src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hdfs.server.common;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.InetSocketAddress;
@@ -470,7 +472,6 @@ public class JspHelper {
                                             Configuration conf
                                            ) throws IOException {
     UserGroupInformation ugi = null;
-    final String RANDOM_USER = "webuser1234";
     if(UserGroupInformation.isSecurityEnabled()) {
       String user = request.getRemoteUser();
       String tokenString = request.getParameter(DELEGATION_PARAMETER_NAME);
@@ -478,19 +479,18 @@ public class JspHelper {
         Token<DelegationTokenIdentifier> token = 
           new Token<DelegationTokenIdentifier>();
         token.decodeFromUrlString(tokenString);
-        if (user == null) {
-          //this really doesn't break any security since we use the 
-          //delegation token for authentication in
-          //the back end.
-          user = RANDOM_USER;
-        }
         InetSocketAddress serviceAddr = NameNode.getAddress(conf);
         LOG.info("Setting service in token: "
             + new Text(serviceAddr.getAddress().getHostAddress() + ":"
                 + serviceAddr.getPort()));
         token.setService(new Text(serviceAddr.getAddress().getHostAddress()
             + ":" + serviceAddr.getPort()));
-        ugi = UserGroupInformation.createRemoteUser(user);
+        ByteArrayInputStream buf = new ByteArrayInputStream(token
+            .getIdentifier());
+        DataInputStream in = new DataInputStream(buf);
+        DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+        id.readFields(in);
+        ugi = id.getUser();
         ugi.addToken(token);
         ugi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
       } else {

src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java

@@ -30,7 +30,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.znerd.xmlenc.XMLOutputter;
 
@@ -47,9 +46,9 @@ public class ContentSummaryServlet extends DfsServlet {
       (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
     final UserGroupInformation ugi = getUGI(request, conf);
     try {
-      ugi.doAs(new PrivilegedExceptionAction<Object>() {
+      ugi.doAs(new PrivilegedExceptionAction<Void>() {
         @Override
-        public Object run() throws Exception {
+        public Void run() throws Exception {
           final String path = request.getPathInfo();
 
           final PrintWriter out = response.getWriter();
@@ -72,8 +71,7 @@ public class ContentSummaryServlet extends DfsServlet {
             }
             xml.endTag();
           } catch(IOException ioe) {
-            new RemoteException(ioe.getClass().getName(), ioe.getMessage()
-                ).writeXml(path, xml);
+            writeXml(ioe, path, xml);
           }
           xml.endDocument();
           return null;

src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java

@@ -36,7 +36,9 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.znerd.xmlenc.XMLOutputter;
 
 /**
  * A base class for the servlets in DFS.
@@ -47,11 +49,36 @@ abstract class DfsServlet extends HttpServlet {
 
   static final Log LOG = LogFactory.getLog(DfsServlet.class.getCanonicalName());
 
+  /** Write the object to XML format */
+  protected void writeXml(Exception except, String path, XMLOutputter doc)
+      throws IOException {
+    doc.startTag(RemoteException.class.getSimpleName());
+    doc.attribute("path", path);
+    if (except instanceof RemoteException) {
+      doc.attribute("class", ((RemoteException) except).getClassName());
+    } else {
+      doc.attribute("class", except.getClass().getName());
+    }
+    String msg = except.getLocalizedMessage();
+    int i = msg.indexOf("\n");
+    if (i >= 0) {
+      msg = msg.substring(0, i);
+    }
+    doc.attribute("message", msg.substring(msg.indexOf(":") + 1).trim());
+    doc.endTag();
+  }
+
   /**
    * Create a {@link NameNode} proxy from the current {@link ServletContext}. 
    */
   protected ClientProtocol createNameNodeProxy() throws IOException {
     ServletContext context = getServletContext();
+    // if we are running in the Name Node, use it directly rather than via 
+    // rpc
+    NameNode nn = (NameNode) context.getAttribute("name.node");
+    if (nn != null) {
+      return nn;
+    }
     InetSocketAddress nnAddr = (InetSocketAddress)context.getAttribute("name.node.address");
     Configuration conf = new HdfsConfiguration(
         (Configuration)context.getAttribute(JspHelper.CURRENT_CONF));

src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java

@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.znerd.xmlenc.XMLOutputter;
@@ -107,12 +106,9 @@ public class FileChecksumServlets {
             filename, nnproxy, socketFactory, socketTimeout);
         MD5MD5CRC32FileChecksum.write(xml, checksum);
       } catch(IOException ioe) {
-        new RemoteException(ioe.getClass().getName(), ioe.getMessage()
-            ).writeXml(filename, xml);
+        writeXml(ioe, filename, xml);
       } catch (InterruptedException e) {
-        new RemoteException(e.getClass().getName(), e.getMessage()
-        ).writeXml(filename, xml);
-        
+        writeXml(e, filename, xml);
       }
       xml.endDocument();
     }

src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java

@@ -94,39 +94,40 @@ public class FileDataServlet extends DfsServlet {
    * GET http://<nn>:<port>/data[/<path>] HTTP/1.1
    * }
    */
-  public void doGet(HttpServletRequest request, HttpServletResponse response)
+  public void doGet(final HttpServletRequest request,
+      final HttpServletResponse response)
       throws IOException {
     final Configuration conf = 
       (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
     final UserGroupInformation ugi = getUGI(request, conf);
 
     try {
-      final ClientProtocol nnproxy = ugi
-          .doAs(new PrivilegedExceptionAction<ClientProtocol>() {
-            @Override
-            public ClientProtocol run() throws IOException {
-              return createNameNodeProxy();
-            }
-          });
-
-      final String path =
-        request.getPathInfo() != null ? request.getPathInfo() : "/";
+      ugi.doAs(new PrivilegedExceptionAction<Void>() {
+        @Override
+        public Void run() throws IOException {
+          ClientProtocol nn = createNameNodeProxy();
+          final String path = request.getPathInfo() != null ? request
+              .getPathInfo() : "/";
 
-      String delegationToken = 
-        request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
+          String delegationToken = request
+              .getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
 
-      HdfsFileStatus info = nnproxy.getFileInfo(path);
-      if ((info != null) && !info.isDir()) {
-        String redirect = createUri(path, info, ugi, nnproxy,
-              request, delegationToken).toURL().toString();
-        response.sendRedirect(redirect);
-      } else if (info == null){
-        response.sendError(400, "cat: File not found " + path);
-      } else {
-        response.sendError(400, "cat: " + path + ": is a directory");
-      }
-    } catch (URISyntaxException e) {
-      response.getWriter().println(e.toString());
+          HdfsFileStatus info = nn.getFileInfo(path);
+          if ((info != null) && !info.isDir()) {
+            try {
+              response.sendRedirect(createUri(path, info, ugi, nn, request,
+                  delegationToken).toURL().toString());
+            } catch (URISyntaxException e) {
+              response.getWriter().println(e.toString());
+            }
+          } else if (info == null) {
+            response.sendError(400, "cat: File not found " + path);
+          } else {
+            response.sendError(400, "cat: " + path + ": is a directory");
+          }
+          return null;
+        }
+      });
     } catch (IOException e) {
       response.sendError(400, e.getMessage());
     } catch (InterruptedException e) {

src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java

@@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.util.VersionInfo;
 
 import org.znerd.xmlenc.*;
@@ -141,60 +140,61 @@ public class ListPathsServlet extends DfsServlet {
       final Configuration conf = 
         (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
       
-      ClientProtocol nnproxy = getUGI(request, conf).doAs
-        (new PrivilegedExceptionAction<ClientProtocol>() {
+      getUGI(request, conf).doAs(new PrivilegedExceptionAction<Void>() {
         @Override
-        public ClientProtocol run() throws IOException {
-          return createNameNodeProxy();
-        }
-      });
+        public Void run() throws IOException {
+          ClientProtocol nn = createNameNodeProxy();
+          doc.declaration();
+          doc.startTag("listing");
+          for (Map.Entry<String, String> m : root.entrySet()) {
+            doc.attribute(m.getKey(), m.getValue());
+          }
 
-      doc.declaration();
-      doc.startTag("listing");
-      for (Map.Entry<String,String> m : root.entrySet()) {
-        doc.attribute(m.getKey(), m.getValue());
-      }
-
-      HdfsFileStatus base = nnproxy.getFileInfo(path);
-      if ((base != null) && base.isDir()) {
-        writeInfo(path, base, doc);
-      }
+          HdfsFileStatus base = nn.getFileInfo(path);
+          if ((base != null) && base.isDir()) {
+            writeInfo(path, base, doc);
+          }
 
-      Stack<String> pathstack = new Stack<String>();
-      pathstack.push(path);
-      while (!pathstack.empty()) {
-        String p = pathstack.pop();
-        try {
-          byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;
-          DirectoryListing thisListing;
-          do {
-            assert lastReturnedName != null;
-            thisListing = nnproxy.getListing(p, lastReturnedName);
-            if (thisListing == null) {
-              if (lastReturnedName.length == 0) {
-                LOG.warn("ListPathsServlet - Path " + p + " does not exist");
-              }
-              break;
+          Stack<String> pathstack = new Stack<String>();
+          pathstack.push(path);
+          while (!pathstack.empty()) {
+            String p = pathstack.pop();
+            try {
+              byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;
+              DirectoryListing thisListing;
+              do {
+                assert lastReturnedName != null;
+                thisListing = nn.getListing(p, lastReturnedName);
+                if (thisListing == null) {
+                  if (lastReturnedName.length == 0) {
+                    LOG
+                        .warn("ListPathsServlet - Path " + p
+                            + " does not exist");
+                  }
+                  break;
+                }
+                HdfsFileStatus[] listing = thisListing.getPartialListing();
+                for (HdfsFileStatus i : listing) {
+                  String localName = i.getLocalName();
+                  if (exclude.matcher(localName).matches()
+                      || !filter.matcher(localName).matches()) {
+                    continue;
+                  }
+                  if (recur && i.isDir()) {
+                    pathstack.push(new Path(p, localName).toUri().getPath());
+                  }
+                  writeInfo(p, i, doc);
+                }
+                lastReturnedName = thisListing.getLastName();
+              } while (thisListing.hasMore());
+            } catch (IOException re) {
+              writeXml(re, p, doc);
             }
-            HdfsFileStatus[] listing = thisListing.getPartialListing();
-            for (HdfsFileStatus i : listing) {
-              String localName = i.getLocalName();
-              if (exclude.matcher(localName).matches()
-                  || !filter.matcher(localName).matches()) {
-                continue;
-              }
-              if (recur && i.isDir()) {
-                pathstack.push(new Path(p, localName).toUri().getPath());
-              }
-              writeInfo(p, i, doc);
-            }
-            lastReturnedName = thisListing.getLastName();
-          } while (thisListing.hasMore());
-        } catch(RemoteException re) {re.writeXml(p, doc);}
-      }
-      if (doc != null) {
-        doc.endDocument();
-      }
+          }
+          doc.endDocument();
+          return null;
+        }
+      });
     } catch (InterruptedException e) {
       LOG.warn("ListPathServlet encountered InterruptedException", e);
       response.sendError(400, e.getMessage());