HADOOP-4284. Support filters that apply to all requests, or global filters,

to HttpServer. Contributed by Kan Zhang.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@706518 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -43,6 +43,9 @@ Trunk (unreleased changes)
     HADOOP-4238. When listing jobs, if scheduling information isn't available 
     print NA instead of empty output. (Sreekanth Ramakrishnan via johan)
 
+    HADOOP-4284. Support filters that apply to all requests, or global filters,
+    to HttpServer. (Kan Zhang via cdouglas)
+
   OPTIMIZATIONS
 
   BUG FIXES

src/core/org/apache/hadoop/http/FilterContainer.java

@@ -30,4 +30,11 @@ public interface FilterContainer {
    * @param parameters a map from parameter names to initial values
    */
   void addFilter(String name, String classname, Map<String, String> parameters);
+  /**
+   * Add a global filter to the container.
+   * @param name filter name
+   * @param classname filter class name
+   * @param parameters a map from parameter names to initial values
+   */
+  void addGlobalFilter(String name, String classname, Map<String, String> parameters);
 }

src/core/org/apache/hadoop/http/HttpServer.java

@@ -24,6 +24,7 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
@@ -35,10 +36,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.log.LogLevel;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.mortbay.http.HttpContext;
 import org.mortbay.http.SocketListener;
 import org.mortbay.http.SslListener;
-import org.mortbay.http.handler.ResourceHandler;
 import org.mortbay.jetty.servlet.Dispatcher;
 import org.mortbay.jetty.servlet.FilterHolder;
 import org.mortbay.jetty.servlet.WebApplicationContext;
@@ -60,6 +59,8 @@ public class HttpServer implements FilterContainer {
 
   protected final org.mortbay.jetty.Server webServer;
   protected final WebApplicationContext webAppContext;
+  protected final Map<WebApplicationContext, Boolean> defaultContexts = 
+      new HashMap<WebApplicationContext, Boolean>();
   protected final boolean findPort;
   protected final SocketListener listener;
   private SslListener sslListener;
@@ -91,6 +92,7 @@ public class HttpServer implements FilterContainer {
 
     final String appDir = getWebAppsPath();
     webAppContext = webServer.addWebApplication("/", appDir + "/" + name);
+    addDefaultApps(appDir);
 
     final FilterInitializer[] initializers = getFilterInitializers(conf); 
     if (initializers != null) {
@@ -98,7 +100,7 @@ public class HttpServer implements FilterContainer {
         c.initFilter(this);
       }
     }
-    addWebapps(appDir);
+    addDefaultServlets();
   }
 
   /** Get an array of FilterConfiguration specified in the conf */
@@ -121,11 +123,11 @@ public class HttpServer implements FilterContainer {
   }
 
   /**
-   * Add webapps and servlets.
+   * Add default apps.
    * @param appDir The application directory
    * @throws IOException
    */
-  protected void addWebapps(final String appDir) throws IOException {
+  protected void addDefaultApps(final String appDir) throws IOException {
     // set up the context for "/logs/" if "hadoop.log.dir" property is defined. 
     String logDir = System.getProperty("hadoop.log.dir");
     if (logDir != null) {
@@ -134,7 +136,12 @@ public class HttpServer implements FilterContainer {
 
     // set up the context for "/static/*"
     addContext("/static/*", appDir + "/static", true);
-
+  }
+  
+  /**
+   * Add default servlets.
+   */
+  protected void addDefaultServlets() {
     // set up default servlets
     addServlet("stacks", "/stacks", StackServlet.class);
     addServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
@@ -145,16 +152,11 @@ public class HttpServer implements FilterContainer {
    * @param pathSpec The path spec for the context
    * @param dir The directory containing the context
    * @param isFiltered if true, the servlet is added to the filter path mapping 
+   * @throws IOException
    */
-  protected void addContext(String pathSpec, String dir, boolean isFiltered) {
-    HttpContext context = new HttpContext();
-    context.setContextPath(pathSpec);
-    context.setResourceBase(dir);
-    context.addHandler(new ResourceHandler());
-    webServer.addContext(context);
-    if (isFiltered) {
-      addFilterPathMapping(pathSpec);
-    }
+  protected void addContext(String pathSpec, String dir, boolean isFiltered) throws IOException {
+    WebApplicationContext webAppCtx = webServer.addWebApplication(pathSpec, dir);
+    defaultContexts.put(webAppCtx, isFiltered);
   }
 
   /**
@@ -176,7 +178,7 @@ public class HttpServer implements FilterContainer {
   public void addServlet(String name, String pathSpec,
       Class<? extends HttpServlet> clazz) {
     addInternalServlet(name, pathSpec, clazz);
-    addFilterPathMapping(pathSpec);
+    addFilterPathMapping(pathSpec, webAppContext);
   }
 
   /**
@@ -207,11 +209,42 @@ public class HttpServer implements FilterContainer {
   /** {@inheritDoc} */
   public void addFilter(String name, String classname,
       Map<String, String> parameters) {
-    WebApplicationHandler handler = webAppContext.getWebApplicationHandler();
 
-    LOG.info("adding " + name + " (class=" + classname + ")");
+    final String[] USER_FACING_URLS = { "*.html", "*.jsp" };
+    defineFilter(webAppContext, name, classname, parameters, USER_FACING_URLS);
+    for (Map.Entry<WebApplicationContext, Boolean> e : defaultContexts
+        .entrySet()) {
+      if (e.getValue()) {
+        WebApplicationContext ctx = e.getKey();
+        defineFilter(ctx, name, classname, parameters, USER_FACING_URLS);
+        WebApplicationHandler handler = ctx.getWebApplicationHandler();
+        handler.addFilterPathMapping(ctx.getContextPath() + "/*", name,
+            Dispatcher.__ALL);
+        LOG.info("Added filter " + name + " (class=" + classname
+            + ") to context path " + ctx.getContextPath() + "/*");
+      }
+    }
     filterNames.add(name);
+  }
 
+  /** {@inheritDoc} */
+  public void addGlobalFilter(String name, String classname,
+      Map<String, String> parameters) {
+    final String[] ALL_URLS = { "/*" };
+    defineFilter(webAppContext, name, classname, parameters, ALL_URLS);
+    for (WebApplicationContext ctx : defaultContexts.keySet()) {
+      defineFilter(ctx, name, classname, parameters, ALL_URLS);
+    }
+    LOG.info("Added global filter" + name + " (class=" + classname + ")");
+  }
+
+  /**
+   * Define a filter for a context and set up default url mappings.
+   */
+  protected void defineFilter(WebApplicationContext ctx, String name,
+      String classname, Map<String, String> parameters, String[] urls) {
+
+    WebApplicationHandler handler = ctx.getWebApplicationHandler();
     FilterHolder holder = handler.defineFilter(name, classname);
     if (parameters != null) {
       for(Map.Entry<String, String> e : parameters.entrySet()) {
@@ -219,8 +252,7 @@ public class HttpServer implements FilterContainer {
       }
     }
 
-    final String[] USER_FACING_URLS = {"*.html", "*.jsp"};
-    for(String url : USER_FACING_URLS) {
+    for (String url : urls) {
       handler.addFilterPathMapping(url, name, Dispatcher.__ALL);
     }
   }
@@ -228,14 +260,15 @@ public class HttpServer implements FilterContainer {
   /**
    * Add the path spec to the filter path mapping.
    * @param pathSpec The path spec
+   * @param webAppCtx The WebApplicationContext to add to
    */
-  protected void addFilterPathMapping(String pathSpec) {
-    WebApplicationHandler handler = webAppContext.getWebApplicationHandler();
+  protected void addFilterPathMapping(String pathSpec, WebApplicationContext webAppCtx) {
+    WebApplicationHandler handler = webAppCtx.getWebApplicationHandler();
     for(String name : filterNames) {
       handler.addFilterPathMapping(pathSpec, name, Dispatcher.__ALL);
     }
   }
-
+  
   /**
    * Get the value in the webapp context.
    * @param name The name of the attribute

src/test/org/apache/hadoop/http/TestGlobalFilter.java

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.http;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+public class TestGlobalFilter extends junit.framework.TestCase {
+  static final Log LOG = LogFactory.getLog(HttpServer.class);
+  static final Map<String, Integer> COUNTS = new TreeMap<String, Integer>(); 
+
+  /** A very simple filter which counts number of accesses for each uri */
+  static public class CountingFilter implements Filter {
+    private FilterConfig filterConfig = null;
+
+    public void init(FilterConfig filterConfig) {
+      this.filterConfig = filterConfig;
+    }
+
+    public void destroy() {
+      this.filterConfig = null;
+    }
+
+    public void doFilter(ServletRequest request, ServletResponse response,
+        FilterChain chain) throws IOException, ServletException {
+      if (filterConfig == null)
+         return;
+
+      String uri = ((HttpServletRequest)request).getRequestURI();
+      LOG.info("filtering " + uri);
+      Integer value = COUNTS.get(uri);
+      value = value == null? 1: value + 1;
+      COUNTS.put(uri, value);
+      chain.doFilter(request, response);
+    }
+
+    /** Configuration for CountingFilter */
+    static public class Initializer extends FilterInitializer {
+      public Initializer() {}
+
+      void initFilter(FilterContainer container) {
+        container.addGlobalFilter("counting", CountingFilter.class.getName(), null);
+      }
+    }
+  }
+  
+  
+  /** access a url, ignoring some IOException such as the page does not exist */
+  static void access(String urlstring) throws IOException {
+    LOG.warn("access " + urlstring);
+    URL url = new URL(urlstring);
+    URLConnection connection = url.openConnection();
+    connection.connect();
+    
+    try {
+      BufferedReader in = new BufferedReader(new InputStreamReader(
+          connection.getInputStream()));
+      try {
+        for(; in.readLine() != null; );
+      } finally {
+        in.close();
+      }
+    } catch(IOException ioe) {
+      LOG.warn("urlstring=" + urlstring, ioe);
+    }
+  }
+
+  public void testServletFilter() throws Exception {
+    Configuration conf = new Configuration();
+    
+    //start a http server with CountingFilter
+    conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+        CountingFilter.Initializer.class.getName());
+    HttpServer http = new HttpServer("datanode", "localhost", 0, true, conf);
+    http.start();
+
+    final String fsckURL = "/fsck";
+    final String stacksURL = "/stacks";
+    final String ajspURL = "/a.jsp";
+    final String listPathsURL = "/listPaths";
+    final String dataURL = "/data";
+    final String streamFile = "/streamFile";
+    final String rootURL = "/";
+    final String allURL = "/*";
+    final String outURL = "/static/a.out";
+    final String logURL = "/logs/a.log";
+
+    final String[] urls = {fsckURL, stacksURL, ajspURL, listPathsURL, 
+        dataURL, streamFile, rootURL, allURL, outURL, logURL};
+    final Random ran = new Random();
+    final int[] sequence = new int[100];
+    final int[] counts = new int[urls.length]; 
+
+    //generate a random sequence and update counts 
+    for(int i = 0; i < sequence.length; i++) {
+      sequence[i] = ran.nextInt(urls.length);
+      counts[sequence[i]]++;
+    }
+
+    //access the urls as the sequence
+    final String prefix = "http://localhost:" + http.getPort();
+    try {
+      for(int i = 0; i < sequence.length; i++) {
+        access(prefix + urls[sequence[i]]);
+      }
+    } finally {
+      http.stop();
+    }
+
+    LOG.info("COUNTS = " + COUNTS);
+    
+    //verify counts
+    for(int i = 0; i < urls.length; i++) {
+      if (counts[i] == 0) {
+        assertFalse(COUNTS.containsKey(urls[i]));
+      } else {
+        assertEquals(counts[i], COUNTS.remove(urls[i]).intValue());
+      }
+    }
+    assertTrue(COUNTS.isEmpty());
+  }
+}