HADOOP-12847. hadoop daemonlog should support https and SPNEGO for Kerberized cluster. (Wei-Chiu Chuang via Yongjun Zhang)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java

@@ -17,20 +17,38 @@
  */
 package org.apache.hadoop.log;
 
-import java.io.*;
-import java.net.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.net.URLConnection;
 import java.util.regex.Pattern;
 
-import javax.servlet.*;
-import javax.servlet.http.*;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Charsets;
-import org.apache.commons.logging.*;
-import org.apache.commons.logging.impl.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Jdk14Logger;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
 
 /**
  * Change log level in runtime.
@@ -38,43 +56,252 @@ import org.apache.hadoop.util.ServletUtil;
 @InterfaceStability.Evolving
 public class LogLevel {
   public static final String USAGES = "\nUsage: General options are:\n"
-      + "\t[-getlevel <host:httpPort> <classname>]\n"
-      + "\t[-setlevel <host:httpPort> <classname> <level>]\n";
+      + "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
+      + "\t[-setlevel <host:port> <classname> <level> "
+      + "[-protocol (http|https)]\n";
 
+  public static final String PROTOCOL_HTTP = "http";
+  public static final String PROTOCOL_HTTPS = "https";
   /**
    * A command line implementation
    */
-  public static void main(String[] args) {
-    if (args.length == 3 && "-getlevel".equals(args[0])) {
-      process("http://" + args[1] + "/logLevel?log=" + args[2]);
-      return;
-    }
-    else if (args.length == 4 && "-setlevel".equals(args[0])) {
-      process("http://" + args[1] + "/logLevel?log=" + args[2]
-              + "&level=" + args[3]);
-      return;
-    }
+  public static void main(String[] args) throws Exception {
+    CLI cli = new CLI(new Configuration());
+    System.exit(cli.run(args));
+  }
+
+  /**
+   * Valid command line options.
+   */
+  private enum Operations {
+    GETLEVEL,
+    SETLEVEL,
+    UNKNOWN
+  }
 
+  private static void printUsage() {
     System.err.println(USAGES);
-    System.exit(-1);
   }
 
-  private static void process(String urlstring) {
-    try {
-      URL url = new URL(urlstring);
-      System.out.println("Connecting to " + url);
-      URLConnection connection = url.openConnection();
+  public static boolean isValidProtocol(String protocol) {
+    return ((protocol.equals(PROTOCOL_HTTP) ||
+      protocol.equals(PROTOCOL_HTTPS)));
+  }
+
+  @VisibleForTesting
+  static class CLI extends Configured implements Tool {
+    private Operations operation = Operations.UNKNOWN;
+    private String protocol;
+    private String hostName;
+    private String className;
+    private String level;
+
+    CLI(Configuration conf) {
+      setConf(conf);
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+      try {
+        parseArguments(args);
+        sendLogLevelRequest();
+      } catch (HadoopIllegalArgumentException e) {
+        printUsage();
+        throw e;
+      }
+      return 0;
+    }
+
+    /**
+     * Send HTTP/HTTPS request to the daemon.
+     * @throws HadoopIllegalArgumentException if arguments are invalid.
+     * @throws Exception if unable to connect
+     */
+    private void sendLogLevelRequest()
+        throws HadoopIllegalArgumentException, Exception {
+      switch (operation) {
+      case GETLEVEL:
+        doGetLevel();
+        break;
+      case SETLEVEL:
+        doSetLevel();
+        break;
+      default:
+        throw new HadoopIllegalArgumentException(
+          "Expect either -getlevel or -setlevel");
+      }
+    }
+
+    public void parseArguments(String[] args) throws
+        HadoopIllegalArgumentException {
+      if (args.length == 0) {
+        throw new HadoopIllegalArgumentException("No arguments specified");
+      }
+      int nextArgIndex = 0;
+      while (nextArgIndex < args.length) {
+        if (args[nextArgIndex].equals("-getlevel")) {
+          nextArgIndex = parseGetLevelArgs(args, nextArgIndex);
+        } else if (args[nextArgIndex].equals("-setlevel")) {
+          nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
+        } else if (args[nextArgIndex].equals("-protocol")) {
+          nextArgIndex = parseProtocolArgs(args, nextArgIndex);
+        } else {
+          throw new HadoopIllegalArgumentException(
+              "Unexpected argument " + args[nextArgIndex]);
+        }
+      }
+
+      // if operation is never specified in the arguments
+      if (operation == Operations.UNKNOWN) {
+        throw new HadoopIllegalArgumentException(
+            "Must specify either -getlevel or -setlevel");
+      }
+
+      // if protocol is unspecified, set it as http.
+      if (protocol == null) {
+        protocol = PROTOCOL_HTTP;
+      }
+    }
+
+    private int parseGetLevelArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // fail if multiple operations are specified in the arguments
+      if (operation != Operations.UNKNOWN) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -getlevel command");
+      }
+      // check number of arguments is sufficient
+      if (index+2 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-getlevel needs two parameters");
+      }
+      operation = Operations.GETLEVEL;
+      hostName = args[index+1];
+      className = args[index+2];
+      return index+3;
+    }
+
+    private int parseSetLevelArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // fail if multiple operations are specified in the arguments
+      if (operation != Operations.UNKNOWN) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -setlevel command");
+      }
+      // check number of arguments is sufficient
+      if (index+3 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-setlevel needs three parameters");
+      }
+      operation = Operations.SETLEVEL;
+      hostName = args[index+1];
+      className = args[index+2];
+      level = args[index+3];
+      return index+4;
+    }
+
+    private int parseProtocolArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // make sure only -protocol is specified
+      if (protocol != null) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -protocol command");
+      }
+      // check number of arguments is sufficient
+      if (index+1 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-protocol needs one parameter");
+      }
+      // check protocol is valid
+      protocol = args[index+1];
+      if (!isValidProtocol(protocol)) {
+        throw new HadoopIllegalArgumentException(
+            "Invalid protocol: " + protocol);
+      }
+      return index+2;
+    }
+
+    /**
+     * Send HTTP/HTTPS request to get log level.
+     *
+     * @throws HadoopIllegalArgumentException if arguments are invalid.
+     * @throws Exception if unable to connect
+     */
+    private void doGetLevel() throws Exception {
+      process(protocol + "://" + hostName + "/logLevel?log=" + className);
+    }
+
+    /**
+     * Send HTTP/HTTPS request to set log level.
+     *
+     * @throws HadoopIllegalArgumentException if arguments are invalid.
+     * @throws Exception if unable to connect
+     */
+    private void doSetLevel() throws Exception {
+      process(protocol + "://" + hostName + "/logLevel?log=" + className
+          + "&level=" + level);
+    }
+
+    /**
+     * Connect to the URL. Supports HTTP/HTTPS and supports SPNEGO
+     * authentication. It falls back to simple authentication if it fails to
+     * initiate SPNEGO.
+     *
+     * @param url the URL address of the daemon servlet
+     * @return a connected connection
+     * @throws Exception if it can not establish a connection.
+     */
+    private URLConnection connect(URL url) throws Exception {
+      AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+      AuthenticatedURL aUrl;
+      SSLFactory clientSslFactory;
+      URLConnection connection;
+      // If https is chosen, configures SSL client.
+      if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
+        clientSslFactory = new SSLFactory(
+            SSLFactory.Mode.CLIENT, this.getConf());
+        clientSslFactory.init();
+        SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+
+        aUrl = new AuthenticatedURL(
+            new KerberosAuthenticator(), clientSslFactory);
+        connection = aUrl.openConnection(url, token);
+        HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+        httpsConn.setSSLSocketFactory(sslSocketF);
+      } else {
+        aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+        connection = aUrl.openConnection(url, token);
+      }
+
       connection.connect();
+      return connection;
+    }
+
+    /**
+     * Configures the client to send HTTP/HTTPS request to the URL.
+     * Supports SPENGO for authentication.
+     * @param urlString URL and query string to the daemon's web UI
+     * @throws Exception if unable to connect
+     */
+    private void process(String urlString) throws Exception {
+      URL url = new URL(urlString);
+      System.out.println("Connecting to " + url);
 
-      BufferedReader in = new BufferedReader(new InputStreamReader(
-          connection.getInputStream(), Charsets.UTF_8));
-      for(String line; (line = in.readLine()) != null; )
+      URLConnection connection = connect(url);
+
+      // read from the servlet
+      BufferedReader in = new BufferedReader(
+          new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
+      for (String line;;) {
+        line = in.readLine();
+        if (line == null) {
+          break;
+        }
         if (line.startsWith(MARKER)) {
           System.out.println(TAG.matcher(line).replaceAll(""));
         }
+      }
       in.close();
-    } catch (IOException ioe) {
-      System.err.println("" + ioe);
     }
   }
 

hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md

@@ -229,17 +229,37 @@ Commands useful for administrators of a hadoop cluster.
 
 Usage:
 
-    hadoop daemonlog -getlevel <host:httpport> <classname>
-    hadoop daemonlog -setlevel <host:httpport> <classname> <level>
+    hadoop daemonlog -getlevel <host:port> <classname> [-protocol (http|https)]
+    hadoop daemonlog -setlevel <host:port> <classname> <level> [-protocol (http|https)]
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
-| `-getlevel` *host:httpport* *classname* | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>` |
-| `-setlevel` *host:httpport* *classname* *level* | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>&level=<level>` |
+| `-getlevel` *host:port* *classname* [-protocol (http|https)] | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*. The `-protocol` flag specifies the protocol for connection. |
+| `-setlevel` *host:port* *classname* *level* [-protocol (http|https)] | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*.  The `-protocol` flag specifies the protocol for connection. |
 
-Get/Set the log level for a Log identified by a qualified class name in the daemon.
+Get/Set the log level for a Log identified by a qualified class name in the daemon dynamically.
+By default, the command sends a HTTP request, but this can be overridden by using argument `-protocol https` to send a HTTPS request.
+
+Example:
+
+    $ bin/hadoop daemonlog -setlevel 127.0.0.1:9870 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
+    $ bin/hadoop daemonlog -getlevel 127.0.0.1:9871 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG -protocol https
+
+Note that the setting is not permanent and will be reset when the daemon is restarted.
+This command works by sending a HTTP/HTTPS request to the daemon's internal Jetty servlet, so it supports the following daemons:
+
+* HDFS
+    * name node
+    * secondary name node
+    * data node
+    * journal node
+* YARN
+    * resource manager
+    * node manager
+    * Timeline server
+
+However, the command does not support KMS server, because its web interface is based on Tomcat, which does not support the servlet.
 
-	Example: $ bin/hadoop daemonlog -setlevel 127.0.0.1:9870 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
 
 Files
 -----

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java

@@ -17,99 +17,404 @@
 */
 package org.apache.hadoop.log;
 
-import java.io.*;
-import java.net.*;
+import java.io.File;
+import java.net.SocketException;
+import java.net.URI;
+import java.util.concurrent.Callable;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.log.LogLevel.CLI;
+import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.KerberosTestUtils;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.junit.Assert;
 
-import junit.framework.TestCase;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
 
-import org.apache.commons.logging.*;
-import org.apache.commons.logging.impl.*;
-import org.apache.log4j.*;
-import org.junit.Assert;
+import javax.net.ssl.SSLException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test LogLevel.
+ */
+public class TestLogLevel extends KerberosSecurityTestcase {
+  private static final File BASEDIR = GenericTestUtils.getRandomizedTestDir();
+  private static String keystoresDir;
+  private static String sslConfDir;
+  private static Configuration conf;
+  private static Configuration sslConf;
+  private final String logName = TestLogLevel.class.getName();
+  private String clientPrincipal;
+  private String serverPrincipal;
+  private final Log testlog = LogFactory.getLog(logName);
+  private final Logger log = ((Log4JLogger)testlog).getLogger();
+  private final static String PRINCIPAL = "loglevel.principal";
+  private final static String KEYTAB  = "loglevel.keytab";
 
-public class TestLogLevel extends TestCase {
-  static final PrintStream out = System.out;
-
-  public void testDynamicLogLevel() throws Exception {
-    String logName = TestLogLevel.class.getName();
-    Log testlog = LogFactory.getLog(logName);
-
-    //only test Log4JLogger
-    if (testlog instanceof Log4JLogger) {
-      Logger log = ((Log4JLogger)testlog).getLogger();
-      log.debug("log.debug1");
-      log.info("log.info1");
-      log.error("log.error1");
-      Assert.assertNotEquals("Get default Log Level which shouldn't be ERROR.",
-          Level.ERROR, log.getEffectiveLevel());
-
-      HttpServer2 server = new HttpServer2.Builder().setName("..")
-          .addEndpoint(new URI("http://localhost:0")).setFindPort(true)
-          .build();
-
-      server.start();
-      String authority = NetUtils.getHostPortString(server
-          .getConnectorAddress(0));
-
-      //servlet
-      URL url = new URL("http://" + authority + "/logLevel?log=" + logName
-          + "&level=" + Level.ERROR);
-      out.println("*** Connecting to " + url);
-      URLConnection connection = url.openConnection();
-      connection.connect();
-
-      BufferedReader in = new BufferedReader(new InputStreamReader(
-          connection.getInputStream()));
-      for(String line; (line = in.readLine()) != null; out.println(line));
-      in.close();
-
-      log.debug("log.debug2");
-      log.info("log.info2");
-      log.error("log.error2");
-      assertEquals("Try setting log level: ERROR from servlet.", Level.ERROR,
-          log.getEffectiveLevel());
-
-      //command line
-      String[] args = {"-setlevel", authority, logName, Level.DEBUG.toString()};
-      LogLevel.main(args);
-      log.debug("log.debug3");
-      log.info("log.info3");
-      log.error("log.error3");
-      assertEquals("Try setting log level: DEBUG via command line", Level.DEBUG,
-          log.getEffectiveLevel());
-
-      // Test mixed upper case and lower case in level string.
-      String[] args2 = {"-setlevel", authority, logName, "Info"};
-      LogLevel.main(args2);
-      log.debug("log.debug4");
-      log.info("log.info4");
-      log.error("log.error4");
-      assertEquals("Try setting log level: Info via command line.", Level.INFO,
-          log.getEffectiveLevel());
-
-      // Test "Error" instead of "ERROR" should work for servlet
-      URL newUrl = new URL("http://" + authority + "/logLevel?log=" + logName
-          + "&level=" + "Error");
-      out.println("*** Connecting to " + newUrl);
-      connection = newUrl.openConnection();
-      connection.connect();
-
-      BufferedReader in2 = new BufferedReader(new InputStreamReader(
-          connection.getInputStream()));
-      for(String line; (line = in2.readLine()) != null; out.println(line));
-      in2.close();
-
-      log.debug("log.debug5");
-      log.info("log.info5");
-      log.error("log.error5");
-      assertEquals("Try setting log level: Error via servlet.", Level.ERROR,
-          log.getEffectiveLevel());
+  @BeforeClass
+  public static void setUp() throws Exception {
+    org.slf4j.Logger logger =
+        LoggerFactory.getLogger(KerberosAuthenticator.class);
+    GenericTestUtils.setLogLevel(logger, Level.DEBUG);
+    FileUtil.fullyDelete(BASEDIR);
+    if (!BASEDIR.mkdirs()) {
+      throw new Exception("unable to create the base directory for testing");
     }
-    else {
-      out.println(testlog.getClass() + " not tested.");
+    conf = new Configuration();
+
+    setupSSL(BASEDIR);
+  }
+
+  static private void setupSSL(File base) throws Exception {
+    keystoresDir = base.getAbsolutePath();
+    sslConfDir = KeyStoreTestUtil.getClasspathDir(TestLogLevel.class);
+    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+
+    sslConf = KeyStoreTestUtil.getSslConfig();
+  }
+
+  @Before
+  public void setupKerberos() throws Exception {
+    File keytabFile = new File(KerberosTestUtils.getKeytabFile());
+    clientPrincipal = KerberosTestUtils.getClientPrincipal();
+    serverPrincipal = KerberosTestUtils.getServerPrincipal();
+    clientPrincipal = clientPrincipal.substring(0,
+        clientPrincipal.lastIndexOf("@"));
+    serverPrincipal = serverPrincipal.substring(0,
+        serverPrincipal.lastIndexOf("@"));
+    getKdc().createPrincipal(keytabFile, clientPrincipal, serverPrincipal);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+    FileUtil.fullyDelete(BASEDIR);
+  }
+
+  /**
+   * Test client command line options. Does not validate server behavior.
+   * @throws Exception
+   */
+  @Test(timeout=120000)
+  public void testCommandOptions() throws Exception {
+    final String className = this.getClass().getName();
+
+    assertFalse(validateCommand(new String[] {"-foo" }));
+    // fail due to insufficient number of arguments
+    assertFalse(validateCommand(new String[] {}));
+    assertFalse(validateCommand(new String[] {"-getlevel" }));
+    assertFalse(validateCommand(new String[] {"-setlevel" }));
+    assertFalse(validateCommand(new String[] {"-getlevel", "foo.bar:8080" }));
+
+    // valid command arguments
+    assertTrue(validateCommand(
+        new String[] {"-getlevel", "foo.bar:8080", className }));
+    assertTrue(validateCommand(
+        new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG" }));
+    assertTrue(validateCommand(
+        new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+            "http" }));
+    assertTrue(validateCommand(
+        new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+            "https" }));
+    assertTrue(validateCommand(
+        new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
+            "-protocol", "http" }));
+    assertTrue(validateCommand(
+        new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
+            "-protocol", "https" }));
+
+    // fail due to the extra argument
+    assertFalse(validateCommand(
+        new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+            "https", "blah" }));
+    assertFalse(validateCommand(
+        new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
+            "-protocol", "https", "blah" }));
+    assertFalse(validateCommand(
+        new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+            "https", "-protocol", "https" }));
+    assertFalse(validateCommand(
+        new String[] {"-getlevel", "foo.bar:8080", className,
+            "-setlevel", "foo.bar:8080", className }));
+  }
+
+  /**
+   * Check to see if a command can be accepted.
+   *
+   * @param args a String array of arguments
+   * @return true if the command can be accepted, false if not.
+   */
+  private boolean validateCommand(String[] args) throws Exception {
+    CLI cli = new CLI(sslConf);
+    try {
+      cli.parseArguments(args);
+    } catch (HadoopIllegalArgumentException e) {
+      return false;
+    } catch (Exception e) {
+      // this is used to verify the command arguments only.
+      // no HadoopIllegalArgumentException = the arguments are good.
+      return true;
+    }
+    return true;
+  }
+
+  /**
+   * Creates and starts a Jetty server binding at an ephemeral port to run
+   * LogLevel servlet.
+   * @param protocol "http" or "https"
+   * @param isSpnego true if SPNEGO is enabled
+   * @return a created HttpServer2 object
+   * @throws Exception if unable to create or start a Jetty server
+   */
+  private HttpServer2 createServer(String protocol, boolean isSpnego)
+      throws Exception {
+    HttpServer2.Builder builder = new HttpServer2.Builder()
+        .setName("..")
+        .addEndpoint(new URI(protocol + "://localhost:0"))
+        .setFindPort(true)
+        .setConf(conf);
+    if (isSpnego) {
+      // Set up server Kerberos credentials.
+      // Since the server may fall back to simple authentication,
+      // use ACL to make sure the connection is Kerberos/SPNEGO authenticated.
+      builder.setSecurityEnabled(true)
+          .setUsernameConfKey(PRINCIPAL)
+          .setKeytabConfKey(KEYTAB)
+          .setACL(new AccessControlList(clientPrincipal));
+    }
+
+    // if using HTTPS, configure keystore/truststore properties.
+    if (protocol.equals(LogLevel.PROTOCOL_HTTPS)) {
+      builder = builder.
+          keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+          .keyStore(sslConf.get("ssl.server.keystore.location"),
+              sslConf.get("ssl.server.keystore.password"),
+              sslConf.get("ssl.server.keystore.type", "jks"))
+          .trustStore(sslConf.get("ssl.server.truststore.location"),
+              sslConf.get("ssl.server.truststore.password"),
+              sslConf.get("ssl.server.truststore.type", "jks"));
+    }
+    HttpServer2 server = builder.build();
+    // Enable SPNEGO for LogLevel servlet
+    if (isSpnego) {
+      server.addInternalServlet("logLevel", "/logLevel", LogLevel.Servlet.class,
+          true);
+    }
+    server.start();
+    return server;
+  }
+
+  private void testDynamicLogLevel(final String bindProtocol,
+      final String connectProtocol, final boolean isSpnego)
+      throws Exception {
+    testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego,
+        Level.DEBUG.toString());
+  }
+
+  /**
+   * Run both client and server using the given protocol.
+   *
+   * @param bindProtocol specify either http or https for server
+   * @param connectProtocol specify either http or https for client
+   * @param isSpnego true if SPNEGO is enabled
+   * @throws Exception
+   */
+  private void testDynamicLogLevel(final String bindProtocol,
+      final String connectProtocol, final boolean isSpnego,
+      final String newLevel) throws Exception {
+    if (!LogLevel.isValidProtocol(bindProtocol)) {
+      throw new Exception("Invalid server protocol " + bindProtocol);
+    }
+    if (!LogLevel.isValidProtocol(connectProtocol)) {
+      throw new Exception("Invalid client protocol " + connectProtocol);
+    }
+    Level oldLevel = log.getEffectiveLevel();
+    Assert.assertNotEquals("Get default Log Level which shouldn't be ERROR.",
+        Level.ERROR, oldLevel);
+
+    // configs needed for SPNEGO at server side
+    if (isSpnego) {
+      conf.set(PRINCIPAL, KerberosTestUtils.getServerPrincipal());
+      conf.set(KEYTAB, KerberosTestUtils.getKeytabFile());
+      conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+          "kerberos");
+      conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
+          true);
+      UserGroupInformation.setConfiguration(conf);
+    }
+
+    final HttpServer2 server = createServer(bindProtocol, isSpnego);
+    // get server port
+    final String authority = NetUtils.getHostPortString(server
+        .getConnectorAddress(0));
+
+    KerberosTestUtils.doAsClient(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        // client command line
+        getLevel(connectProtocol, authority);
+        setLevel(connectProtocol, authority, newLevel);
+        return null;
+      }
+    });
+    server.stop();
+    // restore log level
+    GenericTestUtils.setLogLevel(log, oldLevel);
+  }
+
+  /**
+   * Run LogLevel command line to start a client to get log level of this test
+   * class.
+   *
+   * @param protocol specify either http or https
+   * @param authority daemon's web UI address
+   * @throws Exception if unable to connect
+   */
+  private void getLevel(String protocol, String authority) throws Exception {
+    String[] getLevelArgs = {"-getlevel", authority, logName, "-protocol",
+        protocol};
+    CLI cli = new CLI(sslConf);
+    cli.run(getLevelArgs);
+  }
+
+  /**
+   * Run LogLevel command line to start a client to set log level of this test
+   * class to debug.
+   *
+   * @param protocol specify either http or https
+   * @param authority daemon's web UI address
+   * @throws Exception if unable to run or log level does not change as expected
+   */
+  private void setLevel(String protocol, String authority, String newLevel)
+      throws Exception {
+    String[] setLevelArgs = {"-setlevel", authority, logName,
+        newLevel, "-protocol", protocol};
+    CLI cli = new CLI(sslConf);
+    cli.run(setLevelArgs);
+
+    assertEquals("new level not equal to expected: ", newLevel.toUpperCase(),
+        log.getEffectiveLevel().toString());
+  }
+
+  /**
+   * Test setting log level to "Info".
+   *
+   * @throws Exception
+   */
+  @Test(timeout=60000)
+  public void testInfoLogLevel() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false,
+        "Info");
+  }
+
+  /**
+   * Test setting log level to "Error".
+   *
+   * @throws Exception
+   */
+  @Test(timeout=60000)
+  public void testErrorLogLevel() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false,
+        "Error");
+  }
+
+  /**
+   * Server runs HTTP, no SPNEGO.
+   *
+   * @throws Exception
+   */
+  @Test(timeout=60000)
+  public void testLogLevelByHttp() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+          false);
+      fail("A HTTPS Client should not have succeeded in connecting to a " +
+          "HTTP server");
+    } catch (SSLException e) {
+      GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+    }
+  }
+
+  /**
+   * Server runs HTTP + SPNEGO.
+   *
+   * @throws Exception
+   */
+  @Test(timeout=60000)
+  public void testLogLevelByHttpWithSpnego() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+          true);
+      fail("A HTTPS Client should not have succeeded in connecting to a " +
+          "HTTP server");
+    } catch (SSLException e) {
+      GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+    }
+  }
+
+  /**
+   * Server runs HTTPS, no SPNEGO.
+   *
+   * @throws Exception
+   */
+  @Test(timeout=60000)
+  public void testLogLevelByHttps() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+        false);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+          false);
+      fail("A HTTP Client should not have succeeded in connecting to a " +
+          "HTTPS server");
+    } catch (SocketException e) {
+      GenericTestUtils.assertExceptionContains(
+          "Unexpected end of file from server", e);
+    }
+  }
+
+  /**
+   * Server runs HTTPS + SPNEGO.
+   *
+   * @throws Exception
+   */
+  @Test(timeout=60000)
+  public void testLogLevelByHttpsWithSpnego() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+        true);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+          true);
+      fail("A HTTP Client should not have succeeded in connecting to a " +
+          "HTTPS server");
+    }  catch (SocketException e) {
+      GenericTestUtils.assertExceptionContains(
+          "Unexpected end of file from server", e);
     }
   }
-}
+}