AMBARI-2941. Ambari server gets auth exception while getting JMX metrics from namenode in secure cluster. (Dilli Arumugam via swagle)

ambari-server/conf/unix/ambari-env.sh

@@ -13,4 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 AMBARI_PASSHPHRASE="DEV"
+export AMBARI_JVM_ARGS='-Xms512m -Xmx2048m -Djava.security.auth.login.config=/etc/ambari-server/conf/krb5JAASLogin.conf -Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false'

ambari-server/conf/unix/krb5JAASLogin.conf

@@ -0,0 +1,13 @@
+com.sun.security.jgss.initiate {
+    com.sun.security.auth.module.Krb5LoginModule required 
+    renewTGT=true
+    doNotPrompt=true
+    useKeyTab=true
+    keyTab="/etc/security/keytabs/ambari.keytab"
+    principal="ambari@EXAMPLE.COM"
+    isInitiator=true
+    storeKey=true
+    useTicketCache=true
+    client=true; 
+};
+

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AppCookieManager.java

@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import java.io.IOException;
+import java.net.URI;
+import java.security.Principal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
+import org.apache.http.HeaderElement;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.params.AuthPolicy;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+/**
+ * Handles SPNego authentication as a client of hadoop service, caches
+ * hadoop.auth cookie returned by hadoop service on successful SPNego
+ * authentication. Refreshes hadoop.auth cookie on demand if the cookie has
+ * expired.
+ * 
+ */
+public class AppCookieManager {
+
+  static final String HADOOP_AUTH = "hadoop.auth";
+  private static final String HADOOP_AUTH_EQ = "hadoop.auth=";
+  private static final String SET_COOKIE = "Set-Cookie";
+
+  private static final EmptyJaasCredentials EMPTY_JAAS_CREDENTIALS = new EmptyJaasCredentials();
+
+  private Map<String, String> endpointCookieMap = new ConcurrentHashMap<String, String>();
+  private static Log LOG = LogFactory.getLog(AppCookieManager.class);
+
+  /**
+   * Utility method to exercise AppCookieManager directly
+   * @param args element 0 of args should be a URL to hadoop service protected by SPengo
+   * @throws IOException in case of errors
+   */
+  public static void main(String[] args) throws IOException {
+    new AppCookieManager().getAppCookie(args[0], false);
+  }
+
+  public AppCookieManager() {
+  }
+
+  /**
+   * Returns hadoop.auth cookie, doing needed SPNego authentication
+   * 
+   * @param endpoint
+   *          the URL of the Hadoop service
+   * @param refresh
+   *          flag indicating wehther to refresh the cookie, if
+   *          <code>true</code>, we do a new SPNego authentication and refresh
+   *          the cookie even if the cookie already exists in local cache
+   * @return hadoop.auth cookie value
+   * @throws IOException
+   *           in case of problem getting hadoop.auth cookie
+   */
+  public String getAppCookie(String endpoint, boolean refresh)
+      throws IOException {
+
+    HttpUriRequest outboundRequest = new HttpGet(endpoint);
+    URI uri = outboundRequest.getURI();
+    String scheme = uri.getScheme();
+    String host = uri.getHost();
+    int port = uri.getPort();
+    String path = uri.getPath();
+    if (!refresh) {
+      String appCookie = endpointCookieMap.get(endpoint);
+      if (appCookie != null) {
+        return appCookie;
+      }
+    }
+
+    clearAppCookie(endpoint);
+    
+    DefaultHttpClient client = new DefaultHttpClient();
+    SPNegoSchemeFactory spNegoSF = new SPNegoSchemeFactory(/* stripPort */true);
+    // spNegoSF.setSpengoGenerator(new BouncySpnegoTokenGenerator());
+    client.getAuthSchemes().register(AuthPolicy.SPNEGO, spNegoSF);
+    client.getCredentialsProvider().setCredentials(
+        new AuthScope(/* host */null, /* port */-1, /* realm */null),
+        EMPTY_JAAS_CREDENTIALS);
+
+    String hadoopAuthCookie = null;
+    HttpResponse httpResponse = null;
+    try {
+      HttpHost httpHost = new HttpHost(host, port, scheme);
+      HttpRequest httpRequest = new HttpOptions(path);
+      httpResponse = client.execute(httpHost, httpRequest);
+      Header[] headers = httpResponse.getHeaders(SET_COOKIE);
+      hadoopAuthCookie = getHadoopAuthCookieValue(headers);
+      if (hadoopAuthCookie == null) {
+        LOG.error("SPNego authentication failed, can not get hadoop.auth cookie for URL: " + endpoint);
+        throw new IOException(
+            "SPNego authentication failed, can not get hadoop.auth cookie");
+      }
+    } finally {
+      if (httpResponse != null) {
+        HttpEntity entity = httpResponse.getEntity();
+        if (entity != null) {
+          entity.getContent().close();
+        }
+      }
+
+    }
+ 
+    hadoopAuthCookie = HADOOP_AUTH_EQ + quote(hadoopAuthCookie);
+    setAppCookie(endpoint, hadoopAuthCookie);
+    if (LOG.isInfoEnabled()) {
+      LOG.info("Successful SPNego authentication to URL:" + uri.toString());
+    }
+    return hadoopAuthCookie;
+  }
+
+  
+  /**
+   * Returns the cached app cookie
+   *  @param endpoint the hadoop end point we authenticate to
+   * @return the cached app cookie, can be null
+   */
+  public String getCachedAppCookie(String endpoint) {
+    return endpointCookieMap.get(endpoint);
+  }
+  
+  /**
+   *  Sets the cached app cookie i cache
+   *  @param endpoint the hadoop end point we authenticate to
+   *  @param appCookie the app cookie
+   */
+  private void setAppCookie(String endpoint, String appCookie) {
+    endpointCookieMap.put(endpoint, appCookie);
+  }
+
+  /**
+   *  Clears the cached app cookie
+   *  @param endpoint the hadoop end point we authenticate to
+   */
+  private void clearAppCookie(String endpoint) {
+    endpointCookieMap.remove(endpoint);
+  }
+  
+  static String quote(String s) {
+    return s == null ? s : "\"" + s + "\"";
+  }
+
+  static String getHadoopAuthCookieValue(Header[] headers) {
+    if (headers == null) {
+      return null;
+    }
+    for (Header header : headers) {
+      HeaderElement[] elements = header.getElements();
+      for (HeaderElement element : elements) {
+        String cookieName = element.getName();
+        if (cookieName.equals(HADOOP_AUTH)) {
+          if (element.getValue() != null) {
+            String trimmedVal = element.getValue().trim();
+            if (!trimmedVal.isEmpty()) {
+              return trimmedVal;
+            }
+          }
+        }
+      }
+    }
+    return null;
+  }
+
+
+  private static class EmptyJaasCredentials implements Credentials {
+
+    public String getPassword() {
+      return null;
+    }
+
+    public Principal getUserPrincipal() {
+      return null;
+    }
+
+  }
+
+}

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java

@@ -22,57 +22,112 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
 import java.security.KeyStore;
 
-import org.apache.ambari.server.controller.utilities.StreamProvider;
-
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSocketFactory;
 import javax.net.ssl.TrustManagerFactory;
 
+import org.apache.ambari.server.controller.utilities.StreamProvider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicHeader;
+
 /**
  * URL based implementation of a stream provider.
  */
 public class URLStreamProvider implements StreamProvider {
 
+  private static final String COOKIE = "Cookie";
+  private static final String WWW_AUTHENTICATE = "WWW-Authenticate";
+  private static final String NEGOTIATE = "Negotiate";
+  private static Log LOG = LogFactory.getLog(URLStreamProvider.class);
+
   private final int connTimeout;
   private final int readTimeout;
   private final String path;
   private final String password;
   private final String type;
   private volatile SSLSocketFactory sslSocketFactory = null;
+  private AppCookieManager appCookieManager;
 
   /**
    * Provide the connection timeout for the underlying connection.
-   *
-   * @param connectionTimeout  time, in milliseconds, to attempt a connection
-   * @param readTimeout        the read timeout in milliseconds
+   * 
+   * @param connectionTimeout
+   *          time, in milliseconds, to attempt a connection
+   * @param readTimeout
+   *          the read timeout in milliseconds
    */
-  public URLStreamProvider(int connectionTimeout, int readTimeout,
-                           String path, String password, String type) {
+  public URLStreamProvider(int connectionTimeout, int readTimeout, String path,
+      String password, String type) {
+
     this.connTimeout = connectionTimeout;
     this.readTimeout = readTimeout;
-    this.path        = path;
-    this.password    = password;
-    this.type        = type;
+    this.path = path;
+    this.password = password;
+    this.type = type;
+    appCookieManager = new AppCookieManager();
   }
-  
+
   @Override
   public InputStream readFrom(String spec) throws IOException {
 
-    URLConnection connection = spec.startsWith("https") ?
-        getSSLConnection(spec) : getConnection(spec);
+    HttpURLConnection connection = spec.startsWith("https") ? 
+        (HttpURLConnection)getSSLConnection(spec)
+        : (HttpURLConnection)getConnection(spec);
 
+    String appCookie = appCookieManager.getCachedAppCookie(spec);
+    if (appCookie != null) {
+      if (LOG.isInfoEnabled()) {
+        LOG.info("Using cached app cookie for URL:" + spec);
+      }
+      connection.setRequestProperty(COOKIE, appCookie);
+    }
     connection.setConnectTimeout(connTimeout);
     connection.setReadTimeout(readTimeout);
     connection.setDoOutput(true);
 
-    return connection.getInputStream();
-  }
+    int statusCode = connection.getResponseCode();
+    if (statusCode == HttpStatus.SC_UNAUTHORIZED ) {
+      String wwwAuthHeader = connection.getHeaderField(WWW_AUTHENTICATE);
+      if (LOG.isInfoEnabled()) {
+        LOG.info("Received WWW-Authentication header:" + wwwAuthHeader + ", for URL:" + spec);
+      }
+      if (wwwAuthHeader != null && 
+          wwwAuthHeader.trim().startsWith(NEGOTIATE)) {
+        //connection.getInputStream().close();
+        connection = spec.startsWith("https") ? 
+            (HttpURLConnection)getSSLConnection(spec)
+            : (HttpURLConnection)getConnection(spec);
+        appCookie = appCookieManager.getAppCookie(spec, true);
+        connection.setRequestProperty(COOKIE, appCookie);
+        connection.setConnectTimeout(connTimeout);
+        connection.setReadTimeout(readTimeout);
+        connection.setDoOutput(true);
+        return connection.getInputStream();
+      } else {
+        // no supported authentication type found
+        // we would let the original response propogate
+        LOG.error("Unsupported WWW-Authentication header:" + wwwAuthHeader+ ", for URL:" + spec);
+        return connection.getInputStream();
+      }
+    } else {
+      // not a 401 Unauthorized status code
+      // we would let the original response propogate
+      return connection.getInputStream();
+    }
 
+  }
 
   // ----- helper methods ----------------------------------------------------
 
@@ -88,14 +143,15 @@ public class URLStreamProvider implements StreamProvider {
       synchronized (this) {
         if (sslSocketFactory == null) {
           try {
-            FileInputStream in    = new FileInputStream(new File(path));
-            KeyStore        store = KeyStore.getInstance(type == null ? KeyStore.getDefaultType() : type);
+            FileInputStream in = new FileInputStream(new File(path));
+            KeyStore store = KeyStore.getInstance(type == null ? KeyStore
+                .getDefaultType() : type);
 
             store.load(in, password.toCharArray());
             in.close();
 
-            TrustManagerFactory tmf =
-                TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            TrustManagerFactory tmf = TrustManagerFactory
+                .getInstance(TrustManagerFactory.getDefaultAlgorithm());
 
             tmf.init(store);
             SSLContext context = SSLContext.getInstance("TLS");
@@ -108,7 +164,8 @@ public class URLStreamProvider implements StreamProvider {
         }
       }
     }
-    HttpsURLConnection connection = (HttpsURLConnection)(new URL(spec).openConnection());
+    HttpsURLConnection connection = (HttpsURLConnection) (new URL(spec)
+        .openConnection());
 
     connection.setSSLSocketFactory(sslSocketFactory);
 

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AppCookieManagerTest.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
+import org.junit.Test;
+
+public class AppCookieManagerTest {
+
+  @Test
+  public void getCachedKnoxAppCookie() {
+    assertNull(new AppCookieManager().getCachedAppCookie("http://dummy"));
+  }
+
+  @Test
+  public void getHadoopAuthCookieValueWithNullHeaders() {
+    assertNull(AppCookieManager.getHadoopAuthCookieValue(null));
+  }
+  
+  @Test
+  public void getHadoopAuthCookieValueWitEmptylHeaders() {
+    assertNull(AppCookieManager.getHadoopAuthCookieValue(new Header[0]));
+  }
+  
+  @Test
+  public void getHadoopAuthCookieValueWithValidlHeaders() {
+    Header[] headers = new Header[1];
+    headers[0] = new BasicHeader("Set-Cookie", AppCookieManager.HADOOP_AUTH + "=dummyvalue");
+    assertNotNull(AppCookieManager.getHadoopAuthCookieValue(headers));
+  }
+  
+}