AMBARI-9586 - Alerts: WEB type alerts fail in Kerberos secured cluster (Yurii Shylov via jonathanhurley)

ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py

@@ -212,6 +212,8 @@ class BaseAlert(object):
     https_property_key = None
     https_property_value_key = None
     default_port = None
+    kerberos_keytab = None
+    kerberos_principal = None
     
     if 'http' in uri_structure:
       http_key = self._find_lookup_property(uri_structure['http'])
@@ -228,12 +230,19 @@ class BaseAlert(object):
     if 'default_port' in uri_structure:
       default_port = uri_structure['default_port']
 
+    if 'kerberos_keytab' in uri_structure:
+      kerberos_keytab = self._find_lookup_property(uri_structure['kerberos_keytab'])
+
+    if 'kerberos_principal' in uri_structure:
+      kerberos_principal = self._find_lookup_property(uri_structure['kerberos_principal'])
+
     AlertUriLookupKeys = namedtuple('AlertUriLookupKeys', 
-        'http https https_property https_property_value default_port')
+        'http https https_property https_property_value default_port kerberos_keytab kerberos_principal')
     
     alert_uri_lookup_keys = AlertUriLookupKeys(http=http_key, https=https_key, 
         https_property=https_property_key, 
-        https_property_value=https_property_value_key, default_port=default_port)
+        https_property_value=https_property_value_key, default_port=default_port,
+        kerberos_keytab=kerberos_keytab, kerberos_principal=kerberos_principal)
     
     return alert_uri_lookup_keys
 

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -20,7 +20,9 @@ limitations under the License.
 
 import logging
 import time
-import urllib2
+import subprocess
+import os
+
 from alerts.base_alert import BaseAlert
 from collections import namedtuple
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
@@ -29,6 +31,8 @@ from ambari_commons.inet_utils import resolve_address
 
 logger = logging.getLogger()
 
+CURL_CONNECTION_TIMEOUT = '20'
+
 class WebAlert(BaseAlert):
   
   def __init__(self, alert_meta, alert_source_meta):
@@ -55,9 +59,10 @@ class WebAlert(BaseAlert):
     web_response = self._make_web_request(url)
     status_code = web_response.status_code
     time_seconds = web_response.time_millis / 1000
+    error_message = web_response.error_msg
 
     if status_code == 0:
-      return (self.RESULT_CRITICAL, [status_code, url, time_seconds])
+      return (self.RESULT_CRITICAL, [status_code, url, time_seconds, error_message])
     
     if status_code < 400:
       return (self.RESULT_OK, [status_code, url, time_seconds])
@@ -106,21 +111,49 @@ class WebAlert(BaseAlert):
     Makes an http(s) request to a web resource and returns the http code. If
     there was an error making the request, return 0 for the status code.
     """    
-    WebResponse = namedtuple('WebResponse', 'status_code time_millis')
+    WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg')
     
     time_millis = 0
     
     try:
-      start_time = time.time()      
-      response = urllib2.urlopen(url)
+      kerberos_keytab = None
+      kerberos_principal = None
+
+      if self.uri_property_keys.kerberos_principal is not None:
+        kerberos_principal = self._lookup_property_value(
+          self.uri_property_keys.kerberos_principal)
+
+        if kerberos_principal is not None:
+          # substitute _HOST in kerberos principal with actual fqdn
+          kerberos_principal = kerberos_principal.replace('_HOST', self.host_name)
+
+      if self.uri_property_keys.kerberos_keytab is not None:
+        kerberos_keytab = self._lookup_property_value(self.uri_property_keys.kerberos_keytab)
+
+      if kerberos_principal is not None and kerberos_keytab is not None:
+        os.system("kinit -kt {0} {1} > /dev/null".format(kerberos_keytab, kerberos_principal))
+
+      # substitute 0.0.0.0 in url with actual fqdn
+      url = url.replace('0.0.0.0', self.host_name)
+      start_time = time.time()
+      curl = subprocess.Popen(['curl', '--negotiate', '-u', ':', '-sL', '-w',
+        '%{http_code}', url, '--connect-timeout', CURL_CONNECTION_TIMEOUT,
+        '-o', '/dev/null'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+      out, err = curl.communicate()
+
+      if err != '':
+        raise Exception(err)
+
+      response_code = int(out)
       time_millis = time.time() - start_time
-    except:
+    except Exception, exc:
       if logger.isEnabledFor(logging.DEBUG):
-        logger.exception("[Alert][{0}] Failed to make a web request".format(self.get_name()))
-      
-      return WebResponse(status_code=0, time_millis=0)
-    
-    return WebResponse(status_code=response.getcode(), time_millis=time_millis) 
+        logger.exception("[Alert][{0}] Unable to make a web request.".format(self.get_name()))
+
+      return WebResponse(status_code=0, time_millis=0, error_msg=str(exc))
+
+    return WebResponse(status_code=response_code, time_millis=time_millis, error_msg=None)
 
 
   def _get_reporting_text(self, state):

ambari-agent/src/test/python/ambari_agent/TestAlerts.py

@@ -419,14 +419,14 @@ class TestAlerts(TestCase):
             "text": "(Unit Tests) warning: {0}",
           },
           "critical": {
-            "text": "(Unit Tests) critical: {1}",
+            "text": "(Unit Tests) critical: {1}. {3}",
           }
         }
       }
     }
 
-    WebResponse = namedtuple('WebResponse', 'status_code time_millis')
-    wa_make_web_request_mock.return_value = WebResponse(200,1.234)
+    WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg')
+    wa_make_web_request_mock.return_value = WebResponse(200,1.234,None)
 
     # run the alert and check HTTP 200    
     collector = AlertCollector()
@@ -437,11 +437,11 @@ class TestAlerts(TestCase):
     alerts = collector.alerts()
     self.assertEquals(0, len(collector.alerts()))
 
-    self.assertEquals('OK', alerts[0]['state'])
     self.assertEquals('(Unit Tests) ok: 200', alerts[0]['text'])
+    self.assertEquals('OK', alerts[0]['state'])
 
     # run the alert and check HTTP 500
-    wa_make_web_request_mock.return_value = WebResponse(500,1.234)
+    wa_make_web_request_mock.return_value = WebResponse(500,1.234,None)
     collector = AlertCollector()
     alert = WebAlert(json, json['source'])
     alert.set_helpers(collector, {'hdfs-site/dfs.datanode.http.address': '1.2.3.4:80'})
@@ -454,7 +454,7 @@ class TestAlerts(TestCase):
     self.assertEquals('(Unit Tests) warning: 500', alerts[0]['text'])
 
     # run the alert and check critical
-    wa_make_web_request_mock.return_value = WebResponse(0,0)
+    wa_make_web_request_mock.return_value = WebResponse(0,0,'error message')
      
     collector = AlertCollector()
     alert = WebAlert(json, json['source'])
@@ -466,7 +466,7 @@ class TestAlerts(TestCase):
     
     # http assertion indicating that we properly determined non-SSL
     self.assertEquals('CRITICAL', alerts[0]['state'])
-    self.assertEquals('(Unit Tests) critical: http://1.2.3.4:80', alerts[0]['text'])
+    self.assertEquals('(Unit Tests) critical: http://1.2.3.4:80. error message', alerts[0]['text'])
      
     collector = AlertCollector()
     alert = WebAlert(json, json['source'])
@@ -482,7 +482,7 @@ class TestAlerts(TestCase):
     
     # SSL assertion
     self.assertEquals('CRITICAL', alerts[0]['state'])
-    self.assertEquals('(Unit Tests) critical: https://1.2.3.4:8443', alerts[0]['text'])
+    self.assertEquals('(Unit Tests) critical: https://1.2.3.4:8443. error message', alerts[0]['text'])
 
   def test_reschedule(self):
     test_file_path = os.path.join('ambari_agent', 'dummy_files')

ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java

@@ -55,6 +55,18 @@ public class AlertUri {
   @SerializedName("https_property_value")
   private String m_httpsPropertyValue;
 
+  /**
+   * Kerberos keytab path to use.
+   */
+  @SerializedName("kerberos_keytab")
+  private String m_kerberosKeytab;
+
+  /**
+   * Kerberos principal name to use.
+   */
+  @SerializedName("kerberos_principal")
+  private String m_kerberosPrincipal;
+
   /**
    * A default port to use on the host running the alert if no URLs can be
    * found.
@@ -114,6 +126,24 @@ public class AlertUri {
     return m_httpsPropertyValue;
   }
 
+  /**
+   * The configuration property with kerberos keytab path.
+   *
+   * @return the configuration property, or {@code null} for none.
+   */
+  public String getKerberosKeytab() {
+    return m_kerberosKeytab;
+  }
+
+  /**
+   * The configuration property with kerberos principal name.
+   *
+   * @return the configuration property, or {@code null} for none.
+   */
+  public String getKerberosPrincipal() {
+    return m_kerberosPrincipal;
+  }
+
   /**
    * {@inheritDoc}
    */

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json

@@ -38,7 +38,9 @@
           "type": "WEB",
           "uri": {
             "http": "{{falcon-env/falcon_port}}",
-            "default_port": 15000
+            "default_port": 15000,
+            "kerberos_keytab": "{{core-site/hadoop.http.authentication.kerberos.keytab}}",
+            "kerberos_principal": "{{core-site/hadoop.http.authentication.kerberos.principal}}"
           },
           "reporting": {
             "ok": {
@@ -48,7 +50,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json

@@ -91,7 +91,9 @@
             "http": "{{hdfs-site/dfs.namenode.http-address}}",
             "https": "{{hdfs-site/dfs.namenode.https-address}}",
             "https_property": "{{hdfs-site/dfs.http.policy}}",
-            "https_property_value": "HTTPS_ONLY"
+            "https_property_value": "HTTPS_ONLY",
+            "kerberos_keytab": "{{hdfs-site/dfs.web.authentication.kerberos.keytab}}",
+            "kerberos_principal": "{{hdfs-site/dfs.web.authentication.kerberos.principal}}"
           },
           "reporting": {
             "ok": {
@@ -101,7 +103,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }
@@ -482,7 +484,9 @@
             "http": "{{hdfs-site/dfs.datanode.http.address}}",
             "https": "{{hdfs-site/dfs.datanode.https.address}}",
             "https_property": "{{hdfs-site/dfs.http.policy}}",
-            "https_property_value": "HTTPS_ONLY"
+            "https_property_value": "HTTPS_ONLY",
+            "kerberos_keytab": "{{hdfs-site/dfs.web.authentication.kerberos.keytab}}",
+            "kerberos_principal": "{{hdfs-site/dfs.web.authentication.kerberos.principal}}"
           },
           "reporting": {
             "ok": {
@@ -492,7 +496,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json

@@ -11,7 +11,9 @@
         "source": {
           "type": "WEB",
           "uri": {
-            "http": "{{oozie-site/oozie.base.url}}/oozie"
+            "http": "{{oozie-site/oozie.base.url}}/oozie",
+            "kerberos_keytab": "{{oozie-site/oozie.authentication.kerberos.keytab}}",
+            "kerberos_principal": "{{oozie-site/oozie.authentication.kerberos.principal}}"
           },
           "reporting": {
             "ok": {
@@ -21,7 +23,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json

@@ -61,7 +61,9 @@
         "source": {
           "type": "WEB",
           "uri": {
-            "http": "{{storm-site/ui.port}}"
+            "http": "{{storm-site/ui.port}}",
+            "kerberos_keytab": "{{core-site/hadoop.http.authentication.kerberos.keytab}}",
+            "kerberos_principal": "{{core-site/hadoop.http.authentication.kerberos.principal}}"
           },
           "reporting": {
             "ok": {
@@ -71,7 +73,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json

@@ -14,7 +14,9 @@
             "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
             "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
             "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
-            "https_property_value": "HTTPS_ONLY"
+            "https_property_value": "HTTPS_ONLY",
+            "kerberos_keytab": "{{mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file}}",
+            "kerberos_principal": "{{mapred-site/mapreduce.jobhistory.webapp.spnego-principal}}"
           },
           "reporting": {
             "ok": {
@@ -24,7 +26,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }
@@ -175,7 +177,9 @@
             "https": "{{yarn-site/yarn.nodemanager.webapp.https.address}}",
             "https_property": "{{yarn-site/yarn.http.policy}}",
             "https_property_value": "HTTPS_ONLY",
-            "default_port": 8042
+            "default_port": 8042,
+            "kerberos_keytab": "{{yarn-site/yarn.nodemanager.webapp.spnego-keytab-file}}",
+            "kerberos_principal": "{{yarn-site/yarn.nodemanager.webapp.spnego-principal}}"
           },
           "reporting": {
             "ok": {
@@ -185,7 +189,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }
@@ -216,7 +220,9 @@
             "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
             "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
             "https_property": "{{yarn-site/yarn.http.policy}}",
-            "https_property_value": "HTTPS_ONLY"
+            "https_property_value": "HTTPS_ONLY",
+            "kerberos_keytab": "{{yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file}}",
+            "kerberos_principal": "{{yarn-site/yarn.resourcemanager.webapp.spnego-principal}}"
           },
           "reporting": {
             "ok": {
@@ -226,7 +232,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }
@@ -333,7 +339,9 @@
             "http": "{{yarn-site/yarn.timeline-service.webapp.address}}",
             "https": "{{yarn-site/yarn.timeline-service.webapp.https.address}}",
             "https_property": "{{yarn-site/yarn.http.policy}}",
-            "https_property_value": "HTTPS_ONLY"
+            "https_property_value": "HTTPS_ONLY",
+            "kerberos_keytab": "{{yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab}}",
+            "kerberos_principal": "{{yarn-site/yarn.timeline-service.http-authentication.kerberos.principal}}"
           },
           "reporting": {
             "ok": {
@@ -343,7 +351,7 @@
               "text": "HTTP {0} response in {2:.3f} seconds"
             },
             "critical": {
-              "text": "Connection failed to {1}"
+              "text": "Connection failed to {1}: {3}"
             }
           }
         }