AMBARI-10099 - Web Alerts Create Confusing Errors And Warnings (jonathanhurley)

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -23,6 +23,7 @@ import time
 import subprocess
 import os
 import urllib2
+from urllib2 import HTTPError
 import uuid
 
 from  tempfile import gettempdir
@@ -89,10 +90,12 @@ class WebAlert(BaseAlert):
     if status_code == 0:
       return (self.RESULT_CRITICAL, [status_code, url, time_seconds, error_message])
 
+    # anything that's less than 400 is OK
     if status_code < 400:
       return (self.RESULT_OK, [status_code, url, time_seconds])
 
-    return (self.RESULT_WARNING, [status_code, url, time_seconds])
+    # everything else is WARNING
+    return (self.RESULT_WARNING, [status_code, url, time_seconds, error_message])
 
 
   def _build_web_query(self, alert_uri):
@@ -140,6 +143,7 @@ class WebAlert(BaseAlert):
     """    
 
     try:
+      response_code = 0
       kerberos_keytab = None
       kerberos_principal = None
 
@@ -199,22 +203,26 @@ class WebAlert(BaseAlert):
             '%{http_code}', url, '--connect-timeout', CURL_CONNECTION_TIMEOUT,
             '-o', '/dev/null'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=kerberos_env)
 
-          out, err = curl.communicate()
+          curl_stdout, curl_stderr = curl.communicate()
         finally:
           if os.path.isfile(cookie_file):
             os.remove(cookie_file)
 
-        if err != '':
-          raise Exception(err)
+        # empty quotes evaluates to false
+        if curl_stderr:
+          raise Exception(curl_stderr)
+
+        # empty quotes evaluates to false
+        if curl_stdout:
+          response_code = int(curl_stdout)
 
-        response_code = int(out)
         time_millis = time.time() - start_time
       else:
         # kerberos is not involved; use urllib2
-        response_code, time_millis = self._make_web_request_urllib(url)
+        response_code, time_millis, error_msg = self._make_web_request_urllib(url)
 
       return WebResponse(status_code=response_code, time_millis=time_millis,
-        error_msg=None)
+        error_msg=error_msg)
 
     except Exception, exception:
       if logger.isEnabledFor(logging.DEBUG):
@@ -230,6 +238,8 @@ class WebAlert(BaseAlert):
     :return: a tuple of the response code and the total time in ms
     """
     response = None
+    error_message = None
+
     start_time = time.time()
 
     try:
@@ -237,7 +247,12 @@ class WebAlert(BaseAlert):
       response_code = response.getcode()
       time_millis = time.time() - start_time
 
-      return response_code, time_millis
+      return response_code, time_millis, error_message
+    except HTTPError, httpError:
+      time_millis = time.time() - start_time
+      error_message = str(httpError)
+
+      return httpError.code, time_millis, error_message
     finally:
       if response is not None:
         try:

ambari-agent/src/test/python/ambari_agent/TestAlerts.py

@@ -380,9 +380,7 @@ class TestAlerts(TestCase):
     self.assertEquals('OK', alerts[0]['state'])
 
     # run the alert and check HTTP 500
-
-
-    wa_make_web_request_mock.return_value = WebResponse(500,1.234,None)
+    wa_make_web_request_mock.return_value = WebResponse(500,1.234,"Internal Server Error")
     collector = AlertCollector()
     alert = WebAlert(definition_json, definition_json['source'], None)
     alert.set_helpers(collector, cluster_configuration)
@@ -393,7 +391,7 @@ class TestAlerts(TestCase):
     self.assertEquals(0, len(collector.alerts()))
     
     self.assertEquals('WARNING', alerts[0]['state'])
-    self.assertEquals('(Unit Tests) warning: 500', alerts[0]['text'])
+    self.assertEquals('(Unit Tests) warning: 500 (Internal Server Error)', alerts[0]['text'])
 
     # run the alert and check critical
     wa_make_web_request_mock.return_value = WebResponse(0,0,'error message')
@@ -1067,7 +1065,7 @@ class TestAlerts(TestCase):
             "text": "(Unit Tests) ok: {0}",
           },
           "warning": {
-            "text": "(Unit Tests) warning: {0}",
+            "text": "(Unit Tests) warning: {0} ({3})",
           },
           "critical": {
             "text": "(Unit Tests) critical: {1}. {3}",

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json

@@ -44,10 +44,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json

@@ -103,10 +103,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -506,10 +506,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json

@@ -17,10 +17,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json

@@ -67,10 +67,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json

@@ -20,10 +20,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -183,10 +183,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -231,10 +231,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -360,10 +360,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/alerts.json

@@ -103,10 +103,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -506,10 +506,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/OOZIE/alerts.json

@@ -15,10 +15,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1}"

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/alerts.json

@@ -20,10 +20,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -183,10 +183,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -231,10 +231,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"
@@ -348,10 +348,10 @@
           },
           "reporting": {
             "ok": {
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response in {2:.3f}s"
             },
             "warning":{
-              "text": "HTTP {0} response in {2:.3f} seconds"
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
             },
             "critical": {
               "text": "Connection failed to {1} ({3})"