AMBARI-19963 Deviation alerts fail with "No JSON object could be decoded" (dsen)

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/network.py → ambari-common/src/main/python/ambari_commons/network.py

@@ -35,5 +35,5 @@ def check_ssl_certificate(host, port, ca_certs):
   try:
     ssl.get_server_certificate((host, port), ssl_version=ssl.PROTOCOL_SSLv23, ca_certs=ca_certs)
   except (ssl.SSLError) as ssl_error:
-    raise Fail("Failed to verify the SSL certificate for AMS Collector https://{0}:{1} with CA certificate in {2}"
+    raise Fail("Failed to verify the SSL certificate for https://{0}:{1} with CA certificate in {2}"
                .format(host, port, ca_certs))

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py

@@ -31,7 +31,7 @@ import random
 import time
 import socket
 import ambari_simplejson as json
-import network
+import ambari_commons.network as network
 import os
 
 GRAFANA_CONNECT_TRIES = 15

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py

@@ -28,7 +28,7 @@ from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons.parallel_processing import PrallelProcessResult, execute_in_parallel, SUCCESS
 
 import httplib
-import network
+import ambari_commons.network as network
 import urllib
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
 import os

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py

@@ -24,6 +24,8 @@ import logging
 import urllib
 import time
 import urllib2
+import os
+import ambari_commons.network as network
 
 from resource_management import Environment
 from ambari_commons.aggregate_functions import sample_standard_deviation, mean
@@ -55,6 +57,7 @@ SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
 SMOKEUSER_KEY = '{{cluster-env/smokeuser}}'
 EXECUTABLE_SEARCH_PATHS = '{{kerberos-env/executable_search_paths}}'
 
+AMS_HTTP_POLICY = '{{ams-site/timeline.metrics.service.http.policy}}'
 METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}'
 METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}'
 METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}'
@@ -105,7 +108,7 @@ def get_tokens():
           EXECUTABLE_SEARCH_PATHS, NN_HTTPS_ADDRESS_KEY, SMOKEUSER_KEY,
           KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY,
           METRICS_COLLECTOR_VIP_HOST_KEY, METRICS_COLLECTOR_VIP_PORT_KEY,
-          METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY)
+          METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, AMS_HTTP_POLICY)
 
 def execute(configurations={}, parameters={}, host_name=None):
   """
@@ -310,9 +313,14 @@ def execute(configurations={}, parameters={}, host_name=None):
 
   encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters)
 
+  ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf"
+  metric_truststore_ca_certs='ca.pem'
+  ca_certs = os.path.join(ams_monitor_conf_dir,
+                          metric_truststore_ca_certs)
+  metric_collector_https_enabled = str(configurations[AMS_HTTP_POLICY]) == "HTTPS_ONLY"
+
   try:
-    conn = httplib.HTTPConnection(collector_host, int(collector_port),
-                                  timeout=connection_timeout)
+    conn = network.get_http_connection(collector_host, int(collector_port), metric_collector_https_enabled, ca_certs)
     conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
     response = conn.getresponse()
     data = response.read()

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py

@@ -81,6 +81,7 @@ class TestAlertMetricsDeviation(RMFTestCase):
       '{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470',
       '{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY',
       '{{ams-site/timeline.metrics.service.webapp.address}}': '0.0.0.0:6188',
+      '{{ams-site/timeline.metrics.service.http.policy}}' : 'HTTP_ONLY',
       '{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070',
       '{{cluster-env/security_enabled}}': 'false',
       '{{cluster-env/smokeuser}}': 'ambari-qa',
@@ -105,6 +106,7 @@ class TestAlertMetricsDeviation(RMFTestCase):
       '{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470',
       '{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY',
       '{{ams-site/timeline.metrics.service.webapp.address}}': '0.0.0.0:6188',
+      '{{ams-site/timeline.metrics.service.http.policy}}' : 'HTTP_ONLY',
       '{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070',
       '{{cluster-env/security_enabled}}': 'false',
       '{{cluster-env/smokeuser}}': 'ambari-qa',