AMBARI-20857. After WE is enabled, graphana fails to start with SSL23_GET_SERVER_HELLO:tlsv1 alert protocol version (echekanskiy)

ambari-agent/src/main/python/ambari_agent/AmbariConfig.py

@@ -316,9 +316,23 @@ class AmbariConfig:
         logger.info("Updating config property (%s) with value (%s)", k, v)
     pass
 
-  def get_force_https_protocol(self):
+  def get_force_https_protocol_name(self):
+    """
+    Get forced https protocol name.
+
+    :return: protocol name, PROTOCOL_TLSv1 by default
+    """
     return self.get('security', 'force_https_protocol', default="PROTOCOL_TLSv1")
 
+  def get_force_https_protocol_value(self):
+    """
+    Get forced https protocol value that correspondents to ssl module variable.
+
+    :return: protocol value
+    """
+    import ssl
+    return getattr(ssl, self.get_force_https_protocol_name())
+
 def isSameHostList(hostlist1, hostlist2):
   is_same = True
 

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -81,7 +81,7 @@ class CustomServiceOrchestrator():
   def __init__(self, config, controller):
     self.config = config
     self.tmp_dir = config.get('agent', 'prefix')
-    self.force_https_protocol = config.get_force_https_protocol()
+    self.force_https_protocol = config.get_force_https_protocol_name()
     self.exec_tmp_dir = Constants.AGENT_TMP_DIR
     self.file_cache = FileCache(config)
     self.status_commands_stdout = os.path.join(self.tmp_dir,

ambari-agent/src/main/python/ambari_agent/NetUtil.py

@@ -29,7 +29,7 @@ LOG_REQUEST_MESSAGE = "GET %s -> %s, body: %s"
 
 logger = logging.getLogger(__name__)
 
-ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol())
+ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol_name())
 
 class NetUtil:
 

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -55,7 +55,7 @@ DEFAULT_CONNECTION_TIMEOUT = 5
 
 WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg')
 
-ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol())
+ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol_name())
 
 class WebAlert(BaseAlert):
 

ambari-common/src/main/python/ambari_commons/network.py

@@ -42,30 +42,20 @@ class HTTPSConnectionWithCustomSslVersion(httplib.HTTPSConnection):
     self.sock = ssl.wrap_socket(conn_socket, self.key_file, self.cert_file,
                                 ssl_version=self.ssl_version)
 
-def get_http_connection(host, port, https_enabled=False, ca_certs=None):
+def get_http_connection(host, port, https_enabled=False, ca_certs=None, ssl_version = ssl.PROTOCOL_SSLv23):
   if https_enabled:
-    ssl_version = ssl.PROTOCOL_SSLv23
     if ca_certs:
-      ssl_version = check_ssl_certificate_and_return_ssl_version(host, port, ca_certs)
+      check_ssl_certificate_and_return_ssl_version(host, port, ca_certs, ssl_version)
     return HTTPSConnectionWithCustomSslVersion(host, port, ssl_version)
   else:
     return httplib.HTTPConnection(host, port)
 
-def check_ssl_certificate_and_return_ssl_version(host, port, ca_certs):
+def check_ssl_certificate_and_return_ssl_version(host, port, ca_certs, ssl_version = ssl.PROTOCOL_SSLv23):
   try:
-    # Try with TLSv1 first.
-    ssl_version = ssl.PROTOCOL_TLSv1
     ssl.get_server_certificate((host, port), ssl_version=ssl_version, ca_certs=ca_certs)
   except ssl.SSLError as ssl_error:
-    print_warning_msg("Failed to verify the SSL certificate for https://{0}:{1} with CA certificate in {2} using ssl.PROTOCOL_TLSv1."
-                      " Trying to use less secure ssl.PROTOCOL_SSLv23. Error : {3}".format(host, port, ca_certs, str(ssl_error)))
-    try:
-      # Try with SSLv23 only if TLSv1 failed.
-      ssl_version = ssl.PROTOCOL_SSLv23
-      ssl.get_server_certificate((host, port), ssl_version=ssl_version, ca_certs=ca_certs)
-    except ssl.SSLError as ssl_error:
-      raise Fail("Failed to verify the SSL certificate for https://{0}:{1} with CA certificate in {2}. Error : {3}"
-               .format(host, port, ca_certs, str(ssl_error)))
+    raise Fail("Failed to verify the SSL certificate for https://{0}:{1} with CA certificate in {2}. Error : {3}"
+             .format(host, port, ca_certs, str(ssl_error)))
   return ssl_version
 
 

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -179,7 +179,7 @@ class Script(object):
         json.dump(Script.structuredOut, fp)
     except IOError, err:
       Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
-      
+
   def get_component_name(self):
     """
     To be overridden by subclasses.
@@ -217,7 +217,7 @@ class Script(object):
     """
     stack_name = Script.get_stack_name()
     component_name = self.get_component_name()
-    
+
     if component_name and stack_name:
       component_version = get_component_version(stack_name, component_name)
 
@@ -260,15 +260,15 @@ class Script(object):
     parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
                       help="use this option to enable outputting *.out files of the service pre-start")
     (self.options, args) = parser.parse_args()
-    
+
     self.log_out_files = self.options.log_out_files
-    
+
     # parse arguments
     if len(args) < 6:
      print "Script expects at least 6 arguments"
      print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
      sys.exit(1)
-     
+
     self.command_name = str.lower(sys.argv[1])
     self.command_data_file = sys.argv[2]
     self.basedir = sys.argv[3]
@@ -311,7 +311,7 @@ class Script(object):
 
         if not self.is_hook():
           self.execute_prefix_function(self.command_name, 'pre', env)
-        
+
         method(env)
 
         if not self.is_hook():
@@ -338,10 +338,10 @@ class Script(object):
   def is_hook(self):
     from resource_management.libraries.script.hook import Hook
     return (Hook in self.__class__.__bases__)
-        
+
   def get_log_folder(self):
     return ""
-  
+
   def get_user(self):
     return ""
 
@@ -353,15 +353,15 @@ class Script(object):
     if self.log_out_files:
       log_folder = self.get_log_folder()
       user = self.get_user()
-      
+
       if log_folder == "":
         Logger.logger.warn("Log folder for current script is not defined")
         return
-      
+
       if user == "":
         Logger.logger.warn("User for current script is not defined")
         return
-      
+
       show_logs(log_folder, user, lines_count=COUNT_OF_LAST_LINES_OF_OUT_FILES_LOGGED, mask=OUT_FILES_MASK)
 
 
@@ -400,7 +400,7 @@ class Script(object):
       raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name))
     method = getattr(self, command_name)
     return method
-  
+
   def get_stack_version_before_packages_installed(self):
     """
     This works in a lazy way (calculates the version first time and stores it). 
@@ -417,7 +417,7 @@ class Script(object):
     if not Script.stack_version_from_distro_select and component_name:
       from resource_management.libraries.functions import stack_select
       Script.stack_version_from_distro_select = stack_select.get_stack_version_before_install(component_name)
-      
+
     # If <stack-selector-tool> has not yet been done (situations like first install),
     # we can use <stack-selector-tool> version itself.
     # Wildcards cause a lot of troubles with installing packages, if the version contains wildcards we should try to specify it.
@@ -428,7 +428,7 @@ class Script(object):
               stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME))
 
     return Script.stack_version_from_distro_select
-  
+
   def format_package_name(self, name):
     from resource_management.libraries.functions.default import default
     """
@@ -465,7 +465,7 @@ class Script(object):
       stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name
 
     package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted)
-    
+
     return package_name
 
   @staticmethod
@@ -490,9 +490,24 @@ class Script(object):
     return Script.tmp_dir
 
   @staticmethod
-  def get_force_https_protocol():
+  def get_force_https_protocol_name():
+    """
+    Get forced https protocol name.
+
+    :return: protocol name, PROTOCOL_TLSv1 by default
+    """
     return Script.force_https_protocol
 
+  @staticmethod
+  def get_force_https_protocol_value():
+    """
+    Get forced https protocol value that correspondents to ssl module variable.
+
+    :return: protocol value
+    """
+    import ssl
+    return getattr(ssl, Script.get_force_https_protocol_name())
+
   @staticmethod
   def get_component_from_role(role_directory_map, default_role):
     """
@@ -664,13 +679,13 @@ class Script(object):
                           hadoop_user, self.get_password(hadoop_user),
                           str(config['hostLevelParams']['stack_version']))
       reload_windows_env()
-      
+
   def check_package_condition(self, package):
     condition = package['condition']
-    
+
     if not condition:
       return True
-    
+
     return self.should_install_package(package)
 
   def should_install_package(self, package):
@@ -860,7 +875,7 @@ class Script(object):
     config = self.get_config()
     return {'configurations':config['configurations'][dict],
             'configuration_attributes':config['configuration_attributes'][dict]}
-    
+
   def generate_configs_get_xml_file_dict(self, filename, dict):
     config = self.get_config()
     return config['configurations'][dict]
@@ -872,7 +887,7 @@ class Script(object):
     """
     import params
     env.set_params(params)
-    
+
     config = self.get_config()
 
     xml_configs_list = config['commandParams']['xml_configs_list']

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py

@@ -23,6 +23,7 @@ from ambari_commons.parallel_processing import PrallelProcessResult, execute_in_
 from service_check import post_metrics_to_collector
 from resource_management.core.logger import Logger
 from resource_management.core.base import Fail
+from resource_management.libraries.script.script import Script
 from resource_management import Template
 from collections import namedtuple
 from urlparse import urlparse
@@ -54,9 +55,13 @@ def perform_grafana_get_call(url, server):
 
   for i in xrange(0, GRAFANA_CONNECT_TRIES):
     try:
-      conn = network.get_http_connection(server.host,
-                                         int(server.port),
-                                         grafana_https_enabled, ca_certs)
+      conn = network.get_http_connection(
+        server.host,
+        int(server.port),
+        grafana_https_enabled,
+        ca_certs,
+        ssl_version=Script.get_force_https_protocol_value()
+      )
 
       userAndPass = b64encode('{0}:{1}'.format(server.user, server.password))
       headers = { 'Authorization' : 'Basic %s' %  userAndPass }
@@ -94,7 +99,13 @@ def perform_grafana_put_call(url, id, payload, server):
 
   for i in xrange(0, GRAFANA_CONNECT_TRIES):
     try:
-      conn = network.get_http_connection(server.host, int(server.port), grafana_https_enabled, ca_certs)
+      conn = network.get_http_connection(
+        server.host,
+        int(server.port),
+        grafana_https_enabled,
+        ca_certs,
+        ssl_version=Script.get_force_https_protocol_value()
+      )
       conn.request("PUT", url + "/" + str(id), payload, headers)
       response = conn.getresponse()
       data = response.read()
@@ -130,9 +141,12 @@ def perform_grafana_post_call(url, payload, server):
   for i in xrange(0, GRAFANA_CONNECT_TRIES):
     try:
       Logger.info("Connecting (POST) to %s:%s%s" % (server.host, server.port, url))
-      conn = network.get_http_connection(server.host,
-                                         int(server.port),
-                                         grafana_https_enabled, ca_certs)
+      conn = network.get_http_connection(
+        server.host,
+        int(server.port),
+        grafana_https_enabled, ca_certs,
+        ssl_version=Script.get_force_https_protocol_value()
+      )
       
       conn.request("POST", url, payload, headers)
 
@@ -171,9 +185,12 @@ def perform_grafana_delete_call(url, server):
 
   for i in xrange(0, GRAFANA_CONNECT_TRIES):
     try:
-      conn = network.get_http_connection(server.host,
-                                         int(server.port),
-                                         grafana_https_enabled, ca_certs)
+      conn = network.get_http_connection(
+        server.host,
+        int(server.port),
+        grafana_https_enabled, ca_certs,
+        ssl_version=Script.get_force_https_protocol_value()
+      )
 
       userAndPass = b64encode('{0}:{1}'.format(server.user, server.password))
       headers = { 'Authorization' : 'Basic %s' %  userAndPass }

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py

@@ -91,10 +91,13 @@ class AMSServiceCheck(Script):
                                                    params.metric_collector_port,
                                                    self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters))
       for i in xrange(0, self.AMS_READ_TRIES):
-        conn = network.get_http_connection(metric_collector_host,
-                                           int(params.metric_collector_port),
-                                           params.metric_collector_https_enabled,
-                                           ca_certs)
+        conn = network.get_http_connection(
+          metric_collector_host,
+          int(params.metric_collector_port),
+          params.metric_collector_https_enabled,
+          ca_certs,
+          ssl_version=Script.get_force_https_protocol_value()
+        )
         conn.request("GET", self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
         response = conn.getresponse()
         Logger.info("Http response for host %s : %s %s" % (metric_collector_host, response.status, response.reason))
@@ -164,10 +167,13 @@ def post_metrics_to_collector(ams_metrics_post_url, metric_collector_host, metri
       Logger.info("Connecting (POST) to %s:%s%s" % (metric_collector_host,
                                                     metric_collector_port,
                                                     ams_metrics_post_url))
-      conn = network.get_http_connection(metric_collector_host,
-                                         int(metric_collector_port),
-                                         metric_collector_https_enabled,
-                                         ca_certs)
+      conn = network.get_http_connection(
+        metric_collector_host,
+        int(metric_collector_port),
+        metric_collector_https_enabled,
+        ca_certs,
+        ssl_version=Script.get_force_https_protocol_value()
+      )
       conn.request("POST", ams_metrics_post_url, metric_json, headers)
 
       response = conn.getresponse()

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py

@@ -34,7 +34,7 @@ from resource_management.libraries.functions.curl_krb_request import curl_krb_re
 from resource_management.libraries.functions.curl_krb_request import DEFAULT_KERBEROS_KINIT_TIMER_MS
 from resource_management.libraries.functions.curl_krb_request import KERBEROS_KINIT_TIMER_PARAMETER
 from ambari_commons.ambari_metrics_helper import select_metric_collector_for_sink
-
+from ambari_agent.AmbariConfig import AmbariConfig
 
 RESULT_STATE_OK = 'OK'
 RESULT_STATE_CRITICAL = 'CRITICAL'
@@ -320,7 +320,13 @@ def execute(configurations={}, parameters={}, host_name=None):
   metric_collector_https_enabled = str(configurations[AMS_HTTP_POLICY]) == "HTTPS_ONLY"
 
   try:
-    conn = network.get_http_connection(collector_host, int(collector_port), metric_collector_https_enabled, ca_certs)
+    conn = network.get_http_connection(
+      collector_host,
+      int(collector_port),
+      metric_collector_https_enabled,
+      ca_certs,
+      ssl_version=AmbariConfig.get_resolved_config().get_force_https_protocol_value()
+    )
     conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
     response = conn.getresponse()
     data = response.read()

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py

@@ -26,4 +26,4 @@ else:
 
 nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize']
 retryAble = default("/commandParams/command_retry_enabled", False)
-script_https_protocol = Script.get_force_https_protocol()
+script_https_protocol = Script.get_force_https_protocol_name()

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py

@@ -38,7 +38,7 @@ from resource_management.libraries.functions.show_logs import show_logs
 from ambari_commons.inet_utils import ensure_ssl_using_protocol
 from zkfc_slave import ZkfcSlaveDefault
 
-ensure_ssl_using_protocol(Script.get_force_https_protocol())
+ensure_ssl_using_protocol(Script.get_force_https_protocol_name())
 
 def safe_zkfc_op(action, env):
   """

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py

@@ -241,4 +241,4 @@ ttnode_heapsize = "1024m"
 dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
 mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
 mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
-script_https_protocol = Script.get_force_https_protocol()
+script_https_protocol = Script.get_force_https_protocol_name()

ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py

@@ -43,13 +43,22 @@ class TestRangerKMS(RMFTestCase):
     self.assertTrue(isfile_mock.called)
     self.assertNoMoreResources()
 
+  current_date = datetime.now()
+
+  class DTMOCK(object):
+    """
+    Mock datetime to avoid test failures when test run a little bit slower than usuall.
+    """
+    def now(self):
+      return TestRangerKMS.current_date
+
   @patch("resource_management.libraries.functions.ranger_functions.Rangeradmin.check_ranger_login_urllib2", new=MagicMock(return_value=200))
   @patch("resource_management.libraries.functions.ranger_functions.Rangeradmin.create_ambari_admin_user", new=MagicMock(return_value=200))
   @patch("kms.get_repo")
   @patch("kms.create_repo")
   @patch("os.path.isfile")
+  @patch("kms.datetime", new=DTMOCK())
   def test_start_default(self, get_repo_mock, create_repo_mock, isfile_mock):
-
     get_repo_mock.return_value = True
     create_repo_mock.return_value = True
 
@@ -64,7 +73,7 @@ class TestRangerKMS(RMFTestCase):
 
     # TODO confirm repo call
 
-    current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    current_datetime = self.current_date.strftime("%Y-%m-%d %H:%M:%S")
 
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/ranger-security.xml',
       owner = 'kms',