AMBARI-17666. Ambari agent can't start when TLSv1 is disabled in Java security (dlysnichenko)

ambari-agent/src/main/python/ambari_agent/AmbariConfig.py

@@ -23,7 +23,6 @@ import ConfigParser
 import StringIO
 import hostname
 import ambari_simplejson as json
-from NetUtil import NetUtil
 import os
 
 from ambari_commons import OSConst
@@ -156,7 +155,6 @@ class AmbariConfig:
   def __init__(self):
     global content
     self.config = ConfigParser.RawConfigParser()
-    self.net = NetUtil(self)
     self.config.readfp(StringIO.StringIO(content))
 
   def get(self, section, value, default=None):
@@ -182,13 +180,22 @@ class AmbariConfig:
   def getConfig(self):
     return self.config
 
-  @staticmethod
-  @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
-  def getConfigFile(home_dir=""):
-    if 'AMBARI_AGENT_CONF_DIR' in os.environ:
-      return os.path.join(os.environ['AMBARI_AGENT_CONF_DIR'], "ambari-agent.ini")
-    else:
-      return "ambari-agent.ini"
+  @classmethod
+  def get_resolved_config(cls, home_dir=""):
+    if hasattr(cls, "_conf_cache"):
+      return getattr(cls, "_conf_cache")
+    config = cls()
+    configPath = os.path.abspath(cls.getConfigFile(home_dir))
+    try:
+      if os.path.exists(configPath):
+        config.read(configPath)
+      else:
+        raise Exception("No config found at {0}, use default".format(configPath))
+
+    except Exception, err:
+      logger.warn(err)
+    setattr(cls, "_conf_cache", config)
+    return config
 
   @staticmethod
   @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
@@ -256,7 +263,8 @@ class AmbariConfig:
     self.config.read(filename)
 
   def getServerOption(self, url, name, default=None):
-    status, response = self.net.checkURL(url)
+    from ambari_agent.NetUtil import NetUtil
+    status, response = NetUtil(self).checkURL(url)
     if status is True:
       try:
         data = json.loads(response)
@@ -293,6 +301,9 @@ class AmbariConfig:
         logger.info("Updating config property (%s) with value (%s)", k, v)
     pass
 
+  def get_force_https_protocol(self):
+    return self.get('security', 'force_https_protocol', default="PROTOCOL_TLSv1")
+
 def isSameHostList(hostlist1, hostlist2):
   is_same = True
 

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -78,6 +78,7 @@ class CustomServiceOrchestrator():
   def __init__(self, config, controller):
     self.config = config
     self.tmp_dir = config.get('agent', 'prefix')
+    self.force_https_protocol = config.get_force_https_protocol()
     self.exec_tmp_dir = Constants.AGENT_TMP_DIR
     self.file_cache = FileCache(config)
     self.status_commands_stdout = os.path.join(self.tmp_dir,
@@ -385,7 +386,8 @@ class CustomServiceOrchestrator():
       
       for py_file, current_base_dir in filtered_py_file_list:
         log_info_on_failure = not command_name in self.DONT_DEBUG_FAILURES_FOR_COMMANDS
-        script_params = [command_name, json_path, current_base_dir, tmpstrucoutfile, logger_level, self.exec_tmp_dir]
+        script_params = [command_name, json_path, current_base_dir, tmpstrucoutfile, logger_level, self.exec_tmp_dir,
+                         self.force_https_protocol]
         
         if log_out_files:
           script_params.append("-o")

ambari-agent/src/main/python/ambari_agent/NetUtil.py

@@ -20,6 +20,8 @@ import httplib
 import sys
 from ssl import SSLError
 from HeartbeatHandlers import HeartbeatStopHandlers
+from ambari_agent.AmbariConfig import AmbariConfig
+from ambari_commons.inet_utils import ensure_ssl_using_protocol
 
 ERROR_SSL_WRONG_VERSION = "SSLError: Failed to connect. Please check openssl library versions. \n" +\
               "Refer to: https://bugzilla.redhat.com/show_bug.cgi?id=1022468 for more details."
@@ -27,6 +29,7 @@ LOG_REQUEST_MESSAGE = "GET %s -> %s, body: %s"
 
 logger = logging.getLogger(__name__)
 
+ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol())
 
 class NetUtil:
 

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -33,8 +33,9 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
 from resource_management.libraries.functions.get_path_from_url import get_path_from_url
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
 from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_tls_v1
+from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol
 from ambari_agent import Constants
+from ambari_agent.AmbariConfig import AmbariConfig
 
 # hashlib is supplied as of Python 2.5 as the replacement interface for md5
 # and other secure hashes.  In 2.6, md5 is deprecated.  Import hashlib if
@@ -54,7 +55,7 @@ DEFAULT_CONNECTION_TIMEOUT = 5
 
 WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg')
 
-ensure_ssl_using_tls_v1()
+ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol())
 
 class WebAlert(BaseAlert):
 
@@ -250,4 +251,4 @@ class WebAlert(BaseAlert):
     if state == self.RESULT_CRITICAL:
       return 'Connection failed to {1}'
 
-    return 'HTTP {0} response in {2:.4f} seconds'
+    return 'HTTP {0} response in {2:.4f} seconds'

ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py

@@ -54,8 +54,8 @@ class TestCustomServiceOrchestrator(TestCase):
     # generate sample config
     tmpdir = tempfile.gettempdir()
     exec_tmp_dir = os.path.join(tmpdir, 'tmp')
-    self.config = ConfigParser.RawConfigParser()
-    self.config.get = AmbariConfig().get
+    self.config = AmbariConfig()
+    self.config.config = ConfigParser.RawConfigParser()
     self.config.add_section('agent')
     self.config.set('agent', 'prefix', tmpdir)
     self.config.set('agent', 'cache_dir', "/cachedir")

ambari-common/src/main/python/ambari_commons/inet_utils.py

@@ -183,10 +183,11 @@ def resolve_address(address):
       return '127.0.0.1'
   return address
 
-def ensure_ssl_using_tls_v1():
+def ensure_ssl_using_protocol(protocol):
   """
   Monkey patching ssl module to force it use tls_v1. Do this in common module to avoid problems with
   PythonReflectiveExecutor.
+  :param protocol: one of ("PROTOCOL_SSLv2", "PROTOCOL_SSLv3", "PROTOCOL_SSLv23", "PROTOCOL_TLSv1", "PROTOCOL_TLSv1_1", "PROTOCOL_TLSv1_2")
   :return:
   """
   from functools import wraps
@@ -197,8 +198,8 @@ def ensure_ssl_using_tls_v1():
     @wraps(func)
     def bar(*args, **kw):
       import ssl
-      kw['ssl_version'] = ssl.PROTOCOL_TLSv1
+      kw['ssl_version'] = getattr(ssl, protocol)
       return func(*args, **kw)
     bar._ambari_patched = True
     return bar
-  ssl.wrap_socket = sslwrap(ssl.wrap_socket)
+  ssl.wrap_socket = sslwrap(ssl.wrap_socket)

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -66,7 +66,7 @@ if OSCheck.is_windows_family():
 else:
   from resource_management.libraries.functions.tar_archive import archive_dir
 
-USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL> <TMP_DIR>
+USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL> <TMP_DIR> [PROTOCOL]
 
 <COMMAND> command type (INSTALL/CONFIGURE/START/STOP/SERVICE_CHECK...)
 <JSON_CONFIG> path to command json file. Ex: /var/lib/ambari-agent/data/command-2.json
@@ -74,6 +74,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
 <STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
 <LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
 <TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp
+[PROTOCOL] optional protocol to use during https connections. Ex: see python ssl.PROTOCOL_<PROTO> variables, default PROTOCOL_TLSv1
 """
 
 _PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}
@@ -120,7 +121,8 @@ class Script(object):
 
   # Class variable
   tmp_dir = ""
- 
+  force_https_protocol = "PROTOCOL_TLSv1"
+
   def load_structured_out(self):
     Script.structuredOut = {}
     if os.path.exists(self.stroutfile):
@@ -246,7 +248,10 @@ class Script(object):
     self.load_structured_out()
     self.logging_level = sys.argv[5]
     Script.tmp_dir = sys.argv[6]
-    
+    # optional script argument for forcing https protocol
+    if len(sys.argv) >= 8:
+      Script.force_https_protocol = sys.argv[7]
+
     logging_level_str = logging._levelNames[self.logging_level]
     Logger.initialize_logger(__name__, logging_level=logging_level_str)
 
@@ -407,6 +412,10 @@ class Script(object):
     """
     return Script.tmp_dir
 
+  @staticmethod
+  def get_force_https_protocol():
+    return Script.force_https_protocol
+
   @staticmethod
   def get_component_from_role(role_directory_map, default_role):
     """

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/checkWebUI.py

@@ -23,22 +23,23 @@ import httplib
 import socket
 import ssl
 
-class TLS1HTTPSConnection(httplib.HTTPSConnection):
+class ForcedProtocolHTTPSConnection(httplib.HTTPSConnection):
   """
   Some of python implementations does not work correctly with sslv3 but trying to use it, we need to change protocol to
   tls1.
   """
-  def __init__(self, host, port, **kwargs):
+  def __init__(self, host, port, force_protocol, **kwargs):
     httplib.HTTPSConnection.__init__(self, host, port, **kwargs)
+    self.force_protocol = force_protocol
 
   def connect(self):
     sock = socket.create_connection((self.host, self.port), self.timeout)
     if getattr(self, '_tunnel_host', None):
       self.sock = sock
       self._tunnel()
-    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
+    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=getattr(ssl, self.force_protocol))
 
-def make_connection(host, port, https):
+def make_connection(host, port, https, force_protocol=None):
   try:
     conn = httplib.HTTPConnection(host, port) if not https else httplib.HTTPSConnection(host, port)
     conn.request("GET", "/")
@@ -46,7 +47,7 @@ def make_connection(host, port, https):
   except ssl.SSLError:
     # got ssl error, lets try to use TLS1 protocol, maybe it will work
     try:
-      tls1_conn = TLS1HTTPSConnection(host, port)
+      tls1_conn = ForcedProtocolHTTPSConnection(host, port, force_protocol)
       tls1_conn.request("GET", "/")
       return tls1_conn.getresponse().status
     except Exception as e:
@@ -65,15 +66,17 @@ def main():
   parser.add_option("-m", "--hosts", dest="hosts", help="Comma separated hosts list for WEB UI to check it availability")
   parser.add_option("-p", "--port", dest="port", help="Port of WEB UI to check it availability")
   parser.add_option("-s", "--https", dest="https", help="\"True\" if value of dfs.http.policy is \"HTTPS_ONLY\"")
+  parser.add_option("-o", "--protocol", dest="protocol", help="Protocol to use when executing https request")
 
   (options, args) = parser.parse_args()
   
   hosts = options.hosts.split(',')
   port = options.port
   https = options.https
+  protocol = options.protocol
 
   for host in hosts:
-    httpCode = make_connection(host, port, https.lower() == "true")
+    httpCode = make_connection(host, port, https.lower() == "true", protocol)
 
     if httpCode != 200:
       print "Cannot access WEB UI on: http://" + host + ":" + port if not https.lower() == "true" else "Cannot access WEB UI on: https://" + host + ":" + port

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py

@@ -26,3 +26,4 @@ else:
 
 nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize']
 retryAble = default("/commandParams/command_retry_enabled", False)
+script_https_protocol = Script.get_force_https_protocol()

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py

@@ -86,7 +86,8 @@ class HdfsServiceCheckDefault(HdfsServiceCheck):
         checkWebUIFileName = "checkWebUI.py"
         checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
         comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
-        checkWebUICmd = format("ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only}")
+
+        checkWebUICmd = format("ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only} -o {script_https_protocol}")
         File(checkWebUIFilePath,
              content=StaticFile(checkWebUIFileName),
              mode=0775)

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py

@@ -35,10 +35,10 @@ from resource_management.core.exceptions import Fail
 from resource_management.libraries.functions.namenode_ha_utils import get_namenode_states
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.functions.show_logs import show_logs
-from ambari_commons.inet_utils import ensure_ssl_using_tls_v1
+from ambari_commons.inet_utils import ensure_ssl_using_protocol
 from zkfc_slave import ZkfcSlaveDefault
 
-ensure_ssl_using_tls_v1()
+ensure_ssl_using_protocol(Script.get_force_https_protocol())
 
 def safe_zkfc_op(action, env):
   """

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py

@@ -241,3 +241,4 @@ ttnode_heapsize = "1024m"
 dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
 mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
 mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+script_https_protocol = Script.get_force_https_protocol()

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/service_check.py

@@ -93,7 +93,7 @@ class HdfsServiceCheck(Script):
       comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
       checkWebUICmd = format(
         "su -s /bin/bash - {smoke_test_user} -c 'python {checkWebUIFilePath} -m "
-        "{comma_sep_jn_hosts} -p {journalnode_port}'")
+        "{comma_sep_jn_hosts} -p {journalnode_port} -o {script_https_protocol}'")
       File(checkWebUIFilePath,
            content=StaticFile(checkWebUIFileName))