AMBARI-8797. Zookeeper service components should indicate security state (rlevas)

ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py

@@ -25,6 +25,7 @@ import json
 
 FILE_TYPE_XML = 'XML'
 FILE_TYPE_PROPERTIES = 'PROPERTIES'
+FILE_TYPE_JAAS_CONF = 'JAAS_CONF'
 
 def validate_security_config_properties(params, configuration_rules):
   """
@@ -57,19 +58,20 @@ def validate_security_config_properties(params, configuration_rules):
       rules = rule_sets['value_checks'] if 'value_checks' in rule_sets else None
       if rules:
         for property_name, expected_value in rules.iteritems():
-          actual_value = actual_values[property_name] if property_name in actual_values else ''
+          actual_value = get_value(actual_values, property_name, '')
           if actual_value != expected_value:
-            issues[config_file] = "Property " + property_name + ". Expected/Actual: " + \
-                                  expected_value + "/" + actual_value
+            issues[config_file] = "Property %s contains an unexpected value. " \
+                                  "Expected/Actual: %s/%s" \
+                                  % (property_name, expected_value, actual_value)
 
       # Process Empty Checks
       # The rules are expected to be a list of property names that should not have empty values
       rules = rule_sets['empty_checks'] if 'empty_checks' in rule_sets else None
       if rules:
         for property_name in rules:
-          actual_value = actual_values[property_name] if property_name in actual_values else ''
+          actual_value = get_value(actual_values, property_name, '')
           if not actual_value:
-            issues[config_file] = "Property " + property_name + " must exist and must not be empty!"
+            issues[config_file] = "Property %s must exist and must not be empty" % property_name
 
       # Process Read Checks
       # The rules are expected to be a list of property names that resolve to files names and must
@@ -77,12 +79,13 @@ def validate_security_config_properties(params, configuration_rules):
       rules = rule_sets['read_checks'] if 'read_checks' in rule_sets else None
       if rules:
         for property_name in rules:
-          actual_value = actual_values[property_name] if property_name in actual_values else None
-          if not actual_value or not os.path.isfile(actual_value):
-            issues[
-              config_file] = "Property " + property_name + " points to an inaccessible file or parameter does not exist!"
+          actual_value = get_value(actual_values, property_name, None)
+          if not actual_value:
+            issues[config_file] = "Property %s does not exist" % property_name
+          elif not os.path.isfile(actual_value):
+            issues[config_file] = "Property %s points to an inaccessible file - %s" % (property_name, actual_value)
     except Exception as e:
-      issues[config_file] = "Exception occurred while validating the config file\nCauses: " + str(e)
+      issues[config_file] = "Exception occurred while validating the config file\nCauses: %s" % str(e)
   return issues
 
 
@@ -117,6 +120,8 @@ def get_params_from_filesystem(conf_dir, config_files):
   result = {}
   from xml.etree import ElementTree as ET
   import ConfigParser, StringIO
+  import re
+
   for config_file, file_type in config_files.iteritems():
     file_name, file_ext = os.path.splitext(config_file)
 
@@ -138,6 +143,30 @@ def get_params_from_filesystem(conf_dir, config_files):
       result[file_name] = {}
       for key, value in props:
         result[file_name].update({key : value})
+
+    elif file_type == FILE_TYPE_JAAS_CONF:
+      section_header = re.compile('^(\w+)\s+\{\s*$')
+      section_data = re.compile('(^[^ \s\=\}\{]+)\s*=?\s*"?([^ ";]+)"?;?\s*$')
+      section_footer = re.compile('^\}\s*;?\s*$')
+      section_name = "root"
+      result[file_name] = {}
+      with open(conf_dir + os.sep + config_file, 'r') as f:
+        for line in f:
+          if line:
+            m = section_header.search(line)
+            if m:
+              section_name = m.group(1)
+              if section_name not in result[file_name]:
+                result[file_name][section_name] = {}
+            else:
+              m = section_footer.search(line)
+              if m:
+                section_name = "root"
+              else:
+                m = section_data.search(line)
+                if m:
+                  result[file_name][section_name][m.group(1)] = m.group(2)
+
   return result
 
 
@@ -198,3 +227,15 @@ def new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principa
       json.dump(result, cache_file)
   finally:
     os.remove(temp_kinit_cache_file)
+
+def get_value(values, property_path, default_value):
+  names = property_path.split('/')
+
+  current_dict = values
+  for name in names:
+    if name in current_dict:
+      current_dict = current_dict[name]
+    else:
+      return default_value
+
+  return current_dict

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py

@@ -24,3 +24,11 @@ config = Script.get_config()
 
 zk_pid_dir = config['configurations']['zookeeper-env']['zk_pid_dir']
 zk_pid_file = format("{zk_pid_dir}/zookeeper_server.pid")
+
+# Security related/required params
+hostname = config['hostname']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+tmp_dir = Script.get_tmp_dir()
+config_dir = "/etc/zookeeper/conf"
+zk_user =  config['configurations']['zookeeper-env']['zk_user']

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py

@@ -25,6 +25,9 @@ from resource_management import *
 from resource_management.libraries.functions import get_unique_id_and_date
 from resource_management.libraries.functions.decorator import retry
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_JAAS_CONF
 from resource_management.libraries.functions.format import format
 from resource_management.core.shell import call
 
@@ -96,5 +99,59 @@ class ZookeeperServer(Script):
     env.set_params(status_params)
     check_process_status(status_params.zk_pid_file)
 
+  def security_status(self, env):
+    import status_params
+
+    env.set_params(status_params)
+
+    if status_params.security_enabled:
+      # Expect the following files to be available in status_params.config_dir:
+      #   zookeeper_jaas.conf
+      #   zookeeper_client_jaas.conf
+      try:
+        props_value_check = None
+        props_empty_check = ['Server/keyTab', 'Server/principal']
+        props_read_check = ['Server/keyTab']
+        zk_env_expectations = build_expectations('zookeeper_jaas', props_value_check, props_empty_check,
+                                                 props_read_check)
+
+        zk_expectations = {}
+        zk_expectations.update(zk_env_expectations)
+
+        security_params = get_params_from_filesystem(status_params.config_dir,
+                                                   {'zookeeper_jaas.conf': FILE_TYPE_JAAS_CONF})
+
+        result_issues = validate_security_config_properties(security_params, zk_expectations)
+        if not result_issues:  # If all validations passed successfully
+          # Double check the dict before calling execute
+          if ( 'zookeeper_jaas' not in security_params
+               or 'Server' not in security_params['zookeeper_jaas']
+               or 'keyTab' not in security_params['zookeeper_jaas']['Server']
+               or 'principal' not in security_params['zookeeper_jaas']['Server']):
+            self.put_structured_out({"securityState": "ERROR"})
+            self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
+            return
+
+          cached_kinit_executor(status_params.kinit_path_local,
+                                status_params.zk_user,
+                                security_params['zookeeper_jaas']['Server']['keyTab'],
+                                security_params['zookeeper_jaas']['Server']['principal'],
+                                status_params.hostname,
+                                status_params.tmp_dir,
+                                30)
+          self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+        else:
+          issues = []
+          for cf in result_issues:
+            issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+          self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+          self.put_structured_out({"securityState": "UNSECURED"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      self.put_structured_out({"securityState": "UNSECURED"})
+
+
 if __name__ == "__main__":
   ZookeeperServer().execute()

ambari-server/src/test/python/stacks/2.0.6/ZOOKEEPER/test_zookeeper_server.py

@@ -227,3 +227,103 @@ class TestZookeeperServer(RMFTestCase):
       owner = 'zookeeper',
       group = 'hadoop',
     )
+
+  @patch("resource_management.libraries.functions.security_commons.build_expectations")
+  @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+  @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+    # Test that function works when is called with correct parameters
+    import status_params
+
+    security_params = {}
+    security_params['zookeeper_jaas'] = {}
+    security_params['zookeeper_jaas']['Server'] = {}
+    security_params['zookeeper_jaas']['Server']['keyTab'] = 'path/to/zookeeper/service/keytab'
+    security_params['zookeeper_jaas']['Server']['principal'] = 'zookeeper_keytab'
+    result_issues = []
+    props_value_check = None
+    props_empty_check = ['Server/keyTab', 'Server/principal']
+    props_read_check = ['Server/keyTab']
+
+    get_params_mock.return_value = security_params
+    validate_security_config_mock.return_value = result_issues
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "security_status",
+                       config_file = "secured.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    build_exp_mock.assert_called_with('zookeeper_jaas', props_value_check, props_empty_check, props_read_check)
+    put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+    self.assertTrue(cached_kinit_executor_mock.call_count, 2)
+    cached_kinit_executor_mock.assert_called_with(status_params.kinit_path_local,
+                              status_params.zk_user,
+                              security_params['zookeeper_jaas']['Server']['keyTab'],
+                              security_params['zookeeper_jaas']['Server']['principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+
+    # Testing that the exception throw by cached_executor is caught
+    cached_kinit_executor_mock.reset_mock()
+    cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+    try:
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "security_status",
+                       config_file = "secured.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+      )
+    except:
+      self.assertTrue(True)
+
+    # Testing with a security_params which doesn't contains zookeeper_jaas
+    empty_security_params = {}
+    cached_kinit_executor_mock.reset_mock()
+    get_params_mock.reset_mock()
+    put_structured_out_mock.reset_mock()
+    get_params_mock.return_value = empty_security_params
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "security_status",
+                       config_file = "secured.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
+
+    # Testing with not empty result_issues
+    result_issues_with_params = {}
+    result_issues_with_params['zookeeper_jaas']="Something bad happened"
+
+    validate_security_config_mock.reset_mock()
+    get_params_mock.reset_mock()
+    validate_security_config_mock.return_value = result_issues_with_params
+    get_params_mock.return_value = security_params
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "security_status",
+                       config_file = "secured.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+    # Testing with security_enable = false
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "security_status",
+                       config_file = "default.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})