AMBARI-8479. YARN service components should indicate security state (rlevas)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py

@@ -21,6 +21,9 @@ Ambari Agent
 
 from resource_management import *
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,\
+  FILE_TYPE_XML
 from resource_management.libraries.functions.format import format
 
 from yarn import yarn
@@ -63,5 +66,67 @@ class ApplicationTimelineServer(Script):
             only_if = format("test -e {yarn_historyserver_pid_file_old}", user=status_params.yarn_user))
     functions.check_process_status(status_params.yarn_historyserver_pid_file)
 
+  def security_status(self, env):
+    import status_params
+    env.set_params(status_params)
+
+    props_value_check = {"yarn.timeline-service.enabled": "true",
+                         "yarn.timeline-service.http-authentication.type": "kerberos",
+                         "yarn.acl.enable": "true"}
+    props_empty_check = ["yarn.timeline-service.principal",
+                         "yarn.timeline-service.keytab",
+                         "yarn.timeline-service.http-authentication.kerberos.principal",
+                         "yarn.timeline-service.http-authentication.kerberos.keytab"]
+
+    props_read_check = ["yarn.timeline-service.keytab",
+                        "yarn.timeline-service.http-authentication.kerberos.keytab"]
+    yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
+                                                props_read_check)
+
+    yarn_expectations ={}
+    yarn_expectations.update(yarn_site_props)
+
+    security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+                                                 {'yarn-site.xml': FILE_TYPE_XML})
+    result_issues = validate_security_config_properties(security_params, yarn_expectations)
+    if not result_issues: # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'yarn-site' not in security_params
+             or 'yarn.timeline-service.keytab' not in security_params['yarn-site']
+             or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \
+          or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \
+          or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']:
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal are not set property."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.timeline-service.keytab'],
+                              security_params['yarn-site']['yarn.timeline-service.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'],
+                              security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
+
 if __name__ == "__main__":
   ApplicationTimelineServer().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py

@@ -24,6 +24,9 @@ import nodemanager_upgrade
 from resource_management import *
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_XML
 
 from yarn import yarn
 from service import service
@@ -69,5 +72,66 @@ class Nodemanager(Script):
     env.set_params(status_params)
     check_process_status(status_params.nodemanager_pid_file)
 
+  def security_status(self, env):
+    import status_params
+    env.set_params(status_params)
+
+    props_value_check = {"yarn.timeline-service.enabled": "true",
+                         "yarn.timeline-service.http-authentication.type": "kerberos",
+                         "yarn.acl.enable": "true"}
+    props_empty_check = ["yarn.nodemanager.principal",
+                         "yarn.nodemanager.keytab",
+                         "yarn.nodemanager.webapp.spnego-principal",
+                         "yarn.nodemanager.webapp.spnego-keytab-file"]
+
+    props_read_check = ["yarn.nodemanager.keytab",
+                        "yarn.nodemanager.webapp.spnego-keytab-file"]
+    yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
+                                         props_read_check)
+
+    yarn_expectations ={}
+    yarn_expectations.update(yarn_site_props)
+
+    security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+                                                 {'yarn-site.xml': FILE_TYPE_XML})
+    result_issues = validate_security_config_properties(security_params, yarn_site_props)
+    if not result_issues: # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'yarn-site' not in security_params
+             or 'yarn.nodemanager.keytab' not in security_params['yarn-site']
+             or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \
+          or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \
+          or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']:
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal are not set property."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.nodemanager.keytab'],
+                              security_params['yarn-site']['yarn.nodemanager.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'],
+                              security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
 if __name__ == "__main__":
   Nodemanager().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py

@@ -21,6 +21,9 @@ Ambari Agent
 
 from resource_management import *
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_XML
 
 from yarn import yarn
 from service import service
@@ -69,6 +72,68 @@ class Resourcemanager(Script):
     check_process_status(status_params.resourcemanager_pid_file)
     pass
 
+  def security_status(self, env):
+    import status_params
+    env.set_params(status_params)
+
+    props_value_check = {"yarn.timeline-service.enabled": "true",
+                         "yarn.timeline-service.http-authentication.type": "kerberos",
+                         "yarn.acl.enable": "true"}
+    props_empty_check = ["yarn.resourcemanager.principal",
+                         "yarn.resourcemanager.keytab",
+                         "yarn.resourcemanager.webapp.spnego-principal",
+                         "yarn.resourcemanager.webapp.spnego-keytab-file"]
+
+    props_read_check = ["yarn.resourcemanager.keytab",
+                        "yarn.resourcemanager.webapp.spnego-keytab-file"]
+    yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
+                                         props_read_check)
+
+    yarn_expectations ={}
+    yarn_expectations.update(yarn_site_props)
+
+    security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+                                                 {'yarn-site.xml': FILE_TYPE_XML})
+    result_issues = validate_security_config_properties(security_params, yarn_site_props)
+    if not result_issues: # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'yarn-site' not in security_params
+             or 'yarn.resourcemanager.keytab' not in security_params['yarn-site']
+             or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \
+          or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \
+          or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']:
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal are not set property."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.resourcemanager.keytab'],
+                              security_params['yarn-site']['yarn.resourcemanager.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'],
+                              security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
+
   def refreshqueues(self, env):
     import params
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py

@@ -36,6 +36,8 @@ yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-histor
 yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid")  # *-historyserver.pid is deprecated
 mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
 
+# Security related/required params
 hadoop_conf_dir = "/etc/hadoop/conf"
 hostname = config['hostname']
 kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+security_enabled = config['configurations']['cluster-env']['security_enabled']

ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py

@@ -609,3 +609,101 @@ class TestNodeManager(RMFTestCase):
     except Fail,fail:
       self.assertTrue(process_mock.called)
       self.assertEqual(process_mock.call_count,12)
+
+  @patch("resource_management.libraries.functions.security_commons.build_expectations")
+  @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+  @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+    # Test that function works when is called with correct parameters
+    import status_params
+
+    security_params = {}
+    security_params['yarn-site'] = {}
+    security_params['yarn-site']['yarn.nodemanager.keytab'] = 'path/to/nodemanager/keytab'
+    security_params['yarn-site']['yarn.nodemanager.principal'] = 'nodemanager_principal'
+    security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'] = 'path/to/nodemanager/webapp/keytab'
+    security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'] = 'nodemanager_webapp_principal'
+    result_issues = []
+    props_value_check = {"yarn.timeline-service.enabled": "true",
+                         "yarn.timeline-service.http-authentication.type": "kerberos",
+                         "yarn.acl.enable": "true"}
+    props_empty_check = ["yarn.nodemanager.principal",
+                         "yarn.nodemanager.keytab",
+                         "yarn.nodemanager.webapp.spnego-principal",
+                         "yarn.nodemanager.webapp.spnego-keytab-file"]
+
+    props_read_check = ["yarn.nodemanager.keytab",
+                        "yarn.nodemanager.webapp.spnego-keytab-file"]
+
+    get_params_mock.return_value = security_params
+    validate_security_config_mock.return_value = result_issues
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+                       classname="Nodemanager",
+                       command="security_status",
+                       config_file="secured.json"
+    )
+
+    build_exp_mock.assert_called_with('yarn-site', props_value_check, props_empty_check, props_read_check)
+    put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+    self.assertTrue(cached_kinit_executor_mock.call_count, 2)
+    cached_kinit_executor_mock.assert_called_with(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'],
+                              security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+
+    # Testing that the exception throw by cached_executor is caught
+    cached_kinit_executor_mock.reset_mock()
+    cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+    try:
+          self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+                       classname="Nodemanager",
+                       command="security_status",
+                       config_file="secured.json"
+          )
+    except:
+      self.assertTrue(True)
+
+    # Testing with a security_params which doesn't contains yarn-site
+    empty_security_params = {}
+    cached_kinit_executor_mock.reset_mock()
+    get_params_mock.reset_mock()
+    put_structured_out_mock.reset_mock()
+    get_params_mock.return_value = empty_security_params
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+                       classname="Nodemanager",
+                       command="security_status",
+                       config_file="secured.json"
+    )
+    put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
+
+    # Testing with not empty result_issues
+    result_issues_with_params = {}
+    result_issues_with_params['yarn-site']="Something bad happened"
+
+    validate_security_config_mock.reset_mock()
+    get_params_mock.reset_mock()
+    validate_security_config_mock.return_value = result_issues_with_params
+    get_params_mock.return_value = security_params
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+                       classname="Nodemanager",
+                       command="security_status",
+                       config_file="secured.json"
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+    # Testing with security_enable = false
+    self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+                       classname="Nodemanager",
+                       command="security_status",
+                       config_file="default.json"
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})

ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py

@@ -110,8 +110,8 @@ class TestResourceManager(RMFTestCase):
     self.assertResourceCalled('File', '/var/run/hadoop-yarn/yarn/yarn-yarn-resourcemanager.pid',
                               action=['delete'])
     self.assertNoMoreResources()
-    
-    
+
+
   def test_decommission_default(self):
     self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
                        classname = "Resourcemanager",
@@ -128,7 +128,7 @@ class TestResourceManager(RMFTestCase):
         user = 'yarn',
     )
     self.assertNoMoreResources()
-    
+
   def test_decommission_secured(self):
     self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
                        classname = "Resourcemanager",
@@ -144,7 +144,7 @@ class TestResourceManager(RMFTestCase):
         environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
         user = 'yarn',
     )
-    
+
     self.assertNoMoreResources()
 
   def assert_configure_default(self):
@@ -424,3 +424,101 @@ class TestResourceManager(RMFTestCase):
                               owner = 'mapred',
                               group = 'hadoop',
                               )
+
+  @patch("resource_management.libraries.functions.security_commons.build_expectations")
+  @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+  @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+    # Test that function works when is called with correct parameters
+    import status_params
+
+    security_params = {}
+    security_params['yarn-site'] = {}
+    security_params['yarn-site']['yarn.resourcemanager.keytab'] = '/path/to/resourcemanager/keytab'
+    security_params['yarn-site']['yarn.resourcemanager.principal'] = 'nresourcemanager_principal'
+    security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'] = 'path/to/resourcemanager/webapp/keytab'
+    security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'] = 'resourcemanager_webapp_principal'
+    result_issues = []
+    props_value_check = {"yarn.timeline-service.enabled": "true",
+                         "yarn.timeline-service.http-authentication.type": "kerberos",
+                         "yarn.acl.enable": "true"}
+    props_empty_check = ["yarn.resourcemanager.principal",
+                         "yarn.resourcemanager.keytab",
+                         "yarn.resourcemanager.webapp.spnego-principal",
+                         "yarn.resourcemanager.webapp.spnego-keytab-file"]
+
+    props_read_check = ["yarn.resourcemanager.keytab",
+                        "yarn.resourcemanager.webapp.spnego-keytab-file"]
+
+    get_params_mock.return_value = security_params
+    validate_security_config_mock.return_value = result_issues
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="security_status",
+                       config_file="secured.json"
+    )
+
+    build_exp_mock.assert_called_with('yarn-site', props_value_check, props_empty_check, props_read_check)
+    put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+    self.assertTrue(cached_kinit_executor_mock.call_count, 2)
+    cached_kinit_executor_mock.assert_called_with(status_params.kinit_path_local,
+                              status_params.yarn_user,
+                              security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'],
+                              security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+
+    # Testing that the exception throw by cached_executor is caught
+    cached_kinit_executor_mock.reset_mock()
+    cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+    try:
+          self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="security_status",
+                       config_file="secured.json"
+          )
+    except:
+      self.assertTrue(True)
+
+    # Testing with a security_params which doesn't contains yarn-site
+    empty_security_params = {}
+    cached_kinit_executor_mock.reset_mock()
+    get_params_mock.reset_mock()
+    put_structured_out_mock.reset_mock()
+    get_params_mock.return_value = empty_security_params
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="security_status",
+                       config_file="secured.json"
+    )
+    put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
+
+    # Testing with not empty result_issues
+    result_issues_with_params = {}
+    result_issues_with_params['yarn-site']="Something bad happened"
+
+    validate_security_config_mock.reset_mock()
+    get_params_mock.reset_mock()
+    validate_security_config_mock.return_value = result_issues_with_params
+    get_params_mock.return_value = security_params
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="security_status",
+                       config_file="secured.json"
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+    # Testing with security_enable = false
+    self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="security_status",
+                       config_file="default.json"
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})