AMBARI-8795. MapReduce service components should indicate security state (rlevas)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py

@@ -23,6 +23,9 @@ from resource_management import *
 from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_XML
 
 from yarn import yarn
 from service import service
@@ -62,5 +65,59 @@ class HistoryServer(Script):
     env.set_params(status_params)
     check_process_status(status_params.mapred_historyserver_pid_file)
 
+  def security_status(self, env):
+    import status_params
+    env.set_params(status_params)
+
+    expectations = {}
+    expectations.update(build_expectations('mapred-site',
+                                           None,
+                                           [
+                                             'mapreduce.jobhistory.keytab',
+                                             'mapreduce.jobhistory.principal',
+                                             'mapreduce.jobhistory.webapp.spnego-keytab-file',
+                                             'mapreduce.jobhistory.webapp.spnego-principal'
+                                           ],
+                                           None))
+
+    security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+                                                 {'mapred-site.xml': FILE_TYPE_XML})
+    result_issues = validate_security_config_properties(security_params, expectations)
+    if not result_issues: # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'mapred-site' not in security_params or
+             'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or
+             'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or
+             'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or
+             'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']):
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal not set."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.mapred_user,
+                              security_params['mapred-site']['mapreduce.jobhistory.keytab'],
+                              security_params['mapred-site']['mapreduce.jobhistory.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir)
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.mapred_user,
+                              security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
+                              security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
 if __name__ == "__main__":
   HistoryServer().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py

@@ -21,6 +21,7 @@ limitations under the License.
 from resource_management import *
 
 config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
 
 mapred_user = config['configurations']['mapred-env']['mapred_user']
 yarn_user = config['configurations']['yarn-env']['yarn_user']
@@ -33,4 +34,8 @@ resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanag
 nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid")
 yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid")
 yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid")  # *-historyserver.pid is deprecated
-mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
+mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
+
+hadoop_conf_dir = "/etc/hadoop/conf"
+hostname = config['hostname']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -511,3 +511,96 @@ class TestHistoryServer(RMFTestCase):
                               owner = 'mapred',
                               group = 'hadoop',
                               )
+
+  @patch("resource_management.libraries.functions.security_commons.build_expectations")
+  @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+  @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+    # Test that function works when is called with correct parameters
+
+    security_params = {
+      "mapred-site": {
+        'mapreduce.jobhistory.keytab': "/path/to/keytab1",
+        'mapreduce.jobhistory.principal': "principal1",
+        'mapreduce.jobhistory.webapp.spnego-keytab-file': "/path/to/keytab2",
+        'mapreduce.jobhistory.webapp.spnego-principal': "principal2"
+      }
+    }
+    result_issues = []
+
+    get_params_mock.return_value = security_params
+    validate_security_config_mock.return_value = result_issues
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+                       classname="HistoryServer",
+                       command="security_status",
+                       config_file="secured.json")
+
+    import status_params
+
+    get_params_mock.assert_called_with(status_params.hadoop_conf_dir, {'mapred-site.xml': 'XML'})
+    build_exp_mock.assert_called_with('mapred-site',
+                                      None,
+                                      [
+                                        'mapreduce.jobhistory.keytab',
+                                        'mapreduce.jobhistory.principal',
+                                        'mapreduce.jobhistory.webapp.spnego-keytab-file',
+                                        'mapreduce.jobhistory.webapp.spnego-principal'
+                                        ],
+                                      None)
+    put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+    self.assertTrue(cached_kinit_executor_mock.call_count, 2)
+    cached_kinit_executor_mock.assert_called_with(status_params.kinit_path_local,
+                                                  status_params.mapred_user,
+                                                  security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
+                                                  security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
+                                                  status_params.hostname,
+                                                  status_params.tmp_dir)
+
+    # Testing that the exception throw by cached_executor is caught
+    cached_kinit_executor_mock.reset_mock()
+    cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+    try:
+      self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+                         classname="HistoryServer",
+                         command="security_status",
+                         config_file="secured.json")
+    except:
+      self.assertTrue(True)
+
+    # Testing with a security_params which doesn't contain mapred-site
+    empty_security_params = {}
+    cached_kinit_executor_mock.reset_mock()
+    get_params_mock.reset_mock()
+    put_structured_out_mock.reset_mock()
+    get_params_mock.return_value = empty_security_params
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+                       classname="HistoryServer",
+                       command="security_status",
+                       config_file="secured.json")
+    put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal not set."})
+
+    # Testing with not empty result_issues
+    result_issues_with_params = {'mapred-site': "Something bad happened"}
+
+    validate_security_config_mock.reset_mock()
+    get_params_mock.reset_mock()
+    validate_security_config_mock.return_value = result_issues_with_params
+    get_params_mock.return_value = security_params
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+                       classname="HistoryServer",
+                       command="security_status",
+                       config_file="secured.json")
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+    # Testing with security_enable = false
+    self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+                       classname="HistoryServer",
+                       command="security_status",
+                       config_file="default.json")
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})