AMBARI-8482. HBase service components should indicate security state (rlevas)

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py

@@ -20,7 +20,9 @@ limitations under the License.
 
 import sys
 from resource_management import *
-
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_XML
 from hbase import hbase
 from hbase_service import hbase_service
 from hbase_decommission import hbase_decommission
@@ -65,6 +67,54 @@ class HbaseMaster(Script):
     pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
     check_process_status(pid_file)
 
+  def security_status(self, env):
+    import status_params
+
+    env.set_params(status_params)
+
+    props_value_check = {"hbase.security.authentication" : "kerberos",
+                         "hbase.security.authorization": "true"}
+    props_empty_check = ['hbase.master.keytab.file',
+                         'hbase.master.kerberos.principal']
+    props_read_check = ['hbase.master.keytab.file']
+    hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check,
+                                                props_read_check)
+
+    hbase_expectations = {}
+    hbase_expectations.update(hbase_site_expectations)
+
+    security_params = get_params_from_filesystem(status_params.hbase_conf_dir,
+                                                 {'hbase-site.xml': FILE_TYPE_XML})
+    result_issues = validate_security_config_properties(security_params, hbase_expectations)
+    if not result_issues:  # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'hbase-site' not in security_params
+             or 'hbase.master.keytab.file' not in security_params['hbase-site']
+             or 'hbase.master.kerberos.principal' not in security_params['hbase-site']):
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal are not set property."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.hbase_user,
+                              security_params['hbase-site']['hbase.master.keytab.file'],
+                              security_params['hbase-site']['hbase.master.kerberos.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
   def decommission(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py

@@ -20,7 +20,9 @@ limitations under the License.
 
 import sys
 from resource_management import *
-
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_XML
 from hbase import hbase
 from hbase_service import hbase_service
 import upgrade
@@ -69,7 +71,55 @@ class HbaseRegionServer(Script):
     env.set_params(status_params)
     pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
     check_process_status(pid_file)
-    
+
+  def security_status(self, env):
+    import status_params
+
+    env.set_params(status_params)
+
+    props_value_check = {"hbase.security.authentication" : "kerberos",
+                         "hbase.security.authorization": "true"}
+    props_empty_check = ['hbase.regionserver.keytab.file',
+                         'hbase.regionserver.kerberos.principal']
+    props_read_check = ['hbase.regionserver.keytab.file']
+    hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check,
+                                                 props_read_check)
+
+    hbase_expectations = {}
+    hbase_expectations.update(hbase_site_expectations)
+
+    security_params = get_params_from_filesystem(status_params.hbase_conf_dir,
+                                                 {'hbase-site.xml': FILE_TYPE_XML})
+    result_issues = validate_security_config_properties(security_params, hbase_expectations)
+    if not result_issues:  # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'hbase-site' not in security_params
+             or 'hbase.regionserver.keytab.file' not in security_params['hbase-site']
+             or 'hbase.regionserver.kerberos.principal' not in security_params['hbase-site']):
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal are not set property."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.hbase_user,
+                              security_params['hbase-site']['hbase.regionserver.keytab.file'],
+                              security_params['hbase-site']['hbase.regionserver.kerberos.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
   def decommission(self, env):
     print "Decommission not yet implemented!"
     

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py

@@ -24,3 +24,13 @@ config = Script.get_config()
 
 pid_dir = config['configurations']['hbase-env']['hbase_pid_dir']
 hbase_user = config['configurations']['hbase-env']['hbase_user']
+
+# Security related/required params
+hostname = config['hostname']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+tmp_dir = Script.get_tmp_dir()
+
+
+hbase_conf_dir_prefix = "/etc/hbase"
+hbase_conf_dir = format("{hbase_conf_dir_prefix}/conf")

ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/kerberos.json

@@ -91,9 +91,6 @@
               }
             }
           ]
-        },
-        {
-          "name": "HBASE_CLIENT"
         }
       ]
     }

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py

@@ -541,3 +541,104 @@ class TestHBaseMaster(RMFTestCase):
 
     self.assertNoMoreResources()
 
+  @patch("resource_management.libraries.functions.security_commons.build_expectations")
+  @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+  @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+    # Test that function works when is called with correct parameters
+    import status_params
+
+    security_params = {}
+    security_params['hbase-site'] = {}
+    security_params['hbase-site']['hbase.master.kerberos.principal'] = '/path/to/hbase_keytab'
+    security_params['hbase-site']['hbase.master.keytab.file'] = 'hbase_principal'
+
+    result_issues = []
+    props_value_check = {"hbase.security.authentication": "kerberos",
+                           "hbase.security.authorization": "true"}
+    props_empty_check = ["hbase.master.keytab.file",
+                           "hbase.master.kerberos.principal"]
+
+    props_read_check = ["hbase.master.keytab.file"]
+
+    get_params_mock.return_value = security_params
+    validate_security_config_mock.return_value = result_issues
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
+                   classname = "HbaseMaster",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    build_exp_mock.assert_called_with('hbase-site', props_value_check, props_empty_check, props_read_check)
+    put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+    cached_kinit_executor_mock.called_with(status_params.kinit_path_local,
+                              status_params.hbase_user,
+                              security_params['hbase-site']['hbase.master.keytab.file'],
+                              security_params['hbase-site']['hbase.master.kerberos.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+
+     # Testing that the exception throw by cached_executor is caught
+    cached_kinit_executor_mock.reset_mock()
+    cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+    try:
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
+                   classname = "HbaseMaster",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+      )
+    except:
+      self.assertTrue(True)
+
+    # Testing with a security_params which doesn't contains hbase-site
+    empty_security_params = {}
+    cached_kinit_executor_mock.reset_mock()
+    get_params_mock.reset_mock()
+    put_structured_out_mock.reset_mock()
+    get_params_mock.return_value = empty_security_params
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
+                   classname = "HbaseMaster",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
+
+    # Testing with not empty result_issues
+    result_issues_with_params = {}
+    result_issues_with_params['hbase-site']="Something bad happened"
+
+    validate_security_config_mock.reset_mock()
+    get_params_mock.reset_mock()
+    validate_security_config_mock.return_value = result_issues_with_params
+    get_params_mock.return_value = security_params
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
+                   classname = "HbaseMaster",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+    # Testing with security_enable = false
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
+                   classname = "HbaseMaster",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py

@@ -467,3 +467,105 @@ class TestHbaseRegionServer(RMFTestCase):
       user = 'hbase')
 
     self.assertNoMoreResources()
+
+  @patch("resource_management.libraries.functions.security_commons.build_expectations")
+  @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
+  @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
+    # Test that function works when is called with correct parameters
+    import status_params
+
+    security_params = {}
+    security_params['hbase-site'] = {}
+    security_params['hbase-site']['hbase.regionserver.keytab.file'] = '/path/to/hbase_keytab'
+    security_params['hbase-site']['hbase.regionserver.kerberos.principal'] = 'hbase_principal'
+
+    result_issues = []
+    props_value_check = {"hbase.security.authentication": "kerberos",
+                           "hbase.security.authorization": "true"}
+    props_empty_check = ["hbase.regionserver.keytab.file",
+                           "hbase.regionserver.kerberos.principal"]
+
+    props_read_check = ["hbase.regionserver.keytab.file"]
+
+    get_params_mock.return_value = security_params
+    validate_security_config_mock.return_value = result_issues
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
+                   classname = "HbaseRegionServer",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    build_exp_mock.assert_called_with('hbase-site', props_value_check, props_empty_check, props_read_check)
+    put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
+    cached_kinit_executor_mock.called_with(status_params.kinit_path_local,
+                              status_params.hbase_user,
+                              security_params['hbase-site']['hbase.regionserver.keytab.file'],
+                              security_params['hbase-site']['hbase.regionserver.kerberos.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir,
+                              30)
+
+     # Testing that the exception throw by cached_executor is caught
+    cached_kinit_executor_mock.reset_mock()
+    cached_kinit_executor_mock.side_effect = Exception("Invalid command")
+
+    try:
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
+                   classname = "HbaseRegionServer",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+      )
+    except:
+      self.assertTrue(True)
+
+    # Testing with a security_params which doesn't contains hbase-site
+    empty_security_params = {}
+    cached_kinit_executor_mock.reset_mock()
+    get_params_mock.reset_mock()
+    put_structured_out_mock.reset_mock()
+    get_params_mock.return_value = empty_security_params
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
+                   classname = "HbaseRegionServer",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
+
+    # Testing with not empty result_issues
+    result_issues_with_params = {}
+    result_issues_with_params['hbase-site']="Something bad happened"
+
+    validate_security_config_mock.reset_mock()
+    get_params_mock.reset_mock()
+    validate_security_config_mock.return_value = result_issues_with_params
+    get_params_mock.return_value = security_params
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
+                   classname = "HbaseRegionServer",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+    # Testing with security_enable = false
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
+                   classname = "HbaseRegionServer",
+                   command = "security_status",
+                   config_file="secured.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})