AMBARI-9534 AMS Service Rename (additional patch) (dsen)

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml

@@ -125,10 +125,10 @@ export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Djava.security.auth.login.co
 
 # use embedded native libs
 _HADOOP_NATIVE_LIB="/usr/lib/ams-hbase/lib/hadoop-native/"
-export HBASE_OPTS="$HBASE_OPTS -Djava.library.path=${_HADOOP_NATIVE_LIB}"
-
-#"Unsetting" HADOOP_HOME to avoid importing HADOOP installed cluster related configs like: /usr/hdp/2.2.0.0-2041/hadoop/conf/ 
+{% if disable_hadoop_environment %}
+# Unset HADOOP_HOME to avoid importing HADOOP installed cluster related configs like: /usr/hdp/2.2.0.0-2041/hadoop/conf/
 export HADOOP_HOME=`pwd`
+{% endif %}
     </value>
   </property>
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml

@@ -64,7 +64,7 @@
   </property>
   <property>
     <name>hbase.zookeeper.quorum</name>
-    <value>localhost</value>
+    <value>{{zookeeper_quorum_hosts}}</value>
     <description>Comma separated list of servers in the ZooKeeper Quorum.
       For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
       By default this is set to localhost for local and pseudo-distributed modes

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml

@@ -117,6 +117,7 @@
         <config-type>ams-log4j</config-type>
         <config-type>ams-hbase-policy</config-type>
         <config-type>ams-hbase-site</config-type>
+        <config-type>ams-hbase-security-site</config-type>
         <config-type>ams-hbase-env</config-type>
         <config-type>ams-hbase-log4j</config-type>
       </configuration-dependencies>

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py

@@ -24,6 +24,7 @@ from service_mapping import *
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons.str_utils import compress_backslashes
 import glob
+import os
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
 def ams(name=None):
@@ -49,9 +50,14 @@ def ams(name=None):
               owner=params.ams_user,
     )
 
+    merged_ams_hbase_site = {}
+    merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
+    if params.security_enabled:
+      merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
+
     XmlConfig( "hbase-site.xml",
                conf_dir = params.ams_collector_conf_dir,
-               configurations = params.config['configurations']['ams-hbase-site'],
+               configurations = merged_ams_hbase_site,
                configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
                owner = params.ams_user,
     )
@@ -131,14 +137,24 @@ def ams(name=None):
               group=params.user_group
     )
 
+    merged_ams_hbase_site = {}
+    merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
+    if params.security_enabled:
+      merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
+
     XmlConfig( "hbase-site.xml",
                conf_dir = params.ams_collector_conf_dir,
-               configurations = params.config['configurations']['ams-hbase-site'],
+               configurations = merged_ams_hbase_site,
                configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
                owner = params.ams_user,
                group = params.user_group
     )
 
+    if params.security_enabled:
+      TemplateConfig(os.path.join(params.hbase_conf_dir, "ams_collector_jaas.conf"),
+                     owner = params.ams_user,
+                     template_tag = None)
+
     if (params.log4j_props != None):
       File(format("{params.ams_collector_conf_dir}/log4j.properties"),
            mode=0644,

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py

@@ -116,9 +116,14 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
              recursive = True
   )
 
+  merged_ams_hbase_site = {}
+  merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
+  if params.security_enabled:
+    merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
+
   XmlConfig("hbase-site.xml",
             conf_dir = params.hbase_conf_dir,
-            configurations = params.config['configurations']['ams-hbase-site'],
+            configurations = merged_ams_hbase_site,
             configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
             owner = params.hbase_user,
             group = params.user_group
@@ -159,6 +164,8 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
 
   if params.security_enabled:
     hbase_TemplateConfig( format("hbase_{name}_jaas.conf"), user=params.hbase_user)
+    hbase_TemplateConfig( format("hbase_client_jaas.conf"), user=params.hbase_user)
+    hbase_TemplateConfig( format("ams_zookeeper_jaas.conf"), user=params.hbase_user)
 
   if name in ["master","regionserver"]:
 
@@ -169,6 +176,13 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
                            owner=params.hbase_user,
                            mode=0775
       )
+
+      params.HdfsDirectory(params.hbase_staging_dir,
+                           action="create_delayed",
+                           owner=params.hbase_user,
+                           mode=0711
+      )
+
       params.HdfsDirectory(None, action="create")
 
     else:

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py

@@ -19,6 +19,9 @@ limitations under the License.
 """
 
 from resource_management import *
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_XML
 from ams import ams
 from ams_service import ams_service
 from hbase import hbase
@@ -27,7 +30,7 @@ from status import check_service_status
 class AmsCollector(Script):
   def install(self, env):
     self.install_packages(env)
-    self.configure(env) # for security
+    # self.configure(env) # for security
 
 
   def configure(self, env):
@@ -55,6 +58,66 @@ class AmsCollector(Script):
     env.set_params(status_params)
     check_service_status(name='collector')
 
+  def security_status(self, env):
+    import status_params
+
+    env.set_params(status_params)
+    props_value_check = {"hbase.security.authentication": "kerberos",
+                         "hbase.security.authorization": "true"}
+
+    props_empty_check = ["hbase.zookeeper.property.authProvider.1",
+                         "hbase.master.keytab.file",
+                         "hbase.master.kerberos.principal",
+                         "hbase.regionserver.keytab.file",
+                         "hbase.regionserver.kerberos.principal"
+    ]
+    props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file']
+    ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check,
+                                                     props_empty_check,
+                                                     props_read_check)
+
+    expectations = {}
+    expectations.update(ams_hbase_site_expectations)
+
+    security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir,
+                                                 {'hbase-site.xml': FILE_TYPE_XML})
+
+    is_hbase_distributed = security_params['hbase-site']['hbase.rootdir'].startswith('hdfs://')
+    # for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured
+    if status_params.security_enabled and not is_hbase_distributed:
+      self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      return
+
+    result_issues = validate_security_config_properties(security_params, expectations)
+
+    if not result_issues:  # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ('hbase-site' not in security_params or
+                'hbase.master.keytab.file' not in security_params['hbase-site'] or
+                'hbase.master.kerberos.principal' not in security_params['hbase-site']):
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Keytab file or principal are not set property."})
+          return
+
+        cached_kinit_executor(status_params.kinit_path_local,
+                              status_params.hbase_user,
+                              security_params['hbase-site']['hbase.master.keytab.file'],
+                              security_params['hbase-site']['hbase.master.kerberos.principal'],
+                              status_params.hostname,
+                              status_params.tmp_dir)
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (
+          cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
 
 if __name__ == "__main__":
   AmsCollector().execute()

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -65,7 +65,9 @@ hbase_root_dir = config['configurations']['ams-hbase-site']['hbase.rootdir']
 is_hbase_distributed = hbase_root_dir.startswith('hdfs://')
 
 # security is disabled for embedded mode, when HBase is backed by file
-security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled'] 
+security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']
+# if cluster is secured and embedded we have to disable haddop env
+disable_hadoop_environment = config['configurations']['cluster-env']['security_enabled'] and not is_hbase_distributed
 
 # this is "hadoop-metrics.properties" for 1.x stacks
 metric_prop_file_name = "hadoop-metrics2-hbase.properties"
@@ -83,6 +85,9 @@ regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xm
 # For embedded mode
 hbase_heapsize = master_heapsize
 
+zookeeper_quorum_hosts = ','.join(ams_collector_hosts) if is_hbase_distributed else 'localhost'
+hbase_cluster_distributed = 'true' if is_hbase_distributed else 'false'
+
 ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir']
 hbase_pid_dir = status_params.hbase_pid_dir
 hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir']
@@ -102,21 +107,29 @@ service_check_data = functions.get_unique_id_and_date()
 user_group = config['configurations']['cluster-env']["user_group"]
 hadoop_user = "hadoop"
 
+kinit_cmd = ""
+
 if security_enabled:
   _hostname_lowercase = config['hostname'].lower()
-  master_jaas_princ = default('/configurations/ams-hbase-site/hbase.master.kerberos.principal', 'hbase/_HOST@EXAMPLE.COM').replace('_HOST',_hostname_lowercase)
-  regionserver_jaas_princ = default('/configurations/ams-hbase-site/hbase.regionserver.kerberos.principal', 'hbase/_HOST@EXAMPLE.COM').replace('_HOST',_hostname_lowercase)
-
-  master_keytab_path = config['configurations']['ams-hbase-site']['hbase.master.keytab.file']
-  regionserver_keytab_path = config['configurations']['ams-hbase-site']['hbase.regionserver.keytab.file']
+  client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
   smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
   hbase_user_keytab = config['configurations']['ams-hbase-env']['hbase_user_keytab']
-  kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 
-if security_enabled:
-   kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
-else:
-   kinit_cmd = ""
+  ams_collector_jaas_config_file = format("{hbase_conf_dir}/ams_collector_jaas.conf")
+  ams_collector_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.myclient.keytab']
+  ams_collector_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.myclient.principal'].replace('_HOST',_hostname_lowercase)
+
+  ams_zookeeper_jaas_config_file = format("{hbase_conf_dir}/ams_zookeeper_jaas.conf")
+  ams_zookeeper_keytab = config['configurations']['ams-hbase-security-site']['ams.zookeeper.keytab']
+  ams_zookeeper_principal_name = config['configurations']['ams-hbase-security-site']['ams.zookeeper.principal'].replace('_HOST',_hostname_lowercase)
+
+  master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
+  master_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.master.keytab.file']
+  master_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+
+  regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
+  regionserver_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.regionserver.keytab.file']
+  regionserver_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
 
 #log4j.properties
 if (('ams-hbase-log4j' in config['configurations']) and ('content' in config['configurations']['ams-hbase-log4j'])):

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params_linux.py

@@ -20,12 +20,12 @@ limitations under the License.
 
 from resource_management import *
 from ambari_commons import OSCheck
-import status_params
+
 config = Script.get_config()
 
 ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf"
 ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/"
-ams_user = status_params.ams_user
+ams_user = config['configurations']['ams-env']['ambari_metrics_user']
 #RPM versioning support
 rpm_version = default("/configurations/hadoop-env/rpm_version", None)
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py

@@ -19,12 +19,21 @@ limitations under the License.
 """
 
 from resource_management import *
+from ambari_commons import OSCheck
 
-config = Script.get_config()
+if OSCheck.is_windows_family():
+  from params_windows import *
+else:
+  from params_linux import *
 
 hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir']
-ams_user = config['configurations']['ams-env']['ambari_metrics_user']
 hbase_user = ams_user
 ams_collector_pid_dir = config['configurations']['ams-env']['metrics_collector_pid_dir']
 ams_monitor_pid_dir = config['configurations']['ams-env']['metrics_monitor_pid_dir']
 
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+ams_hbase_conf_dir = format("{hbase_conf_dir}")
+
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+hostname = config['hostname']
+tmp_dir = Script.get_tmp_dir()

pom.xml

@@ -271,7 +271,8 @@
             <exclude>ambari-metrics/ambari-metrics-timelineservice/src/test/resources/lib/org/apache/phoenix/phoenix-core-tests/4.2.0/phoenix-core-tests-4.2.0.pom</exclude>
             <exclude>ambari-metrics/ambari-metrics-timelineservice/src/test/resources/lib/org/apache/phoenix/phoenix-core-tests/maven-metadata-local.xml</exclude>
             <exclude>ambari-metrics/*/target/**</exclude>
-
+            <!-- ignore .settings and .project  -->
+            <exclude>ambari-metrics/**/.*/**</exclude>
           </excludes>
         </configuration>
         <executions>