AMBARI-4410. Failing unit tests (aonishuk)

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/functions.py

@@ -38,30 +38,3 @@ def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
   
   result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
   return str(result_xmn_val) + heapsize_unit
-
-def get_unique_id_and_date():
-    code, out = checked_call("hostid")
-    id = out.strip()
-    
-    now = datetime.datetime.now()
-    date = now.strftime("%M%d%y")
-
-    return "id{id}_date{date}".format(id=id, date=date)
-  
-def get_kinit_path(pathes_list):
-  """
-  @param pathes: comma separated list
-  """
-  kinit_path = ""
-  
-  for x in pathes_list:
-    if not x:
-      continue
-    
-    path = os.path.join(x,"kinit")
-
-    if os.path.isfile(path):
-      kinit_path = path
-      break
-    
-  return kinit_path

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/params.py

@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 
+from functions import calc_xmn_from_xms
 from resource_management import *
 import functions
 import status_params
@@ -43,7 +44,7 @@ log_dir = config['configurations']['global']['hbase_log_dir']
 master_heapsize = config['configurations']['global']['hbase_master_heapsize']
 
 regionserver_heapsize = config['configurations']['global']['hbase_regionserver_heapsize']
-regionserver_xmn_size = functions.calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
 
 pid_dir = status_params.pid_dir
 tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HDFS/package/scripts/params.py

@@ -50,7 +50,7 @@ dfs_journalnode_kerberos_internal_spnego_principal = config['configurations']['h
 hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
 exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
 
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 #hosts
 hostname = config["hostname"]
 rm_host = default("/clusterHostInfo/rm_host", [])

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HIVE/package/scripts/params.py

@@ -56,7 +56,7 @@ smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 
 security_enabled = config['configurations']['global']['security_enabled']
 
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 hive_metastore_keytab_path =  config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file']
 
 #hive_env

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/MAPREDUCE/package/scripts/params.py

@@ -46,7 +46,7 @@ hadoop_jar_location = "/usr/lib/hadoop/"
 smokeuser = config['configurations']['global']['smokeuser']
 security_enabled = config['configurations']['global']['security_enabled']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 
 #exclude file
 mr_exclude_hosts = default("/clusterHostInfo/decom_tt_hosts", [])

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/NAGIOS/package/scripts/params.py

@@ -20,8 +20,8 @@ Ambari Agent
 
 """
 
-from resource_management import *
 from functions import get_port_from_url
+from resource_management import *
 import status_params
 
 # server configurations
@@ -82,7 +82,7 @@ java64_home = config['hostLevelParams']['java_home']
 security_enabled = config['configurations']['global']['security_enabled']
 
 nagios_keytab_path = default("nagios_keytab_path", "/etc/security/keytabs/nagios.service.keytab")
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 
 dfs_ha_enabled = False
 dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.nameservices", None)

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/OOZIE/package/scripts/params.py

@@ -24,7 +24,7 @@ oozie_libext_dir = "/usr/lib/oozie/libext"
 lzo_enabled = config['configurations']['global']['lzo_enabled']
 security_enabled = config['configurations']['global']['security_enabled']
 
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file']
 oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal']
 smokeuser_keytab = config['configurations']['global']['smokeuser_keytab']

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/SQOOP/package/scripts/params.py

@@ -33,4 +33,4 @@ sqoop_user = "sqoop"
 
 keytab_path = config['configurations']['global']['keytab_path']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/WEBHCAT/package/scripts/params.py

@@ -48,4 +48,4 @@ webhcat_apps_dir = "/apps/webhcat"
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
 security_enabled = config['configurations']['global']['security_enabled']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/ZOOKEEPER/package/scripts/params.py

@@ -68,4 +68,4 @@ security_enabled = config['configurations']['global']['security_enabled']
 
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/functions.py

@@ -38,30 +38,3 @@ def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
   
   result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
   return str(result_xmn_val) + heapsize_unit
-
-def get_unique_id_and_date():
-    code, out = checked_call("hostid")
-    id = out.strip()
-    
-    now = datetime.datetime.now()
-    date = now.strftime("%M%d%y")
-
-    return "id{id}_date{date}".format(id=id, date=date)
-  
-def get_kinit_path(pathes_list):
-  """
-  @param pathes: comma separated list
-  """
-  kinit_path = ""
-  
-  for x in pathes_list:
-    if not x:
-      continue
-    
-    path = os.path.join(x,"kinit")
-
-    if os.path.isfile(path):
-      kinit_path = path
-      break
-    
-  return kinit_path

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/params.py

@@ -18,8 +18,8 @@ limitations under the License.
 
 """
 
+from functions import calc_xmn_from_xms
 from resource_management import *
-import functions
 import status_params
 
 # server configurations
@@ -43,7 +43,7 @@ log_dir = config['configurations']['global']['hbase_log_dir']
 master_heapsize = config['configurations']['global']['hbase_master_heapsize']
 
 regionserver_heapsize = config['configurations']['global']['hbase_regionserver_heapsize']
-regionserver_xmn_size = functions.calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
 
 pid_dir = status_params.pid_dir
 tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
@@ -81,4 +81,4 @@ master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab
 regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 hbase_user_keytab = config['configurations']['global']['hbase_user_keytab']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HDFS/package/scripts/params.py

@@ -51,7 +51,7 @@ dfs_journalnode_kerberos_internal_spnego_principal = config['configurations']['h
 hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
 exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
 
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 #hosts
 hostname = config["hostname"]
 rm_host = default("/clusterHostInfo/rm_host", [])

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/package/scripts/params.py

@@ -56,7 +56,7 @@ smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 
 security_enabled = config['configurations']['global']['security_enabled']
 
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 hive_metastore_keytab_path =  config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file']
 
 #hive_env

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/NAGIOS/package/scripts/params.py

@@ -20,8 +20,8 @@ Ambari Agent
 
 """
 
-from resource_management import *
 from functions import get_port_from_url
+from resource_management import *
 import status_params
 
 # server configurations
@@ -76,7 +76,7 @@ java64_home = config['hostLevelParams']['java_home']
 security_enabled = config['configurations']['global']['security_enabled']
 
 nagios_keytab_path = default("nagios_keytab_path", "/etc/security/keytabs/nagios.service.keytab")
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 
 dfs_ha_enabled = False
 dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.nameservices", None)

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/OOZIE/package/scripts/params.py

@@ -44,7 +44,7 @@ oozie_libext_dir = "/usr/lib/oozie/libext"
 lzo_enabled = config['configurations']['global']['lzo_enabled']
 security_enabled = config['configurations']['global']['security_enabled']
 
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file']
 oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal']
 smokeuser_keytab = config['configurations']['global']['smokeuser_keytab']

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/SQOOP/package/scripts/params.py

@@ -34,4 +34,4 @@ sqoop_user = "sqoop"
 
 keytab_path = config['configurations']['global']['keytab_path']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/WEBHCAT/package/scripts/params.py

@@ -48,4 +48,4 @@ webhcat_apps_dir = "/apps/webhcat"
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
 security_enabled = config['configurations']['global']['security_enabled']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/YARN/package/scripts/params.py

@@ -36,7 +36,7 @@ smokeuser = config['configurations']['global']['smokeuser']
 security_enabled = config['configurations']['global']['security_enabled']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 rm_host = config['clusterHostInfo']['rm_host'][0]
 rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
 rm_https_port = "8090"

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/ZOOKEEPER/package/scripts/params.py

@@ -68,4 +68,4 @@ security_enabled = config['configurations']['global']['security_enabled']
 
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
-kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/test/python/stacks/1.3.3/WEBHCAT/test_webhcat_server.py

@@ -167,12 +167,12 @@ class TestWebHCatServer(RMFTestCase):
       owner = 'hcat',
       group = 'hadoop',
     )
-    self.assertResourceCalled('Execute', ' -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
       path = ['/bin'],
       user = 'hcat',
     )
     self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop-mapreduce/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar',
-      not_if = ' -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
+      not_if = '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
       user = 'hcat',
       conf_dir = '/etc/hadoop/conf',
     )

ambari-server/src/test/python/stacks/2.1.1/WEBHCAT/test_webhcat_server.py

@@ -20,6 +20,9 @@ limitations under the License.
 from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
 
+from resource_management.libraries import functions
+import json
+
 class TestWebHCatServer(RMFTestCase):
 
   def test_configure_default(self):
@@ -167,12 +170,12 @@ class TestWebHCatServer(RMFTestCase):
       owner = 'hcat',
       group = 'hadoop',
     )
-    self.assertResourceCalled('Execute', ' -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
       path = ['/bin'],
       user = 'hcat',
     )
     self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop-mapreduce/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar',
-      not_if = ' -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
+      not_if = '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
       user = 'hcat',
       conf_dir = '/etc/hadoop/conf',
     )

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -35,8 +35,12 @@ PATH_TO_STACKS = os.path.normpath("main/resources/stacks/HDP")
 PATH_TO_STACK_TESTS = os.path.normpath("test/python/stacks/")
 
 class RMFTestCase(TestCase):
-  def executeScript(self, path, classname=None, command=None, config_file=None, 
-                    shell_mock_value = (0, "OK."), os_type=('Suse','11','Final')):
+  def executeScript(self, path, classname=None, command=None, config_file=None,
+                    # common mocks for all the scripts 
+                    shell_mock_value = (0, "OK."), 
+                    os_type=('Suse','11','Final'),
+                    kinit_path_local="/usr/bin/kinit"
+                    ):
     norm_path = os.path.normpath(path)
     src_dir = RMFTestCase._getSrcFolder()
     stack_version = norm_path.split(os.sep)[0]
@@ -75,7 +79,8 @@ class RMFTestCase(TestCase):
       with patch('resource_management.core.shell.checked_call', return_value=shell_mock_value): # we must always mock any shell calls
         with patch.object(Script, 'get_config', return_value=self.config_dict): # mocking configurations
           with patch.object(Script, 'install_packages'):
-            method(RMFTestCase.env)
+            with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local):
+              method(RMFTestCase.env)
   
   def getConfig(self):
     return self.config_dict