AMBARI-4400. unit test failures in ambari-server (aonishuk)

ambari-agent/src/main/python/resource_management/__init__.py

@@ -23,3 +23,4 @@ Ambari Agent
 from resource_management.libraries import *
 from resource_management.core import *
 
+

ambari-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py

@@ -22,10 +22,10 @@ Ambari Agent
 
 __all__ = ["get_unique_id_and_date"]
 import datetime
-from resource_management.core.shell import checked_call
+from resource_management.core import shell
 
 def get_unique_id_and_date():
-    code, out = checked_call("hostid")
+    out = shell.checked_call("hostid")[1]
     id = out.strip()
 
     now = datetime.datetime.now()

ambari-server/src/test/python/stacks/1.3.3/HBASE/test_hbase_regionserver.py

@@ -21,6 +21,7 @@ from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
 
 class TestHbaseRegionServer(RMFTestCase):
+  
   def test_configure_default(self):
     self.executeScript("1.3.3/services/HBASE/package/scripts/hbase_regionserver.py",
                    classname = "HbaseRegionServer",

ambari-server/src/test/python/stacks/1.3.3/HIVE/test_mysql_server.py

@@ -21,7 +21,6 @@ from mock.mock import MagicMock, call, patch
 from stacks.utils.RMFTestCase import *
 
 class TestMySqlServer(RMFTestCase):
-
   def test_configure_default(self):
     self.executeScript("2.1.1/services/HIVE/package/scripts/mysql_server.py",
                        classname = "MysqlServer",
@@ -38,7 +37,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="default.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
                        logoutput = True,
                        path = ['/usr/local/bin/:/bin/:/sbin/'],
                        tries = 1,
@@ -52,7 +51,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="default.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
                               logoutput = True,
                               path = ['/usr/local/bin/:/bin/:/sbin/'],
                               tries = 1,
@@ -76,7 +75,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="secured.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
                               logoutput = True,
                               path = ['/usr/local/bin/:/bin/:/sbin/'],
                               tries = 1,
@@ -90,7 +89,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="secured.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
                               logoutput = True,
                               path = ['/usr/local/bin/:/bin/:/sbin/'],
                               tries = 1,
@@ -98,7 +97,7 @@ class TestMySqlServer(RMFTestCase):
     self.assertNoMoreResources()
 
   def assert_configure_default(self):
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,
@@ -107,20 +106,20 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysqld', u'hive', 'asd', u'c6402.ambari.apache.org'),
+    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', u'hive', 'asd', u'c6402.ambari.apache.org'),
       logoutput = True,
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
     )
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,
     )
 
   def assert_configure_secured(self):
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,
@@ -129,13 +128,13 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysqld', u'hive', 'asd', u'c6402.ambari.apache.org'),
+    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', u'hive', 'asd', u'c6402.ambari.apache.org'),
       logoutput = True,
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
     )
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,

ambari-server/src/test/python/stacks/2.1.1/HIVE/test_mysql_server.py

@@ -38,7 +38,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="default.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
                        logoutput = True,
                        path = ['/usr/local/bin/:/bin/:/sbin/'],
                        tries = 1,
@@ -52,7 +52,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="default.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
                               logoutput = True,
                               path = ['/usr/local/bin/:/bin/:/sbin/'],
                               tries = 1,
@@ -76,7 +76,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="secured.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
                               logoutput = True,
                               path = ['/usr/local/bin/:/bin/:/sbin/'],
                               tries = 1,
@@ -90,7 +90,7 @@ class TestMySqlServer(RMFTestCase):
                        config_file="secured.json"
     )
 
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
                               logoutput = True,
                               path = ['/usr/local/bin/:/bin/:/sbin/'],
                               tries = 1,
@@ -98,7 +98,7 @@ class TestMySqlServer(RMFTestCase):
     self.assertNoMoreResources()
 
   def assert_configure_default(self):
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,
@@ -107,20 +107,20 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysqld', u'hive', 'asd', u'c6402.ambari.apache.org'),
+    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', u'hive', 'asd', u'c6402.ambari.apache.org'),
       logoutput = True,
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
     )
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,
     )
 
   def assert_configure_secured(self):
-    self.assertResourceCalled('Execute', 'service mysqld start',
+    self.assertResourceCalled('Execute', 'service mysql start',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,
@@ -129,13 +129,13 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysqld', u'hive', 'asd', u'c6402.ambari.apache.org'),
+    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', u'hive', 'asd', u'c6402.ambari.apache.org'),
       logoutput = True,
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
     )
-    self.assertResourceCalled('Execute', 'service mysqld stop',
+    self.assertResourceCalled('Execute', 'service mysql stop',
       logoutput = True,
       path = ['/usr/local/bin/:/bin/:/sbin/'],
       tries = 1,

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -29,14 +29,20 @@ from mock.mock import MagicMock, patch
 from resource_management.core.environment import Environment
 from resource_management.libraries.script.config_dictionary import ConfigDictionary
 from resource_management.libraries.script.script import Script
+import platform
+
+PATH_TO_STACKS = os.path.normpath("main/resources/stacks/HDP")
+PATH_TO_STACK_TESTS = os.path.normpath("test/python/stacks/")
 
 class RMFTestCase(TestCase):
-  def executeScript(self, path, classname=None, command=None, config_file=None):
+  def executeScript(self, path, classname=None, command=None, config_file=None, 
+                    shell_mock_value = (0, "OK."), os_type=('Suse','11','Final')):
+    norm_path = os.path.normpath(path)
     src_dir = RMFTestCase._getSrcFolder()
-    stack_version = path.split(os.sep)[0]
-    stacks_path = os.path.join(src_dir,"main/resources/stacks/HDP")
-    configs_path = os.path.join(src_dir, "test/python/stacks/", stack_version, "configs")
-    script_path = os.path.join(stacks_path, path)
+    stack_version = norm_path.split(os.sep)[0]
+    stacks_path = os.path.join(src_dir, PATH_TO_STACKS)
+    configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
+    script_path = os.path.join(stacks_path, norm_path)
     config_file_path = os.path.join(configs_path, config_file)
     
     try:
@@ -52,9 +58,10 @@ class RMFTestCase(TestCase):
     
     # get method to execute
     try:
-      script_module = imp.load_source(classname, script_path)
+      with patch.object(platform, 'linux_distribution', return_value=os_type):
+        script_module = imp.load_source(classname, script_path)
     except IOError:
-      raise RuntimeError("Cannot load class %s from %s",classname, path)
+      raise RuntimeError("Cannot load class %s from %s",classname, norm_path)
     
     script_class_inst = RMFTestCase._get_attr(script_module, classname)()
     method = RMFTestCase._get_attr(script_class_inst, command)
@@ -65,16 +72,17 @@ class RMFTestCase(TestCase):
     
     # run
     with Environment(basedir, test_mode=True) as RMFTestCase.env:
-      with patch.object(Script, 'install_packages', return_value=MagicMock()):
-        with patch.object(Script, 'get_config', return_value=self.config_dict):
-          method(RMFTestCase.env)
+      with patch('resource_management.core.shell.checked_call', return_value=shell_mock_value): # we must always mock any shell calls
+        with patch.object(Script, 'get_config', return_value=self.config_dict): # mocking configurations
+          with patch.object(Script, 'install_packages'):
+            method(RMFTestCase.env)
   
   def getConfig(self):
     return self.config_dict
           
   @staticmethod
   def _getSrcFolder():
-    return os.path.join(os.path.abspath(os.path.dirname(__file__)),"../../../../")
+    return os.path.join(os.path.abspath(os.path.dirname(__file__)),os.path.normpath("../../../../"))
       
   @staticmethod
   def _get_attr(module, attr):

ambari-server/src/test/python/unitTests.py

@@ -114,10 +114,12 @@ def main():
   pwd = os.path.abspath(os.path.dirname(__file__))
 
   ambari_server_folder = get_parent_path(pwd,'ambari-server')
-  ambari_agent_folder = os.path.join(ambari_server_folder,"../ambari-agent")
-  ambari_common_folder = os.path.join(ambari_server_folder,"../ambari-common")
-  sys.path.append(ambari_common_folder + "/src/main/python/jinja2")
-  sys.path.append(ambari_agent_folder + "/src/main/python")
+  ambari_agent_folder = os.path.join(ambari_server_folder,os.path.normpath("../ambari-agent"))
+  ambari_common_folder = os.path.join(ambari_server_folder,os.path.normpath("../ambari-common"))
+  # append pythonpath (for running from IDE)
+  sys.path.append(ambari_common_folder + os.path.normpath("/src/test/python"))
+  sys.path.append(ambari_common_folder + os.path.normpath("/src/main/python/jinja2"))
+  sys.path.append(ambari_agent_folder + os.path.normpath("/src/main/python"))
 
   stacks_folder = pwd+'/stacks'
   #generate test variants(path, service, stack)