AMBARI-12647. Agent errors out during cache invalidation if cache directories are missing (dlysnichenko)

ambari-agent/src/main/python/ambari_agent/FileCache.py

@@ -220,11 +220,12 @@ class FileCache():
     """
     logger.debug("Invalidating directory {0}".format(directory))
     try:
-      if os.path.isfile(directory): # It would be a strange situation
-        os.unlink(directory)
-      elif os.path.isdir(directory):
-        shutil.rmtree(directory)
-      # create directory itself and any parent directories
+      if os.path.exists(directory):
+        if os.path.isfile(directory): # It would be a strange situation
+          os.unlink(directory)
+        elif os.path.isdir(directory):
+          shutil.rmtree(directory)
+        # create directory itself and any parent directories
       os.makedirs(directory)
     except Exception, err:
       raise CachingException("Can not invalidate cache directory {0}: {1}",

ambari-agent/src/test/python/ambari_agent/TestFileCache.py

@@ -274,17 +274,20 @@ class TestFileCache(TestCase):
         self.fail('Unexpected exception thrown:' + str(e))
 
 
+  @patch("os.path.exists")
   @patch("os.path.isfile")
   @patch("os.path.isdir")
   @patch("os.unlink")
   @patch("shutil.rmtree")
   @patch("os.makedirs")
   def test_invalidate_directory(self, makedirs_mock, rmtree_mock,
-                                unlink_mock, isdir_mock, isfile_mock):
+                                unlink_mock, isdir_mock, isfile_mock,
+                                exists_mock):
     fileCache = FileCache(self.config)
     # Test execution flow if path points to file
     isfile_mock.return_value = True
     isdir_mock.return_value = False
+    exists_mock.return_value = True
 
     fileCache.invalidate_directory("dummy-dir")
 
@@ -299,6 +302,7 @@ class TestFileCache(TestCase):
     # Test execution flow if path points to dir
     isfile_mock.return_value = False
     isdir_mock.return_value = True
+    exists_mock.return_value = True
 
     fileCache.invalidate_directory("dummy-dir")
 
@@ -310,6 +314,21 @@ class TestFileCache(TestCase):
     rmtree_mock.reset_mock()
     makedirs_mock.reset_mock()
 
+    # Test execution flow if path points nowhere
+    isfile_mock.return_value = False
+    isdir_mock.return_value = False
+    exists_mock.return_value = False
+
+    fileCache.invalidate_directory("dummy-dir")
+
+    self.assertFalse(unlink_mock.called)
+    self.assertFalse(rmtree_mock.called)
+    self.assertTrue(makedirs_mock.called)
+
+    unlink_mock.reset_mock()
+    rmtree_mock.reset_mock()
+    makedirs_mock.reset_mock()
+
     # Test exception handling
     makedirs_mock.side_effect = self.exc_side_effect
     try:

ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py

@@ -139,9 +139,11 @@ class TestLiveStatus(TestCase):
     # enable stdout
     sys.stdout = sys.__stdout__
 
+  @patch("os.path.isdir")
   @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
   @patch.object(ActualConfigHandler.ActualConfigHandler, "read_actual_component")
-  def test_build(self, read_actual_component_mock):
+  def test_build(self, read_actual_component_mock, isdir_mock):
+    isdir_mock.return_value = False
     for component in LiveStatus.COMPONENTS:
       config = AmbariConfig().getConfig()
       config.set('agent', 'prefix', "ambari_agent" + os.sep + "dummy_files")

ambari-agent/src/test/python/unitTests.py

@@ -25,6 +25,9 @@ python unitTests.py
 python unitTests.py NameOfFile.py
 python unitTests.py NameOfFileWithoutExtension  (this will append .* to the end, so it can match other file names too)
 
+prepend _ to test file name(s) and run "python unitTests.py": execute only
+  test files whose name begins with _ (useful for quick debug)
+
 SETUP:
 To run in Linux from command line,
 cd to this same directory. Then make sure PYTHONPATH is correct.
@@ -38,6 +41,7 @@ $(pwd)/ambari-agent/src/test/python/resource_management:
 $(pwd)/ambari-common/src/main/python/ambari_jinja2
 """
 
+import re
 import unittest
 import fnmatch
 from os.path import isdir
@@ -51,7 +55,6 @@ SELECTED_PREFIX = "_"
 PY_EXT='.py'
 
 
-TEST_MASK = '[Tt]est*.py'
 class TestAgent(unittest.TestSuite):
   def run(self, result):
     run = unittest.TestSuite.run
@@ -75,7 +78,7 @@ def get_test_files(path, mask=None, recursive=True):
   """
   # Must convert mask so it can match a file
   if mask and mask != "" and not mask.endswith("*"):
-    mask = mask + "*"
+    mask=mask+"*"
 
   file_list = []
   directory_items = os.listdir(path)
@@ -84,9 +87,10 @@ def get_test_files(path, mask=None, recursive=True):
     add_to_pythonpath = False
     p = os.path.join(path, item)
     if os.path.isfile(p):
-      if fnmatch.fnmatch(item, mask):
-        add_to_pythonpath = True
-        file_list.append(item)
+      if mask is not None and fnmatch.fnmatch(item, mask) or \
+        mask is None and re.search(r"^_?[Tt]est.*\.py$", item):
+          add_to_pythonpath = True
+          file_list.append(item)
     elif os.path.isdir(p):
       if recursive:
         file_list.extend(get_test_files(p, mask=mask))
@@ -97,7 +101,7 @@ def get_test_files(path, mask=None, recursive=True):
 
 
 def all_tests_suite(custom_test_mask):
-  test_mask = custom_test_mask if custom_test_mask else TEST_MASK
+  test_mask = custom_test_mask if custom_test_mask else None
 
   src_dir = os.getcwd()
   files_list = get_test_files(src_dir, mask=test_mask)