AMBARI-3810. Unittests for File resource an all it's attributes (Eugene Chekanskiy via dlysnichenko)

.gitignore

@@ -18,4 +18,4 @@ target
 .hgtags
 derby.log
 pass.txt
-ambari-agent/src/test/python/dummy_files/current-stack
+ambari-agent/src/test/python/ambari_agent/dummy_files/current-stack

ambari-agent/pom.xml

@@ -124,7 +124,7 @@
                 <argument>unitTests.py</argument>
               </arguments>
               <environmentVariables>
-                <PYTHONPATH>${project.basedir}/../ambari-common/src/test/python:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/src/main/python/ambari_agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/main/puppet/modules/hdp-hadoop/files:$PYTHONPATH</PYTHONPATH>
+                <PYTHONPATH>${project.basedir}/../ambari-common/src/test/python:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/src/main/python/ambari_agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/ambari_agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/src/main/python:${project.basedir}/../ambari-agent/src/main/puppet/modules/hdp-hadoop/files:$PYTHONPATH</PYTHONPATH>
               </environmentVariables>
               <skip>${skipTests}</skip>
             </configuration>

ambari-agent/src/test/python/TestAmbariAgent.py → ambari-agent/src/test/python/ambari_agent/TestAmbariAgent.py

@@ -50,4 +50,4 @@ class TestAmbariAgent(unittest.TestCase):
     self.assertTrue(facter2.communicate.called)
     self.assertTrue(os_path_isfile_mock.called)
     self.assertTrue(os_path_isfile_mock.call_count == 2)
-    self.assertTrue(os_remove_mock.called)
+    self.assertTrue(os_remove_mock.called)

ambari-agent/src/test/python/TestGrep.py → ambari-agent/src/test/python/ambari_agent/TestGrep.py

@@ -32,8 +32,8 @@ class TestGrep(TestCase):
   grep = Grep()
 
   def setUp(self):
-    self.string_good = open('dummy_puppet_output_good.txt', 'r').read().replace("\n", os.linesep)
-    self.string_bad = open('dummy_puppet_output_error.txt', 'r').read().replace("\n", os.linesep)
+    self.string_good = open('ambari_agent' + os.sep + 'dummy_puppet_output_good.txt', 'r').read().replace("\n", os.linesep)
+    self.string_bad = open('ambari_agent' + os.sep + 'dummy_puppet_output_error.txt', 'r').read().replace("\n", os.linesep)
     pass
 
   def test_grep_many_lines(self):

ambari-agent/src/test/python/TestHostCheckReportFileHandler.py → ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py

@@ -158,4 +158,4 @@ class TestHostCheckReportFileHandler(TestCase):
     self.assertEquals(items1Str, items2Str)
 
 if __name__ == "__main__":
-  unittest.main(verbosity=2)
+  unittest.main(verbosity=2)

ambari-agent/src/test/python/TestLiveStatus.py → ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py

@@ -42,9 +42,9 @@ class TestLiveStatus(TestCase):
   def test_build(self, read_actual_component_mock):
     for component in LiveStatus.COMPONENTS:
       config = AmbariConfig().getConfig()
-      config.set('agent', 'prefix', "dummy_files")
+      config.set('agent', 'prefix', "ambari_agent" + os.sep + "dummy_files")
       livestatus = LiveStatus('', component['serviceName'], component['componentName'], {}, config)
-      livestatus.versionsHandler.versionsFilePath = os.path.join("dummy_files","dummy_current_stack")
+      livestatus.versionsHandler.versionsFilePath = "ambari_agent" + os.sep + "dummy_files" + os.sep + "dummy_current_stack"
       result = livestatus.build()
       print "LiveStatus of {0}: {1}".format(component['serviceName'], str(result))
       self.assertEquals(len(result) > 0, True, 'Livestatus should not be empty')

ambari-agent/src/test/python/TestPuppetExecutor.py → ambari-agent/src/test/python/ambari_agent/TestPuppetExecutor.py

@@ -172,7 +172,7 @@ class TestPuppetExecutor(TestCase):
     puppetexecutor.grep = grep
     grep.ERROR_LAST_LINES_BEFORE = 2
     grep.ERROR_LAST_LINES_AFTER = 3
-    string_err = open('dummy_puppet_output_error2.txt', 'r').read().replace("\n", os.linesep)
+    string_err = open('ambari_agent' + os.sep + 'dummy_puppet_output_error2.txt', 'r').read().replace("\n", os.linesep)
     result = puppetexecutor.condenseOutput(string_err, '', 1)
     stripped_string = string_err.strip()
     lines = stripped_string.splitlines(True)
@@ -189,7 +189,7 @@ class TestPuppetExecutor(TestCase):
     puppetexecutor = PuppetExecutor("/tmp", "/x", "/y", "/z", AmbariConfig().getConfig())
     grep = Grep()
     puppetexecutor.grep = grep
-    string_err = open('dummy_puppet_output_error3.txt', 'r').read().replace("\n", os.linesep)
+    string_err = open('ambari_agent' + os.sep + 'dummy_puppet_output_error3.txt', 'r').read().replace("\n", os.linesep)
     result = puppetexecutor.condenseOutput(string_err, '', 1)
     stripped_string = string_err.strip()
     lines = stripped_string.splitlines(True)
@@ -208,7 +208,7 @@ class TestPuppetExecutor(TestCase):
     grep = Grep()
     puppetexecutor.grep = grep
     grep.OUTPUT_LAST_LINES = 2
-    string_good = open('dummy_puppet_output_good.txt', 'r').read().replace("\n", os.linesep)
+    string_good = open('ambari_agent' + os.sep + 'dummy_puppet_output_good.txt', 'r').read().replace("\n", os.linesep)
     result = puppetexecutor.condenseOutput(string_good, PuppetExecutor.NO_ERROR, 0)
     stripped_string = string_good.strip()
     lines = stripped_string.splitlines(True)

ambari-agent/src/test/python/TestRegistration.py → ambari-agent/src/test/python/ambari_agent/TestRegistration.py

@@ -55,4 +55,4 @@ class TestRegistration(TestCase):
     self.assertEquals(data['currentPingPort'] == 33777, True, "current ping port should be 33777")
     self.assertEquals(len(data), 8)
 
-    os.remove(ver_file)
+    os.remove(ver_file)

ambari-agent/src/test/python/TestStackVersionsFileHandler.py → ambari-agent/src/test/python/ambari_agent/TestStackVersionsFileHandler.py

@@ -32,7 +32,7 @@ import logging
 
 stackVersionsFileHandler = \
       StackVersionsFileHandler.StackVersionsFileHandler("/tmp")
-dummyVersionsFile = os.path.join('dummy_files', 'dummy_current_stack')
+dummyVersionsFile = 'ambari_agent' + os.sep + 'dummy_files' + os.sep + 'dummy_current_stack'
 
 class TestStackVersionsFileHandler(TestCase):
 
@@ -113,4 +113,4 @@ class TestStackVersionsFileHandler(TestCase):
     stackVersionsFileHandler.VER_FILE = oldVerFile
 
 if __name__ == "__main__":
-  unittest.main(verbosity=2)
+  unittest.main(verbosity=2)

ambari-agent/src/test/python/dummy_puppet_output_error3.txt → ambari-agent/src/test/python/ambari_agent/dummy_puppet_output_error3.txt

@@ -73,4 +73,4 @@ Package::Process_pkg[nagios-addons]/Anchor[hdp::package::nagios-addons::end]: Sk
 \u001B[0;33mwarning: /Stage[2]/Hdp-monitor-webserver/Hdp::Exec[monitor webserver restart]/Exec[monitor webserver restart]: Skipping because of failed dependencies\u001B[0m
 \u001B[0;36mnotice: /Stage[2]/Hdp-monitor-webserver/Hdp::Exec[monitor webserver restart]/Anchor[hdp::exec::monitor webserver restart::end]: Dependency Package[hdp_mon_nagios_addons] has failures: true\u001B[0m
 \u001B[0;33mwarning: /Stage[2]/Hdp-monitor-webserver/Hdp::Exec[monitor webserver restart]/Anchor[hdp::exec::monitor webserver restart::end]: Skipping because of failed dependencies\u001B[0m
-\u001B[0;36mnotice: Finished catalog run in 49.63
+\u001B[0;36mnotice: Finished catalog run in 49.63

ambari-agent/src/test/python/dummy_puppet_output_good.txt → ambari-agent/src/test/python/ambari_agent/dummy_puppet_output_good.txt

@@ -44,4 +44,4 @@ debug: Stored state in 0.01 seconds
 notice: Finished catalog run in 0.59 seconds
 debug: Finishing transaction 70060456663980
 debug: Received report to process from ambari-dmi
-debug: Processing report from ambari-dmi with processor Puppet::Reports::Store
+debug: Processing report from ambari-dmi with processor Puppet::Reports::Store

ambari-agent/src/test/python/resource_management/TestFileResource.py

@@ -0,0 +1,332 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+import os
+import sys
+from resource_management.core import Environment, Fail
+from resource_management.core.resources import File
+from resource_management.core.system import System
+import resource_management.core.providers.system
+import resource_management
+
+
+@patch.object(System, "platform", new = 'redhat')
+class TestFileResource(TestCase):
+  @patch.object(os.path, "dirname")
+  @patch.object(os.path, "isdir")
+  def test_action_create_dir_exist(self, isdir_mock, dirname_mock):
+    """
+    Tests if 'create' action fails when path is existent directory
+    """
+    isdir_mock.side_effect = [True, False]
+    try:
+      with Environment('/') as env:
+        File('/existent_directory',
+             action='create',
+             mode=0777,
+             content='file-content'
+        )
+      env.run()
+      self.fail("Must fail when directory with name 'path' exist")
+    except Fail as e:
+      self.assertEqual("Applying File['/existent_directory'] failed, directory with name /existent_directory exists",
+                       str(e))
+    self.assertFalse(dirname_mock.called)
+
+  @patch.object(os.path, "dirname")
+  @patch.object(os.path, "isdir")
+  def test_action_create_parent_dir_non_exist(self, isdir_mock, dirname_mock):
+    """
+    Tests if 'create' action fails when parent directory of path
+    doesn't exist
+    """
+    isdir_mock.side_effect = [False, False]
+    dirname_mock.return_value = "/non_existent_directory"
+    try:
+      with Environment('/') as env:
+        File('/non_existent_directory/file',
+             action='create',
+             mode=0777,
+             content='file-content'
+        )
+      env.run()
+      self.fail('Must fail on non existent parent directory')
+    except Fail as e:
+      self.assertEqual(
+        "Applying File['/non_existent_directory/file'] failed, parent directory /non_existent_directory doesn't exist",
+        str(e))
+    self.assertTrue(dirname_mock.called)
+
+  @patch("resource_management.core.providers.system._ensure_metadata")
+  @patch("__builtin__.open")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_action_create_non_existent_file(self, isdir_mock, exists_mock, open_mock, ensure_mock):
+    """
+    Tests if 'create' action create new non existent file and write proper data
+    """
+    isdir_mock.side_effect = [False, True]
+    exists_mock.return_value = False
+    new_file = MagicMock()
+    open_mock.return_value = new_file
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           content='file-content'
+      )
+    env.run()
+
+    open_mock.assert_called_with('/directory/file', 'wb')
+    new_file.__enter__().write.assert_called_with('file-content')
+    self.assertEqual(open_mock.call_count, 1)
+    ensure_mock.assert_called()
+
+
+  @patch("resource_management.core.providers.system._ensure_metadata")
+  @patch("__builtin__.open")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_action_create_replace(self, isdir_mock, exists_mock, open_mock, ensure_mock):
+    """
+    Tests if 'create' action rewrite existent file with new data
+    """
+    isdir_mock.side_effect = [False, True]
+    old_file, new_file = MagicMock(), MagicMock()
+    open_mock.side_effect = [old_file, new_file]
+    old_file.read.return_value = 'old-content'
+    exists_mock.return_value = True
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           backup=False,
+           content='new-content'
+      )
+
+    env.run()
+    old_file.read.assert_called()
+    new_file.__enter__().write.assert_called_with('new-content')
+    ensure_mock.assert_called()
+    self.assertEqual(open_mock.call_count, 2)
+    open_mock.assert_any_call('/directory/file', 'rb')
+    open_mock.assert_any_call('/directory/file', 'wb')
+
+
+  @patch.object(os, "unlink")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_action_delete_is_directory(self, isdir_mock, exist_mock, unlink_mock):
+    """
+    Tests if 'delete' action fails when path is directory
+    """
+    isdir_mock.return_value = True
+
+    try:
+      with Environment('/') as env:
+        File('/directory/file',
+             action='delete',
+             mode=0777,
+             backup=False,
+             content='new-content'
+        )
+      env.run()
+      self.fail("Should fail when deleting directory")
+    except Fail:
+      pass
+
+    self.assertEqual(isdir_mock.call_count, 1)
+    self.assertEqual(exist_mock.call_count, 0)
+    self.assertEqual(unlink_mock.call_count, 0)
+
+  @patch.object(os, "unlink")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_action_delete(self, isdir_mock, exist_mock, unlink_mock):
+    """
+    Tests if 'delete' action removes file
+    """
+    isdir_mock.return_value = False
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='delete',
+           mode=0777,
+           backup=False,
+           content='new-content'
+      )
+    env.run()
+
+    self.assertEqual(isdir_mock.call_count, 1)
+    self.assertEqual(exist_mock.call_count, 1)
+    self.assertEqual(unlink_mock.call_count, 1)
+
+
+  @patch.object(os.path, "isdir")
+  def test_attribute_path(self, isdir_mock):
+    """
+    Tests 'path' attribute
+    """
+    isdir_mock.side_effect = [True, False]
+
+    try:
+      with Environment('/') as env:
+        File('/existent_directory',
+             action='create',
+             mode=0777,
+             content='file-content'
+        )
+      env.run()
+      self.fail("Must fail when directory with name 'path' exist")
+    except Fail as e:
+      pass
+
+    isdir_mock.assert_called_with('/existent_directory')
+
+  @patch.object(resource_management.core.Environment, "backup_file")
+  @patch("resource_management.core.providers.system._ensure_metadata")
+  @patch("__builtin__.open")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_attribute_backup(self, isdir_mock, exists_mock, open_mock, ensure_mock, backup_file_mock):
+    """
+    Tests 'backup' attribute
+    """
+    isdir_mock.side_effect = [False, True, False, True]
+    open_mock.return_value = MagicMock()
+    exists_mock.return_value = True
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           backup=False,
+           content='new-content'
+      )
+    env.run()
+
+    self.assertEqual(backup_file_mock.call_count, 0)
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           backup=True,
+           content='new-content'
+      )
+    env.run()
+
+    self.assertEqual(backup_file_mock.call_count, 1)
+    backup_file_mock.assert_called_with('/directory/file')
+
+
+  @patch("resource_management.core.providers.system._ensure_metadata")
+  @patch("__builtin__.open")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_attribute_replace(self, isdir_mock, exists_mock, open_mock, ensure_mock):
+    """
+    Tests 'replace' attribute
+    """
+    isdir_mock.side_effect = [False, True]
+    old_file, new_file = MagicMock(), MagicMock()
+    open_mock.side_effect = [old_file, new_file]
+    old_file.read.return_value = 'old-content'
+    exists_mock.return_value = True
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           backup=False,
+           content='new-content',
+           replace=False
+      )
+
+    env.run()
+    old_file.read.assert_called()
+    self.assertEqual(new_file.__enter__().write.call_count, 0)
+    ensure_mock.assert_called()
+    self.assertEqual(open_mock.call_count, 0)
+
+
+  @patch("resource_management.core.providers.system._coerce_uid")
+  @patch("resource_management.core.providers.system._coerce_gid")
+  @patch.object(os, "chown")
+  @patch.object(os, "chmod")
+  @patch.object(os, "stat")
+  @patch("__builtin__.open")
+  @patch.object(os.path, "exists")
+  @patch.object(os.path, "isdir")
+  def test_ensure_metadata(self, isdir_mock, exists_mock, open_mock, stat_mock, chmod_mock, chown_mock, gid_mock,
+                           uid_mock):
+    """
+    Tests if _ensure_metadata changes owner, usergroup and permissions of file to proper values
+    """
+    isdir_mock.side_effect = [False, True, False, True]
+    exists_mock.return_value = False
+
+    class stat():
+      def __init__(self):
+        self.st_mode = 0666
+        self.st_uid = 1
+        self.st_gid = 1
+
+    stat_mock.return_value = stat()
+    gid_mock.return_value = 0
+    uid_mock.return_value = 0
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           content='file-content',
+           owner='root',
+           group='hdfs'
+      )
+    env.run()
+
+    open_mock.assert_called_with('/directory/file', 'wb')
+    self.assertEqual(open_mock.call_count, 1)
+    stat_mock.assert_called_with('/directory/file')
+    self.assertEqual(chmod_mock.call_count, 1)
+    self.assertEqual(chown_mock.call_count, 2)
+    gid_mock.assert_called_once_with('hdfs')
+    uid_mock.assert_called_once_with('root')
+
+    chmod_mock.reset_mock()
+    chown_mock.reset_mock()
+    gid_mock.return_value = 1
+    uid_mock.return_value = 1
+
+    with Environment('/') as env:
+      File('/directory/file',
+           action='create',
+           mode=0777,
+           content='file-content',
+           owner='root',
+           group='hdfs'
+      )
+    env.run()
+
+    self.assertEqual(chmod_mock.call_count, 1)
+    self.assertEqual(chown_mock.call_count, 0)

ambari-agent/src/test/python/unitTests.py

@@ -51,7 +51,10 @@ def all_tests_suite():
 
 
   src_dir = os.getcwd()
-  files_list=os.listdir(src_dir)
+  files_list = []
+  for directory in os.listdir(src_dir):
+    if os.path.isdir(directory):
+      files_list += os.listdir(src_dir + os.sep + directory)
   shuffle(files_list)
   tests_list = []