AMBARI-10444. RU Hacks and Technical Debt - HDFS unit tests (dlysnichenko)

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java

@@ -78,7 +78,6 @@ public class DistributeRepositoriesActionListener {
   }
 
   @Subscribe
-  // @AllowConcurrentEvents //TODO: is it thread safe?
   public void onActionFinished(ActionFinalReportReceivedEvent event) {
     // Check if it is "Distribute repositories/install packages" action.
     if (! event.getRole().equals(INSTALL_PACKAGES)) {

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_upgrade.py

@@ -22,7 +22,7 @@ from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.default import default
-from resource_management.core.shell import call
+from resource_management.core import shell
 from resource_management.libraries.functions import Direction, SafeMode
 from resource_management.core.exceptions import Fail
 
@@ -48,7 +48,7 @@ def reach_safemode_state(user, safemode_state, in_ha):
 
   grep_pattern = format("Safe mode is {safemode_state} in {hostname}") if in_ha else format("Safe mode is {safemode_state}")
   safemode_check_with_grep = format("su - {user} -c 'hdfs dfsadmin -safemode get | grep \"{grep_pattern}\"'")
-  code, out = call(safemode_check)
+  code, out = shell.call(safemode_check)
   Logger.info("Command: %s\nCode: %d." % (safemode_check, code))
   if code == 0 and out is not None:
     Logger.info(out)

ambari-server/src/test/python/custom_actions/TestInstallPackages.py

@@ -260,7 +260,7 @@ class TestInstallPackages(RMFTestCase):
     allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
     is_suse_family_mock.return_value = True
 
-    config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json"
+    config_file = self.get_src_folder()+"/test/python/custom_actions/configs/install_packages_config.json"
     with open(config_file, "r") as f:
       config_dict = json.load(f)
 
@@ -315,7 +315,7 @@ class TestInstallPackages(RMFTestCase):
     self.assertNoMoreResources()
 
     # Check case when LZO is enabled
-    config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json"
+    config_file = self.get_src_folder()+"/test/python/custom_actions/configs/install_packages_config.json"
     with open(config_file, "r") as f:
       config_dict = json.load(f)
     config_dict['configurations']['core-site'] = {

ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py

@@ -119,7 +119,7 @@ class TestFlumeHandler(RMFTestCase):
   def test_struct_out(self):
    from resource_management.libraries.script import Script
 
-   configs_path = os.path.join(RMFTestCase._getSrcFolder(),
+   configs_path = os.path.join(RMFTestCase.get_src_folder(),
      "test/python/stacks", self.STACK_VERSION, "configs")
 
    script = Script()

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -149,7 +149,7 @@ class TestDatanode(RMFTestCase):
     self.assertNoMoreResources()
 
   def test_start_secured_HDP22_root(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
       secured_json = json.load(f)
 
@@ -187,7 +187,7 @@ class TestDatanode(RMFTestCase):
     self.assertNoMoreResources()
 
   def test_start_secured_HDP22_non_root_https_only(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
       secured_json = json.load(f)
 
@@ -265,7 +265,7 @@ class TestDatanode(RMFTestCase):
 
   @patch("os.path.exists", new = MagicMock(return_value=False))
   def test_stop_secured_HDP22_root(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
       secured_json = json.load(f)
 
@@ -306,7 +306,7 @@ class TestDatanode(RMFTestCase):
 
   @patch("os.path.exists", new = MagicMock(return_value=False))
   def test_stop_secured_HDP22_non_root_https_only(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
       secured_json = json.load(f)
 
@@ -441,6 +441,23 @@ class TestDatanode(RMFTestCase):
                               )
 
 
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
+                       classname = "DataNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-hdfs-datanode %s' % version,)
+    self.assertNoMoreResources()
+
+
   @patch('time.sleep')
   @patch.object(shell, "call")
   def test_post_rolling_restart(self, process_mock, time_mock):

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
 from mock.mock import MagicMock, patch
 import tempfile
 import tarfile
@@ -178,3 +179,19 @@ class Test(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py",
+                       classname = "HdfsClient",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', 'hdp-select set hadoop-client %s' % version,)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
 import os
 from stacks.utils.RMFTestCase import *
 from mock.mock import MagicMock, patch
@@ -436,4 +437,21 @@ class TestJournalnode(RMFTestCase):
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
-    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
+                       classname = "JournalNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-hdfs-journalnode %s' % version,)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -18,6 +18,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 from ambari_commons import OSCheck
 '''
+import json
 import os
 import tempfile
 from stacks.utils.RMFTestCase import *
@@ -27,7 +28,6 @@ from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 
 
-@patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
 class TestNamenode(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
   STACK_VERSION = "2.0.6"
@@ -43,6 +43,7 @@ class TestNamenode(RMFTestCase):
     self.assert_configure_default()
     self.assertNoMoreResources()
 
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_default_alt_fs(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
@@ -132,6 +133,7 @@ class TestNamenode(RMFTestCase):
     self.assertNoMoreResources()
     pass
 
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
@@ -252,6 +254,8 @@ class TestNamenode(RMFTestCase):
     self.assert_configure_secured()
     self.assertNoMoreResources()
 
+
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_secured(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
@@ -610,6 +614,7 @@ class TestNamenode(RMFTestCase):
   # tests namenode start command when NameNode HA is enabled, and
   # the HA cluster is started initially, rather than using the UI Wizard
   # this test verifies the startup of a "standby" namenode
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_ha_bootstrap_standby_from_blueprint(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
@@ -1039,6 +1044,113 @@ class TestNamenode(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
 
 
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-hdfs-namenode %s' % version,)
+    self.assertNoMoreResources()
+
+
+  def test_post_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "post_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -report -live',
+                              user = 'hdfs',
+                              )
+    self.assertNoMoreResources()
+
+
+  @patch.object(shell, "call")
+  def test_prepare_rolling_upgrade__upgrade(self, shell_call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    json_content['commandParams']['upgrade_direction'] = 'upgrade'
+
+    # Mock safemode_check call
+    shell_call_mock.return_value = 0, "Safe mode is OFF in c6401.ambari.apache.org"
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "prepare_rolling_upgrade",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',)
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade prepare',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade query',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertNoMoreResources()
+
+
+  @patch.object(shell, "call")
+  def test_prepare_rolling_upgrade__downgrade(self, shell_call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    json_content['commandParams']['upgrade_direction'] = 'downgrade'
+
+    # Mock safemode_check call
+    shell_call_mock.return_value = 0, "Safe mode is OFF in c6401.ambari.apache.org"
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "prepare_rolling_upgrade",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',)
+    self.assertNoMoreResources()
+
+
+  def test_finalize_rolling_upgrade(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "finalize_rolling_upgrade",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade query',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade finalize',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade query',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertNoMoreResources()
+
+
 class Popen_Mock:
   return_value = 1
   lines = ['Time Stamp               Iteration#  Bytes Already Moved  Bytes Left To Move  Bytes Being Moved\n',

ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py

@@ -59,7 +59,7 @@ class TestSqoop(RMFTestCase):
     self.assertNoMoreResources()
 
   def test_configure_add_jdbc(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
       loaded_json = json.load(f)
 

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -68,7 +68,7 @@ class RMFTestCase(TestCase):
                     target=TARGET_STACKS
                     ):
     norm_path = os.path.normpath(path)
-    src_dir = RMFTestCase._getSrcFolder()
+    src_dir = RMFTestCase.get_src_folder()
     if target == self.TARGET_STACKS:
       stack_version = norm_path.split(os.sep)[0]
       base_path = os.path.join(src_dir, PATH_TO_STACKS)
@@ -145,16 +145,16 @@ class RMFTestCase(TestCase):
     return self.config_dict
           
   @staticmethod
-  def _getSrcFolder():
+  def get_src_folder():
     return os.path.join(os.path.abspath(os.path.dirname(__file__)),os.path.normpath("../../../../"))
   
   @staticmethod
   def _getCommonServicesFolder():
-    return os.path.join(RMFTestCase._getSrcFolder(), PATH_TO_COMMON_SERVICES)
+    return os.path.join(RMFTestCase.get_src_folder(), PATH_TO_COMMON_SERVICES)
 
   @staticmethod
   def _getStackTestsFolder():
-    return os.path.join(RMFTestCase._getSrcFolder(), PATH_TO_STACK_TESTS)
+    return os.path.join(RMFTestCase.get_src_folder(), PATH_TO_STACK_TESTS)
 
   @staticmethod
   def _get_attr(module, attr):