Selaa lähdekoodia

AMBARI-10444. RU Hacks and Technical Debt - HDFS unit tests (dlysnichenko)

Lisnichenko Dmitro 10 vuotta sitten
vanhempi
commit
aa492e1732

+ 0 - 1
ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java

@@ -78,7 +78,6 @@ public class DistributeRepositoriesActionListener {
   }
   }
 
 
   @Subscribe
   @Subscribe
-  // @AllowConcurrentEvents //TODO: is it thread safe?
   public void onActionFinished(ActionFinalReportReceivedEvent event) {
   public void onActionFinished(ActionFinalReportReceivedEvent event) {
     // Check if it is "Distribute repositories/install packages" action.
     // Check if it is "Distribute repositories/install packages" action.
     if (! event.getRole().equals(INSTALL_PACKAGES)) {
     if (! event.getRole().equals(INSTALL_PACKAGES)) {

+ 2 - 2
ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_upgrade.py

@@ -22,7 +22,7 @@ from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.default import default
-from resource_management.core.shell import call
+from resource_management.core import shell
 from resource_management.libraries.functions import Direction, SafeMode
 from resource_management.libraries.functions import Direction, SafeMode
 from resource_management.core.exceptions import Fail
 from resource_management.core.exceptions import Fail
 
 
@@ -48,7 +48,7 @@ def reach_safemode_state(user, safemode_state, in_ha):
 
 
   grep_pattern = format("Safe mode is {safemode_state} in {hostname}") if in_ha else format("Safe mode is {safemode_state}")
   grep_pattern = format("Safe mode is {safemode_state} in {hostname}") if in_ha else format("Safe mode is {safemode_state}")
   safemode_check_with_grep = format("su - {user} -c 'hdfs dfsadmin -safemode get | grep \"{grep_pattern}\"'")
   safemode_check_with_grep = format("su - {user} -c 'hdfs dfsadmin -safemode get | grep \"{grep_pattern}\"'")
-  code, out = call(safemode_check)
+  code, out = shell.call(safemode_check)
   Logger.info("Command: %s\nCode: %d." % (safemode_check, code))
   Logger.info("Command: %s\nCode: %d." % (safemode_check, code))
   if code == 0 and out is not None:
   if code == 0 and out is not None:
     Logger.info(out)
     Logger.info(out)

+ 2 - 2
ambari-server/src/test/python/custom_actions/TestInstallPackages.py

@@ -260,7 +260,7 @@ class TestInstallPackages(RMFTestCase):
     allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
     allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
     is_suse_family_mock.return_value = True
     is_suse_family_mock.return_value = True
 
 
-    config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json"
+    config_file = self.get_src_folder()+"/test/python/custom_actions/configs/install_packages_config.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       config_dict = json.load(f)
       config_dict = json.load(f)
 
 
@@ -315,7 +315,7 @@ class TestInstallPackages(RMFTestCase):
     self.assertNoMoreResources()
     self.assertNoMoreResources()
 
 
     # Check case when LZO is enabled
     # Check case when LZO is enabled
-    config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json"
+    config_file = self.get_src_folder()+"/test/python/custom_actions/configs/install_packages_config.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       config_dict = json.load(f)
       config_dict = json.load(f)
     config_dict['configurations']['core-site'] = {
     config_dict['configurations']['core-site'] = {

+ 1 - 1
ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py

@@ -119,7 +119,7 @@ class TestFlumeHandler(RMFTestCase):
   def test_struct_out(self):
   def test_struct_out(self):
    from resource_management.libraries.script import Script
    from resource_management.libraries.script import Script
 
 
-   configs_path = os.path.join(RMFTestCase._getSrcFolder(),
+   configs_path = os.path.join(RMFTestCase.get_src_folder(),
      "test/python/stacks", self.STACK_VERSION, "configs")
      "test/python/stacks", self.STACK_VERSION, "configs")
 
 
    script = Script()
    script = Script()

+ 21 - 4
ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -149,7 +149,7 @@ class TestDatanode(RMFTestCase):
     self.assertNoMoreResources()
     self.assertNoMoreResources()
 
 
   def test_start_secured_HDP22_root(self):
   def test_start_secured_HDP22_root(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       secured_json = json.load(f)
       secured_json = json.load(f)
 
 
@@ -187,7 +187,7 @@ class TestDatanode(RMFTestCase):
     self.assertNoMoreResources()
     self.assertNoMoreResources()
 
 
   def test_start_secured_HDP22_non_root_https_only(self):
   def test_start_secured_HDP22_non_root_https_only(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       secured_json = json.load(f)
       secured_json = json.load(f)
 
 
@@ -265,7 +265,7 @@ class TestDatanode(RMFTestCase):
 
 
   @patch("os.path.exists", new = MagicMock(return_value=False))
   @patch("os.path.exists", new = MagicMock(return_value=False))
   def test_stop_secured_HDP22_root(self):
   def test_stop_secured_HDP22_root(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       secured_json = json.load(f)
       secured_json = json.load(f)
 
 
@@ -306,7 +306,7 @@ class TestDatanode(RMFTestCase):
 
 
   @patch("os.path.exists", new = MagicMock(return_value=False))
   @patch("os.path.exists", new = MagicMock(return_value=False))
   def test_stop_secured_HDP22_non_root_https_only(self):
   def test_stop_secured_HDP22_non_root_https_only(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       secured_json = json.load(f)
       secured_json = json.load(f)
 
 
@@ -441,6 +441,23 @@ class TestDatanode(RMFTestCase):
                               )
                               )
 
 
 
 
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
+                       classname = "DataNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-hdfs-datanode %s' % version,)
+    self.assertNoMoreResources()
+
+
   @patch('time.sleep')
   @patch('time.sleep')
   @patch.object(shell, "call")
   @patch.object(shell, "call")
   def test_post_rolling_restart(self, process_mock, time_mock):
   def test_post_rolling_restart(self, process_mock, time_mock):

+ 17 - 0
ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 See the License for the specific language governing permissions and
 limitations under the License.
 limitations under the License.
 '''
 '''
+import json
 from mock.mock import MagicMock, patch
 from mock.mock import MagicMock, patch
 import tempfile
 import tempfile
 import tarfile
 import tarfile
@@ -178,3 +179,19 @@ class Test(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     )
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py",
+                       classname = "HdfsClient",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', 'hdp-select set hadoop-client %s' % version,)
+    self.assertNoMoreResources()

+ 19 - 1
ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 See the License for the specific language governing permissions and
 limitations under the License.
 limitations under the License.
 '''
 '''
+import json
 import os
 import os
 from stacks.utils.RMFTestCase import *
 from stacks.utils.RMFTestCase import *
 from mock.mock import MagicMock, patch
 from mock.mock import MagicMock, patch
@@ -436,4 +437,21 @@ class TestJournalnode(RMFTestCase):
                        hdp_stack_version = self.STACK_VERSION,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     )
-    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
+                       classname = "JournalNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-hdfs-journalnode %s' % version,)
+    self.assertNoMoreResources()

+ 113 - 1
ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -18,6 +18,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 limitations under the License.
 from ambari_commons import OSCheck
 from ambari_commons import OSCheck
 '''
 '''
+import json
 import os
 import os
 import tempfile
 import tempfile
 from stacks.utils.RMFTestCase import *
 from stacks.utils.RMFTestCase import *
@@ -27,7 +28,6 @@ from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 from resource_management.core.exceptions import Fail
 
 
 
 
-@patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
 class TestNamenode(RMFTestCase):
 class TestNamenode(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
   COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
   STACK_VERSION = "2.0.6"
   STACK_VERSION = "2.0.6"
@@ -43,6 +43,7 @@ class TestNamenode(RMFTestCase):
     self.assert_configure_default()
     self.assert_configure_default()
     self.assertNoMoreResources()
     self.assertNoMoreResources()
 
 
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_default_alt_fs(self):
   def test_start_default_alt_fs(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
                        classname = "NameNode",
@@ -132,6 +133,7 @@ class TestNamenode(RMFTestCase):
     self.assertNoMoreResources()
     self.assertNoMoreResources()
     pass
     pass
 
 
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_default(self):
   def test_start_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
                        classname = "NameNode",
@@ -252,6 +254,8 @@ class TestNamenode(RMFTestCase):
     self.assert_configure_secured()
     self.assert_configure_secured()
     self.assertNoMoreResources()
     self.assertNoMoreResources()
 
 
+
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_secured(self):
   def test_start_secured(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
                        classname = "NameNode",
@@ -610,6 +614,7 @@ class TestNamenode(RMFTestCase):
   # tests namenode start command when NameNode HA is enabled, and
   # tests namenode start command when NameNode HA is enabled, and
   # the HA cluster is started initially, rather than using the UI Wizard
   # the HA cluster is started initially, rather than using the UI Wizard
   # this test verifies the startup of a "standby" namenode
   # this test verifies the startup of a "standby" namenode
+  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
   def test_start_ha_bootstrap_standby_from_blueprint(self):
   def test_start_ha_bootstrap_standby_from_blueprint(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
                        classname = "NameNode",
@@ -1039,6 +1044,113 @@ class TestNamenode(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
 
 
 
 
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-hdfs-namenode %s' % version,)
+    self.assertNoMoreResources()
+
+
+  def test_post_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "post_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -report -live',
+                              user = 'hdfs',
+                              )
+    self.assertNoMoreResources()
+
+
+  @patch.object(shell, "call")
+  def test_prepare_rolling_upgrade__upgrade(self, shell_call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    json_content['commandParams']['upgrade_direction'] = 'upgrade'
+
+    # Mock safemode_check call
+    shell_call_mock.return_value = 0, "Safe mode is OFF in c6401.ambari.apache.org"
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "prepare_rolling_upgrade",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',)
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade prepare',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade query',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertNoMoreResources()
+
+
+  @patch.object(shell, "call")
+  def test_prepare_rolling_upgrade__downgrade(self, shell_call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    json_content['commandParams']['upgrade_direction'] = 'downgrade'
+
+    # Mock safemode_check call
+    shell_call_mock.return_value = 0, "Safe mode is OFF in c6401.ambari.apache.org"
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "prepare_rolling_upgrade",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',)
+    self.assertNoMoreResources()
+
+
+  def test_finalize_rolling_upgrade(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "finalize_rolling_upgrade",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade query',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade finalize',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertResourceCalled('Execute', 'hdfs dfsadmin -rollingUpgrade query',
+                              logoutput = True,
+                              user = 'hdfs',
+                              )
+    self.assertNoMoreResources()
+
+
 class Popen_Mock:
 class Popen_Mock:
   return_value = 1
   return_value = 1
   lines = ['Time Stamp               Iteration#  Bytes Already Moved  Bytes Left To Move  Bytes Being Moved\n',
   lines = ['Time Stamp               Iteration#  Bytes Already Moved  Bytes Left To Move  Bytes Being Moved\n',

+ 1 - 1
ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py

@@ -59,7 +59,7 @@ class TestSqoop(RMFTestCase):
     self.assertNoMoreResources()
     self.assertNoMoreResources()
 
 
   def test_configure_add_jdbc(self):
   def test_configure_add_jdbc(self):
-    config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
     with open(config_file, "r") as f:
     with open(config_file, "r") as f:
       loaded_json = json.load(f)
       loaded_json = json.load(f)
 
 

+ 4 - 4
ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -68,7 +68,7 @@ class RMFTestCase(TestCase):
                     target=TARGET_STACKS
                     target=TARGET_STACKS
                     ):
                     ):
     norm_path = os.path.normpath(path)
     norm_path = os.path.normpath(path)
-    src_dir = RMFTestCase._getSrcFolder()
+    src_dir = RMFTestCase.get_src_folder()
     if target == self.TARGET_STACKS:
     if target == self.TARGET_STACKS:
       stack_version = norm_path.split(os.sep)[0]
       stack_version = norm_path.split(os.sep)[0]
       base_path = os.path.join(src_dir, PATH_TO_STACKS)
       base_path = os.path.join(src_dir, PATH_TO_STACKS)
@@ -145,16 +145,16 @@ class RMFTestCase(TestCase):
     return self.config_dict
     return self.config_dict
           
           
   @staticmethod
   @staticmethod
-  def _getSrcFolder():
+  def get_src_folder():
     return os.path.join(os.path.abspath(os.path.dirname(__file__)),os.path.normpath("../../../../"))
     return os.path.join(os.path.abspath(os.path.dirname(__file__)),os.path.normpath("../../../../"))
   
   
   @staticmethod
   @staticmethod
   def _getCommonServicesFolder():
   def _getCommonServicesFolder():
-    return os.path.join(RMFTestCase._getSrcFolder(), PATH_TO_COMMON_SERVICES)
+    return os.path.join(RMFTestCase.get_src_folder(), PATH_TO_COMMON_SERVICES)
 
 
   @staticmethod
   @staticmethod
   def _getStackTestsFolder():
   def _getStackTestsFolder():
-    return os.path.join(RMFTestCase._getSrcFolder(), PATH_TO_STACK_TESTS)
+    return os.path.join(RMFTestCase.get_src_folder(), PATH_TO_STACK_TESTS)
 
 
   @staticmethod
   @staticmethod
   def _get_attr(module, attr):
   def _get_attr(module, attr):