AMBARI-10444. RU Hacks and Technical Debt - Unit Test for (pre|post)_rolling_restart across several services. Last patch (dlysnichenko)

ambari-common/src/main/python/resource_management/libraries/functions/validate.py

@@ -20,7 +20,7 @@ limitations under the License.
 import re
 
 from resource_management.libraries.functions.decorator import retry
-from resource_management.core.shell import call
+from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 
 
@@ -31,6 +31,6 @@ def call_and_match_output(command, regex_expression, err_message):
   :param command: Command to call
   :param regex_expression: Regex expression to search in the output
   """
-  code, out = call(command, logoutput=True)
+  code, out = shell.call(command, logoutput=True)
   if not (out and re.search(regex_expression, out, re.IGNORECASE)):
     raise Fail(err_message)

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py

@@ -27,7 +27,7 @@ from resource_management.libraries.functions.version import compare_versions, fo
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_JAAS_CONF
-from resource_management.core.shell import call
+from resource_management.core import shell
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.check_process_status import check_process_status
@@ -91,11 +91,11 @@ class ZookeeperServerLinux(ZookeeperServer):
     quorum_err_message = "Failed to establish zookeeper quorum"
     call_and_match_output(create_command, 'Created', quorum_err_message)
     call_and_match_output(list_command, r"\[.*?" + unique + ".*?\]", quorum_err_message)
-    call(delete_command)
+    shell.call(delete_command)
 
     if params.client_port:
       check_leader_command = format("echo stat | nc localhost {client_port} | grep Mode")
-      code, out = call(check_leader_command, logoutput=False)
+      code, out = shell.call(check_leader_command, logoutput=False)
       if code == 0 and out:
         Logger.info(out)
 

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -1076,7 +1076,7 @@ class TestNamenode(RMFTestCase):
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
     self.assertResourceCalled('Execute',
-                              'hdp-select set hadoop-hdfs-namenode %s' % version,)
+                              'hdp-select set hadoop-hdfs-namenode %s' % version)
     self.assertNoMoreResources()
 
 

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py

@@ -194,3 +194,19 @@ class TestOozieClient(RMFTestCase):
                               group = 'hadoop',
                               )
     self.assertNoMoreResources()
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_client.py",
+                       classname = "OozieClient",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set oozie-client %s' % version)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -906,4 +906,3 @@ class TestOozieServer(RMFTestCase):
     except Fail,f:
       pass
 
-

ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py

@@ -131,4 +131,20 @@ class TestPigClient(RMFTestCase):
                               mode = 0644,
                               content = 'log4jproperties\nline2'
     )
+    self.assertNoMoreResources()
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pig_client.py",
+                       classname = "PigClient",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set hadoop-client %s' % version)
     self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/ZOOKEEPER/test_zookeeper_client.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
 from mock.mock import MagicMock, call, patch
 from stacks.utils.RMFTestCase import *
 
@@ -151,3 +152,20 @@ class TestZookeeperClient(RMFTestCase):
       group = 'hadoop',
     )
     self.assertNoMoreResources()
+
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_client.py",
+                       classname = "ZookeeperClient",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set zookeeper-client %s' % version)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/ZOOKEEPER/test_zookeeper_server.py

@@ -17,8 +17,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
 from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
+import resource_management.libraries.functions.get_unique_id_and_date
 
 @patch("os.path.exists", new = MagicMock(return_value=True))
 @patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
@@ -338,3 +340,47 @@ class TestZookeeperServer(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set zookeeper-server %s' % version)
+    self.assertNoMoreResources()
+
+  @patch.object(resource_management.libraries.functions, "get_unique_id_and_date")
+  def test_post_rolling_restart(self, get_unique_id_and_date_mock):
+    unique_value = "unique1"
+    get_unique_id_and_date_mock.return_value = unique_value
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+
+    mocks_dict = {}
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_server.py",
+                       classname = "ZookeeperServer",
+                       command = "post_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES,
+                       call_mocks = [
+                         (0, 'Created'),
+                         (0, '[ Unique %s]' % unique_value),
+                         (0, 'stdout'),
+                         (0, 'stdout')
+                       ],
+                       mocks_dict = mocks_dict)
+    self.assertEqual(mocks_dict['call'].call_count, 4)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.2/KAFKA/test_kafka_broker.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
 from stacks.utils.RMFTestCase import *
 
 
@@ -56,3 +57,19 @@ class TestKafkaBroker(RMFTestCase):
                               mode = 0755,
                               cd_access = 'a'
     )
+
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kafka_broker.py",
+                       classname = "KafkaBroker",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              'hdp-select set kafka-broker %s' % version,)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
 from resource_management import *
 from stacks.utils.RMFTestCase import *
 from mock.mock import patch
@@ -213,14 +214,19 @@ class TestKnoxGateway(RMFTestCase):
   @patch("os.path.isdir")
   def test_pre_rolling_restart(self, isdir_mock, tarfile_open_mock):
     isdir_mock.return_value = True
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
                        classname = "KnoxGateway",
                        command = "pre_rolling_restart",
-                       config_file="default.json",
+                       config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
 
     self.assertTrue(tarfile_open_mock.called)
 
-    self.assertResourceCalled("Execute", "hdp-select set knox-server 2.2.1.0-2067")
+    self.assertResourceCalled("Execute", "hdp-select set knox-server %s" % version)

ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py

@@ -47,3 +47,18 @@ class TestMahoutClient(RMFTestCase):
                               )
     self.assertNoMoreResources()
 
+  def test_pre_rolling_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.1.0-3242'
+    json_content['commandParams']['version'] = version
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
+                       classname = "MahoutClient",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+    self.assertResourceCalled('Execute',
+                              ('hdp-select', 'set', 'mahout-client', version), sudo = True)
+    self.assertNoMoreResources()