ambari/ambari-server/src/test/python/TestUpgradeScript_HDP2.py

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import subprocess
from mock.mock import MagicMock, call, patch
from unittest import TestCase
from mock.mock import create_autospec
import sys
import unittest
import UpgradeHelper_HDP2
import StringIO
import logging


class TestUpgradeHDP2Script(TestCase):
  def setUp(self):
    UpgradeHelper_HDP2.logger = MagicMock()
    out = StringIO.StringIO()
    sys.stdout = out


  def tearDown(self):
    sys.stdout = sys.__stdout__


  @patch.object(UpgradeHelper_HDP2, 'backup_single_config_type')
  def test_backup_configs(self, backup_config_mock):
    UpgradeHelper_HDP2.backup_configs(None)
    self.assertTrue(backup_config_mock.called)

  @patch.object(UpgradeHelper_HDP2, 'update_config')
  @patch.object(UpgradeHelper_HDP2, 'get_config')
  @patch('optparse.Values')
  def test_update_with_append(self, optparse_mock, get_config_mock, update_config_mock):
    opm = optparse_mock.return_value
    update_config_mock.return_value = None
    options = MagicMock()
    args = ["save-configs"]
    opm.parse_args.return_value = (options, args)
    get_config_mock.return_value = {"a1": "va1", "a2": "va2", "b1": "vb1", "b2": "vb2", "c1": "vc1", "d1": "d1"}
    site_template = {"y1": "vy1", "a1": "REPLACE_WITH_", "a2": "REPLACE_WITH_", "nb1": "REPLACE_WITH_b1",
                     "nb2": "REPLACE_WITH_b2", "d1": "DELETE_OLD", "b1" : "DELETE_OLD","c1": "vc2"}
    expected_site = {"y1": "vy1", "a1": "va1", "a2": "va2", "nb1": "vb1", "nb2": "vb2", "c1": "vc2"}
    UpgradeHelper_HDP2.update_config_using_existing(opm, "global", site_template)
    get_config_mock.assert_called_once_with(opm, "global")
    update_config_mock.assert_called_once_with(opm, expected_site, "global")
    pass

  @patch.object(UpgradeHelper_HDP2, 'update_config')
  @patch.object(UpgradeHelper_HDP2, 'get_config')
  @patch('optparse.Values')
  def test_update_with_appen_II(self, optparse_mock, get_config_mock, update_config_mock):
    opm = optparse_mock.return_value
    update_config_mock.return_value = None
    options = MagicMock()
    args = ["save-configs"]
    opm.parse_args.return_value = (options, args)
    get_config_mock.return_value = {"a1": "va1", "a2": "va2", "b1": "vb1", "b2": "vb2", "c1": "vc1", "x1": "x1",
                                    "X1": "X1"}
    site_template = {"y1": "vy1", "a1": "REPLACE_WITH_", "a2": "REPLACE_WITH_", "nb1": "REPLACE_WITH_b1",
                     "nb2": "REPLACE_WITH_b2", "x1": "DELETE_OLD", "X1": "DELETE"}
    expected_site = {"y1": "vy1", "a1": "va1", "a2": "va2", "nb1": "vb1", "nb2": "vb2", "c1": "vc1","X1": "DELETE"}
    UpgradeHelper_HDP2.update_config_using_existing(opm, "global", site_template)
    get_config_mock.assert_called_once_with(opm, "global")
    update_config_mock.assert_called_once_with(opm, expected_site, "global")
    pass

  @patch.object(logging.FileHandler, 'setFormatter')
  @patch.object(logging, 'basicConfig')
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, 'write_config')
  @patch.object(UpgradeHelper_HDP2, 'get_config_resp')
  @patch("os.remove")
  @patch("shutil.copyfile")
  @patch("os.path.exists")
  @patch('optparse.OptionParser')
  def test_save_configs(self, option_parser_mock, path_exists_mock, shutil_copy_mock, os_remove_mock,
                        get_config_resp_mock, write_config_mock, file_handler_mock, logging_mock, set_formatter_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    path_exists_mock.return_value = True
    shutil_copy_mock = MagicMock()
    options = self.get_mock_options()
    args = ["backup-configs"]
    opm.parse_args.return_value = (options, args)

    def get_config_resp_side_effect(ops, type, error_if_na):
      if type == "global":
        return ("version1", "")
      elif type == "core-site":
        return ("version1", "")
      elif type == "hdfs-site":
        return ("version1", "")
      elif type == "mapred-site":
        return ("version2", "")
      elif type == "hbase-site":
        return ("version2", "")
      else:
        return (None, None)

    get_config_resp_mock.side_effect = get_config_resp_side_effect
    UpgradeHelper_HDP2.main()
    shutil_copy_mock.assert_called_once()
    os_remove_mock.assert_called_once_with("logfile")
    write_expected = [call("", "global", "version1"), call("", "core-site", "version1"),
                      call("", "hdfs-site", "version1"), call("", "mapred-site", "version2"),
                      call("", "hbase-site", "version2")]
    write_config_mock.assert_has_calls(write_expected, any_order=True)
    get_config_expected = [call(options, "global", True), call(options, "core-site", True),
                           call(options, "hdfs-site", True), call(options, "mapred-site", True),
                           call(options, "hbase-site", False), call(options, "oozie-site", False),
                           call(options, "webhcat-site", False), call(options, "hive-site", False)]
    get_config_resp_mock.assert_has_calls(get_config_expected, any_order=True)
    pass


  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'write_mapping')
  @patch("json.loads")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_save_mr_mapping(self, option_parser_mock, curl_mock, json_loads_mock, write_mapping_mock,
                           backup_file_mock, file_handler_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = self.get_mock_options()
    args = ["save-mr-mapping"]
    opm.parse_args.return_value = (options, args)
    curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "', '"href" : "']
    json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
    UpgradeHelper_HDP2.main()
    expected_curl_calls = [
      call(False, "-u", "admin:admin",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
      call(False, "-u", "admin:admin",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
      call(False, "-u", "admin:admin",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
    self.assertTrue(write_mapping_mock.called)
    write_call = write_mapping_mock.call_args
    args, kargs = write_call
    self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
    self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
    self.assertTrue('TASKTRACKER' in args[0].keys())
    self.assertTrue('TASKTRACKER' in args[0].keys())
    pass


  @patch.object(UpgradeHelper_HDP2, "get_YN_input")
  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_delete_mr(self, option_parser_mock, curl_mock,
                     backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = self.get_mock_options()
    args = ["delete-mr"]
    opm.parse_args.return_value = (options, args)
    curl_mock.return_value = ''
    get_yn_mock.return_value = True
    read_mapping_mock.return_value = {
      "TASKTRACKER": ["c6401", "c6402"],
      "JOBTRACKER": ["c6401"],
      "MAPREDUCE_CLIENT": ["c6401"]}
    UpgradeHelper_HDP2.main()
    expected_curl_calls = [
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "DELETE",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
    pass


  @patch.object(UpgradeHelper_HDP2, "has_component_in_stack_def")
  @patch.object(UpgradeHelper_HDP2, "get_cluster_stackname")
  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_add_yarn_mr_with_ATS(self, option_parser_mock, curl_mock,
                       backup_file_mock, file_handler_mock, read_mapping_mock, get_stack_mock,  has_comp_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = self.get_mock_options()
    args = ["add-yarn-mr2"]
    opm.parse_args.return_value = (options, args)
    curl_mock.return_value = ''
    has_comp_mock.return_value = True
    read_mapping_mock.return_value = {
      "TASKTRACKER": ["c6401", "c6402"],
      "JOBTRACKER": ["c6401"],
      "HISTORYSERVER": ["c6401"],
      "MAPREDUCE_CLIENT": ["c6403"]}
    UpgradeHelper_HDP2.main()
    expected_curl_calls = [
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
      call(False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari', '-X', 'POST',
           'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER")]
    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
    pass
  
  @patch.object(UpgradeHelper_HDP2, "has_component_in_stack_def")
  @patch.object(UpgradeHelper_HDP2, "get_cluster_stackname")
  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_add_yarn_mr_without_ATS(self, option_parser_mock, curl_mock,
                       backup_file_mock, file_handler_mock, read_mapping_mock, get_stack_mock,  has_comp_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = self.get_mock_options()
    args = ["add-yarn-mr2"]
    opm.parse_args.return_value = (options, args)
    curl_mock.return_value = ''
    has_comp_mock.return_value = False
    read_mapping_mock.return_value = {
      "TASKTRACKER": ["c6401", "c6402"],
      "JOBTRACKER": ["c6401"],
      "HISTORYSERVER": ["c6401"],
      "MAPREDUCE_CLIENT": ["c6403"]}
    UpgradeHelper_HDP2.main()
    expected_curl_calls = [
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
      call(False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari', '-X', 'POST',
           'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"),
      ]
    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
    
    # assert no ATS was added
    self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
        "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")     
    self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
        "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")  
    pass


  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_install_yarn_mr2(self, option_parser_mock, curl_mock,
                            backup_file_mock, file_handler_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = self.get_mock_options()
    args = ["install-yarn-mr2"]
    opm.parse_args.return_value = (options, args)
    curl_mock.return_value = '"href" : "'
    UpgradeHelper_HDP2.main()
    expected_curl_calls = [
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "PUT", "-d",
           """{"RequestInfo":{"context":"Install MapReduce2"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "PUT", "-d",
           """{"RequestInfo":{"context":"Install YARN"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
           "http://localhost:8080/api/v1/clusters/c1/services/YARN")]
    curl_mock.assert_has_calls(expected_curl_calls, any_order=False)
    pass


  @patch.object(UpgradeHelper_HDP2, "get_config")
  @patch.object(UpgradeHelper_HDP2, "rename_all_properties")
  @patch.object(UpgradeHelper_HDP2, "update_config_using_existing_properties")
  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_update_single_configs(self, option_parser_mock, curl_mock,
                                 backup_file_mock, file_handler_mock, read_mapping_mock,
                                 update_config_mock, rename_all_prop_mock, get_config_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = MagicMock()
    args = ["update-configs", "hdfs-site"]
    opm.parse_args.return_value = (options, args)
    curl_mock.side_effect = ['', '', '', '', '', '', '']
    read_mapping_mock.return_value = {"JOBTRACKER": ["c6401"]}
    update_config_mock.side_effect = [None]
    get_config_mock.return_value = {}
    prop_to_move = {"dfs.namenode.checkpoint.edits.dir": "a1",
                    "dfs.namenode.checkpoint.dir": "a2",
                    "dfs.namenode.checkpoint.period": "a3"}
    rename_all_prop_mock.side_effect = [
      prop_to_move,
      {}, {}]
    UpgradeHelper_HDP2.main()
    self.assertTrue(update_config_mock.call_count == 1)
    args, kargs = update_config_mock.call_args_list[0]
    self.assertEqual("hdfs-site", args[1])
    for key in prop_to_move.keys():
      self.assertEqual(prop_to_move[key], args[3][key])
    pass


  @patch.object(UpgradeHelper_HDP2, "get_config_resp")
  @patch.object(UpgradeHelper_HDP2, "get_config")
  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_no_hbase(self, option_parser_mock, curl_mock,
                          backup_file_mock, file_handler_mock, read_mapping_mock,
                          get_config_mock, get_config_resp_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = MagicMock()
    args = ["update-configs"]
    opm.parse_args.return_value = (options, args)
    options.logfile = "logfile"
    options.user = "admin"
    options.password = "admin"
    options.hostname = "localhost"
    options.clustername = "c1"
    curl_mock.side_effect = ['', '', '', '', '', '', '']
    read_mapping_mock.return_value = {
      "TASKTRACKER": ["c6401", "c6402"],
      "JOBTRACKER": ["c6401"],
      "MAPREDUCE_CLIENT": ["c6403"]}
    get_config_resp_mock.return_value = "hbase-site", None
    get_config_mock.return_value = {
      "mapred.hosts": "an_old_value",
      "mapred.hosts.exclude": "an_old_value",
      "mapred.jobtracker.maxtasks.per.job": "an_old_value",
      "mapred.jobtracker.taskScheduler": "an_old_value",
      "dfs.df.interval": "an_old_value",
      "mapred.userlog.retain.hours": "will_not_be_stored",
      "global1": "global11"
    }
    UpgradeHelper_HDP2.main()
    self.assertEqual(6, len(curl_mock.call_args_list))


  @patch.object(UpgradeHelper_HDP2, "get_config_resp")
  @patch.object(UpgradeHelper_HDP2, "get_config")
  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_update_configs(self, option_parser_mock, curl_mock,
                          backup_file_mock, file_handler_mock, read_mapping_mock,
                          get_config_mock, get_config_resp_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    opm = option_parser_mock.return_value
    options = MagicMock()
    args = ["update-configs"]
    opm.parse_args.return_value = (options, args)
    options.logfile = "logfile"
    options.user = "admin"
    options.password = "admin"
    options.hostname = "localhost"
    options.clustername = "c1"
    curl_mock.side_effect = ['', '', '', '', '', '', '', '']
    read_mapping_mock.return_value = {
      "TASKTRACKER": ["c6401", "c6402"],
      "JOBTRACKER": ["c6401"],
      "MAPREDUCE_CLIENT": ["c6403"]}
    get_config_resp_mock.return_value = "hbase-site", {}
    site_properties = {
      "mapred.hosts": "an_old_value",
      "mapred.hosts.exclude": "an_old_value",
      "mapred.jobtracker.maxtasks.per.job": "an_old_value",
      "hbase.rpc.engine": "an_old_value",
      "dfs.df.interval": "an_old_value",
      "mapred.userlog.retain.hours": "will_not_be_stored",
      "global1": "global11"
    }
    get_config_mock.side_effect = [
      site_properties.copy(), site_properties.copy(), site_properties.copy(),
      site_properties.copy(), site_properties.copy(), site_properties.copy()]

    saved_global = UpgradeHelper_HDP2.GLOBAL
    saved_hdfs = UpgradeHelper_HDP2.HDFS_SITE
    saved_core = UpgradeHelper_HDP2.CORE_SITE
    saved_habse = UpgradeHelper_HDP2.HBASE_SITE
    saved_hive = UpgradeHelper_HDP2.HIVE_SITE
    saved_mapred = UpgradeHelper_HDP2.MAPRED_SITE
    try:
      UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
      UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
      UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
      UpgradeHelper_HDP2.main()
    finally:
      UpgradeHelper_HDP2.GLOBAL = saved_global
      UpgradeHelper_HDP2.HDFS_SITE = saved_hdfs
      UpgradeHelper_HDP2.CORE_SITE = saved_core
      UpgradeHelper_HDP2.MAPRED_SITE = saved_mapred

    self.assertEqual(8, len(curl_mock.call_args_list))
    self.validate_update_config_call(curl_mock.call_args_list[0], "capacity-scheduler")
    self.validate_update_config_call(curl_mock.call_args_list[1], "yarn-site")
    self.validate_update_config_call(curl_mock.call_args_list[3], "mapred-site")
    self.validate_update_config_call(curl_mock.call_args_list[2], "global")
    self.validate_config_replacememt(curl_mock.call_args_list[1], "yarn-site")
    self.validate_config_replacememt(curl_mock.call_args_list[2], "global")
    self.validate_config_replacememt(curl_mock.call_args_list[3], "mapred-site")
    self.validate_config_replacememt(curl_mock.call_args_list[4], "hdfs-site")
    self.validate_config_replacememt(curl_mock.call_args_list[5], "core-site")
    self.validate_config_replacememt(curl_mock.call_args_list[6], "hbase-site")
    pass

  @patch.object(UpgradeHelper_HDP2, "read_mapping")
  @patch("subprocess.Popen")
  @patch.object(UpgradeHelper_HDP2, "get_YN_input")
  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  @patch('optparse.OptionParser')
  def test_print_only(self, option_parser_mock, curl_mock,
                      backup_file_mock, file_handler_mock, get_yn_mock, popen_mock, read_mapping_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    options = self.get_mock_options(True)
    get_yn_mock.return_value = True
    read_mapping_mock.return_value = {
      "TASKTRACKER": ["c6401", "c6402"]}
    UpgradeHelper_HDP2.delete_mr(options)
    self.assertFalse(popen_mock.called)
    pass

  @patch.object(logging, 'FileHandler')
  @patch.object(UpgradeHelper_HDP2, "backup_file")
  @patch.object(UpgradeHelper_HDP2, 'curl')
  def test_get_and_parse_properties(self, curl_mock,
                                    backup_file_mock, file_handler_mock):
    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
    options = self.get_mock_options()
    curl_mock.side_effect = [
      """{ "href" : "http://localhost:8080/api/v1/clusters/c1",
           "Clusters" : {
              "cluster_id" : 2,
              "cluster_name" : "c1",
              "version" : "HDP-2.0.5",
              "desired_configs" : {
                "capacity-scheduler" : {
                  "user" : "admin",
                  "tag" : "version137"
                },
                "core-site" : {
                   "user" : "admin",
                   "tag" : "version138"
                }}}}""",
      """{
           "href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
           "items" : [
             {
               "href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
               "tag" : "version1378850572477",
               "type" : "core-site",
               "Config" : {
                 "cluster_name" : "c1"
               },
               "properties" : {
                 "name1" : "value1",
                 "name2" : "value2",
                 "name3" : "value3"
               }}]}""",
      """{ "href" : "http://localhost:8080/api/v1/clusters/c1",
           "Clusters" : {
              "desired_configs" : {
                "capacity-scheduler" : {
                  "user" : "admin",
                  "tag" : "version137"
                }}}}"""]
    properties = UpgradeHelper_HDP2.get_config(options, "core-site")
    self.assertTrue(len(properties.keys()) == 3)
    self.assertTrue(properties["name1"] == "value1")
    self.assertTrue(properties["name2"] == "value2")
    self.assertTrue(properties["name3"] == "value3")
    try:
      UpgradeHelper_HDP2.get_config(options, "hdfs-site")
    except Exception, e:
      self.assertTrue('Unable to get the current version for config type hdfs-site' in e.reason)
      pass
    pass

  def test_rename_all_properties(self):
    site_properties = {
      "mapred.task.is.map": "mapreduce.task.ismap",
      "mapred.task.partition": "mapreduce.task.partition",
      "mapred.task.profile": "mapreduce.task.profile",
      "abc": "abc"
    }
    site_properties = \
      UpgradeHelper_HDP2.rename_all_properties(site_properties, UpgradeHelper_HDP2.PROPERTY_MAPPING)
    for key in site_properties.keys():
      self.assertEqual(key, site_properties[key])
    self.assertEqual(4, len(site_properties))
    pass

  def test_tags_count(self):
    def count_tags(template):
      deleted = 0
      replaced = 0
      for key in template.keys():
        value = template[key]
        if value == UpgradeHelper_HDP2.DELETE_OLD_TAG:
          deleted += 1
          continue
        if value.find(UpgradeHelper_HDP2.REPLACE_WITH_TAG) == 0:
          replaced += 1
          continue
        pass
      return deleted, replaced

    deleted, replaced = count_tags(UpgradeHelper_HDP2.GLOBAL)
    self.assertEqual(6, replaced)
    self.assertEqual(25, deleted)

    deleted, replaced = count_tags(UpgradeHelper_HDP2.MAPRED_SITE)
    self.assertEqual(0, replaced)
    self.assertEqual(95, deleted)

    deleted, replaced = count_tags(UpgradeHelper_HDP2.CORE_SITE)
    self.assertEqual(0, replaced)
    self.assertEqual(2, deleted)

    deleted, replaced = count_tags(UpgradeHelper_HDP2.HDFS_SITE)
    self.assertEqual(0, replaced)
    self.assertEqual(8, deleted)
    pass

  def validate_update_config_call(self, call, type):
    args, kargs = call
    self.assertTrue(args[6] == 'PUT')
    self.assertTrue(type in args[8])
    pass

  def validate_config_replacememt(self, call, type):
    args, kargs = call
    self.assertFalse("REPLACE_WITH_" in args[8])
    self.assertFalse("REPLACE_JH_HOST" in args[8])
    self.assertFalse("REPLACE_RM_HOST" in args[8])
    if type == "yarn-site":
      self.assertTrue("c6401" in args[8])
      self.assertFalse("an_old_value" in args[8])
    elif type == "mapred-site":
      self.assertFalse("will_not_be_stored" in args[8])
      self.assertTrue("fs.df.interval" in args[8])
      self.assertFalse("dfs.df.interval" in args[8])
    elif type == "global":
      self.assertTrue("global11" in args[8])
      self.assertTrue("an_old_value" in args[8])
      self.assertTrue("mapred.hosts.exclude" in args[8])
    elif (type == "core-site") or (type == "hdfs-site"):
      self.assertTrue("global11" in args[8])
      self.assertTrue("global2" in args[8])
      self.assertTrue("hbase.rpc.engine" in args[8])
    elif type == "hbase-site":
      self.assertTrue("global11" in args[8])
      self.assertTrue("hbase.hstore.blockingStoreFiles" in args[8])
      self.assertTrue("dfs.df.interval" in args[8])
      self.assertFalse("hbase.rpc.engine" in args[8])
    pass

  def get_mock_options(self, printonly=False):
    options = MagicMock()
    options.logfile = "logfile"
    options.user = "admin"
    options.password = "admin"
    options.hostname = "localhost"
    options.clustername = "c1"
    options.printonly = printonly
    return options


if __name__ == "__main__":
  unittest.main()