AMBARI-3180. Helper script for HDP upgrade.

ambari-server/src/main/python/UpgradeHelper_HDP2.py

@@ -77,12 +77,12 @@ MAPRED_QUEUE_ACLS = {
   "mapred.queue.default.acl-administer-jobs": "*", "mapred.queue.default.acl-submit-job": "*"}
 
 MAPRED_SITE = {
-  "mapred.hosts": "/etc/hadoop/conf/mapred.include",
-  "mapred.hosts.exclude": "/etc/hadoop/conf/mapred.exclude",
-  "mapred.jobtracker.maxtasks.per.job": "-1",
-  "mapred.jobtracker.taskScheduler": "org.apache.hadoop.mapred.CapacityTaskScheduler",
-  "mapred.task.tracker.task-controller": "org.apache.hadoop.mapred.DefaultTaskController",
-  "mapred.userlog.retain.hours": "24",
+  "mapred.hosts": "REPLACE_WITH_",
+  "mapred.hosts.exclude": "REPLACE_WITH_",
+  "mapred.jobtracker.maxtasks.per.job": "REPLACE_WITH_",
+  "mapred.jobtracker.taskScheduler": "REPLACE_WITH_",
+  "mapred.task.tracker.task-controller": "REPLACE_WITH_",
+  "mapred.userlog.retain.hours": "REPLACE_WITH_",
   "mapreduce.admin.map.child.java.opts": "-Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN",
   "mapreduce.admin.reduce.child.java.opts": "-Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN",
   "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/`$JAVA_HOME/bin/java -d32 -version &> /dev/null;if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`",
@@ -315,7 +315,7 @@ def backup_file(filePath):
       shutil.copyfile(filePath, filePath + "." + timestamp.strftime(format))
       os.remove(filePath)
     except (Exception), e:
-      logger.warn('Could not backup file "%s": %s' % (str(filePath, e)))
+      logger.warn('Could not backup file "%s": %s' % (filePath, str(e)))
   return 0
 
 
@@ -371,7 +371,31 @@ def get_mr1_mapping(options):
   write_mapping(hostmapping)
 
 
+def get_YN_input(prompt, default):
+  yes = set(['yes', 'ye', 'y'])
+  no = set(['no', 'n'])
+  return get_choice_string_input(prompt, default, yes, no)
+
+
+def get_choice_string_input(prompt, default, firstChoice, secondChoice):
+  choice = raw_input(prompt).lower()
+  if choice in firstChoice:
+    return True
+  elif choice in secondChoice:
+    return False
+  elif choice is "": # Just enter pressed
+    return default
+  else:
+    print "input not recognized, please try again: "
+    return get_choice_string_input(prompt, default, firstChoice, secondChoice)
+
+
 def delete_mr(options):
+  saved_mr_mapping = get_YN_input("Have you saved MR host mapping using action save-mr-mapping [y/n] (n)? ", False)
+  if not saved_mr_mapping:
+    raise FatalException(1, "Ensure MAPREDUCE host component mapping is saved before deleting it. Use action "
+                            "save-mr-mapping.")
+
   SERVICE_URL_FORMAT = URL_FORMAT + '/services/MAPREDUCE'
   COMPONENT_URL_FORMAT = URL_FORMAT + '/hosts/{2}/host_components/{3}'
   NON_CLIENTS = ["JOBTRACKER", "TASKTRACKER"]
@@ -511,30 +535,51 @@ def get_config_resp(options, type, error_if_na=True):
   pass
 
 
-def modify_configs(options):
+def modify_configs(options, config_type):
   hostmapping = read_mapping()
   # Add capacity-scheduler, mapred-queue-acls, yarn-site
-  update_config(options, CAPACITY_SCHEDULER, CAPACITY_SCHEDULER_TAG)
-  update_config(options, MAPRED_QUEUE_ACLS, MAPRED_QUEUE_ACLS_TAG)
+  if (config_type is None) or (config_type == CAPACITY_SCHEDULER_TAG):
+    update_config(options, CAPACITY_SCHEDULER, CAPACITY_SCHEDULER_TAG)
+    pass
+
+  if (config_type is None) or (config_type == MAPRED_QUEUE_ACLS_TAG):
+    update_config(options, MAPRED_QUEUE_ACLS, MAPRED_QUEUE_ACLS_TAG)
+    pass
   jt_host = hostmapping["JOBTRACKER"][0]
 
-  for key in YARN_SITE.keys():
-    if REPLACE_JH_HOST_NAME_TAG in YARN_SITE[key]:
-      YARN_SITE[key] = YARN_SITE[key].replace(REPLACE_JH_HOST_NAME_TAG, jt_host, 1)
-    if REPLACE_RM_HOST_NAME_TAG in YARN_SITE[key]:
-      YARN_SITE[key] = YARN_SITE[key].replace(REPLACE_RM_HOST_NAME_TAG, jt_host, 1)
-  update_config(options, YARN_SITE, YARN_SITE_TAG)
+  if (config_type is None) or (config_type == YARN_SITE_TAG):
+    for key in YARN_SITE.keys():
+      if REPLACE_JH_HOST_NAME_TAG in YARN_SITE[key]:
+        YARN_SITE[key] = YARN_SITE[key].replace(REPLACE_JH_HOST_NAME_TAG, jt_host, 1)
+      if REPLACE_RM_HOST_NAME_TAG in YARN_SITE[key]:
+        YARN_SITE[key] = YARN_SITE[key].replace(REPLACE_RM_HOST_NAME_TAG, jt_host, 1)
+        pass
+      pass
+    pass
+    update_config(options, YARN_SITE, YARN_SITE_TAG)
+    pass
 
   # Update mapred-site config
-  for key in MAPRED_SITE.keys():
-    if REPLACE_JH_HOST_NAME_TAG in MAPRED_SITE[key]:
-      MAPRED_SITE[key] = MAPRED_SITE[key].replace(REPLACE_JH_HOST_NAME_TAG, jt_host, 1)
-  update_config(options, MAPRED_SITE, MAPRED_SITE_TAG)
+  if (config_type is None) or (config_type == MAPRED_SITE_TAG):
+    for key in MAPRED_SITE.keys():
+      if REPLACE_JH_HOST_NAME_TAG in MAPRED_SITE[key]:
+        MAPRED_SITE[key] = MAPRED_SITE[key].replace(REPLACE_JH_HOST_NAME_TAG, jt_host, 1)
+        pass
+      pass
+    pass
+    update_config_using_existing(options, MAPRED_SITE_TAG, MAPRED_SITE)
+    pass
 
   # Update global config, hdfs-site, core-site
-  update_config_using_existing(options, GLOBAL_TAG, GLOBAL, True)
-  update_config_using_existing(options, HDFS_SITE_TAG, HDFS_SITE)
-  update_config_using_existing(options, CORE_SITE_TAG, CORE_SITE)
+  if (config_type is None) or (config_type == GLOBAL_TAG):
+    update_config_using_existing(options, GLOBAL_TAG, GLOBAL, True)
+    pass
+  if (config_type is None) or (config_type == HDFS_SITE_TAG):
+    update_config_using_existing(options, HDFS_SITE_TAG, HDFS_SITE)
+    pass
+  if (config_type is None) or (config_type == CORE_SITE_TAG):
+    update_config_using_existing(options, CORE_SITE_TAG, CORE_SITE)
+    pass
   pass
 
 
@@ -633,7 +678,8 @@ def curl(*args):
 # Main.
 #
 def main():
-  parser = optparse.OptionParser(usage="usage: %prog [options] action\n  Valid actions: " + VALID_ACTIONS, )
+  parser = optparse.OptionParser(usage="usage: %prog [options] action\n  Valid actions: " + VALID_ACTIONS
+                                 + "\n  update-configs accepts type, e.g. hdfs-site to update specific configs",)
 
   parser.add_option("-n", "--printonly",
                     action="store_true", dest="printonly", default=False,
@@ -684,12 +730,17 @@ def main():
   try:
     if action == GET_MR_MAPPING_ACTION:
       get_mr1_mapping(options)
+      pprint("File mr_mapping contains the host mapping for mapreduce components. This file is critical for later "
+             "steps.")
     elif action == DELETE_MR_ACTION:
       delete_mr(options)
     elif action == ADD_YARN_MR2_ACTION:
       add_services(options)
     elif action == MODIFY_CONFIG_ACTION:
-      modify_configs(options)
+      config_type = None
+      if len(args) > 1:
+        config_type = args[1]
+      modify_configs(options, config_type)
     elif action == INSTALL_YARN_MR2_ACTION:
       install_services(options)
     elif action == BACKUP_CONFIG_ACTION:
@@ -699,7 +750,9 @@ def main():
 
   except FatalException as e:
     if e.reason is not None:
-      pprint("ERROR: Exiting with exit code {0}. Reason: {1}".format(e.code, e.reason))
+      error = "ERROR: Exiting with exit code {0}. Reason: {1}".format(e.code, e.reason)
+      pprint(error)
+      logger.error(error)
     sys.exit(e.code)
 
   if options.exit_message is not None:

ambari-server/src/test/python/TestUpgradeScript_HDP2.py

@@ -16,17 +16,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 import subprocess
-from mock.mock import MagicMock
+from mock.mock import MagicMock, call, patch
 from unittest import TestCase
-from mock.mock import patch
+from mock.mock import create_autospec
 import sys
 import unittest
 import UpgradeHelper_HDP2
 import StringIO
+import logging
 
 
 class TestUpgradeHDP2Script(TestCase):
   def setUp(self):
+    UpgradeHelper_HDP2.logger = MagicMock()
     out = StringIO.StringIO()
     sys.stdout = out
 
@@ -56,7 +58,7 @@ class TestUpgradeHDP2Script(TestCase):
     UpgradeHelper_HDP2.update_config_using_existing(opm, "global", site_template, True)
     get_config_mock.assert_called_once_with(opm, "global")
     update_config_mock.assert_called_once_with(opm, expected_site, "global")
-
+    pass
 
   @patch.object(UpgradeHelper_HDP2, 'update_config')
   @patch.object(UpgradeHelper_HDP2, 'get_config')
@@ -74,6 +76,376 @@ class TestUpgradeHDP2Script(TestCase):
     UpgradeHelper_HDP2.update_config_using_existing(opm, "global", site_template)
     get_config_mock.assert_called_once_with(opm, "global")
     update_config_mock.assert_called_once_with(opm, expected_site, "global")
+    pass
+
+  @patch.object(logging.FileHandler, 'setFormatter')
+  @patch.object(logging, 'basicConfig')
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, 'write_config')
+  @patch.object(UpgradeHelper_HDP2, 'get_config_resp')
+  @patch("os.remove")
+  @patch("shutil.copyfile")
+  @patch("os.path.exists")
+  @patch('optparse.OptionParser')
+  def test_save_configs(self, option_parser_mock, path_exists_mock, shutil_copy_mock, os_remove_mock,
+                        get_config_resp_mock, write_config_mock, file_handler_mock, logging_mock, set_formatter_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    path_exists_mock.return_value = True
+    shutil_copy_mock = MagicMock()
+    options = MagicMock()
+    args = ["backup-configs"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+
+    def get_config_resp_side_effect(ops, type, error_if_na):
+      if type == "global":
+        return ("version1", "")
+      elif type == "core-site":
+        return ("version1", "")
+      elif type == "hdfs-site":
+        return ("version1", "")
+      elif type == "mapred-site":
+        return ("version2", "")
+      elif type == "hbase-site":
+        return ("version2", "")
+      else:
+        return (None, None)
+
+    get_config_resp_mock.side_effect = get_config_resp_side_effect
+    UpgradeHelper_HDP2.main()
+    shutil_copy_mock.assert_called_once()
+    os_remove_mock.assert_called_once_with("logfile")
+    write_expected = [call("", "global", "version1"), call("", "core-site", "version1"),
+                      call("", "hdfs-site", "version1"), call("", "mapred-site", "version2"),
+                      call("", "hbase-site", "version2")]
+    write_config_mock.assert_has_calls(write_expected, any_order=True)
+    get_config_expected = [call(options, "global", True), call(options, "core-site", True),
+                           call(options, "hdfs-site", True), call(options, "mapred-site", True),
+                           call(options, "hbase-site", False), call(options, "oozie-site", False),
+                           call(options, "webhcat-site", False), call(options, "hive-site", False)]
+    get_config_resp_mock.assert_has_calls(get_config_expected, any_order=True)
+    pass
+
+
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'write_mapping')
+  @patch("json.loads")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_save_mr_mapping(self, option_parser_mock, curl_mock, json_loads_mock, write_mapping_mock,
+                           backup_file_mock, file_handler_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["save-mr-mapping"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "']
+    json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
+    UpgradeHelper_HDP2.main()
+    expected_curl_calls = [
+      call("-u", "admin:admin",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
+      call("-u", "admin:admin",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
+      call("-u", "admin:admin",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
+    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
+    self.assertTrue(write_mapping_mock.called)
+    write_call = write_mapping_mock.call_args
+    args, kargs = write_call
+    self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
+    self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
+    self.assertTrue('TASKTRACKER' in args[0].keys())
+    self.assertTrue('TASKTRACKER' in args[0].keys())
+    pass
+
+
+  @patch.object(UpgradeHelper_HDP2, "get_YN_input")
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_delete_mr(self, option_parser_mock, curl_mock,
+                     backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["delete-mr"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.return_value = ''
+    get_yn_mock.return_value = True
+    read_mapping_mock.return_value = {
+      "TASKTRACKER": ["c6401", "c6402"],
+      "JOBTRACKER": ["c6401"],
+      "MAPREDUCE_CLIENT": ["c6401"]}
+    UpgradeHelper_HDP2.main()
+    expected_curl_calls = [
+      call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
+      call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
+      call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
+      call("-u", "admin:admin", "-X", "DELETE",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
+    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
+    pass
+
+
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_add_yarn_mr(self, option_parser_mock, curl_mock,
+                       backup_file_mock, file_handler_mock, read_mapping_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["add-yarn-mr2"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.return_value = ''
+    read_mapping_mock.return_value = {
+      "TASKTRACKER": ["c6401", "c6402"],
+      "JOBTRACKER": ["c6401"],
+      "MAPREDUCE_CLIENT": ["c6403"]}
+    UpgradeHelper_HDP2.main()
+    expected_curl_calls = [
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
+      call("-u", "admin:admin", "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER")]
+    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
+    pass
+
+
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_install_yarn_mr2(self, option_parser_mock, curl_mock,
+                            backup_file_mock, file_handler_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["install-yarn-mr2"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.return_value = '"href" : "'
+    UpgradeHelper_HDP2.main()
+    expected_curl_calls = [
+      call("-u", "admin:admin", "-X", "PUT", "-d",
+           """{"ServiceInfo": {"state": "INSTALLED"}}""",
+           "http://localhost:8080/api/v1/clusters/c1/services?ServiceInfo/state=INIT")]
+    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
+    pass
+
+
+  @patch.object(UpgradeHelper_HDP2, "update_config_using_existing")
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_update_single_configs(self, option_parser_mock, curl_mock,
+                                 backup_file_mock, file_handler_mock, read_mapping_mock,
+                                 update_config_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["update-configs", "hdfs-site"]
+    opm.parse_args.return_value = (options, args)
+    curl_mock.side_effect = ['', '', '', '', '', '', '']
+    read_mapping_mock.return_value = {"JOBTRACKER": ["c6401"]}
+    update_config_mock.side_effect = [None]
+    UpgradeHelper_HDP2.main()
+    self.assertTrue(update_config_mock.call_count == 1)
+    pass
+
+  @patch.object(UpgradeHelper_HDP2, "get_config")
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_update_configs(self, option_parser_mock, curl_mock,
+                          backup_file_mock, file_handler_mock, read_mapping_mock,
+                          get_config_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["update-configs"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.side_effect = ['', '', '', '', '', '', '']
+    read_mapping_mock.return_value = {
+      "TASKTRACKER": ["c6401", "c6402"],
+      "JOBTRACKER": ["c6401"],
+      "MAPREDUCE_CLIENT": ["c6403"]}
+    get_config_mock.return_value = {
+      "mapred.hosts": "an_old_value",
+      "mapred.hosts.exclude": "an_old_value",
+      "mapred.jobtracker.maxtasks.per.job": "an_old_value",
+      "mapred.jobtracker.taskScheduler": "an_old_value",
+      "mapred.task.tracker.task-controller": "an_old_value",
+      "mapred.userlog.retain.hours": "an_old_value",
+      "global1": "global11"
+    }
+    UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
+    UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
+    UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
+    UpgradeHelper_HDP2.main()
+    self.validate_update_config_call(curl_mock.call_args_list[0], "capacity-scheduler")
+    self.validate_update_config_call(curl_mock.call_args_list[1], "mapred-queue-acls")
+    self.validate_update_config_call(curl_mock.call_args_list[2], "yarn-site")
+    self.validate_update_config_call(curl_mock.call_args_list[3], "mapred-site")
+    self.validate_update_config_call(curl_mock.call_args_list[4], "global")
+    self.validate_config_replacememt(curl_mock.call_args_list[2], "yarn-site")
+    self.validate_config_replacememt(curl_mock.call_args_list[3], "mapred-site")
+    self.validate_config_replacememt(curl_mock.call_args_list[4], "global")
+    pass
+
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_get_and_parse_properties(self, option_parser_mock, curl_mock,
+                                    backup_file_mock, file_handler_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["update-configs"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.side_effect = [
+      """{ "href" : "http://localhost:8080/api/v1/clusters/c1",
+           "Clusters" : {
+              "cluster_id" : 2,
+              "cluster_name" : "c1",
+              "version" : "HDP-2.0.5",
+              "desired_configs" : {
+                "capacity-scheduler" : {
+                  "user" : "admin",
+                  "tag" : "version137"
+                },
+                "core-site" : {
+                   "user" : "admin",
+                   "tag" : "version138"
+                }}}}""",
+      """{
+           "href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
+           "items" : [
+             {
+               "href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
+               "tag" : "version1378850572477",
+               "type" : "core-site",
+               "Config" : {
+                 "cluster_name" : "c1"
+               },
+               "properties" : {
+                 "name1" : "value1",
+                 "name2" : "value2",
+                 "name3" : "value3"
+               }}]}""",
+      """{ "href" : "http://localhost:8080/api/v1/clusters/c1",
+           "Clusters" : {
+              "desired_configs" : {
+                "capacity-scheduler" : {
+                  "user" : "admin",
+                  "tag" : "version137"
+                }}}}"""]
+    properties = UpgradeHelper_HDP2.get_config(options, "core-site")
+    self.assertTrue(len(properties.keys()) == 3)
+    self.assertTrue(properties["name1"] == "value1")
+    self.assertTrue(properties["name2"] == "value2")
+    self.assertTrue(properties["name3"] == "value3")
+    try:
+      UpgradeHelper_HDP2.get_config(options, "hdfs-site")
+    except Exception, e:
+      self.assertTrue('Unable to get the current version for config type hdfs-site' in e.reason)
+      pass
+    pass
+
+  def validate_update_config_call(self, call, type):
+    args, kargs = call
+    self.assertTrue(args[3] == 'PUT')
+    self.assertTrue(type in args[5])
+    pass
+
+  def validate_config_replacememt(self, call, type):
+    args, kargs = call
+    self.assertFalse("REPLACE_WITH_" in args[5])
+    self.assertFalse("REPLACE_JH_HOST" in args[5])
+    self.assertFalse("REPLACE_RM_HOST" in args[5])
+    if type == "yarn-site":
+      self.assertTrue("c6401" in args[5])
+      self.assertFalse("an_old_value" in args[5])
+    elif type == "mapred-site":
+      self.assertTrue("an_old_value" in args[5])
+    elif type == "global":
+      self.assertTrue("global11" in args[5])
+      self.assertTrue("an_old_value" in args[5])
+      self.assertTrue("mapred.hosts.exclude" in args[5])
+    elif (type == "core-site") or (type == "hdfs-site"):
+      self.assertTrue("global11" in args[5])
+      self.assertFalse("an_old_value" in args[5])
+      self.assertFalse("mapred.hosts.exclude" in args[5])
+    pass
+
+  def test_read_config(self):
+    pass
 
 
 if __name__ == "__main__":