AMBARI-3342. Update upgrade script with latest config mapping.

ambari-server/src/main/python/UpgradeHelper_HDP2.py

@@ -49,6 +49,7 @@ GLOBAL_TAG = "global"
 HDFS_SITE_TAG = "hdfs-site"
 CORE_SITE_TAG = "core-site"
 YARN_SITE_TAG = "yarn-site"
+HBASE_SITE_TAG = "hbase-site"
 REPLACE_JH_HOST_NAME_TAG = "REPLACE_JH_HOST"
 REPLACE_RM_HOST_NAME_TAG = "REPLACE_RM_HOST"
 REPLACE_WITH_TAG = "REPLACE_WITH_"
@@ -403,9 +404,9 @@ MAPRED_SITE = {
   "mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad": "DELETE_OLD",
   "mapreduce.jobtracker.tasktracker.maxblacklists": "DELETE_OLD",
   "mapreduce.jobtracker.webinterface.trusted": "DELETE_OLD",
-  "mapreduce.map.java.opts": "-Xmx320m",
+  "mapreduce.map.java.opts": "-Xmx756m",
   "mapreduce.map.log.level": "INFO",
-  "mapreduce.map.sort.spill.percent": "0.1",
+  "mapreduce.map.sort.spill.percent": "0.7",
   "mapreduce.reduce.input.limit": "DELETE_OLD",
   "mapreduce.reduce.java.opts": "-Xmx756m",
   "mapreduce.reduce.log.level": "INFO",
@@ -438,9 +439,9 @@ MAPRED_SITE = {
   "mapreduce.tasktracker.taskmemorymanager.monitoringinterval": "DELETE_OLD",
   "mapreduce.tasktracker.tasks.sleeptimebeforesigkill": "DELETE_OLD",
   "yarn.app.mapreduce.am.admin-command-opts": "-Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN",
-  "yarn.app.mapreduce.am.command-opts": "-Xmx756m",
+  "yarn.app.mapreduce.am.command-opts": "-Xmx312m",
   "yarn.app.mapreduce.am.log.level": "INFO",
-  "yarn.app.mapreduce.am.resource.mb": "1024",
+  "yarn.app.mapreduce.am.resource.mb": "512",
   "yarn.app.mapreduce.am.staging-dir": "/user"
 }
 
@@ -540,7 +541,7 @@ YARN_SITE = {
   "yarn.nodemanager.log.retain-second": "604800",
   "yarn.nodemanager.remote-app-log-dir": "/app-logs",
   "yarn.nodemanager.remote-app-log-dir-suffix": "logs",
-  "yarn.nodemanager.resource.memory-mb": "10240",
+  "yarn.nodemanager.resource.memory-mb": "5120",
   "yarn.nodemanager.vmem-check-enabled": "false",
   "yarn.nodemanager.vmem-pmem-ratio": "2.1",
   "yarn.resourcemanager.address": "REPLACE_RM_HOST:8050",
@@ -551,11 +552,24 @@ YARN_SITE = {
   "yarn.resourcemanager.scheduler.address": "REPLACE_RM_HOST:8030",
   "yarn.resourcemanager.scheduler.class": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler",
   "yarn.resourcemanager.webapp.address": "REPLACE_RM_HOST:8088",
-  "yarn.scheduler.maximum-allocation-mb": "6144",
+  "yarn.scheduler.maximum-allocation-mb": "2048",
   "yarn.scheduler.minimum-allocation-mb": "512"
 }
 
 
+HBASE_SITE = {
+  "dfs.support.append": "DELETE_OLD",
+  "hbase.hregion.majorcompaction": "604800000",
+  "hbase.hregion.max.filesize": "10737418240",
+  "hbase.hstore.blockingStoreFiles": "10",
+  "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+  "hbase.regionserver.handler.count": "60",
+  "hbase.rpc.engine": "DELETE_OLD",
+  "hfile.block.cache.size": "0.40",
+  "zookeeper.session.timeout": "30"
+}
+
+
 class FatalException(Exception):
   def __init__(self, code, reason):
     self.code = code
@@ -825,7 +839,7 @@ def modify_configs(options, config_type):
 
   # Update global config
   if (config_type is None) or (config_type == GLOBAL_TAG):
-    update_config_using_existing(options, GLOBAL_TAG, GLOBAL, True)
+    update_config_using_existing(options, GLOBAL_TAG, GLOBAL.copy(), True)
     pass
 
   core_site_latest = rename_all_properties(get_config(options, CORE_SITE_TAG), PROPERTY_MAPPING)
@@ -839,23 +853,33 @@ def modify_configs(options, config_type):
     pass
 
   # Update mapred-site config
+  mapred_updated = MAPRED_SITE.copy()
   if (config_type is None) or (config_type == MAPRED_SITE_TAG):
-    for key in MAPRED_SITE.keys():
-      if REPLACE_JH_HOST_NAME_TAG in MAPRED_SITE[key]:
-        MAPRED_SITE[key] = MAPRED_SITE[key].replace(REPLACE_JH_HOST_NAME_TAG, jt_host, 1)
+    for key in mapred_updated.keys():
+      if REPLACE_JH_HOST_NAME_TAG in mapred_updated[key]:
+        mapred_updated[key] = mapred_updated[key].replace(REPLACE_JH_HOST_NAME_TAG, jt_host, 1)
         pass
       pass
     pass
-    update_config_using_existing_properties(options, MAPRED_SITE_TAG, MAPRED_SITE, mapred_site_latest, True)
+    update_config_using_existing_properties(options, MAPRED_SITE_TAG, mapred_updated, mapred_site_latest, True)
     pass
 
   # Update hdfs-site, core-site
   if (config_type is None) or (config_type == HDFS_SITE_TAG):
-    update_config_using_existing_properties(options, HDFS_SITE_TAG, HDFS_SITE, hdfs_site_latest, True)
+    update_config_using_existing_properties(options, HDFS_SITE_TAG, HDFS_SITE.copy(), hdfs_site_latest, True)
     pass
   if (config_type is None) or (config_type == CORE_SITE_TAG):
-    update_config_using_existing_properties(options, CORE_SITE_TAG, CORE_SITE, core_site_latest, True)
+    update_config_using_existing_properties(options, CORE_SITE_TAG, CORE_SITE.copy(), core_site_latest, True)
     pass
+
+  # Update hbase-site if exists
+  if (config_type is None) or (config_type == HBASE_SITE_TAG):
+    tag, structured_resp = get_config_resp(options, HBASE_SITE_TAG, False)
+    if structured_resp is not None:
+      update_config_using_existing(options, HBASE_SITE_TAG, HBASE_SITE.copy(), True)
+      pass
+    pass
+
   pass
 
 

ambari-server/src/test/python/TestUpgradeScript_HDP2.py

@@ -302,15 +302,17 @@ class TestUpgradeHDP2Script(TestCase):
       self.assertEqual(prop_to_move[key], args[3][key])
     pass
 
+
+  @patch.object(UpgradeHelper_HDP2, "get_config_resp")
   @patch.object(UpgradeHelper_HDP2, "get_config")
   @patch.object(UpgradeHelper_HDP2, "read_mapping")
   @patch.object(logging, 'FileHandler')
   @patch.object(UpgradeHelper_HDP2, "backup_file")
   @patch.object(UpgradeHelper_HDP2, 'curl')
   @patch('optparse.OptionParser')
-  def test_update_configs(self, option_parser_mock, curl_mock,
+  def test_no_hbase(self, option_parser_mock, curl_mock,
                           backup_file_mock, file_handler_mock, read_mapping_mock,
-                          get_config_mock):
+                          get_config_mock, get_config_resp_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     opm = option_parser_mock.return_value
     options = MagicMock()
@@ -326,26 +328,86 @@ class TestUpgradeHDP2Script(TestCase):
       "TASKTRACKER": ["c6401", "c6402"],
       "JOBTRACKER": ["c6401"],
       "MAPREDUCE_CLIENT": ["c6403"]}
+    get_config_resp_mock.return_value = "hbase-site", None
     get_config_mock.return_value = {
       "mapred.hosts": "an_old_value",
       "mapred.hosts.exclude": "an_old_value",
       "mapred.jobtracker.maxtasks.per.job": "an_old_value",
       "mapred.jobtracker.taskScheduler": "an_old_value",
-      "mapred.task.tracker.task-controller": "an_old_value",
+      "dfs.df.interval": "an_old_value",
       "mapred.userlog.retain.hours": "will_not_be_stored",
       "global1": "global11"
     }
-    UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
-    UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
-    UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
     UpgradeHelper_HDP2.main()
+    self.assertEqual(6, len(curl_mock.call_args_list))
+
+
+  @patch.object(UpgradeHelper_HDP2, "get_config_resp")
+  @patch.object(UpgradeHelper_HDP2, "get_config")
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_update_configs(self, option_parser_mock, curl_mock,
+                          backup_file_mock, file_handler_mock, read_mapping_mock,
+                          get_config_mock, get_config_resp_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = MagicMock()
+    args = ["update-configs"]
+    opm.parse_args.return_value = (options, args)
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    curl_mock.side_effect = ['', '', '', '', '', '', '', '']
+    read_mapping_mock.return_value = {
+      "TASKTRACKER": ["c6401", "c6402"],
+      "JOBTRACKER": ["c6401"],
+      "MAPREDUCE_CLIENT": ["c6403"]}
+    get_config_resp_mock.return_value = "hbase-site", {}
+    site_properties = {
+      "mapred.hosts": "an_old_value",
+      "mapred.hosts.exclude": "an_old_value",
+      "mapred.jobtracker.maxtasks.per.job": "an_old_value",
+      "hbase.rpc.engine": "an_old_value",
+      "dfs.df.interval": "an_old_value",
+      "mapred.userlog.retain.hours": "will_not_be_stored",
+      "global1": "global11"
+    }
+    get_config_mock.side_effect = [
+      site_properties.copy(), site_properties.copy(), site_properties.copy(),
+      site_properties.copy(), site_properties.copy(), site_properties.copy()]
+
+    saved_global = UpgradeHelper_HDP2.GLOBAL
+    saved_hdfs = UpgradeHelper_HDP2.HDFS_SITE
+    saved_core = UpgradeHelper_HDP2.CORE_SITE
+    saved_habse = UpgradeHelper_HDP2.HBASE_SITE
+    saved_mapred = UpgradeHelper_HDP2.MAPRED_SITE
+    try:
+      UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
+      UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
+      UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
+      UpgradeHelper_HDP2.main()
+    finally:
+      UpgradeHelper_HDP2.GLOBAL = saved_global
+      UpgradeHelper_HDP2.HDFS_SITE = saved_hdfs
+      UpgradeHelper_HDP2.CORE_SITE = saved_core
+      UpgradeHelper_HDP2.MAPRED_SITE = saved_mapred
+
+    self.assertEqual(7, len(curl_mock.call_args_list))
     self.validate_update_config_call(curl_mock.call_args_list[0], "capacity-scheduler")
     self.validate_update_config_call(curl_mock.call_args_list[1], "yarn-site")
     self.validate_update_config_call(curl_mock.call_args_list[3], "mapred-site")
     self.validate_update_config_call(curl_mock.call_args_list[2], "global")
     self.validate_config_replacememt(curl_mock.call_args_list[1], "yarn-site")
-    self.validate_config_replacememt(curl_mock.call_args_list[3], "mapred-site")
     self.validate_config_replacememt(curl_mock.call_args_list[2], "global")
+    self.validate_config_replacememt(curl_mock.call_args_list[3], "mapred-site")
+    self.validate_config_replacememt(curl_mock.call_args_list[4], "hdfs-site")
+    self.validate_config_replacememt(curl_mock.call_args_list[5], "core-site")
+    self.validate_config_replacememt(curl_mock.call_args_list[6], "hbase-site")
     pass
 
   @patch.object(UpgradeHelper_HDP2, "read_mapping")
@@ -484,14 +546,21 @@ class TestUpgradeHDP2Script(TestCase):
       self.assertFalse("an_old_value" in args[6])
     elif type == "mapred-site":
       self.assertFalse("will_not_be_stored" in args[6])
+      self.assertTrue("fs.df.interval" in args[6])
+      self.assertFalse("dfs.df.interval" in args[6])
     elif type == "global":
       self.assertTrue("global11" in args[6])
       self.assertTrue("an_old_value" in args[6])
       self.assertTrue("mapred.hosts.exclude" in args[6])
     elif (type == "core-site") or (type == "hdfs-site"):
       self.assertTrue("global11" in args[6])
-      self.assertFalse("an_old_value" in args[6])
-      self.assertFalse("mapred.hosts.exclude" in args[6])
+      self.assertTrue("global2" in args[6])
+      self.assertTrue("hbase.rpc.engine" in args[6])
+    elif type == "hbase-site":
+      self.assertTrue("global11" in args[6])
+      self.assertTrue("hbase.hstore.blockingStoreFiles" in args[6])
+      self.assertTrue("dfs.df.interval" in args[6])
+      self.assertFalse("hbase.rpc.engine" in args[6])
     pass
 
   def get_mock_options(self, printonly=False):