AMBARI-12362. Manual upgrade fails - YARN RM restart failed due to yarn node labels error.

ambari-server/src/main/python/upgradeHelper.py

@@ -66,6 +66,10 @@ Example:
            "value": "{TEMPLATE_TAG}",
            "template": "yes",
            "required-services": ["HDFS", "YARN"]
+          },
+          "test_property": {
+           "value": "new value",
+           "value-required": "old value"  (optional, property would be set if the required value is present)
           }
         }
       },
@@ -287,6 +291,7 @@ class CatConst(Const):
   ITEMS_TAG = "items"
   TYPE_TAG = "type"
   TRUE_TAG = "yes"
+  VALUE_REQUIRED_TAG = "value-required"
   STACK_PROPERTIES_MAPPING_LIST_TAG = "property-mapping"
   VALUE_TEMPLATE_TAG = "template"
   SEARCH_PATTERN = "(\{[^\{\}]+\})"  # {XXXXX}
@@ -520,10 +525,12 @@ class UpgradeCatalog(object):
       properties[catalog_item_name] = catalog_property_item[CatConst.PROPERTY_VALUE_TAG]
     return properties
 
-  def __can_handler_execute(self, catalog_options, property_item):
+  def __can_handler_execute(self, catalog_options, catalog_property_item, property_item, properties):
     """
     :type catalog_options dict
+    :type catalog_property_item str
     :type property_item dict
+    :type properties dict
     """
     can_process = True
 
@@ -541,6 +548,10 @@ class UpgradeCatalog(object):
     if required_list is not None:
       can_process = can_process and is_services_exists(required_list)
 
+    if CatConst.VALUE_REQUIRED_TAG in property_item and property_item[CatConst.VALUE_REQUIRED_TAG] is not None and\
+      CatConst.PROPERTY_VALUE_TAG in property_item and catalog_property_item in properties:
+      can_process = properties[catalog_property_item] == property_item[CatConst.VALUE_REQUIRED_TAG]
+
     return can_process
 
   def process_simple_transformations(self, name, properties):
@@ -560,7 +571,7 @@ class UpgradeCatalog(object):
     catalog_item = self._properties_catalog[name]
     for catalog_property_item in catalog_item.keys():
       catalog_options = self.options[name] if name in self.options else {}
-      if self.__can_handler_execute(catalog_options, catalog_item[catalog_property_item]):
+      if self.__can_handler_execute(catalog_options, catalog_property_item, catalog_item[catalog_property_item], properties):
         for handler in tag_handlers:
           handler(catalog_property_item, catalog_item[catalog_property_item], properties)
 

ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json

@@ -16,6 +16,10 @@
           "yarn-site": {
             "merged-copy": "yes"
           },
+          "capacity-scheduler": {
+            "merged-copy": "yes",
+            "required-services": ["YARN"]
+          },
           "mapred-site": {
             "merged-copy": "yes"
           },
@@ -328,6 +332,17 @@
           "yarn.timeline-service.recovery.enabled": "true",
           "yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
           "yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
+        },
+        "capacity-scheduler": {
+          "yarn.scheduler.capacity.root.default-node-label-expression": "",
+          "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {
+            "value": "10",
+            "value-required": "-1"
+          },
+          "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {
+            "value": "100",
+            "value-required": "-1"
+          }
         }
       },
       "property-mapping": {
@@ -344,4 +359,4 @@
       }
     }
   ]
-}
+}

ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json

@@ -35,6 +35,10 @@
                       "merged-copy": "yes",
                       "required-services": ["YARN"]
                     },
+                    "capacity-scheduler": {
+                      "merged-copy": "yes",
+                      "required-services": ["YARN"]
+                    },
                     "kafka-broker": {
                       "merged-copy": "yes",
                       "required-services": ["KAFKA"]
@@ -254,6 +258,17 @@
                   "yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
                   "yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
                 },
+                "capacity-scheduler": {
+                  "yarn.scheduler.capacity.root.default-node-label-expression": "",
+                  "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {
+                    "value": "10",
+                    "value-required": "-1"
+                  },
+                  "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {
+                    "value": "100",
+                    "value-required": "-1"
+                  }
+                },
                 "kafka-broker": {
                   "listeners": {
                     "value": "{KAFKA_LISTENERS}",
@@ -1693,4 +1708,4 @@
      }
     }
   ]
-}
+}

ambari-server/src/test/python/TestUpgradeHelper.py

@@ -881,8 +881,6 @@ class TestUpgradeHelper(TestCase):
     self.assertEqual(1, open_mock.call_count)
 
 
-
-
   def test_report_formatter(self):
     file = StringIO()
     cfg_item = self.catalog_cfg_type
@@ -921,5 +919,68 @@ class TestUpgradeHelper(TestCase):
 
     self.assertEqual(expected_output, file.getvalue())
 
+  @patch.object(upgradeHelper, "get_config_resp_all")
+  def test_conditional_replace(self, get_config_resp_all_mock):
+    test_catalog = """
+        {
+      "version": "1.0",
+      "stacks": [
+        {
+          "name": "HDP",
+          "old-version": "1.0",
+          "target-version": "1.1",
+          "options": {
+            "config-types":{
+              "test": {
+                "merged-copy": "yes"
+              }
+            }
+          },
+          "properties": {
+             "test": {
+               "test": {
+                 "value": "10",
+                 "value-required": "-1"
+               },
+               "test2": {
+                 "value": "10",
+                 "value-required": "-2"
+               }
+            }
+          },
+          "property-mapping": {
+          }
+        }
+      ]
+    }
+    """
+
+    expected_properties = {"test":"10", "test2":"15"}
+
+    old_opt = upgradeHelper.Options.OPTIONS
+    options = lambda: ""
+    options.from_stack = "1.0"
+    options.to_stack = "1.1"
+    options.upgrade_json = ""
+
+    upgradeHelper.Options.OPTIONS = options
+    upgradeHelper.Options.SERVICES = [self.required_service]
+    get_config_resp_all_mock.return_value = {
+      "test": {
+        "properties": {"test":"-1", "test2":"15"}
+      }
+    }
+
+    ucf = UpgradeCatalogFactoryMock(test_catalog)
+    scf = upgradeHelper.ServerConfigFactory()
+
+    cfg = scf.get_config("test")
+    ucfg = ucf.get_catalog("1.0", "1.1")
+
+    cfg.merge(ucfg)
+    upgradeHelper.Options.OPTIONS = old_opt
+
+    self.assertEqual(expected_properties, cfg.properties)
+
 if __name__ == "__main__":
   unittest.main()