AMBARI-19404 Warning by default by stack advisor (dsen)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -413,12 +413,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     self.updateMountProperties("hdfs-site", hdfs_mount_properties, configurations, services, hosts)
 
-    if hdfsSiteProperties and "dfs.datanode.data.dir" in hdfsSiteProperties and\
-      hdfsSiteProperties["dfs.datanode.data.dir"] is not None:
+    if configurations and "hdfs-site" in configurations and \
+            "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"] and \
+                    configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"] is not None:
+      dataDirs = configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"].split(",")
 
+    elif hdfsSiteProperties and "dfs.datanode.data.dir" in hdfsSiteProperties and \
+                    hdfsSiteProperties["dfs.datanode.data.dir"] is not None:
       dataDirs = hdfsSiteProperties["dfs.datanode.data.dir"].split(",")
-    else:
-      dataDirs = configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"].split(",")
 
     # dfs.datanode.du.reserved should be set to 10-15% of volume size
     # For each host selects maximum size of the volume. Then gets minimum for all hosts.

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -100,17 +100,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
       putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
 
-      # Above is the default calculated 'maximum' values derived purely from hosts.
-      # However, there are 'maximum' and other attributes that actually change based on the values
-      #  of other configs. We need to update those values.
-      if ("yarn-site" in services["configurations"]):
-        if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
-          putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
-          putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
-        if ("yarn.nodemanager.resource.cpu-vcores" in services["configurations"]["yarn-site"]["properties"]):
-          putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
-          putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
-
       kerberos_authentication_enabled = self.isSecurityEnabled(services)
       if kerberos_authentication_enabled:
         putYarnProperty('yarn.nodemanager.container-executor.class',

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -1507,6 +1507,60 @@ class TestHDP206StackAdvisor(TestCase):
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts)
     self.assertEquals(configurations, expected)
 
+    hosts = {
+      "items": [
+        {
+          "href": "/api/v1/hosts/host1",
+          "Hosts": {
+            "cpu_count": 1,
+            "host_name": "c6401.ambari.apache.org",
+            "os_arch": "x86_64",
+            "os_type": "centos6",
+            "ph_cpu_count": 1,
+            "public_host_name": "public.c6401.ambari.apache.org",
+            "rack_info": "/default-rack",
+            "total_mem": 2097152,
+            "disk_info": [{
+              "available" : "21039512",
+              "device" : "/dev/vda1",
+              "used" : "3316924",
+              "percent" : "14%",
+              "size" : "25666616",
+              "type" : "ext4",
+              "mountpoint" : "/"
+            },
+              {
+                "available" : "244732200",
+                "device" : "/dev/vdb",
+                "used" : "60508",
+                "percent" : "1%",
+                "size" : "257899908",
+                "type" : "ext4",
+                "mountpoint" : "/grid/0"
+              }]
+          }
+        }
+      ]}
+
+    expected["hdfs-site"] = {
+      'properties': {
+        'dfs.datanode.du.reserved': '33011188224',
+        'dfs.internal.nameservices': 'mycluster',
+        'dfs.ha.namenodes.mycluster': 'nn1,nn2',
+        'dfs.datanode.data.dir': '/hadoop/hdfs/data,/grid/0/hadoop/hdfs/data',
+        'dfs.namenode.name.dir': '/hadoop/hdfs/namenode,/grid/0/hadoop/hdfs/namenode',
+        'dfs.namenode.checkpoint.dir': '/hadoop/hdfs/namesecondary',
+      },
+      'property_attributes': {
+        'dfs.namenode.rpc-address': {
+          'delete': 'true'
+        }
+      }
+    }
+    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts)
+    self.assertEquals(configurations, expected)
+
+
 
   def test_recommendAmsConfigurations(self):
     configurations = {

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1168,17 +1168,36 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
-    # Test - with no 'changed-configurations', we should get updated 'maximum's.
     services.pop("changed-configurations", None)
+    services["changed-configurations"] = [{
+        "type": "yarn-site",
+        "name": "yarn.nodemanager.resource.memory-mb",
+        "old_value": "1280"
+    }]
     services.pop("configurations", None)
-    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}}
-    expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9'
-    expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9'
+    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321'}}}
+
+    expected["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] = '4321'
+    expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"] = '4321'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-mb"]["maximum"] = '4321'
     self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
+    services["changed-configurations"].append({
+        "type": "yarn-site",
+        "name": "yarn.nodemanager.resource.cpu-vcores",
+        "old_value": "7"
+    })
+    services.pop("configurations", None)
+    services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.cpu-vcores": '9', "yarn.nodemanager.resource.memory-mb": '4321'}}}
+    expected["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"] = '9'
+    expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-vcores"] = '9'
+    expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9'
+    expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9'
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations, expected)
+
   def test_multipleDependsOn(self):
     configurations = {
       "yarn-env": {