AMBARI-10332. BE: Extend stack-advisor to recommend property_value_attributes for HDFS (dsen via srimanth)

ambari-server/src/main/java/org/apache/ambari/server/agent/DiskInfo.java

@@ -117,7 +117,17 @@ public class DiskInfo {
   public void setSize(String size) {
     this.size = size;
   }
-  
+
+  @JsonProperty("device")
+  public String getDevice() {
+    return device;
+  }
+
+  @JsonProperty("device")
+  public void setDevice(String device) {
+    this.device = device;
+  }
+
   @Override
   public String toString() {
     return "available=" + this.available + ",mountpoint=" + this.mountpoint

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java

@@ -70,7 +70,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
   protected static Log LOG = LogFactory.getLog(StackAdvisorCommand.class);
 
   private static final String GET_HOSTS_INFO_URI = "/api/v1/hosts"
-      + "?fields=Hosts&Hosts/host_name.in(%s)";
+      + "?fields=Hosts/*&Hosts/host_name.in(%s)";
   private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s/"
       + "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version"
       + ",services/StackServices/service_name,services/StackServices/service_version"

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml

@@ -61,6 +61,12 @@
       <maximum>2</maximum>
       <increment-step>1</increment-step>
     </value-attributes>
+    <depends-on>
+      <property>
+        <type>hdfs-site</type>
+        <name>dfs.datanode.data.dir</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -139,11 +139,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
  
   def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
-    putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+    putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
     putHDFSProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024))
-    putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+    putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
     putHDFSProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128))
-    putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+    putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
     putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
 
   def recommendHbaseEnvConfigurations(self, configurations, clusterData, services, hosts):
@@ -152,7 +152,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024)
 
   def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
-    putAmsEnvProperty = self.putProperty(configurations, "ams-env")
     putAmsEnvProperty = self.putProperty(configurations, "ams-env")
     putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site")
     putTimelineServiceProperty = self.putProperty(configurations, "ams-site")
@@ -209,16 +208,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
 
 
-  def getHostWithComponent(self, serviceName, componentName, services, hosts):
+  def getHostsWithComponent(self, serviceName, componentName, services, hosts):
     if services is not None and hosts is not None and serviceName in [service["StackServices"]["service_name"] for service in services["services"]]:
       service = [serviceEntry for serviceEntry in services["services"] if serviceEntry["StackServices"]["service_name"] == serviceName][0]
       components = [componentEntry for componentEntry in service["components"] if componentEntry["StackServiceComponents"]["component_name"] == componentName]
       if (len(components) > 0 and len(components[0]["StackServiceComponents"]["hostnames"]) > 0):
-        # NodeManager available - determine hosts and memory
+        # component available - determine hosts and memory
         componentHostname = components[0]["StackServiceComponents"]["hostnames"][0]
         componentHosts = [host for host in hosts["items"] if host["Hosts"]["host_name"] == componentHostname]
-        if (componentHosts is not None and len(componentHosts) > 0):
-          return componentHosts[0]
+        return componentHosts
+    return []
+
+  def getHostWithComponent(self, serviceName, componentName, services, hosts):
+    componentHosts = self.getHostsWithComponent(serviceName, componentName, services, hosts)
+    if (len(componentHosts) > 0):
+      return componentHosts[0]
+    return None
 
   def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
 

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -53,19 +53,60 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
 
   def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
-    putHdfsProperty = self.putProperty(configurations, "hdfs-site")
-    putHdfsProperty("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096)
-    putHDFSProperty = self.putProperty(configurations, "hadoop-env")
-    putHDFSProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024))
-    putHDFSProperty = self.putProperty(configurations, "hadoop-env")
-    putHDFSProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128))
-    putHDFSProperty = self.putProperty(configurations, "hadoop-env")
-    putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
+    putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services)
+    putHdfsSiteProperty("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096)
+    dataDirsCount = 1
+    if "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"]:
+      dataDirsCount = len(str(configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"]).split(","))
+    if dataDirsCount <= 2:
+      failedVolumesTolerated = 0
+    elif dataDirsCount <= 4:
+      failedVolumesTolerated = 1
+    else:
+      failedVolumesTolerated = 2
+    putHdfsSiteProperty("dfs.datanode.failed.volumes.tolerated", failedVolumesTolerated)
+
+    namenodeHosts = self.getHostsWithComponent("HDFS", "NAMENODE", services, hosts)
+
+    # 25 * # of cores on NameNode
+    nameNodeCores = 4
+    if namenodeHosts is not None and len(namenodeHosts):
+      nameNodeCores = int(namenodeHosts[0]['Hosts']['cpu_count'])
+    putHdfsSiteProperty("dfs.namenode.handler.count", 25*nameNodeCores)
+
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     if ('ranger-hdfs-plugin-properties' in services['configurations']) and ('ranger-hdfs-plugin-enabled' in services['configurations']['ranger-hdfs-plugin-properties']['properties']):
       rangerPluginEnabled = services['configurations']['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled']
       if ("RANGER" in servicesList) and (rangerPluginEnabled.lower() == 'Yes'.lower()):
-        putHDFSProperty("dfs.permissions.enabled",'true')
+        putHdfsSiteProperty("dfs.permissions.enabled",'true')
+
+    putHdfsSiteProperty("dfs.namenode.safemode.threshold-pct", "0.99f" if len(namenodeHosts) > 1 else "1.0f")
+
+    putHdfsEnvProperty = self.putProperty(configurations, "hadoop-env", services)
+    putHdfsEnvProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024))
+    putHdfsEnvProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128))
+    putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
+
+    # Property Attributes
+    putHdfsEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hadoop-env")
+    if (namenodeHosts is not None and len(namenodeHosts) > 0):
+      if len(namenodeHosts) > 1:
+        namenode_heapsize = min(int(namenodeHosts[0]["Hosts"]["total_mem"]), int(namenodeHosts[1]["Hosts"]["total_mem"])) / 1024
+      else:
+        namenode_heapsize = int(namenodeHosts[0]["Hosts"]["total_mem"] / 1024) # total_mem in kb
+
+      putHdfsEnvPropertyAttribute('namenode_heapsize', 'max', namenode_heapsize)
+
+    datanodeHosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts)
+    if (datanodeHosts is not None and len(datanodeHosts)>0):
+      min_datanode_ram_kb = 1073741824 # 1 TB
+      for datanode in datanodeHosts:
+        ram_kb = datanode['Hosts']['total_mem']
+        min_datanode_ram_kb = min(min_datanode_ram_kb, ram_kb)
+      putHdfsEnvPropertyAttribute('dtnode_heapsize', 'max', int(min_datanode_ram_kb/1024))
+
+    putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site")
+    putHdfsSitePropertyAttribute('dfs.datanode.failed.volumes.tolerated', 'max', dataDirsCount)
 
   def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP22StackAdvisor, self).recommendHiveConfigurations(configurations, clusterData, services, hosts)

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -135,7 +135,6 @@ class TestHDP22StackAdvisor(TestCase):
 
 
   def test_validateHDFSConfigurations(self):
-    self.maxDiff = None
     recommendedDefaults = None
 
     unsecure_cluster_core_site = {
@@ -1152,7 +1151,14 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(configurations, expected)
 
   def test_recommendHDFSConfigurations(self):
-    configurations = {}
+    configurations = {
+      'ranger-hdfs-plugin-properties':{
+        "properties": {"ranger-hdfs-plugin-enabled":"Yes"}
+      },
+      'hdfs-site': {
+        "properties": {"dfs.datanode.data.dir": "/path/1,/path/2,/path/3,/path/4"}
+      }
+    }
     clusterData = {
       "totalAvailableRam": 2048,
       "hBaseInstalled": 111
@@ -1163,27 +1169,178 @@ class TestHDP22StackAdvisor(TestCase):
           'namenode_heapsize': '1024',
           'namenode_opt_newsize' : '256',
           'namenode_opt_maxnewsize' : '256'
+        },
+        'property_attributes': {
+          'dtnode_heapsize': {'max': '2048'},
+          'namenode_heapsize': {'max': '1024'}
         }
       },
       'hdfs-site': {
         'properties': {
-          'dfs.datanode.max.transfer.threads': '16384'
+          'dfs.datanode.max.transfer.threads': '16384',
+          'dfs.namenode.safemode.threshold-pct': '1.0f',
+          'dfs.datanode.failed.volumes.tolerated': '1',
+          'dfs.namenode.handler.count': '25',
+          'dfs.datanode.data.dir': '/path/1,/path/2,/path/3,/path/4'
         },
+        'property_attributes': {
+          'dfs.datanode.failed.volumes.tolerated': {'max': '4'}
+        }
+      },
+      'ranger-hdfs-plugin-properties': {
+        'properties': {
+          'ranger-hdfs-plugin-enabled': 'Yes'
+        }
       }
     }
     services = {"services":
                     [{"StackServices":
                           {"service_name" : "HDFS",
-                           "service_version" : "2.6.0.2.2",
-                           }
-                     }],
-                "configurations": {
-                    'ranger-hdfs-plugin-properties':{
-                        "properties": {"ranger-hdfs-plugin-enabled":"Yes"}
-                    }
-                }
+                           "service_version" : "2.6.0.2.2"
+                           },
+                      "components":[
+                        {
+                          "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/DATANODE",
+                          "StackServiceComponents":{
+                            "advertise_version":"true",
+                            "cardinality":"1+",
+                            "component_category":"SLAVE",
+                            "component_name":"DATANODE",
+                            "custom_commands":[
+
+                            ],
+                            "display_name":"DataNode",
+                            "is_client":"false",
+                            "is_master":"false",
+                            "service_name":"HDFS",
+                            "stack_name":"HDP",
+                            "stack_version":"2.2",
+                            "hostnames":[
+                              "host1"
+                            ]
+                          },
+                          "dependencies":[
+
+                          ]
+                        },
+                        {
+                          "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/JOURNALNODE",
+                          "StackServiceComponents":{
+                            "advertise_version":"true",
+                            "cardinality":"0+",
+                            "component_category":"SLAVE",
+                            "component_name":"JOURNALNODE",
+                            "custom_commands":[
+
+                            ],
+                            "display_name":"JournalNode",
+                            "is_client":"false",
+                            "is_master":"false",
+                            "service_name":"HDFS",
+                            "stack_name":"HDP",
+                            "stack_version":"2.2",
+                            "hostnames":[
+                              "host1"
+                            ]
+                          },
+                          "dependencies":[
+                            {
+                              "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/JOURNALNODE/dependencies/HDFS_CLIENT",
+                              "Dependencies":{
+                                "component_name":"HDFS_CLIENT",
+                                "dependent_component_name":"JOURNALNODE",
+                                "dependent_service_name":"HDFS",
+                                "stack_name":"HDP",
+                                "stack_version":"2.2"
+                              }
+                            }
+                          ]
+                        },
+                        {
+                          "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/NAMENODE",
+                          "StackServiceComponents":{
+                            "advertise_version":"true",
+                            "cardinality":"1-2",
+                            "component_category":"MASTER",
+                            "component_name":"NAMENODE",
+                            "custom_commands":[
+                              "DECOMMISSION",
+                              "REBALANCEHDFS"
+                            ],
+                            "display_name":"NameNode",
+                            "is_client":"false",
+                            "is_master":"true",
+                            "service_name":"HDFS",
+                            "stack_name":"HDP",
+                            "stack_version":"2.2",
+                            "hostnames":[
+                              "host2"
+                            ]
+                          },
+                          "dependencies":[
+
+                          ]
+                        },
+                        {
+                          "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/SECONDARY_NAMENODE",
+                          "StackServiceComponents":{
+                            "advertise_version":"true",
+                            "cardinality":"1",
+                            "component_category":"MASTER",
+                            "component_name":"SECONDARY_NAMENODE",
+                            "custom_commands":[
+
+                            ],
+                            "display_name":"SNameNode",
+                            "is_client":"false",
+                            "is_master":"true",
+                            "service_name":"HDFS",
+                            "stack_name":"HDP",
+                            "stack_version":"2.2",
+                            "hostnames":[
+                              "host1"
+                            ]
+                          },
+                          "dependencies":[
+
+                          ]
+                        },
+                      ],
+                    }],
+                "configurations": configurations
                 }
-    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, '')
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/host1",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "host1",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "host1",
+            "rack_info" : "/default-rack",
+            "total_mem" : 2097152
+          }
+        },
+        {
+          "href" : "/api/v1/hosts/host2",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "host2",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "host2",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1048576
+          }
+        },
+      ]
+    }
+
+    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
   def test_validateHDFSConfigurationsEnv(self):