AMBARI-14524 HDFS Recommendation: dfs.datanode.du.reserved should be set to 10%-15% of volume size (dsen)

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml

@@ -278,6 +278,12 @@
       <type>int</type>
       <unit>bytes</unit>
     </value-attributes>
+    <depends-on>
+      <property>
+        <type>hdfs-site</type>
+        <name>dfs.datanode.data.dir</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -239,6 +239,27 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       if len(namenodes.split(',')) > 1:
         putHDFSSitePropertyAttributes("dfs.namenode.rpc-address", "delete", "true")
 
+    #Initialize default 'dfs.datanode.data.dir' if needed
+    if (not hdfsSiteProperties) or ('dfs.datanode.data.dir' not in hdfsSiteProperties):
+      putHDFSSiteProperty('dfs.datanode.data.dir', '/hadoop/hdfs/data')
+    #dfs.datanode.du.reserved should be set to 10-15% of volume size
+    mountPoints = []
+    mountPointDiskAvailableSpace = [] #kBytes
+    for host in hosts["items"]:
+      for diskInfo in host["Hosts"]["disk_info"]:
+        mountPoints.append(diskInfo["mountpoint"])
+        mountPointDiskAvailableSpace.append(long(diskInfo["size"]))
+    maxFreeVolumeSize = 0l #kBytes
+    dataDirs = hdfsSiteProperties['dfs.datanode.data.dir'].split(",")
+    for dataDir in dataDirs:
+      mp = getMountPointForDir(dataDir, mountPoints)
+      for i in range(len(mountPoints)):
+        if mp == mountPoints[i]:
+          if mountPointDiskAvailableSpace[i] > maxFreeVolumeSize:
+            maxFreeVolumeSize = mountPointDiskAvailableSpace[i]
+      
+    putHDFSSiteProperty('dfs.datanode.du.reserved', maxFreeVolumeSize * 1024 / 8) #Bytes
+    
     # recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site
     self.recommendHadoopProxyUsers(configurations, services, hosts)
 
@@ -787,7 +808,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
   def getServiceConfigurationValidators(self):
     return {
-      "HDFS": {"hadoop-env": self.validateHDFSConfigurationsEnv},
+      "HDFS": { "hdfs-site": self.validateHDFSConfigurations,
+                "hadoop-env": self.validateHDFSConfigurationsEnv},
       "MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
       "YARN": {"yarn-site": self.validateYARNConfigurations},
       "HBASE": {"hbase-env": self.validateHbaseEnvConfigurations},
@@ -1291,6 +1313,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
                         {"config-name": "hbase_user", "item": self.validatorEqualsPropertyItem(properties, "hbase_user", hbase_site, "hbase.superuser")} ]
     return self.toConfigurationValidationProblems(validationItems, "hbase-env")
 
+  def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+    validationItems = [{"config-name": 'dfs.datanode.du.reserved', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'dfs.datanode.du.reserved')}]
+    return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
+
   def validateHDFSConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'namenode_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_heapsize')},
                         {"config-name": 'namenode_opt_newsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_newsize')},

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -1042,6 +1042,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
 
 
   def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+    parentValidationProblems = super(HDP22StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
     # We can not access property hadoop.security.authentication from the
     # other config (core-site). That's why we are using another heuristics here
     hdfs_site = properties
@@ -1179,7 +1180,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                                   "item": self.getWarnItem(
                                     "Invalid property value: {0}. Valid values are {1}.".format(
                                       data_transfer_protection_value, VALID_TRANSFER_PROTECTION_VALUES))})
-    return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
+    validationProblems = self.toConfigurationValidationProblems(validationItems, "hdfs-site")
+    validationProblems.extend(parentValidationProblems)
+    return validationProblems
 
   def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
     hive_server2 = properties

ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py

@@ -603,7 +603,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     return parentValidators
 
   def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
-    super(HDP23StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    parentValidationProblems = super(HDP23StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
 
     # We can not access property hadoop.security.authentication from the
     # other config (core-site). That's why we are using another heuristics here
@@ -618,7 +618,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         validationItems.append({"config-name": 'dfs.namenode.inode.attributes.provider.class',
                                     "item": self.getWarnItem(
                                       "dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")})
-    return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
+    validationProblems = self.toConfigurationValidationProblems(validationItems, "hdfs-site")
+    validationProblems.extend(parentValidationProblems)
+    return validationProblems
 
 
   def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -961,7 +961,11 @@ class TestHDP206StackAdvisor(TestCase):
             "ph_cpu_count": 1,
             "public_host_name": "c6401.ambari.apache.org",
             "rack_info": "/default-rack",
-            "total_mem": 2097152
+            "total_mem": 2097152,
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
           }
         }]}
 
@@ -1041,12 +1045,14 @@ class TestHDP206StackAdvisor(TestCase):
                 'falcon-env':
                   {'properties':
                      {'falcon_user': 'falcon'}},
+                'hdfs-site':
+                  {'properties': 
+                     {'dfs.datanode.data.dir': '/hadoop/hdfs/data',
+                      'dfs.datanode.du.reserved': '1024'}},
                 'hive-env':
                   {'properties':
                      {'hive_user': 'hive',
                       'webhcat_user': 'webhcat'}},
-                'hdfs-site':
-                  {'properties': {}},
                 'hadoop-env':
                   {'properties':
                      {'hdfs_user': 'hdfs',
@@ -1094,7 +1100,9 @@ class TestHDP206StackAdvisor(TestCase):
                      {'hive_user': 'hive',
                       'webhcat_user': 'webhcat'}},
                 'hdfs-site':
-                  {'properties': {}},
+                  {'properties': 
+                     {'dfs.datanode.data.dir': '/hadoop/hdfs/data',
+                      'dfs.datanode.du.reserved': '1024'}},
                 'hadoop-env':
                   {'properties':
                      {'hdfs_user': 'hdfs1',
@@ -1110,8 +1118,11 @@ class TestHDP206StackAdvisor(TestCase):
     configurations["hdfs-site"]["properties"]['dfs.nameservices'] = "mycluster"
     configurations["hdfs-site"]["properties"]['dfs.ha.namenodes.mycluster'] = "nn1,nn2"
     services['configurations'] = configurations
+
     expected["hdfs-site"] = {
       'properties': {
+        'dfs.datanode.data.dir': '/hadoop/hdfs/data',
+        'dfs.datanode.du.reserved': '1024',
         'dfs.nameservices': 'mycluster',
         'dfs.ha.namenodes.mycluster': 'nn1,nn2'
       },
@@ -1188,6 +1199,29 @@ class TestHDP206StackAdvisor(TestCase):
     expected = "zk.host1:2183,zk.host2:2183,zk.host3:2183"
     self.assertEquals(result, expected)
 
+  def test_validateHDFSConfigurations(self):
+    configurations = {}
+    services = ''
+    hosts = ''
+    #Default configuration
+    recommendedDefaults = {'dfs.datanode.du.reserved': '1024'}
+    properties = {'dfs.datanode.du.reserved': '1024'}
+    res = self.stackAdvisor.validateHDFSConfigurations(properties, 
+                    recommendedDefaults, configurations, services, hosts)
+    self.assertFalse(res)
+    #Value is less then expected
+    recommendedDefaults = {'dfs.datanode.du.reserved': '1024'}
+    properties = {'dfs.datanode.du.reserved': '512'}
+    res = self.stackAdvisor.validateHDFSConfigurations(properties, 
+                    recommendedDefaults, configurations, services, hosts)
+    self.assertTrue(res)
+    #Value is begger then expected
+    recommendedDefaults = {'dfs.datanode.du.reserved': '1024'}
+    properties = {'dfs.datanode.du.reserved': '2048'}
+    res = self.stackAdvisor.validateHDFSConfigurations(properties, 
+                    recommendedDefaults, configurations, services, hosts)
+    self.assertFalse(res)
+
   def test_validateHDFSConfigurationsEnv(self):
     configurations = {}
 

ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py

@@ -251,7 +251,16 @@ class TestHDP21StackAdvisor(TestCase):
         }
       }
     }
-
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
+          }
+        }]}
     services = {
       "services": [
         {
@@ -282,11 +291,13 @@ class TestHDP21StackAdvisor(TestCase):
       },
       "hdfs-site": {
         "properties": {
+          'dfs.datanode.data.dir': '/hadoop/hdfs/data',
+          'dfs.datanode.du.reserved': '1024'
         }
       }
     }
 
-    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, '')
+    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
   def test_validateHDFSConfigurationsEnv(self):

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -206,7 +206,9 @@ class TestHDP22StackAdvisor(TestCase):
 
 
   def test_validateHDFSConfigurations(self):
-    recommendedDefaults = None
+    recommendedDefaults = {
+      'dfs.datanode.du.reserved': '1024'
+    }
 
     unsecure_cluster_core_site = {
       'hadoop.security.authentication': 'simple',
@@ -220,6 +222,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Unsecured cluster, secure ports
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
     }
@@ -247,6 +250,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Unsecured cluster, unsecure ports
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.datanode.address': '0.0.0.0:55555',
                     'dfs.datanode.http.address': '0.0.0.0:55555',
                     }
@@ -276,6 +280,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, invalid dfs.http.policy value
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'WRONG_VALUE',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -310,6 +315,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address not defined
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     }
@@ -339,6 +345,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and secure
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.https.address': '0.0.0.0:1022',
@@ -369,6 +376,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and non secure
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.https.address': '0.0.0.0:50475',
@@ -399,6 +407,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https property not defined
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                  }
@@ -450,6 +459,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https defined and secure
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                     'dfs.datanode.https.address': '0.0.0.0:1022',
@@ -499,6 +509,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, valid non-root configuration
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                     'dfs.datanode.https.address': '0.0.0.0:50475',
@@ -523,6 +534,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, insecure port
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTP_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:50475',
@@ -564,6 +576,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, valid configuration
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTP_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -587,6 +600,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, absent dfs.http.policy (typical situation)
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
                     }
@@ -609,6 +623,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, misusage of dfs.data.transfer.protection warning
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTP_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -638,6 +653,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, wrong dfs.data.transfer.protection value
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                     'dfs.datanode.https.address': '0.0.0.0:50475',
@@ -667,6 +683,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Hadoop wire encryption enabled
 
     properties = {  # hdfs-site
+                    'dfs.datanode.du.reserved': '1024',
                     'dfs.encrypt.data.transfer': 'true',  # Wire encryption
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -2753,6 +2770,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       'hdfs-site': {
         'properties': {
+          'dfs.datanode.du.reserved': '1024',
           'dfs.datanode.max.transfer.threads': '16384',
           'dfs.namenode.safemode.threshold-pct': '1.000',
           'dfs.datanode.failed.volumes.tolerated': '1',
@@ -2879,7 +2897,11 @@ class TestHDP22StackAdvisor(TestCase):
             "ph_cpu_count" : 1,
             "public_host_name" : "host1",
             "rack_info" : "/default-rack",
-            "total_mem" : 2097152
+            "total_mem" : 2097152,
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
           }
         },
         {
@@ -2892,7 +2914,11 @@ class TestHDP22StackAdvisor(TestCase):
             "ph_cpu_count" : 1,
             "public_host_name" : "host2",
             "rack_info" : "/default-rack",
-            "total_mem" : 10485760
+            "total_mem" : 10485760,
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
           }
         },
       ]
@@ -2916,7 +2942,11 @@ class TestHDP22StackAdvisor(TestCase):
             "ph_cpu_count" : 1,
             "public_host_name" : hostname,
             "rack_info" : "/default-rack",
-            "total_mem" : 2097152
+            "total_mem" : 2097152,
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
           }
         }
       )
@@ -2939,7 +2969,11 @@ class TestHDP22StackAdvisor(TestCase):
             "ph_cpu_count" : 1,
             "public_host_name" : hostname,
             "rack_info" : "/default-rack",
-            "total_mem" : 2097152
+            "total_mem" : 2097152,
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
           }
         }
       )
@@ -2964,7 +2998,11 @@ class TestHDP22StackAdvisor(TestCase):
             "ph_cpu_count" : 1,
             "public_host_name" : hostname,
             "rack_info" : "/default-rack",
-            "total_mem" : 2097152
+            "total_mem" : 2097152,
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
           }
         }
       )

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -284,13 +284,34 @@ class TestHDP23StackAdvisor(TestCase):
 
 
   def test_recommendHDFSConfigurations(self):
-    configurations = {}
+    configurations = {
+      "hdfs-site": {
+        "properties": {
+          "dfs.namenode.inode.attributes.provider.class": "org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer",
+        }
+      },
+      "ranger-hdfs-plugin-properties": {
+        "properties": {
+          "ranger-hdfs-plugin-enabled": "No"
+        }
+      }
+    }
     clusterData = {
       "totalAvailableRam": 2048,
       "hBaseInstalled": True,
       "hbaseRam": 112,
       "reservedRam": 128
     }
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "disk_info": [{
+              "size": '8',
+              "mountpoint": "/"
+            }]
+          }
+        }]}
     services = {
       "services":
         [
@@ -298,35 +319,26 @@ class TestHDP23StackAdvisor(TestCase):
             "StackServices": {
               "service_name" : "HDFS",
               "service_version" : "2.6.0.2.2"
-            }
+            },
+            "components": [
+            ]
           }
         ],
       "Versions": {
         "stack_version": "2.3"
       },
-      "configurations": {
-        "hdfs-site": {
-          "properties": {
-            "dfs.namenode.inode.attributes.provider.class": "org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer"
-          }
-        },
-        "ranger-hdfs-plugin-properties": {
-          "properties": {
-            "ranger-hdfs-plugin-enabled": "No"
-          }
-        }
-      }
+      "configurations": configurations
     }
 
     # Test with Ranger HDFS plugin disabled
-    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, None)
+    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations['hdfs-site']['property_attributes']['dfs.namenode.inode.attributes.provider.class'], {'delete': 'true'}, "Test with Ranger HDFS plugin is disabled")
 
     # Test with Ranger HDFS plugin is enabled
     configurations['hdfs-site']['properties'] = {}
     configurations['hdfs-site']['property_attributes'] = {}
     services['configurations']['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled'] = 'Yes'
-    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, None)
+    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations['hdfs-site']['properties']['dfs.namenode.inode.attributes.provider.class'], 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer', "Test with Ranger HDFS plugin is enabled")
 
   def test_recommendYARNConfigurations(self):