AMBARI-17550. While changing NN, DN directories from UI, proper warning should be present for invalid values (aonishuk)

ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py

@@ -162,3 +162,7 @@ class TestDatanodeHelper(TestCase):
       print args[0]
 
     self.assertEquals(0, log_error.call_count)
+
+  def test_get_mounts_with_multiple_data_dirs(self):
+    self.assertEquals([], mounted_dirs_helper.get_mounts_with_multiple_data_dirs(["/", "/hodoop", "/tmp"], "/hadoop/data,/tmp"))
+    self.assertEquals([("/", ["/hadoop/data", "/tmp"])], mounted_dirs_helper.get_mounts_with_multiple_data_dirs(["/"], "/hadoop/data,/tmp"))

ambari-agent/src/test/python/resource_management/TestFileSystem.py

@@ -165,3 +165,22 @@ class TestFileSystem(TestCase):
 
     mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1")
     self.assertEqual(mount_point, "/hadoop/hdfs/data1")
+
+  @patch.object(Logger, "info")
+  @patch.object(Logger, "error")
+  @patch('resource_management.core.providers.mount.get_mounted')
+  def test_get_mount_point_for_dir_with_mounsts(self, mounted_mock, log_error, log_info):
+      """
+      Testing get_mount_point_for_dir when mounsts are provided.
+      """
+      mounted_mock.return_value = self._get_mount(self.MOUNT_TYPE.SAME_PREFIX_MOUNTS)
+
+      # refresh cached mounts
+      file_system.get_and_cache_mount_points(True)
+
+      mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1")
+      self.assertEqual(mount_point, "/hadoop/hdfs/data1")
+
+      # Should use provided mounts, not fetch via get_and_cache_mount_points
+      mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1", ["/", "/hadoop"])
+      self.assertEqual(mount_point, "/hadoop")

ambari-common/src/main/python/resource_management/libraries/functions/file_system.py

@@ -47,7 +47,7 @@ def get_and_cache_mount_points(refresh=False):
     return mounts
 
 
-def get_mount_point_for_dir(dir):
+def get_mount_point_for_dir(dir, mount_points = None):
   """
   :param dir: Directory to check, even if it doesn't exist.
   :return: Returns the closest mount point as a string for the directory. if the "dir" variable is None, will return None.
@@ -57,7 +57,7 @@ def get_mount_point_for_dir(dir):
   if dir:
     dir = dir.strip()
 
-    cached_mounts = get_and_cache_mount_points()
+    cached_mounts = [m['mount_point'] for m in get_and_cache_mount_points()] if mount_points is None else mount_points
 
     # If the path is "/hadoop/hdfs/data", then possible matches for mounts could be
     # "/", "/hadoop/hdfs", and "/hadoop/hdfs/data".
@@ -65,11 +65,11 @@ def get_mount_point_for_dir(dir):
     for m in cached_mounts:
       # Ensure that the mount path and the dir path ends with "/"
       # The mount point "/hadoop" should not match the path "/hadoop1"
-      if os.path.join(dir, "").startswith(os.path.join(m['mount_point'], "")):
+      if os.path.join(dir, "").startswith(os.path.join(m, "")):
         if best_mount_found is None:
-          best_mount_found = m["mount_point"]
-        elif os.path.join(best_mount_found, "").count(os.path.sep) < os.path.join(m["mount_point"], "").count(os.path.sep):
-          best_mount_found = m["mount_point"]
+          best_mount_found = m
+        elif os.path.join(best_mount_found, "").count(os.path.sep) < os.path.join(m, "").count(os.path.sep):
+          best_mount_found = m
 
   Logger.info("Mount point for directory %s is %s" % (str(dir), str(best_mount_found)))
   return best_mount_found

ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py

@@ -19,9 +19,10 @@ limitations under the License.
 Ambari Agent
 
 """
-__all__ = ["handle_mounted_dirs", ]
+__all__ = ["handle_mounted_dirs", "get_mounts_with_multiple_data_dirs"]
 import os
 import re
+from collections import defaultdict
 
 from resource_management.libraries.functions.file_system import get_mount_point_for_dir, get_and_cache_mount_points
 from resource_management.core.logger import Logger
@@ -198,3 +199,19 @@ def handle_mounted_dirs(func, dirs_string, history_filename, update_cache=True):
 
   return dir_to_mount
 
+def get_mounts_with_multiple_data_dirs(mount_points, dirs):
+  """
+  Returns a list with (mount, dir_list) for mounts with multiple dirs.
+  Currently is used in the stack_advisor.
+  """
+  mount_dirs = defaultdict(list)
+  for dir in [raw_dir.strip() for raw_dir in dirs.split(",")]:
+    mount_point = get_mount_point_for_dir(dir, mount_points)
+    mount_dirs[mount_point].append(dir)
+
+  partition_mounts_list = []
+  for mount_point, dir_list in mount_dirs.iteritems():
+    if len(dir_list) > 1:
+      partition_mounts_list.append((mount_point, dir_list))
+
+  return partition_mounts_list

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -25,6 +25,7 @@ import socket
 from math import ceil, floor
 
 from resource_management.core.logger import Logger
+from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs
 
 from stack_advisor import DefaultStackAdvisor
 
@@ -1438,7 +1439,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     return self.toConfigurationValidationProblems(validationItems, "hbase-env")
 
   def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
-    validationItems = [{"config-name": 'dfs.datanode.du.reserved', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'dfs.datanode.du.reserved')}]
+    clusterEnv = getSiteProperties(configurations, "cluster-env")
+    validationItems = [{"config-name": 'dfs.datanode.du.reserved', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'dfs.datanode.du.reserved')},
+                       {"config-name": 'dfs.datanode.data.dir', "item": self.validatorOneDataDirPerPartition(properties, 'dfs.datanode.data.dir', services, hosts, clusterEnv)}]
     return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
 
   def validateHDFSConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
@@ -1447,6 +1450,45 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
                         {"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
     return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
 
+  def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):
+    if not propertyName in properties:
+      return self.getErrorItem("Value should be set")
+    dirs = properties[propertyName]
+
+    if not (clusterEnv and "one_dir_per_partition" in clusterEnv and clusterEnv["one_dir_per_partition"].lower() == "true"):
+      return None
+
+    dataNodeHosts = self.getDataNodeHosts(services, hosts)
+
+    warnings = set()
+    for host in dataNodeHosts:
+      hostName = host["Hosts"]["host_name"]
+
+      mountPoints = []
+      for diskInfo in host["Hosts"]["disk_info"]:
+        mountPoints.append(diskInfo["mountpoint"])
+
+      if get_mounts_with_multiple_data_dirs(mountPoints, dirs):
+        # A detailed message can be too long on large clusters:
+        # warnings.append("Host: " + hostName + "; Mount: " + mountPoint + "; Data directories: " + ", ".join(dirList))
+        warnings.add(hostName)
+        break;
+
+    if len(warnings) > 0:
+      return self.getWarnItem("cluster-env/one_dir_per_partition is enabled but there are multiple data directories on the same mount. Affected hosts: {0}".format(", ".join(sorted(warnings))))
+
+    return None
+
+  """
+  Returns the list of Data Node hosts.
+  """
+  def getDataNodeHosts(self, services, hosts):
+    if len(hosts["items"]) > 0:
+      dataNodeHosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts)
+      if dataNodeHosts is not None:
+        return dataNodeHosts
+    return []
+
   def getMastersWithMultipleInstances(self):
     return ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
 

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -1261,23 +1261,26 @@ class TestHDP206StackAdvisor(TestCase):
 
   def test_validateHDFSConfigurations(self):
     configurations = {}
-    services = ''
+    services = {'configurations': {}}
     hosts = ''
     #Default configuration
     recommendedDefaults = {'dfs.datanode.du.reserved': '1024'}
-    properties = {'dfs.datanode.du.reserved': '1024'}
+    properties = {'dfs.datanode.du.reserved': '1024',
+                  'dfs.datanode.data.dir': '/hadoop/hdfs/data'}
     res = self.stackAdvisor.validateHDFSConfigurations(properties, 
                     recommendedDefaults, configurations, services, hosts)
     self.assertFalse(res)
     #Value is less then expected
     recommendedDefaults = {'dfs.datanode.du.reserved': '1024'}
-    properties = {'dfs.datanode.du.reserved': '512'}
+    properties = {'dfs.datanode.du.reserved': '512',
+                  'dfs.datanode.data.dir': '/hadoop/hdfs/data'}
     res = self.stackAdvisor.validateHDFSConfigurations(properties, 
                     recommendedDefaults, configurations, services, hosts)
     self.assertTrue(res)
     #Value is begger then expected
     recommendedDefaults = {'dfs.datanode.du.reserved': '1024'}
-    properties = {'dfs.datanode.du.reserved': '2048'}
+    properties = {'dfs.datanode.du.reserved': '2048',
+                  'dfs.datanode.data.dir': '/hadoop/hdfs/data'}
     res = self.stackAdvisor.validateHDFSConfigurations(properties, 
                     recommendedDefaults, configurations, services, hosts)
     self.assertFalse(res)
@@ -1314,6 +1317,79 @@ class TestHDP206StackAdvisor(TestCase):
     res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
     self.assertEquals(res, res_expected)
 
+  def test_validateOneDataDirPerPartition(self):
+    recommendedDefaults = {
+      'dfs.datanode.du.reserved': '1024'
+    }
+
+    properties = {
+                  'dfs.datanode.du.reserved': '1024',
+                  'dfs.datanode.data.dir': '/hadoop/hdfs/data,/hadoop/hdfs/data2',
+                 }
+    configurations = {
+      'hdfs-site': {
+          'properties': properties,
+      },
+      'cluster-env': {
+          'properties': {'one_dir_per_partition': 'true'}
+      },
+    }
+    services = {"services":
+                 [{"StackServices":
+                   {"service_name" : "HDFS",
+                     "service_version" : "2.6.0.2.2",
+                   },
+                   "components": [
+                     {
+                       "StackServiceComponents": {
+                         "component_name": "DATANODE",
+                         "hostnames": ["host1", "host2"]
+                       }
+                     }
+                   ]
+                 }],
+                "configurations": configurations,
+               }
+    host1 = {
+      "Hosts" : {
+        "host_name" : "host1",
+        "disk_info": [
+          {"mountpoint" : "/hadoop/hdfs/data"},
+          {"mountpoint" : "/hadoop/hdfs/data2"}
+        ]
+      }
+    }
+    host2 = {
+      "Hosts" : {
+        "host_name" : "host2",
+        "disk_info": [
+          {"mountpoint": "/"},
+          {"mountpoint" : "/hadoop"}
+        ]
+      }
+    }
+    hosts = {
+        "items" : [
+            host1,
+            host2
+        ]
+    }
+
+    # Multiple data directories on the same mount. A warning is expected.
+    expected = [{'config-name': 'dfs.datanode.data.dir',
+               'config-type': 'hdfs-site',
+               'level': 'WARN',
+               'message': "cluster-env/one_dir_per_partition is enabled but there are multiple data directories on the same mount. Affected hosts: host2",
+               'type': 'configuration'}]
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEquals(validation_problems, expected)
+
+    # One data directory.
+    properties['dfs.datanode.data.dir'] = '/hadoop/hdfs/data'
+    expected = []
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEquals(validation_problems, expected)
+
   def test_validateAmsHbaseSiteConfigurations(self):
     configurations = {
       "hdfs-site": {

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -358,6 +358,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Unsecured cluster, secure ports
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
     }
@@ -377,7 +378,8 @@ class TestHDP22StackAdvisor(TestCase):
                           {"service_name" : "HDFS",
                            "service_version" : "2.6.0.2.2",
                            }
-                     }]
+                     }],
+                "configurations": {}
                 }
     expected = []  # No warnings
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None)
@@ -386,6 +388,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Unsecured cluster, unsecure ports
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.datanode.address': '0.0.0.0:55555',
                     'dfs.datanode.http.address': '0.0.0.0:55555',
                     }
@@ -407,8 +410,9 @@ class TestHDP22StackAdvisor(TestCase):
                 [{"StackServices":
                       {"service_name" : "HDFS",
                        "service_version" : "2.6.0.2.2",
-                       }
-                 }]
+                       },
+                 }],
+                "configurations": {}
             }
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None)
     self.assertEquals(validation_problems, expected)
@@ -416,6 +420,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, invalid dfs.http.policy value
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'WRONG_VALUE',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -442,8 +447,9 @@ class TestHDP22StackAdvisor(TestCase):
             [{"StackServices":
                   {"service_name" : "HDFS",
                    "service_version" : "2.6.0.2.2",
-                   }
-             }]
+                   },
+             }],
+             "configurations": {}
         }
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None)
     self.assertEquals(validation_problems, expected)
@@ -451,6 +457,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address not defined
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     }
@@ -473,7 +480,8 @@ class TestHDP22StackAdvisor(TestCase):
                   {"service_name" : "HDFS",
                    "service_version" : "2.6.0.2.2",
                    }
-             }]
+             }],
+            "configurations": {}
         }
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None)
     self.assertEquals(validation_problems, expected)
@@ -481,6 +489,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and secure
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.https.address': '0.0.0.0:1022',
@@ -504,7 +513,8 @@ class TestHDP22StackAdvisor(TestCase):
                   {"service_name" : "HDFS",
                    "service_version" : "2.6.0.2.2",
                    }
-             }]
+             }],
+             "configurations": {}
         }
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None)
     self.assertEquals(validation_problems, expected)
@@ -512,6 +522,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and non secure
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.https.address': '0.0.0.0:50475',
@@ -535,7 +546,8 @@ class TestHDP22StackAdvisor(TestCase):
                   {"service_name" : "HDFS",
                    "service_version" : "2.6.0.2.2",
                    }
-             }]
+             }],
+             "configurations": {}
         }
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, None)
     self.assertEquals(validation_problems, expected)
@@ -543,6 +555,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https property not defined
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                  }
@@ -595,6 +608,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https defined and secure
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                     'dfs.datanode.https.address': '0.0.0.0:1022',
@@ -645,6 +659,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, valid non-root configuration
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                     'dfs.datanode.https.address': '0.0.0.0:50475',
@@ -670,6 +685,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, insecure port
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTP_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:50475',
@@ -712,6 +728,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, valid configuration
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTP_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -736,6 +753,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, absent dfs.http.policy (typical situation)
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
                     }
@@ -759,6 +777,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, misusage of dfs.data.transfer.protection warning
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTP_ONLY',
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',
@@ -789,6 +808,7 @@ class TestHDP22StackAdvisor(TestCase):
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, wrong dfs.data.transfer.protection value
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.http.policy': 'HTTPS_ONLY',
                     'dfs.datanode.address': '0.0.0.0:50010',
                     'dfs.datanode.https.address': '0.0.0.0:50475',
@@ -819,6 +839,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     properties = {  # hdfs-site
                     'dfs.datanode.du.reserved': '1024',
+                    'dfs.datanode.data.dir': '/hadoop/hdfs/data',
                     'dfs.encrypt.data.transfer': 'true',  # Wire encryption
                     'dfs.datanode.address': '0.0.0.0:1019',
                     'dfs.datanode.http.address': '0.0.0.0:1022',

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -1512,7 +1512,7 @@ class TestHDP23StackAdvisor(TestCase):
     self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE')
 
   def test_validateHDFSConfigurations(self):
-    properties = {}
+    properties = {'dfs.datanode.data.dir': '/hadoop/hdfs/data'}
     recommendedDefaults = {}
     configurations = {
       "core-site": {"properties": {}},