AMBARI-19121. Add ability for users to provide mount exclusion list before cluster deployment (Dmytro Grinenko via smohanty)

ambari-agent/conf/unix/ambari-agent.ini

@@ -39,6 +39,7 @@ alert_kinit_timeout=14400000
 system_resource_overrides=/etc/resource_overrides
 ; memory_threshold_soft_mb=400
 ; memory_threshold_hard_mb=1000
+; ignore_mount_points=/mnt/custom1,/mnt/custom2
 
 [security]
 keysdir=/var/lib/ambari-agent/keys

ambari-agent/src/main/python/ambari_agent/Hardware.py

@@ -41,6 +41,7 @@ class Hardware:
   CHECK_REMOTE_MOUNTS_TIMEOUT_DEFAULT = '10'
   IGNORE_ROOT_MOUNTS = ["proc", "dev", "sys"]
   IGNORE_DEVICES = ["proc", "tmpfs", "cgroup", "mqueue", "shm"]
+  LINUX_PATH_SEP = "/"
 
   def __init__(self, config):
     self.hardware = {
@@ -87,6 +88,37 @@ class Hardware:
 
     return True
 
+  @classmethod
+  def _is_mount_blacklisted(cls, blacklist, mount_point):
+    """
+    Verify if particular mount point is in the black list.
+
+    :return True if mount_point or a part of mount point is in the blacklist, otherwise return False
+
+     Example:
+       Mounts: /, /mnt/my_mount, /mnt/my_mount/sub_mount
+       Blacklist: /mnt/my_mount
+       Result: /
+
+    :type blacklist list
+    :type mount_point str
+    :rtype bool
+    """
+
+    if not blacklist or not mount_point:
+      return False
+
+    mount_point_elements = mount_point.split(cls.LINUX_PATH_SEP)
+
+    for el in blacklist:
+      el_list = el.split(cls.LINUX_PATH_SEP)
+      # making patch elements comparision
+      if el_list == mount_point_elements[:len(el_list)]:
+        return True
+
+    return False
+
+
   @classmethod
   @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
   def osdisks(cls, config=None):
@@ -95,6 +127,11 @@ class Hardware:
     and any mounts with spaces. """
     timeout = cls._get_mount_check_timeout(config)
     command = ["timeout", timeout, "df", "-kPT"]
+    blacklisted_mount_points = []
+
+    if config:
+      ignore_mount_value = config.get("agent", "ignore_mount_points", default="")
+      blacklisted_mount_points = [item.strip() for item in ignore_mount_value.split(",")]
 
     if not cls._check_remote_mounts(config):
       command.append("-l")
@@ -103,6 +140,7 @@ class Hardware:
     dfdata = df.communicate()[0]
     mounts = [cls._parse_df_line(line) for line in dfdata.splitlines() if line]
     result_mounts = []
+    ignored_mounts = []
 
     for mount in mounts:
       if not mount:
@@ -113,13 +151,21 @@ class Hardware:
        - mounted device is not in the ignored list
        - is accessible to user under which current process running
        - it is not file-mount (docker environment)
+       - mount path or a part of mount path is not in the blacklist
       """
-      if mount["device"] not in cls.IGNORE_DEVICES and \
+      if mount["device"] not in cls.IGNORE_DEVICES and\
          mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\
-         cls._chk_writable_mount(mount['mountpoint']) and \
-         not path_isfile(mount["mountpoint"]):
+         cls._chk_writable_mount(mount['mountpoint']) and\
+         not path_isfile(mount["mountpoint"]) and\
+         not cls._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]):
 
         result_mounts.append(mount)
+      else:
+        ignored_mounts.append(mount)
+
+      if len(ignored_mounts) > 0:
+        ignore_list = [el["mountpoint"] for el in ignored_mounts]
+        logger.info("Some mount points was ignored: {0}".format(', '.join(ignore_list)))
 
     return result_mounts
 

ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py

@@ -68,7 +68,7 @@ class TestCustomServiceOrchestrator(TestCase):
   def test_add_reg_listener_to_controller(self, FileCache_mock):
     FileCache_mock.return_value = None
     dummy_controller = MagicMock()
-    config = AmbariConfig().getConfig()
+    config = AmbariConfig()
     tempdir = tempfile.gettempdir()
     config.set('agent', 'prefix', tempdir)
     CustomServiceOrchestrator(config, dummy_controller)
@@ -204,7 +204,7 @@ class TestCustomServiceOrchestrator(TestCase):
   def test_resolve_script_path(self, FileCache_mock, exists_mock):
     FileCache_mock.return_value = None
     dummy_controller = MagicMock()
-    config = AmbariConfig().getConfig()
+    config = AmbariConfig()
     orchestrator = CustomServiceOrchestrator(config, dummy_controller)
     # Testing existing path
     exists_mock.return_value = True

ambari-agent/src/test/python/ambari_agent/TestHardware.py

@@ -25,6 +25,7 @@ from mock.mock import patch, MagicMock, Mock
 import unittest
 import platform
 import socket
+import os
 from only_for_platform import not_for_platform, PLATFORM_WINDOWS
 from ambari_agent import hostname
 from ambari_agent.Hardware import Hardware
@@ -373,6 +374,75 @@ SwapFree:        1598676 kB
     self.assertEquals(2, json_mock.call_count)
     self.assertEquals('value', result['key'])
 
+  @patch.object(Hardware, "_chk_writable_mount")
+  @patch("ambari_agent.Hardware.path_isfile")
+  def test_osdisks_blacklist(self, isfile_mock, chk_writable_mount_mock):
+    df_output = \
+      """Filesystem                                                                                        Type  1024-blocks     Used Available Capacity Mounted on
+      /dev/mapper/docker-253:0-4980899-d45c264d37ab18c8ed14f890f4d59ac2b81e1c52919eb36a79419787209515f3 xfs      31447040  1282384  30164656       5% /
+      tmpfs                                                                                             tmpfs    32938336        4  32938332       1% /dev
+      tmpfs                                                                                             tmpfs    32938336        0  32938336       0% /sys/fs/cgroup
+      /dev/mapper/fedora-root                                                                           ext4    224161316 12849696 199901804       7% /etc/resolv.conf
+      /dev/mapper/fedora-root                                                                           ext4    224161316 12849696 199901804       7% /etc/hostname
+      /dev/mapper/fedora-root                                                                           ext4    224161316 12849696 199901804       7% /etc/hosts
+      shm                                                                                               tmpfs       65536        0     65536       0% /dev/shm
+      /dev/mapper/fedora-root                                                                           ext4    224161316 12849696 199901804       7% /run/secrets
+      /dev/mapper/fedora-root                                                                           ext4    224161316 12849696 199901804       7% /mnt/blacklisted_mount
+      /dev/mapper/fedora-root                                                                           ext4    224161316 12849696 199901804       7% /mnt/blacklisted_mount/sub-dir
+      """
+
+    def isfile_side_effect(path):
+      assume_files = ["/etc/resolv.conf", "/etc/hostname", "/etc/hosts"]
+      return path in assume_files
+
+    def chk_writable_mount_side_effect(path):
+      assume_read_only = ["/run/secrets"]
+      return path not in assume_read_only
+
+    isfile_mock.side_effect = isfile_side_effect
+    chk_writable_mount_mock.side_effect = chk_writable_mount_side_effect
+
+    config_dict = {
+      "agent": {
+        "ignore_mount_points": "/mnt/blacklisted_mount"
+      }
+    }
+
+    with patch("subprocess.Popen") as open_mock:
+      proc_mock = Mock()
+      attr = {
+        'communicate.return_value': [
+          df_output
+        ]
+      }
+      proc_mock.configure_mock(**attr)
+      open_mock.return_value = proc_mock
+
+      def conf_get(section, key, default=""):
+        if section in config_dict and key in config_dict[section]:
+          return config_dict[section][key]
+
+        return default
+
+      def has_option(section, key):
+        return section in config_dict and key in config_dict[section]
+
+      conf = Mock()
+      attr = {
+        'get.side_effect': conf_get,
+        'has_option.side_effect': has_option
+      }
+      conf.configure_mock(**attr)
+
+      result = Hardware.osdisks(conf)
+
+    self.assertEquals(1, len(result))
+
+    expected_mounts_left = ["/"]
+    mounts_left = [item["mountpoint"] for item in result]
+
+    self.assertEquals(expected_mounts_left, mounts_left)
+
 
 if __name__ == "__main__":
   unittest.main()