AMBARI-3047. Enhance host clean up to handle tmp files and folders.

ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py

@@ -47,10 +47,13 @@ class HostCheckReportFileHandler:
 
       if 'existingUsers' in hostInfo.keys():
         items = []
+        items2 = []
         for itemDetail in hostInfo['existingUsers']:
           items.append(itemDetail['name'])
+          items2.append(itemDetail['homeDir'])
         config.add_section('users')
         config.set('users', 'usr_list', ','.join(items))
+        config.set('users', 'usr_homedir_list', ','.join(items2))
 
       if 'alternatives' in hostInfo.keys():
         items = []

ambari-agent/src/main/python/ambari_agent/HostCleanup.py

@@ -30,6 +30,7 @@ import shlex
 import sys
 import datetime
 import AmbariConfig
+from pwd import getpwnam
 
 logger = logging.getLogger()
 configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
@@ -37,6 +38,7 @@ configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
 PACKAGE_ERASE_CMD_RHEL = "yum erase -y {0}"
 PACKAGE_ERASE_CMD_SUSE = "zypper -n -q remove {0}"
 USER_ERASE_CMD = "userdel -rf {0}"
+GROUP_ERASE_CMD = "groupdel {0}"
 PROC_KILL_CMD = "kill -9 {0}"
 ALT_DISP_CMD = "alternatives --display {0}"
 ALT_ERASE_CMD = "alternatives --remove {0} {1}"
@@ -51,6 +53,8 @@ PACKAGE_SECTION = "packages"
 PACKAGE_KEY = "pkg_list"
 USER_SECTION = "users"
 USER_KEY = "usr_list"
+USER_HOMEDIR_KEY = "usr_homedir_list"
+USER_HOMEDIR_SECTION = "usr_homedir"
 REPO_SECTION = "repositories"
 REPOS_KEY = "repo_list"
 DIR_SECTION = "directories"
@@ -59,6 +63,8 @@ PROCESS_SECTION = "processes"
 PROCESS_KEY = "proc_list"
 ALT_SECTION = "alternatives"
 ALT_KEYS = ["symlink_list", "target_list"]
+HADOOP_GROUP = "hadoop"
+FOLDER_LIST = ["/tmp"]
 
 # resources that should not be cleaned
 REPOSITORY_BLACK_LIST = ["ambari.repo"]
@@ -83,11 +89,14 @@ class HostCleanup:
     if argMap:
       packageList = argMap.get(PACKAGE_SECTION)
       userList = argMap.get(USER_SECTION)
+      homeDirList = argMap.get(USER_HOMEDIR_SECTION)
       dirList = argMap.get(DIR_SECTION)
       repoList = argMap.get(REPO_SECTION)
       procList = argMap.get(PROCESS_SECTION)
       alt_map = argMap.get(ALT_SECTION)
 
+      if userList and not USER_SECTION in SKIP_LIST:
+        userIds = self.get_user_ids(userList)
       if procList and not PROCESS_SECTION in SKIP_LIST:
         logger.info("\n" + "Killing pid's: " + str(procList) + "\n")
         self.do_kill_processes(procList)
@@ -97,6 +106,8 @@ class HostCleanup:
       if userList and not USER_SECTION in SKIP_LIST:
         logger.info("\n" + "Deleting users: " + str(userList))
         self.do_delete_users(userList)
+        self.do_erase_dir_silent(homeDirList)
+        self.do_delete_by_owner(userIds, FOLDER_LIST)
       if dirList and not DIR_SECTION in SKIP_LIST:
         logger.info("\n" + "Deleting directories: " + str(dirList))
         self.do_erase_dir_silent(dirList)
@@ -138,6 +149,12 @@ class HostCleanup:
     except:
       logger.warn("Cannot read user list: " + str(sys.exc_info()[0]))
 
+    try:
+      if config.has_option(USER_SECTION, USER_HOMEDIR_KEY):
+        propertyMap[USER_HOMEDIR_SECTION] = config.get(USER_SECTION, USER_HOMEDIR_KEY).split(',')
+    except:
+      logger.warn("Cannot read user homedir list: " + str(sys.exc_info()[0]))
+
     try:
       if config.has_option(REPO_SECTION, REPOS_KEY):
         propertyMap[REPO_SECTION] = config.get(REPO_SECTION, REPOS_KEY).split(',')
@@ -327,6 +344,34 @@ class HostCleanup:
           logger.info("File doesn't exists: " + path)
     return 0
 
+  def do_delete_group(self):
+    groupDelCommand = GROUP_ERASE_CMD.format(HADOOP_GROUP)
+    (returncode, stdoutdata, stderrdata) = self.run_os_command(groupDelCommand)
+    if returncode != 0:
+      logger.warn("Cannot delete group : " + HADOOP_GROUP + ", " + stderrdata)
+    else:
+      logger.info("Successfully deleted group: " + HADOOP_GROUP)
+
+  def do_delete_by_owner(self, userIds, folders):
+    for folder in folders:
+      for filename in os.listdir(folder):
+        fileToCheck = os.path.join(folder, filename)
+        stat = os.stat(fileToCheck)
+        if stat.st_uid in userIds:
+          self.do_erase_dir_silent([fileToCheck])
+          logger.info("Deleting file/folder: " + fileToCheck)
+
+  def get_user_ids(self, userList):
+    userIds = []
+    if userList:
+      for user in userList:
+        if user:
+          try:
+            userIds.append(getpwnam(user).pw_uid)
+          except Exception:
+            logger.warn("Cannot find user : " + user)
+    return userIds
+
   def do_delete_users(self, userList):
     if userList:
       for user in userList:
@@ -337,6 +382,7 @@ class HostCleanup:
             logger.warn("Cannot delete user : " + user + ", " + stderrdata)
           else:
             logger.info("Successfully deleted user: " + user)
+      self.do_delete_group()
     return 0
 
   def is_current_user_root(self):
@@ -401,7 +447,7 @@ def main():
     help="log file to store results.", metavar="FILE")
   parser.add_option("-k", "--skip", dest="skip",
     help="(packages|users|directories|repositories|processes|alternatives)." +\
-         " separator = ,")
+         " Use , as separator. Use empty string to clean all (by default users are skipped)")
 
   (options, args) = parser.parse_args()
   # set output file

ambari-agent/src/main/python/ambari_agent/HostInfo.py

@@ -41,7 +41,8 @@ class HostInfo:
     "hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios",
     "oozie", "sqoop", "hue", "zookeeper", "mapred", "hdfs", "flume",
     "ambari_qa", "hadoop_deploy", "rrdcached", "hcat", "ambari-qa",
-    "sqoop-ambari-qa", "sqoop-ambari_qa"
+    "sqoop-ambari-qa", "sqoop-ambari_qa", "webhcat", "hadoop-hdfs", "hadoop-yarn",
+    "hadoop-mapreduce"
   ]
 
   # List of live services checked for on the host, takes a map of plan strings
@@ -219,22 +220,6 @@ class HostInfo:
     except:
       pass
 
-  def checkFoldersBasedOnNames(self, basePaths, projectNames, existingUsers, dirs):
-    foldersToIgnore = []
-    for user in existingUsers:
-      foldersToIgnore.append(user['homeDir'])
-    try:
-      for dirName in basePaths:
-        for project in projectNames:
-          path = dirName.strip() + project.strip()
-          if not path in foldersToIgnore and os.path.exists(path):
-            obj = {}
-            obj['type'] = self.dirType(path)
-            obj['name'] = path
-            dirs.append(obj)
-    except:
-      pass
-
   def javaProcs(self, list):
     try:
       pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
@@ -321,7 +306,6 @@ class HostInfo:
 
       dirs = []
       self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
-      self.checkFoldersBasedOnNames(self.DIRNAME_PATTERNS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
       dict['stackFoldersAndFiles'] = dirs
 
       installedPackages = []

ambari-agent/src/test/python/TestHostCheckReportFileHandler.py

@@ -71,6 +71,7 @@ class TestHostCheckReportFileHandler(TestCase):
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
     configValidator.read(configPath)
     users = configValidator.get('users', 'usr_list')
+    users = configValidator.get('users', 'usr_homedir_list')
     self.assertEquals(users, '')
     names = configValidator.get('alternatives', 'symlink_list')
     targets = configValidator.get('alternatives', 'target_list')
@@ -124,7 +125,9 @@ class TestHostCheckReportFileHandler(TestCase):
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
     configValidator.read(configPath)
     users = configValidator.get('users', 'usr_list')
+    homedirs = configValidator.get('users', 'usr_homedir_list')
     self.assertEquals(users, 'user1')
+    self.assertEquals(homedirs, '/var/log')
 
     names = configValidator.get('alternatives', 'symlink_list')
     targets = configValidator.get('alternatives', 'target_list')

ambari-agent/src/test/python/TestHostCleanup.py

@@ -42,6 +42,7 @@ PROCESS_KEY = "proc_list"
 ALT_SECTION = "alternatives"
 ALT_KEYS = ["symlink_list", "target_list"]
 ALT_ERASE_CMD = "alternatives --remove {0} {1}"
+USER_HOMEDIR_SECTION = "usr_homedir"
 
 class TestHostCleanup(TestCase):
 
@@ -148,7 +149,7 @@ created = 2013-07-02 20:39:22.162757"""
                    REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'],
                    PROCESS_SECTION:['abcd', 'pqrst'],
                    ALT_SECTION:{ALT_KEYS[0]:['alt1','alt2'], ALT_KEYS[1]:[
-                     'dir1']}}
+                     'dir1']}, USER_HOMEDIR_SECTION:['decf']}
     get_os_type_method.return_value = 'redhat'
     find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst']
 
@@ -160,7 +161,8 @@ created = 2013-07-02 20:39:22.162757"""
     self.assertTrue(do_erase_packages_method.called)
     self.assertTrue(do_kill_processes_method.called)
     self.assertTrue(do_erase_alternatives_method.called)
-    do_erase_dir_silent_method.assert_called_once_with(['abcd', 'pqrst'])
+    calls = [call(['decf']), call(['abcd', 'pqrst'])]
+    do_erase_dir_silent_method.assert_has_calls(calls)
     do_erase_packages_method.assert_called_once_with(['abcd', 'pqrst'])
     do_erase_files_silent_method.assert_called_once_with(['abcd', 'pqrst'])
     do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
@@ -172,6 +174,8 @@ created = 2013-07-02 20:39:22.162757"""
     sys.stdout = sys.__stdout__
 
 
+  @patch.object(HostCleanup.HostCleanup, 'do_delete_by_owner')
+  @patch.object(HostCleanup.HostCleanup, 'get_user_ids')
   @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives')
   @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')
   @patch.object(HostCleanup.HostCleanup, 'get_os_type')
@@ -184,7 +188,8 @@ created = 2013-07-02 20:39:22.162757"""
                       do_erase_dir_silent_method,
                       do_erase_files_silent_method, do_kill_processes_method,
                       get_os_type_method, find_repo_files_for_repos_method,
-                      do_erase_alternatives_method):
+                      do_erase_alternatives_method, get_user_ids_method,
+                      do_delete_by_owner_method):
 
     out = StringIO.StringIO()
     sys.stdout = out
@@ -198,6 +203,8 @@ created = 2013-07-02 20:39:22.162757"""
 
     self.hostcleanup.do_cleanup(propertyMap)
 
+    self.assertFalse(do_delete_by_owner_method.called)
+    self.assertFalse(get_user_ids_method.called)
     self.assertFalse(do_delete_users_method.called)
     self.assertTrue(do_erase_dir_silent_method.called)
     self.assertTrue(do_erase_files_silent_method.called)
@@ -235,12 +242,35 @@ created = 2013-07-02 20:39:22.162757"""
     self.assertFalse(do_erase_files_silent_method.called)
     self.assertFalse(do_erase_packages_method.called)
     self.assertTrue(do_kill_processes_method.called)
-    do_erase_dir_silent_method.assert_called_once_with(['abcd', 'pqrst'])
+    calls = [call(None), call(['abcd', 'pqrst'])]
+    do_erase_dir_silent_method.assert_has_calls(calls)
     do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
     do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst'])
 
     sys.stdout = sys.__stdout__
 
+  @patch.object(HostCleanup.HostCleanup, 'do_erase_dir_silent')
+  @patch("os.stat")
+  @patch("os.path.join")
+  @patch("os.listdir")
+  def test_do_delete_by_owner(self, listdir_mock, join_mock, stat_mock, do_erase_dir_silent_method):
+    listdir_mock.return_value = ["k", "j"]
+    join_mock.return_value = "path"
+    response = MagicMock()
+    response.st_uid = 1
+    stat_mock.return_value = response
+    self.hostcleanup.do_delete_by_owner([1, 2], ["a"])
+    self.assertTrue(do_erase_dir_silent_method.called)
+    calls = [call(["path"]), call(["path"])]
+    do_erase_dir_silent_method.assert_has_calls(calls)
+
+  @patch.object(HostCleanup.HostCleanup, 'run_os_command')
+  def test_do_delete_users(self, run_os_command_mock):
+    run_os_command_mock.return_value = (1, "", "")
+    self.hostcleanup.do_delete_users(["a", "b"])
+    self.assertTrue(run_os_command_mock.called)
+    calls = [call('userdel -rf a'), call('userdel -rf b'), call('groupdel hadoop')]
+    run_os_command_mock.assert_has_calls(calls)
 
   @patch("ConfigParser.RawConfigParser")
   @patch("__builtin__.open")

ambari-agent/src/test/python/TestHostInfo.py

@@ -196,18 +196,6 @@ class TestHostInfo(TestCase):
     packageAnalyzer.allInstalledPackages(installedPackages)
     self.assertEqual(installedPackages, [])
 
-  @patch('os.path.exists')
-  def test_checkFoldersBasedOnNames(self, path_mock):
-    path_mock.return_value = True
-    hostInfo = HostInfo()
-    results = []
-    existingUsers = [{'name':'a1', 'homeDir':'/home/a1'}, {'name':'b1', 'homeDir':'/home/b1'}]
-    hostInfo.checkFoldersBasedOnNames(["/etc/conf", "/var/lib_", "/home/"], ["a1", "b1"], existingUsers, results)
-    self.assertEqual(4, len(results))
-    names = [i['name'] for i in results]
-    for item in ['/etc/confa1', '/var/lib_a1', '/etc/confb1', '/var/lib_b1']:
-      self.assertTrue(item in names)
-
 
   @patch('os.path.exists')
   def test_checkFolders(self, path_mock):