AMBARI-13518 Host cleanup should remove only stack directories from /usr/hdp (dsen)

ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -16,15 +16,19 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 import datetime
 import os.path
 import logging
+import re
 import traceback
 from AmbariConfig import AmbariConfig
-import ConfigParser;
+import ConfigParser
 
+HADOOP_ROOT_DIR = "/usr/hdp"
+HADOOP_PERM_REMOVE_LIST = ["current"]
+HADOOP_ITEMDIR_REGEX = "(\d\.){3}\d-\d{4}"
 logger = logging.getLogger(__name__)
 
 class HostCheckReportFileHandler:
@@ -83,6 +87,37 @@ class HostCheckReportFileHandler:
       logger.error("Can't write host check file at %s :%s " % (self.hostCheckCustomActionsFilePath, err.message))
       traceback.print_exc()
 
+  def _hdp_list_directory(self):
+    """
+    Return filtered list of /usr/hdp directory allowed to be removed
+    :rtype list
+    """
+
+    if not os.path.exists(HADOOP_ROOT_DIR):
+      return []
+
+    matcher = re.compile(HADOOP_ITEMDIR_REGEX)  # pre-compile regexp
+    folder_content = os.listdir(HADOOP_ROOT_DIR)
+    remove_list = []
+
+    remlist_items_count = 0
+
+    for item in folder_content:
+      full_path = "%s%s%s" % (HADOOP_ROOT_DIR, os.path.sep, item)
+      if item in HADOOP_PERM_REMOVE_LIST:
+        remove_list.append(full_path)
+        remlist_items_count += 1
+
+      if matcher.match(item) is not None:
+        remove_list.append(full_path)
+        remlist_items_count += 1
+
+    # if remove list have same length as target folder, assume that they are identical
+    if remlist_items_count == len(folder_content):
+      remove_list.append(HADOOP_ROOT_DIR)
+
+    return remove_list
+
   def writeHostCheckFile(self, hostInfo):
     if self.hostCheckFilePath is None:
       return
@@ -117,8 +152,7 @@ class HostCheckReportFileHandler:
         items = []
         for itemDetail in hostInfo['stackFoldersAndFiles']:
           items.append(itemDetail['name'])
-        if os.path.exists('/usr/hdp'):
-          items.append('/usr/hdp')
+        items += self._hdp_list_directory()
         config.add_section('directories')
         config.set('directories', 'dir_list', ','.join(items))
 

ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 from unittest import TestCase
 import unittest
+from mock.mock import patch
 import os
 import tempfile
 from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
@@ -38,8 +39,8 @@ class TestHostCheckReportFileHandler(TestCase):
     config.set('agent', 'prefix', os.path.dirname(tmpfile))
 
     handler = HostCheckReportFileHandler(config)
-    dict = {}
-    handler.writeHostCheckFile(dict)
+    mydict = {}
+    handler.writeHostCheckFile(mydict)
 
     configValidator = ConfigParser.RawConfigParser()
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
@@ -56,16 +57,16 @@ class TestHostCheckReportFileHandler(TestCase):
     config.set('agent', 'prefix', os.path.dirname(tmpfile))
 
     handler = HostCheckReportFileHandler(config)
-    dict = {}
-    dict['hostHealth'] = {}
-    dict['existingUsers'] = []
-    dict['alternatives'] = []
-    dict['stackFoldersAndFiles'] = []
-    dict['hostHealth']['activeJavaProcs'] = []
-    dict['installedPackages'] = []
-    dict['existingRepos'] = []
+    mydict = {}
+    mydict['hostHealth'] = {}
+    mydict['existingUsers'] = []
+    mydict['alternatives'] = []
+    mydict['stackFoldersAndFiles'] = []
+    mydict['hostHealth']['activeJavaProcs'] = []
+    mydict['installedPackages'] = []
+    mydict['existingRepos'] = []
 
-    handler.writeHostCheckFile(dict)
+    handler.writeHostCheckFile(mydict)
 
     configValidator = ConfigParser.RawConfigParser()
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
@@ -96,19 +97,19 @@ class TestHostCheckReportFileHandler(TestCase):
 
     handler = HostCheckReportFileHandler(config)
 
-    dict = {}
-    dict['hostHealth'] = {}
-    dict['existingUsers'] = [{'name':'user1', 'homeDir':'/var/log', 'status':'Exists'}]
-    dict['alternatives'] = [
+    mydict = {}
+    mydict['hostHealth'] = {}
+    mydict['existingUsers'] = [{'name':'user1', 'homeDir':'/var/log', 'status':'Exists'}]
+    mydict['alternatives'] = [
       {'name':'/etc/alternatives/hadoop-conf', 'target':'/etc/hadoop/conf.dist'},
       {'name':'/etc/alternatives/hbase-conf', 'target':'/etc/hbase/conf.1'}
     ]
-    dict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt', 'type':'file'}]
-    dict['hostHealth']['activeJavaProcs'] = [
+    mydict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt', 'type':'file'}]
+    mydict['hostHealth']['activeJavaProcs'] = [
       {'pid':355,'hadoop':True,'command':'some command','user':'root'},
       {'pid':455,'hadoop':True,'command':'some command','user':'hdfs'}
     ]
-    handler.writeHostCheckFile(dict)
+    handler.writeHostCheckFile(mydict)
 
     configValidator = ConfigParser.RawConfigParser()
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
@@ -130,13 +131,13 @@ class TestHostCheckReportFileHandler(TestCase):
     self.chkItemsEqual(procs, ['455', '355'])
 
 
-    dict['installed_packages'] = [
+    mydict['installed_packages'] = [
       {'name':'hadoop','version':'3.2.3','repoName':'HDP'},
       {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'}
     ]
-    dict['existing_repos'] = ['HDP', 'HDP-epel']
+    mydict['existing_repos'] = ['HDP', 'HDP-epel']
     
-    handler.writeHostChecksCustomActionsFile(dict)
+    handler.writeHostChecksCustomActionsFile(mydict)
     configValidator = ConfigParser.RawConfigParser()
     configPath_ca = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_CUSTOM_ACTIONS_FILE)
     configValidator.read(configPath_ca)
@@ -150,13 +151,40 @@ class TestHostCheckReportFileHandler(TestCase):
     time = configValidator.get('metadata', 'created')
     self.assertTrue(time != None)
 
+  @patch("os.path.exists")
+  @patch("os.listdir")
+  def test_write_host_stack_list(self, list_mock, exists_mock):
+    exists_mock.return_value = True
+    list_mock.return_value = ["1.1.1.1-1234", "current", "test"]
+
+    tmpfile = tempfile.mktemp()
+
+    config = ConfigParser.RawConfigParser()
+    config.add_section('agent')
+    config.set('agent', 'prefix', os.path.dirname(tmpfile))
+
+    handler = HostCheckReportFileHandler(config)
+
+    mydict = {}
+    mydict['hostHealth'] = {}
+    mydict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt', 'type':'file'}]
+
+    handler.writeHostCheckFile(mydict)
+
+    configValidator = ConfigParser.RawConfigParser()
+    configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
+    configValidator.read(configPath)
+
+    paths = configValidator.get('directories', 'dir_list')
+    self.chkItemsEqual(paths, ['/a/b', '/a/b.txt', '/usr/hdp/1.1.1.1-1234', '/usr/hdp/current'])
+
   def chkItemsEqual(self, commaDelimited, items):
     items1 = commaDelimited.split(',')
     items1.sort()
     items.sort()
     items1Str = ','.join(items1)
     items2Str = ','.join(items)
-    self.assertEquals(items1Str, items2Str)
+    self.assertEquals(items2Str, items1Str)
 
 if __name__ == "__main__":
   unittest.main(verbosity=2)