AMBARI-3189. Review host check filters for various OS types.

ambari-agent/src/main/python/ambari_agent/HostCleanup.py

@@ -45,7 +45,7 @@ ALT_ERASE_CMD = "alternatives --remove {0} {1}"
 
 REPO_PATH_RHEL = "/etc/yum.repos.d"
 REPO_PATH_SUSE = "/etc/zypp/repos.d/"
-SKIP_LIST = ["users"]
+SKIP_LIST = []
 HOST_CHECK_FILE_NAME = "hostcheck.result"
 OUTPUT_FILE_NAME = "hostcleanup.result"
 
@@ -70,8 +70,8 @@ FOLDER_LIST = ["/tmp"]
 REPOSITORY_BLACK_LIST = ["ambari.repo"]
 PACKAGES_BLACK_LIST = ["ambari-server", "ambari-agent"]
 
-class HostCleanup:
 
+class HostCleanup:
   def resolve_ambari_config(self):
     try:
       config = AmbariConfig.config
@@ -124,7 +124,8 @@ class HostCleanup:
   def read_host_check_file(self, config_file_path):
     propertyMap = {}
     try:
-      with open(config_file_path, 'r'): pass
+      with open(config_file_path, 'r'):
+        pass
     except Exception, e:
       logger.error("Host check result not found at: " + str(config_file_path))
       return None
@@ -196,7 +197,7 @@ class HostCleanup:
       out = p2.communicate()[0]
       logger.debug('alternatives --display ' + alt_name + '\n, out = ' + out)
     except:
-      logger.warn('Cannot process alternative named: ' + alt_name + ',' +\
+      logger.warn('Cannot process alternative named: ' + alt_name + ',' + \
                   'error: ' + str(sys.exc_info()[0]))
 
     return out
@@ -222,8 +223,8 @@ class HostCleanup:
                 for entry in alternates:
                   if entry:
                     alt_path = entry.split()[0]
-                    logger.debug('Erasing alternative named: ' + alt_name + ', '\
-                                  'path: ' + alt_path)
+                    logger.debug('Erasing alternative named: ' + alt_name + ', ' \
+                                                                            'path: ' + alt_path)
 
                     command = ALT_ERASE_CMD.format(alt_name, alt_path)
                     (returncode, stdoutdata, stderrdata) = self.run_os_command(command)
@@ -308,7 +309,7 @@ class HostCleanup:
         logger.warn("Unsupported OS type, cannot remove package.")
 
       if command != '':
-        logger.debug('Executing: '+ str(command))
+        logger.debug('Executing: ' + str(command))
         (returncode, stdoutdata, stderrdata) = self.run_os_command(command)
         if returncode != 0:
           logger.warn("Erasing packages failed: " + stderrdata)
@@ -390,7 +391,7 @@ class HostCleanup:
 
   def get_os_type(self):
     os_info = platform.linux_distribution(
-      None, None, None, ['SuSE', 'redhat' ], 0
+      None, None, None, ['SuSE', 'redhat'], 0
     )
     return os_info[0].lower()
 
@@ -403,9 +404,9 @@ class HostCleanup:
     if type(cmd) == str:
       cmd = shlex.split(cmd)
     process = subprocess.Popen(cmd,
-      stdout=subprocess.PIPE,
-      stdin=subprocess.PIPE,
-      stderr=subprocess.PIPE
+                               stdout=subprocess.PIPE,
+                               stdin=subprocess.PIPE,
+                               stderr=subprocess.PIPE
     )
     (stdoutdata, stderrdata) = process.communicate()
     return process.returncode, stdoutdata, stderrdata
@@ -429,6 +430,27 @@ def backup_file(filePath):
       logger.warn('Could not backup file "%s": %s' % (str(filePath, e)))
   return 0
 
+
+def get_YN_input(prompt, default):
+  yes = set(['yes', 'ye', 'y'])
+  no = set(['no', 'n'])
+  return get_choice_string_input(prompt, default, yes, no)
+
+
+def get_choice_string_input(prompt, default, firstChoice, secondChoice):
+  choice = raw_input(prompt).lower()
+  if choice in firstChoice:
+    return True
+  elif choice in secondChoice:
+    return False
+  elif choice is "": # Just enter pressed
+    return default
+  else:
+    print "input not recognized, please try again: "
+    return get_choice_string_input(prompt, default, firstChoice, secondChoice)
+  pass
+
+
 def main():
   h = HostCleanup()
   config = h.resolve_ambari_config()
@@ -438,16 +460,16 @@ def main():
 
   parser = optparse.OptionParser()
   parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
-    default=False, help="output verbosity.")
+                    default=False, help="output verbosity.")
   parser.add_option("-f", "--file", dest="inputfile",
-    default=hostCheckFilePath,
-    help="host check result file to read.", metavar="FILE")
+                    default=hostCheckFilePath,
+                    help="host check result file to read.", metavar="FILE")
   parser.add_option("-o", "--out", dest="outputfile",
-    default=hostCheckResultPath,
-    help="log file to store results.", metavar="FILE")
+                    default=hostCheckResultPath,
+                    help="log file to store results.", metavar="FILE")
   parser.add_option("-k", "--skip", dest="skip",
-    help="(packages|users|directories|repositories|processes|alternatives)." +\
-         " Use , as separator. Use empty string to clean all (by default users are skipped)")
+                    help="(packages|users|directories|repositories|processes|alternatives)." + \
+                         " Use , as separator.")
 
   (options, args) = parser.parse_args()
   # set output file
@@ -469,11 +491,17 @@ def main():
     global SKIP_LIST
     SKIP_LIST = options.skip.split(',')
 
-
   is_root = h.is_current_user_root()
   if not is_root:
     raise RuntimeError('HostCleanup needs to be run as root.')
 
+  if "users" not in SKIP_LIST:
+    delete_users = get_YN_input('You have elected to remove all users as well. If it is not intended then use '
+                               'option --skip \"users\". Do you want to continue [y/n] (n)', False)
+    if not delete_users:
+      print 'Exiting. Use option --skip="users" to skip deleting users'
+      sys.exit(1)
+
   hostcheckfile = options.inputfile
   propMap = h.read_host_check_file(hostcheckfile)
 

ambari-agent/src/main/python/ambari_agent/HostInfo.py

@@ -80,7 +80,7 @@ class HostInfo:
   # Additional packages to look for (search packages that start with these)
   ADDITIONAL_PACKAGES = [
     "rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb", "libconfuse", "ambari-log4j",
-    "hadoop", "zookeeper"
+    "hadoop", "zookeeper", "oozie", "webhcat"
   ]
 
   # ignore packages from repos whose names start with these strings
@@ -263,8 +263,15 @@ class HostInfo:
      return self.current_umask
     else:
      return self.current_umask
- 
- 
+
+
+  def get_os_type(self):
+    os_info = platform.linux_distribution(
+      None, None, None, ['SuSE', 'redhat' ], 0
+    )
+    return os_info[0].lower()
+
+
   """ Return various details about the host
   componentsMapped: indicates if any components are mapped to this host
   commandsInProgress: indicates if any commands are in progress
@@ -278,17 +285,18 @@ class HostInfo:
 
     dict['hostHealth']['diskStatus'] = [self.osdiskAvailableSpace("/")]
 
-    dict['rpms'] = []
-
     liveSvcs = []
     self.checkLiveServices(self.DEFAULT_LIVE_SERVICES, liveSvcs)
     dict['hostHealth']['liveServices'] = liveSvcs
     
     dict['umask'] = str(self.getUMask())
 
+    # detailed host check is not available for Suse
+    isSuse =  'suse' == self.get_os_type()
+
     # If commands are in progress or components are already mapped to this host
     # Then do not perform certain expensive host checks
-    if componentsMapped or commandsInProgress:
+    if componentsMapped or commandsInProgress or isSuse:
       dict['existingRepos'] = [self.RESULT_UNAVAILABLE]
       dict['installedPackages'] = []
       dict['alternatives'] = []

ambari-agent/src/test/python/TestHostCleanup.py

@@ -103,18 +103,21 @@ created = 2013-07-02 20:39:22.162757"""
       self.inputfile = inputfile
       self.skip = skip
       self.verbose = False
-      
-  @patch.object(HostCleanup.HostCleanup, 'do_cleanup')  
+
+  @patch.object(HostCleanup, 'get_YN_input')
+  @patch.object(HostCleanup.HostCleanup, 'do_cleanup')
   @patch.object(HostCleanup.HostCleanup, 'is_current_user_root')
   @patch.object(logging.FileHandler, 'setFormatter')
   @patch.object(HostCleanup.HostCleanup,'read_host_check_file')
   @patch.object(logging,'basicConfig')
   @patch.object(logging, 'FileHandler')
   @patch.object(optparse.OptionParser, 'parse_args')
-  def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, set_formatter_mock, user_root_mock, do_cleanup_mock):
+  def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
+                   set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock):
     parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False), [])
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     user_root_mock.return_value = True
+    get_yn_input_mock.return_value = True
     HostCleanup.main()
     
     # test --out
@@ -140,9 +143,6 @@ created = 2013-07-02 20:39:22.162757"""
                       do_erase_files_silent_method, do_kill_processes_method,
                       get_os_type_method, find_repo_files_for_repos_method,
                       do_erase_alternatives_method):
-    global SKIP_LIST
-    oldSkipList = HostCleanup.SKIP_LIST
-    HostCleanup.SKIP_LIST = []
     out = StringIO.StringIO()
     sys.stdout = out
     propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'],
@@ -170,7 +170,6 @@ created = 2013-07-02 20:39:22.162757"""
     do_erase_alternatives_method.assert_called_once_with({ALT_KEYS[0]:['alt1',
                                               'alt2'], ALT_KEYS[1]:['dir1']})
 
-    HostCleanup.SKIP_LIST = oldSkipList
     sys.stdout = sys.__stdout__
 
 
@@ -191,6 +190,9 @@ created = 2013-07-02 20:39:22.162757"""
                       do_erase_alternatives_method, get_user_ids_method,
                       do_delete_by_owner_method):
 
+    global SKIP_LIST
+    oldSkipList = HostCleanup.SKIP_LIST
+    HostCleanup.SKIP_LIST = ["users"]
     out = StringIO.StringIO()
     sys.stdout = out
     propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'],
@@ -211,6 +213,7 @@ created = 2013-07-02 20:39:22.162757"""
     self.assertTrue(do_erase_packages_method.called)
     self.assertTrue(do_kill_processes_method.called)
     self.assertTrue(do_erase_alternatives_method.called)
+    HostCleanup.SKIP_LIST = oldSkipList
     sys.stdout = sys.__stdout__
 
   @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')

ambari-agent/src/test/python/TestHostInfo.py

@@ -230,6 +230,51 @@ class TestHostInfo(TestCase):
     self.assertTrue(newlist[1]['status'], "Invalid home directory")
 
 
+  @patch.object(HostInfo, 'get_os_type')
+  @patch('os.umask')
+  @patch.object(HostInfo, 'osdiskAvailableSpace')
+  @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile')
+  @patch.object(PackagesAnalyzer, 'allAvailablePackages')
+  @patch.object(PackagesAnalyzer, 'allInstalledPackages')
+  @patch.object(PackagesAnalyzer, 'getPackageDetails')
+  @patch.object(PackagesAnalyzer, 'getInstalledPkgsByNames')
+  @patch.object(PackagesAnalyzer, 'getInstalledPkgsByRepo')
+  @patch.object(PackagesAnalyzer, 'getInstalledRepos')
+  @patch.object(HostInfo, 'checkUsers')
+  @patch.object(HostInfo, 'checkLiveServices')
+  @patch.object(HostInfo, 'javaProcs')
+  @patch.object(HostInfo, 'checkFolders')
+  @patch.object(HostInfo, 'etcAlternativesConf')
+  @patch.object(HostInfo, 'hadoopVarRunCount')
+  @patch.object(HostInfo, 'hadoopVarLogCount')
+  def test_hostinfo_register_suse(self, hvlc_mock, hvrc_mock, eac_mock, cf_mock, jp_mock,
+                             cls_mock, cu_mock, gir_mock, gipbr_mock, gipbn_mock,
+                             gpd_mock, aip_mock, aap_mock, whcf_mock, odas_mock,
+                             os_umask_mock, get_os_type_mock):
+    hvlc_mock.return_value = 1
+    hvrc_mock.return_value = 1
+    gipbr_mock.return_value = ["pkg1"]
+    gipbn_mock.return_value = ["pkg2"]
+    gpd_mock.return_value = ["pkg1", "pkg2"]
+    odas_mock.return_value = [{'name':'name1'}]
+    get_os_type_mock.return_value = "suse"
+
+    hostInfo = HostInfo()
+    dict = {}
+    hostInfo.register(dict, False, False)
+    self.assertFalse(gir_mock.called)
+    self.assertFalse(gpd_mock.called)
+    self.assertFalse(aip_mock.called)
+    self.assertFalse(aap_mock.called)
+    self.assertTrue(odas_mock.called)
+    self.assertTrue(os_umask_mock.called)
+    self.assertFalse(whcf_mock.called)
+
+    self.assertTrue(0 == len(dict['installedPackages']))
+    self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth'])
+
+
+  @patch.object(HostInfo, 'get_os_type')
   @patch('os.umask')
   @patch.object(HostInfo, 'osdiskAvailableSpace')
   @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile')
@@ -249,13 +294,14 @@ class TestHostInfo(TestCase):
   def test_hostinfo_register(self, hvlc_mock, hvrc_mock, eac_mock, cf_mock, jp_mock,
                              cls_mock, cu_mock, gir_mock, gipbr_mock, gipbn_mock,
                              gpd_mock, aip_mock, aap_mock, whcf_mock, odas_mock,
-                             os_umask_mock):
+                             os_umask_mock, get_os_type_mock):
     hvlc_mock.return_value = 1
     hvrc_mock.return_value = 1
     gipbr_mock.return_value = ["pkg1"]
     gipbn_mock.return_value = ["pkg2"]
     gpd_mock.return_value = ["pkg1", "pkg2"]
     odas_mock.return_value = [{'name':'name1'}]
+    get_os_type_mock.return_value = "redhat"
 
     hostInfo = HostInfo()
     dict = {}

ambari-server/src/main/java/org/apache/ambari/server/agent/AgentEnv.java

@@ -31,11 +31,6 @@ public class AgentEnv {
    */
   private Directory[] stackFoldersAndFiles = new Directory[0];
 
-  /**
-   * Various RPM package versions.
-   */
-  private Rpm[] rpms = new Rpm[0];
-
   /**
    * Directories that match name <code>/etc/alternatives/*conf</code>
    */
@@ -79,14 +74,6 @@ public class AgentEnv {
     stackFoldersAndFiles = dirs;
   }
   
-  public void setRpms(Rpm[] rpm) {
-    rpms = rpm;
-  }
-  
-  public Rpm[] getRpms() {
-    return rpms;
-  }
-  
   public void setExistingUsers(ExistingUser[] users) {
     existingUsers = users;
   }
@@ -199,43 +186,6 @@ public class AgentEnv {
     }
   }
 
-  /**
-   * Represents information about rpm-installed packages
-   */
-  public static class Rpm {
-    @SerializedName("name")
-    private String rpmName;
-    @SerializedName("installed")
-    private boolean rpmInstalled = false;
-    @SerializedName("version")
-    private String rpmVersion;
-    
-    public void setName(String name) {
-      rpmName = name;
-    }
-    
-    public String getName() {
-      return rpmName;
-    }
-    
-    public void setInstalled(boolean installed) {
-      rpmInstalled = installed;
-    }
-    
-    public boolean isInstalled() {
-      return rpmInstalled;
-    }
-    
-    public void setVersion(String version) {
-      rpmVersion = version;
-    }
-    
-    @JsonSerialize(include=Inclusion.NON_NULL)
-    public String getVersion() {
-      return rpmVersion;
-    }
-  }
-
   public static class PackageDetail {
     @SerializedName("name")
     private String pkgName;