AMBARI-2724. Improve host check to look for packages that are installed by HDP-UTILS-1.1.0.15. (smohanty)

ambari-agent/src/main/python/ambari_agent/Heartbeat.py

@@ -64,6 +64,11 @@ class Heartbeat:
         # There may be IN_PROGRESS tasks
         commandsInProgress = True
       pass
+
+    # For first request/heartbeat assume no components are mapped
+    if int(id) == 0:
+      componentsMapped = False
+
     logger.info("Sending heartbeat with response id: " + str(id) + " and "
                 "timestamp: " + str(timestamp) +
                 ". Command(s) in progress: " + repr(commandsInProgress) +

ambari-agent/src/main/python/ambari_agent/HostCleanup.py

@@ -421,7 +421,7 @@ def main():
 
   if options.skip:
     global SKIP_LIST
-    SKIP_LIST = options.skip.splt(',')
+    SKIP_LIST = options.skip.split(',')
 
 
   is_root = h.is_current_user_root()

ambari-agent/src/main/python/ambari_agent/HostInfo.py

@@ -39,7 +39,8 @@ class HostInfo:
   DEFAULT_PROJECT_NAMES = [
     "hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios",
     "oozie", "sqoop", "hue", "zookeeper", "mapred", "hdfs", "flume",
-    "ambari_qa", "hadoop_deploy", "rrdcached", "hcat"
+    "ambari_qa", "hadoop_deploy", "rrdcached", "hcat", "ambari-qa",
+    "sqoop-ambari-qa", "sqoop-ambari_qa"
   ]
 
   # List of live services checked for on the host
@@ -59,9 +60,14 @@ class HostInfo:
     "hadoop", "zookeeper"
   ]
 
+  # Additional path patterns to find existing directory
+  DIRNAME_PATTERNS = [
+    "/tmp/hadoop-", "/tmp/hsperfdata_"
+  ]
+
   # Default set of directories that are checked for existence of files and folders
   DEFAULT_DIRS = [
-    "/etc", "/var/run", "/var/log", "/usr/lib", "/var/lib", "/var/tmp", "/tmp", "/var"
+    "/etc", "/var/run", "/var/log", "/usr/lib", "/var/lib", "/var/tmp", "/tmp", "/var", "/hadoop"
   ]
 
   # Packages that are used to find repos (then repos are used to find other packages)
@@ -71,14 +77,21 @@ class HostInfo:
 
   # Additional packages to look for (search packages that start with these)
   ADDITIONAL_PACKAGES = [
-    "rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb", "libconfuse", "ambari-log4j"
+    "rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb", "libconfuse", "ambari-log4j",
+    "mysql", "hadoop", "zookeeper"
   ]
 
-  # ignores packages from repos whose names start with these strings
+  # ignore packages from repos whose names start with these strings
   IGNORE_PACKAGES_FROM_REPOS = [
     "ambari", "installed"
   ]
 
+  # ignore required packages
+  IGNORE_PACKAGES = [
+    "epel-release"
+  ]
+
+  # ignore repos from the list of repos to be cleaned
   IGNORE_REPOS = [
     "ambari", "HDP-UTILS"
   ]
@@ -105,30 +118,6 @@ class HostInfo:
       return 'file'
     return 'unknown'
 
-  def rpmInfo(self, rpmList):
-    config = AmbariConfig.config
-
-    try:
-      for rpmName in config.get('heartbeat', 'rpms').split(','):
-        rpmName = rpmName.strip()
-        rpm = {}
-        rpm['name'] = rpmName
-
-        try:
-          osStat = subprocess.Popen(["rpm", "-q", rpmName], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-          out, err = osStat.communicate()
-          if (0 != osStat.returncode or 0 == len(out.strip())):
-            rpm['installed'] = False
-          else:
-            rpm['installed'] = True
-            rpm['version'] = out.strip()
-        except:
-          rpm['available'] = False
-
-        rpmList.append(rpm)
-    except:
-      pass
-
   def hadoopVarRunCount(self):
     if not os.path.exists('/var/run/hadoop'):
       return 0
@@ -216,6 +205,22 @@ class HostInfo:
     except:
       pass
 
+  def checkFoldersBasedOnNames(self, basePaths, projectNames, existingUsers, dirs):
+    foldersToIgnore = []
+    for user in existingUsers:
+      foldersToIgnore.append(user['homeDir'])
+    try:
+      for dirName in basePaths:
+        for project in projectNames:
+          path = dirName.strip() + project.strip()
+          if not path in foldersToIgnore and os.path.exists(path):
+            obj = {}
+            obj['type'] = self.dirType(path)
+            obj['name'] = path
+            dirs.append(obj)
+    except:
+      pass
+
   def javaProcs(self, list):
     try:
       pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
@@ -274,9 +279,7 @@ class HostInfo:
 
     dict['hostHealth']['diskStatus'] = [self.osdiskAvailableSpace("/")]
 
-    rpms = []
-    self.rpmInfo(rpms)
-    dict['rpms'] = rpms
+    dict['rpms'] = []
 
     liveSvcs = []
     self.checkLiveServices(self.DEFAULT_LIVE_SERVICES, liveSvcs)
@@ -304,6 +307,7 @@ class HostInfo:
 
       dirs = []
       self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
+      self.checkFoldersBasedOnNames(self.DIRNAME_PATTERNS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
       dict['stackFoldersAndFiles'] = dirs
 
       installedPackages = []
@@ -314,14 +318,15 @@ class HostInfo:
       repos = []
       self.packages.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
                                       self.IGNORE_PACKAGES_FROM_REPOS, repos)
-      repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
-      dict['existingRepos'] = repos
-
-      packagesInstalled = self.packages.getInstalledPkgsByRepo(repos, installedPackages)
+      packagesInstalled = self.packages.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages)
       additionalPkgsInstalled = self.packages.getInstalledPkgsByNames(
         self.ADDITIONAL_PACKAGES, installedPackages)
       allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
       dict['installedPackages'] = self.packages.getPackageDetails(installedPackages, allPackages)
+
+      repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
+      dict['existingRepos'] = repos
+
       self.reportFileHandler.writeHostCheckFile(dict)
       pass
 

ambari-agent/src/main/python/ambari_agent/PackagesAnalyzer.py

@@ -31,7 +31,7 @@ logger = logging.getLogger()
 class PackagesAnalyzer:
 
   # default timeout for async invoked processes
-  TIMEOUT_SECONDS = 60
+  TIMEOUT_SECONDS = 30
   event = threading.Event()
 
   def launch_subprocess(self, command):
@@ -41,7 +41,7 @@ class PackagesAnalyzer:
     self.event.wait(self.TIMEOUT_SECONDS)
     if command.returncode is None:
       logger.error("Task timed out and will be killed")
-      shell.killprocessgrp(command.pid)
+      shell.kill_process_with_children(command.pid)
     pass
 
   def subprocessWithTimeout(self, command):
@@ -175,15 +175,25 @@ class PackagesAnalyzer:
         repoList.append(repo)
 
   # Get all the installed packages from the repos listed in repos
-  def getInstalledPkgsByRepo(self, repos, installedPackages):
+  def getInstalledPkgsByRepo(self, repos, ignorePackages, installedPackages):
     packagesFromRepo = []
+    packagesToRemove = []
     for repo in repos:
       subResult = []
       for item in installedPackages:
         if repo == item[2]:
           subResult.append(item[0])
       packagesFromRepo = list(set(packagesFromRepo + subResult))
-    return packagesFromRepo
+
+    for package in packagesFromRepo:
+      keepPackage = True
+      for ignorePackage in ignorePackages:
+        if self.nameMatch(ignorePackage, package):
+          keepPackage = False
+          break
+      if keepPackage:
+        packagesToRemove.append(package)
+    return packagesToRemove
 
   # Gets all installed packages that start with names in pkgNames
   def getInstalledPkgsByNames(self, pkgNames, installedPackages):

ambari-agent/src/main/python/ambari_agent/shell.py

@@ -59,10 +59,6 @@ def killstaleprocesses():
       os.unlink(os.path.join(prefix,file))
   logger.info ("Killed stale processes")
 
-
-def killprocessgrp(pid):
-  run_kill_function(os.killpg, pid)
-
 def kill_process_with_children(parent_pid):
   def kill_tree_function(pid, signal):
     '''

ambari-agent/src/test/python/TestHostInfo.py

@@ -107,9 +107,9 @@ class TestHostInfo(TestCase):
     for repo in expected:
       self.assertTrue(repo in repos)
 
-    packagesInstalled = packageAnalyzer.getInstalledPkgsByRepo(repos, installedPackages)
-    self.assertEqual(7, len(packagesInstalled))
-    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "epel", "epel-2", "def-def.x86"]
+    packagesInstalled = packageAnalyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages)
+    self.assertEqual(5, len(packagesInstalled))
+    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"]
     for repo in expected:
       self.assertTrue(repo in packagesInstalled)
 
@@ -121,8 +121,8 @@ class TestHostInfo(TestCase):
       self.assertTrue(additionalPkg in additionalPkgsInstalled)
 
     allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
-    self.assertEqual(9, len(allPackages))
-    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "epel", "epel-2", "nagios", "rrd", "def-def.x86"]
+    self.assertEqual(7, len(allPackages))
+    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "nagios", "rrd", "def-def.x86"]
     for package in expected:
       self.assertTrue(package in allPackages)
 
@@ -196,6 +196,19 @@ class TestHostInfo(TestCase):
     packageAnalyzer.allInstalledPackages(installedPackages)
     self.assertEqual(installedPackages, [])
 
+  @patch('os.path.exists')
+  def test_checkFoldersBasedOnNames(self, path_mock):
+    path_mock.return_value = True
+    hostInfo = HostInfo()
+    results = []
+    existingUsers = [{'name':'a1', 'homeDir':'/home/a1'}, {'name':'b1', 'homeDir':'/home/b1'}]
+    hostInfo.checkFoldersBasedOnNames(["/etc/conf", "/var/lib_", "/home/"], ["a1", "b1"], existingUsers, results)
+    self.assertEqual(4, len(results))
+    names = [i['name'] for i in results]
+    for item in ['/etc/confa1', '/var/lib_a1', '/etc/confb1', '/var/lib_b1']:
+      self.assertTrue(item in names)
+
+
   @patch('os.path.exists')
   def test_checkFolders(self, path_mock):
     path_mock.return_value = True
@@ -459,26 +472,5 @@ class TestHostInfo(TestCase):
     self.assertEquals(result[0]['target'], 'real_path_to_conf')
 
 
-  @patch("subprocess.Popen")
-  def test_rpmInfo(self, subproc_popen):
-    hostInfo = HostInfo()
-    p = MagicMock()
-    p.returncode = 0
-    p.communicate.side_effect = [('out', 'err'), ('', 'err') , ('out', 'err', 'fail')]
-    subproc_popen.return_value = p
-    rpmList = []
-    hostInfo.rpmInfo(rpmList)
-
-    self.assertEquals(rpmList[0]['version'], 'out')
-    self.assertEquals(rpmList[0]['name'], 'glusterfs')
-    self.assertTrue(rpmList[0]['installed'])
-
-    self.assertEquals(rpmList[1]['name'], 'openssl')
-    self.assertFalse(rpmList[1]['installed'])
-
-    self.assertFalse(rpmList[2]['available'])
-    self.assertEquals(rpmList[2]['name'], 'wget')
-
-
 if __name__ == "__main__":
-  unittest.main(verbosity=2)
+  unittest.main()

ambari-agent/src/test/python/TestShell.py

@@ -86,9 +86,3 @@ class TestShell(unittest.TestCase):
     else:
       # Do not run under other systems
       pass
-
-
-  @patch("os.killpg")
-  def test_killprocessgrp(self, killpg_grp):
-    shell.killprocessgrp(999)
-    self.assertTrue(killpg_grp.called)