AMBARI-9795. [Monarch] Cluster create failed with timeout on the client side (aonishuk)

ambari-agent/pom.xml

@@ -657,6 +657,7 @@
         <artifactId>apache-rat-plugin</artifactId>
         <configuration>
           <excludes>
+          	<exclude>src/test/python/tmp_hostcheck.result</exclude>
             <exclude>src/examples/*</exclude>
             <exclude>src/test/python/dummy*.txt</exclude>
             <exclude>src/test/python/ambari_agent/dummy_files/*</exclude>

ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py

@@ -22,6 +22,7 @@ import datetime
 import os.path
 import logging
 import traceback
+from AmbariConfig import AmbariConfig
 import ConfigParser;
 
 logger = logging.getLogger()
@@ -29,12 +30,58 @@ logger = logging.getLogger()
 class HostCheckReportFileHandler:
 
   HOST_CHECK_FILE = "hostcheck.result"
+  HOST_CHECK_CUSTOM_ACTIONS_FILE = "hostcheck_custom_actions.result"
 
-  def __init__(self, config):
+  def __init__(self, config=None):
     self.hostCheckFilePath = None
-    if config is not None:
-      hostCheckFileDir = config.get('agent', 'prefix')
-      self.hostCheckFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_FILE)
+    
+    if config is None:
+      config = self.resolve_ambari_config()
+      
+    hostCheckFileDir = config.get('agent', 'prefix')
+    self.hostCheckFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_FILE)
+    self.hostCheckCustomActionsFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_CUSTOM_ACTIONS_FILE)
+    
+  def resolve_ambari_config(self):
+    try:
+      config = AmbariConfig()
+      if os.path.exists(AmbariConfig.getConfigFile()):
+        config.read(AmbariConfig.getConfigFile())
+      else:
+        raise Exception("No config found, use default")
+
+    except Exception, err:
+      logger.warn(err)
+    return config
+    
+  def writeHostChecksCustomActionsFile(self, structuredOutput):
+    if self.hostCheckCustomActionsFilePath is None:
+      return
+    
+    try:
+      logger.info("Host check custom action report at " + self.hostCheckCustomActionsFilePath)
+      config = ConfigParser.RawConfigParser()
+      config.add_section('metadata')
+      config.set('metadata', 'created', str(datetime.datetime.now()))
+      
+      if 'installed_packages' in structuredOutput.keys():
+        items = []
+        for itemDetail in structuredOutput['installed_packages']:
+          items.append(itemDetail['name'])
+        config.add_section('packages')
+        config.set('packages', 'pkg_list', ','.join(map(str, items)))
+
+      if 'existing_repos' in structuredOutput.keys():
+        config.add_section('repositories')
+        config.set('repositories', 'repo_list', ','.join(structuredOutput['existing_repos']))
+        
+      self.removeFile(self.hostCheckCustomActionsFilePath)
+      self.touchFile(self.hostCheckCustomActionsFilePath)
+      with open(self.hostCheckCustomActionsFilePath, 'wb') as configfile:
+        config.write(configfile)
+    except Exception, err:
+      logger.error("Can't write host check file at %s :%s " % (self.hostCheckFilePath, err.message))
+      traceback.print_exc()
 
   def writeHostCheckFile(self, hostInfo):
     if self.hostCheckFilePath is None:
@@ -81,33 +128,22 @@ class HostCheckReportFileHandler:
           config.add_section('processes')
           config.set('processes', 'proc_list', ','.join(map(str, items)))
 
-      if 'installedPackages' in hostInfo.keys():
-        items = []
-        for itemDetail in hostInfo['installedPackages']:
-          items.append(itemDetail['name'])
-        config.add_section('packages')
-        config.set('packages', 'pkg_list', ','.join(map(str, items)))
-
-      if 'existingRepos' in hostInfo.keys():
-        config.add_section('repositories')
-        config.set('repositories', 'repo_list', ','.join(hostInfo['existingRepos']))
-
-      self.removeFile()
-      self.touchFile()
+      self.removeFile(self.hostCheckFilePath)
+      self.touchFile(self.hostCheckFilePath)
       with open(self.hostCheckFilePath, 'wb') as configfile:
         config.write(configfile)
     except Exception, err:
       logger.error("Can't write host check file at %s :%s " % (self.hostCheckFilePath, err.message))
       traceback.print_exc()
 
-  def removeFile(self):
-    if os.path.isfile(self.hostCheckFilePath):
-      logger.info("Removing old host check file at %s" % self.hostCheckFilePath)
-      os.remove(self.hostCheckFilePath)
+  def removeFile(self, path):
+    if os.path.isfile(path):
+      logger.info("Removing old host check file at %s" % path)
+      os.remove(path)
 
-  def touchFile(self):
-    if not os.path.isfile(self.hostCheckFilePath):
-      logger.info("Creating host check file at %s" % self.hostCheckFilePath)
-      open(self.hostCheckFilePath, 'w').close()
+  def touchFile(self, path):
+    if not os.path.isfile(path):
+      logger.info("Creating host check file at %s" % path)
+      open(path, 'w').close()
 
 

ambari-agent/src/main/python/ambari_agent/HostCleanup.py

@@ -55,7 +55,9 @@ ALT_ERASE_CMD = "alternatives --remove {0} {1}"
 REPO_PATH_RHEL = "/etc/yum.repos.d"
 REPO_PATH_SUSE = "/etc/zypp/repos.d/"
 SKIP_LIST = []
+TMP_HOST_CHECK_FILE_NAME = "tmp_hostcheck.result"
 HOST_CHECK_FILE_NAME = "hostcheck.result"
+HOST_CHECK_CUSTOM_ACTIONS_FILE = "hostcheck_custom_actions.result"
 OUTPUT_FILE_NAME = "hostcleanup.result"
 
 PACKAGE_SECTION = "packages"
@@ -523,13 +525,15 @@ def main():
   config = h.resolve_ambari_config()
   hostCheckFileDir = config.get('agent', 'prefix')
   hostCheckFilePath = os.path.join(hostCheckFileDir, HOST_CHECK_FILE_NAME)
+  hostCheckCustomActionsFilePath = os.path.join(hostCheckFileDir, HOST_CHECK_CUSTOM_ACTIONS_FILE)
+  hostCheckFilesPaths = hostCheckFilePath + "," + hostCheckCustomActionsFilePath
   hostCheckResultPath = os.path.join(hostCheckFileDir, OUTPUT_FILE_NAME)
 
   parser = optparse.OptionParser()
   parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
                     default=False, help="output verbosity.")
-  parser.add_option("-f", "--file", dest="inputfile",
-                    default=hostCheckFilePath,
+  parser.add_option("-f", "--file", dest="inputfiles",
+                    default=hostCheckFilesPaths,
                     help="host check result file to read.", metavar="FILE")
   parser.add_option("-o", "--out", dest="outputfile",
                     default=hostCheckResultPath,
@@ -575,8 +579,15 @@ def main():
         print 'Exiting. Use option --skip="users" to skip deleting users'
         sys.exit(1)
 
-  hostcheckfile = options.inputfile
-  propMap = h.read_host_check_file(hostcheckfile)
+  hostcheckfile, hostcheckfileca  = options.inputfiles.split(",")
+  
+  with open(TMP_HOST_CHECK_FILE_NAME, "wb") as tmp_f:
+    with open(hostcheckfile, "rb") as f1:
+      with open(hostcheckfileca, "rb") as f2:
+        tmp_f.write(f1.read())
+        tmp_f.write(f2.read())
+  
+  propMap = h.read_host_check_file(TMP_HOST_CHECK_FILE_NAME)
 
   if propMap:
     h.do_cleanup(propMap)

ambari-agent/src/main/python/ambari_agent/HostInfo.py

@@ -147,33 +147,6 @@ class HostInfoLinux(HostInfo):
     "/hadoop", "/usr/hdp"
   ]
 
-  # Packages that are used to find repos (then repos are used to find other packages)
-  PACKAGES = [
-    "hadoop_2_2_*", "hadoop-2-2-.*", "zookeeper_2_2_*", "zookeeper-2-2-.*",
-    "hadoop", "zookeeper", "webhcat", "*-manager-server-db", "*-manager-daemons"
-  ]
-
-  # Additional packages to look for (search packages that start with these)
-  ADDITIONAL_PACKAGES = [
-    "rrdtool", "rrdtool-python", "ganglia", "gmond", "gweb", "libconfuse",
-    "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat"
-  ]
-
-  # ignore packages from repos whose names start with these strings
-  IGNORE_PACKAGES_FROM_REPOS = [
-    "ambari", "installed"
-  ]
-
-  # ignore required packages
-  IGNORE_PACKAGES = [
-    "epel-release"
-  ]
-
-  # ignore repos from the list of repos to be cleaned
-  IGNORE_REPOS = [
-    "ambari", "HDP-UTILS"
-  ]
-
   DEFAULT_SERVICE_NAME = "ntpd"
   SERVICE_STATUS_CMD = "%s %s status" % (SERVICE_CMD, DEFAULT_SERVICE_NAME)
 
@@ -202,8 +175,6 @@ class HostInfoLinux(HostInfo):
         result['name'] = fields[0]
         result['homeDir'] = fields[5]
         result['status'] = "Available"
-        if not os.path.exists(homeDir):
-          result['status'] = "Invalid home directory"
         results.append(result)
 
   def checkFolders(self, basePaths, projectNames, existingUsers, dirs):
@@ -248,18 +219,6 @@ class HostInfoLinux(HostInfo):
       pass
     pass
 
-  def getReposToRemove(self, repos, ignoreList):
-    reposToRemove = []
-    for repo in repos:
-      addToRemoveList = True
-      for ignoreRepo in ignoreList:
-        if packages_analyzer.nameMatch(ignoreRepo, repo):
-          addToRemoveList = False
-          continue
-      if addToRemoveList:
-        reposToRemove.append(repo)
-    return reposToRemove
-
   def getTransparentHugePage(self):
     # This file exist only on redhat 6
     thp_regex = "\[(.+)\]"
@@ -323,8 +282,6 @@ class HostInfoLinux(HostInfo):
     # If commands are in progress or components are already mapped to this host
     # Then do not perform certain expensive host checks
     if componentsMapped or commandsInProgress:
-      dict['existingRepos'] = [self.RESULT_UNAVAILABLE]
-      dict['installedPackages'] = []
       dict['alternatives'] = []
       dict['stackFoldersAndFiles'] = []
       dict['existingUsers'] = []
@@ -342,23 +299,6 @@ class HostInfoLinux(HostInfo):
       self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
       dict['stackFoldersAndFiles'] = dirs
 
-      installedPackages = []
-      availablePackages = []
-      packages_analyzer.allInstalledPackages(installedPackages)
-      packages_analyzer.allAvailablePackages(availablePackages)
-
-      repos = []
-      packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
-                                      self.IGNORE_PACKAGES_FROM_REPOS, repos)
-      packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages)
-      additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
-        self.ADDITIONAL_PACKAGES, installedPackages)
-      allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
-      dict['installedPackages'] = packages_analyzer.getPackageDetails(installedPackages, allPackages)
-
-      repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
-      dict['existingRepos'] = repos
-
       self.reportFileHandler.writeHostCheckFile(dict)
       pass
 
@@ -469,8 +409,6 @@ class HostInfoWindows(HostInfo):
     # If commands are in progress or components are already mapped to this host
     # Then do not perform certain expensive host checks
     if componentsMapped or commandsInProgress:
-      dict['existingRepos'] = [self.RESULT_UNAVAILABLE]
-      dict['installedPackages'] = []
       dict['alternatives'] = []
       dict['stackFoldersAndFiles'] = []
       dict['existingUsers'] = []

ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py

@@ -84,12 +84,6 @@ class TestHostCheckReportFileHandler(TestCase):
     procs = configValidator.get('processes', 'proc_list')
     self.assertEquals(procs, '')
 
-    pkgs = configValidator.get('packages', 'pkg_list')
-    self.assertEquals(pkgs, '')
-
-    repos = configValidator.get('repositories', 'repo_list')
-    self.assertEquals(repos, '')
-
     time = configValidator.get('metadata', 'created')
     self.assertTrue(time != None)
 
@@ -114,11 +108,6 @@ class TestHostCheckReportFileHandler(TestCase):
       {'pid':355,'hadoop':True,'command':'some command','user':'root'},
       {'pid':455,'hadoop':True,'command':'some command','user':'hdfs'}
     ]
-    dict['installedPackages'] = [
-      {'name':'hadoop','version':'3.2.3','repoName':'HDP'},
-      {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'}
-    ]
-    dict['existingRepos'] = ['HDP', 'HDP-epel']
     handler.writeHostCheckFile(dict)
 
     configValidator = ConfigParser.RawConfigParser()
@@ -140,6 +129,18 @@ class TestHostCheckReportFileHandler(TestCase):
     procs = configValidator.get('processes', 'proc_list')
     self.chkItemsEqual(procs, ['455', '355'])
 
+
+    dict['installed_packages'] = [
+      {'name':'hadoop','version':'3.2.3','repoName':'HDP'},
+      {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'}
+    ]
+    dict['existing_repos'] = ['HDP', 'HDP-epel']
+    
+    handler.writeHostChecksCustomActionsFile(dict)
+    configValidator = ConfigParser.RawConfigParser()
+    configPath_ca = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_CUSTOM_ACTIONS_FILE)
+    configValidator.read(configPath_ca)
+    
     pkgs = configValidator.get('packages', 'pkg_list')
     self.chkItemsEqual(pkgs, ['hadoop', 'hadoop-lib'])
 

ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py

@@ -112,9 +112,9 @@ class TestHostCleanup(TestCase):
     sys.stdout = sys.__stdout__
 
   class HostCleanupOptions:
-    def __init__(self, outputfile, inputfile, skip, verbose, silent, java_home):
+    def __init__(self, outputfile, inputfiles, skip, verbose, silent, java_home):
       self.outputfile = outputfile
-      self.inputfile = inputfile
+      self.inputfiles = inputfiles
       self.skip = skip
       self.verbose = verbose
       self.silent = silent
@@ -131,7 +131,9 @@ class TestHostCleanup(TestCase):
   @patch.object(optparse.OptionParser, 'parse_args')
   def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
                    set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock):
-    parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False,
+    open('/tmp/someinputfile1', 'a').close()
+    open('/tmp/someinputfile2', 'a').close()
+    parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/tmp/someinputfile1,/tmp/someinputfile2', '', False,
                                                                    False, 'java_home'), [])
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     user_root_mock.return_value = True
@@ -145,7 +147,7 @@ class TestHostCleanup(TestCase):
     #test --verbose
     logging_mock.assert_called_with(level=logging.INFO)
     # test --in
-    read_host_check_file_mock.assert_called_with('/someinputfile')
+    read_host_check_file_mock.assert_called_with('tmp_hostcheck.result')
     self.assertTrue(get_yn_input_mock.called)
 
   @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir')
@@ -179,7 +181,9 @@ class TestHostCleanup(TestCase):
   @patch.object(optparse.OptionParser, 'parse_args')
   def test_options_silent(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
                    set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock):
-    parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False,
+    open('/tmp/someinputfile1', 'a').close()
+    open('/tmp/someinputfile2', 'a').close()
+    parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/tmp/someinputfile1,/tmp/someinputfile2', '', False,
                                                                    True, 'java_home'), [])
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     user_root_mock.return_value = True
@@ -193,7 +197,7 @@ class TestHostCleanup(TestCase):
     #test --verbose
     logging_mock.assert_called_with(level=logging.INFO)
     # test --in
-    read_host_check_file_mock.assert_called_with('/someinputfile')
+    read_host_check_file_mock.assert_called_with('tmp_hostcheck.result')
     self.assertFalse(get_yn_input_mock.called)
 
   @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')

ambari-agent/src/test/python/ambari_agent/TestHostInfo.py

@@ -82,20 +82,6 @@ class TestHostInfo(TestCase):
     self.assertTrue(installedPackages[3][2], "HDP")
     self.assertTrue(installedPackages[6][1], "11-38.13.9")
 
-  @not_for_platform(PLATFORM_WINDOWS)
-  def test_getReposToRemove(self):
-    l1 = ["Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15", "Ambari 1.x", "HDP"]
-    l2 = ["Ambari", "HDP-UTIL"]
-    hostInfo = HostInfoLinux()
-    l3 = hostInfo.getReposToRemove(l1, l2)
-    self.assertTrue(1, len(l3))
-    self.assertEqual(l3[0], "HDP")
-
-    l1 = ["AMBARI.dev-1.x", "HDP-1.3.0"]
-    l3 = hostInfo.getReposToRemove(l1, l2)
-    self.assertTrue(1, len(l3))
-    self.assertEqual(l3[0], "HDP-1.3.0")
-
   @not_for_platform(PLATFORM_WINDOWS)
   def test_perform_package_analysis(self):
     installedPackages = [
@@ -286,14 +272,9 @@ class TestHostInfo(TestCase):
     dict = {}
     hostInfo.register(dict, False, False)
     self.assertTrue(cit_mock.called)
-    self.assertTrue(gir_mock.called)
-    self.assertTrue(gpd_mock.called)
-    self.assertTrue(aip_mock.called)
-    self.assertTrue(aap_mock.called)
     self.assertTrue(os_umask_mock.called)
     self.assertTrue(whcf_mock.called)
 
-    self.assertTrue(0 < len(dict['installedPackages']))
     self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth'])
 
   @not_for_platform(PLATFORM_WINDOWS)
@@ -342,25 +323,14 @@ class TestHostInfo(TestCase):
     hostInfo = HostInfoLinux()
     dict = {}
     hostInfo.register(dict, False, False)
-    self.assertTrue(gir_mock.called)
-    self.assertTrue(gpd_mock.called)
-    self.assertTrue(aip_mock.called)
     self.assertTrue(cit_mock.called)
     self.assertEqual(1, cit_mock.call_count)
 
-    for existingPkg in ["pkg1", "pkg2"]:
-      self.assertTrue(existingPkg in dict['installedPackages'])
-    args, kwargs = gpd_mock.call_args_list[0]
-    for existingPkg in ["pkg1", "pkg2"]:
-      self.assertTrue(existingPkg in args[1])
-
   def verifyReturnedValues(self, dict):
     hostInfo = HostInfoLinux()
     self.assertEqual(dict['alternatives'], [])
     self.assertEqual(dict['stackFoldersAndFiles'], [])
     self.assertEqual(dict['existingUsers'], [])
-    self.assertEqual(dict['existingRepos'][0], hostInfo.RESULT_UNAVAILABLE)
-    self.assertEqual(dict['installedPackages'], [])
     self.assertTrue(dict['iptablesIsRunning'])
 
   @patch("os.path.exists")

ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py

@@ -261,3 +261,15 @@ def getPackageDetails(installedPackages, foundPackages):
         pkgDetail['repoName'] = installedPackage[2]
     packageDetails.append(pkgDetail)
   return packageDetails
+
+def getReposToRemove(repos, ignoreList):
+  reposToRemove = []
+  for repo in repos:
+    addToRemoveList = True
+    for ignoreRepo in ignoreList:
+      if nameMatch(ignoreRepo, repo):
+        addToRemoveList = False
+        continue
+    if addToRemoveList:
+      reposToRemove.append(repo)
+  return reposToRemove

ambari-server/src/main/resources/custom_actions/scripts/check_host.py

@@ -24,16 +24,20 @@ import os
 import subprocess
 import socket
 
+from resource_management.libraries.functions import packages_analyzer
 from ambari_commons import os_utils
 from ambari_commons.os_check import OSCheck, OSConst
 from ambari_commons.inet_utils import download_file
 from resource_management import Script, Execute, format
 from ambari_agent.HostInfo import HostInfo
+from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
 
 CHECK_JAVA_HOME = "java_home_check"
 CHECK_DB_CONNECTION = "db_connection_check"
 CHECK_HOST_RESOLUTION = "host_resolution_check"
 CHECK_LAST_AGENT_ENV = "last_agent_env_check"
+CHECK_INSTALLED_PACKAGES = "installed_packages"
+CHECK_EXISTING_REPOS = "existing_repos"
 
 DB_MYSQL = "mysql"
 DB_ORACLE = "oracle"
@@ -52,9 +56,42 @@ JDBC_DRIVER_SYMLINK_MSSQL = "sqljdbc4.jar"
 JDBC_AUTH_SYMLINK_MSSQL = "sqljdbc_auth.dll"
 
 class CheckHost(Script):
+  # Packages that are used to find repos (then repos are used to find other packages)
+  PACKAGES = [
+    "hadoop_2_2_*", "hadoop-2-2-.*", "zookeeper_2_2_*", "zookeeper-2-2-.*",
+    "hadoop", "zookeeper", "webhcat", "*-manager-server-db", "*-manager-daemons"
+  ]
+  
+
+  # ignore packages from repos whose names start with these strings
+  IGNORE_PACKAGES_FROM_REPOS = [
+    "ambari", "installed"
+  ]
+  
+
+  # ignore required packages
+  IGNORE_PACKAGES = [
+    "epel-release"
+  ]
+  
+  # Additional packages to look for (search packages that start with these)
+  ADDITIONAL_PACKAGES = [
+    "rrdtool", "rrdtool-python", "ganglia", "gmond", "gweb", "libconfuse",
+    "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat"
+  ]
+  
+  # ignore repos from the list of repos to be cleaned
+  IGNORE_REPOS = [
+    "ambari", "HDP-UTILS"
+  ]
+  
+  def __init__(self):
+    self.reportFileHandler = HostCheckReportFileHandler()
+  
   def actionexecute(self, env):
     config = Script.get_config()
     tmp_dir = Script.get_tmp_dir()
+    report_file_handler_dict = {}
 
     #print "CONFIG: " + str(config)
 
@@ -93,9 +130,43 @@ class CheckHost(Script):
       except Exception, exception :
         print "There was an unknown error while checking last host environment details: " + str(exception)
         structured_output[CHECK_LAST_AGENT_ENV] = {"exit_code" : 1, "message": str(exception)}
-
+        
+    # CHECK_INSTALLED_PACKAGES and CHECK_EXISTING_REPOS required to run together for
+    # reasons of not doing the same common work twice for them as it takes some time, especially on Ubuntu.
+    if CHECK_INSTALLED_PACKAGES in check_execute_list and CHECK_EXISTING_REPOS in check_execute_list:
+      try :
+        installed_packages, repos = self.execute_existing_repos_and_installed_packages_check(config)
+        structured_output[CHECK_INSTALLED_PACKAGES] = installed_packages
+        structured_output[CHECK_EXISTING_REPOS] = repos
+      except Exception, exception :
+        print "There was an unknown error while checking installed packages and existing repositories: " + str(exception)
+        structured_output[CHECK_INSTALLED_PACKAGES] = {"exit_code" : 1, "message": str(exception)}
+        structured_output[CHECK_EXISTING_REPOS] = {"exit_code" : 1, "message": str(exception)}
+        
+    # this is necessary for HostCleanup to know later what were the results.
+    self.reportFileHandler.writeHostChecksCustomActionsFile(structured_output)
+    
     self.put_structured_out(structured_output)
 
+  def execute_existing_repos_and_installed_packages_check(self, config):
+      installedPackages = []
+      availablePackages = []
+      packages_analyzer.allInstalledPackages(installedPackages)
+      packages_analyzer.allAvailablePackages(availablePackages)
+
+      repos = []
+      packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
+                                      self.IGNORE_PACKAGES_FROM_REPOS, repos)
+      packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages)
+      additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
+        self.ADDITIONAL_PACKAGES, installedPackages)
+      allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
+      
+      installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages)
+      repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS)
+
+      return installedPackages, repos
+
 
   def execute_java_home_available_check(self, config):
     print "Java home check started."

ambari-server/src/test/python/custom_actions/TestCheckHost.py

@@ -55,7 +55,8 @@ class TestCheckHost(TestCase):
     checkHost = CheckHost()
     checkHost.actionexecute(None)
 
-    self.assertEquals(os_isfile_mock.call_args[0][0], 'test_java_home/bin/java')
+    print os_isfile_mock.call_args
+    self.assertEquals(os_isfile_mock.call_args[0][0], '/tmp/ambari-agent/hostcheck_custom_actions.result')
     self.assertEquals(structured_out_mock.call_args[0][0], {'java_home_check': {'message': 'Java home exists!',
                                                                                 'exit_code': 0}})
     # test, java home doesn't exist
@@ -64,7 +65,7 @@ class TestCheckHost(TestCase):
 
     checkHost.actionexecute(None)
 
-    self.assertEquals(os_isfile_mock.call_args[0][0], 'test_java_home/bin/java')
+    self.assertEquals(os_isfile_mock.call_args[0][0], '/tmp/ambari-agent/hostcheck_custom_actions.result')
     self.assertEquals(structured_out_mock.call_args[0][0], {'java_home_check': {"message": "Java home doesn't exist!",
                                                                                 "exit_code" : 1}})
 
@@ -282,8 +283,6 @@ class TestCheckHost(TestCase):
     self.assertTrue('alternatives' in last_agent_env_check_result['last_agent_env_check'])
     self.assertTrue('umask' in last_agent_env_check_result['last_agent_env_check'])
     self.assertTrue('stackFoldersAndFiles' in last_agent_env_check_result['last_agent_env_check'])
-    self.assertTrue('existingRepos' in last_agent_env_check_result['last_agent_env_check'])
-    self.assertTrue('installedPackages' in last_agent_env_check_result['last_agent_env_check'])
     self.assertTrue('existingUsers' in last_agent_env_check_result['last_agent_env_check'])
 
     # try it now with errors