AMBARI-17120. Ambari warns the user about a bunch of rpm packages that are installed on barebone CentOS 7 (aonishuk)

ambari-agent/src/test/python/ambari_agent/TestHostInfo.py

@@ -75,52 +75,6 @@ class TestHostInfo(TestCase):
     self.assertTrue(installedPackages[3][2], "HDP")
     self.assertTrue(installedPackages[6][1], "11-38.13.9")
 
-  def test_perform_package_analysis(self):
-    installedPackages = [
-      ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"],
-      ["hadoop-b", "2.3", "HDP-epel"], ["epel", "3.1", "HDP-epel"], ["epel-2", "3.1", "HDP-epel"],
-      ["hadoop-c", "2.3", "Ambari"], ["ambari-s", "3.1", "Ambari"],
-      ["ganglia", "2.3", "GANGLIA"], ["rrd", "3.1", "RRD"],
-      ["keeper-1", "2.3", "GANGLIA"], ["keeper-2", "3.1", "base"],["def-def.x86", "2.2", "DEF.3"],
-      ["def.1", "1.2", "NewDEF"]
-    ]
-    availablePackages = [
-      ["hadoop-d", "2.3", "HDP"], ["zk-2", "3.1", "HDP"], ["pig", "3.1", "HDP"],
-      ["epel-3", "2.3", "HDP-epel"], ["hadoop-e", "3.1", "HDP-epel"],
-      ["ambari-a", "3.1", "Ambari"],
-      ["keeper-3", "3.1", "base"]
-    ]
-
-    packagesToLook = ["webhcat", "hadoop", "*-def"]
-    reposToIgnore = ["ambari"]
-    additionalPackages = ["ganglia", "rrd"]
-
-    repos = []
-    packages_analyzer.getInstalledRepos(packagesToLook, installedPackages + availablePackages, reposToIgnore, repos)
-    self.assertEqual(3, len(repos))
-    expected = ["HDP", "HDP-epel", "DEF.3"]
-    for repo in expected:
-      self.assertTrue(repo in repos)
-
-    packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages)
-    self.assertEqual(5, len(packagesInstalled))
-    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"]
-    for repo in expected:
-      self.assertTrue(repo in packagesInstalled)
-
-    additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
-        additionalPackages, installedPackages)
-    self.assertEqual(2, len(additionalPkgsInstalled))
-    expected = ["ganglia", "rrd"]
-    for additionalPkg in expected:
-      self.assertTrue(additionalPkg in additionalPkgsInstalled)
-
-    allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
-    self.assertEqual(7, len(allPackages))
-    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "ganglia", "rrd", "def-def.x86"]
-    for package in expected:
-      self.assertTrue(package in allPackages)
-
   @patch.object(OSCheck, 'get_os_family')
   @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout')
   def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock):

ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py

@@ -37,4 +37,50 @@ class TestPackagesAnalyzer(TestCase):
     checked_call_mock.return_value = (0, '0.0.1-SNAPSHOT','')
     result = packages_analyzer.getInstalledPackageVersion("package1")
     self.assertEqual(result, '0.0.1-SNAPSHOT')
-    self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)])
+    self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)])
+
+  def test_perform_package_analysis(self):
+    installedPackages = [
+      ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"],
+      ["hadoop-b", "2.3", "HDP-epel"], ["epel", "3.1", "HDP-epel"], ["epel-2", "3.1", "HDP-epel"],
+      ["hadoop-c", "2.3", "Ambari"], ["ambari-s", "3.1", "Ambari"],
+      ["ganglia", "2.3", "GANGLIA"], ["rrd", "3.1", "RRD"],
+      ["keeper-1", "2.3", "GANGLIA"], ["keeper-2", "3.1", "base"],["def-def.x86", "2.2", "DEF.3"],
+      ["def.1", "1.2", "NewDEF"]
+    ]
+    availablePackages = [
+      ["hadoop-d", "2.3", "HDP"], ["zk-2", "3.1", "HDP"], ["pig", "3.1", "HDP"],
+      ["epel-3", "2.3", "HDP-epel"], ["hadoop-e", "3.1", "HDP-epel"],
+      ["ambari-a", "3.1", "Ambari"],
+      ["keeper-3", "3.1", "base"]
+    ]
+
+    packagesToLook = ["^webhcat.*$", "^hadoop.*$", "^.+-def.*$"]
+    reposToIgnore = ["ambari"]
+    additionalPackages = ["ganglia", "rrd"]
+
+    repos = []
+    packages_analyzer.getInstalledRepos(packagesToLook, installedPackages + availablePackages, reposToIgnore, repos)
+    self.assertEqual(3, len(repos))
+    expected = ["HDP", "HDP-epel", "DEF.3"]
+    for repo in expected:
+      self.assertTrue(repo in repos)
+
+    packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages)
+    self.assertEqual(5, len(packagesInstalled))
+    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"]
+    for repo in expected:
+      self.assertTrue(repo in packagesInstalled)
+
+    additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
+      additionalPackages, installedPackages)
+    self.assertEqual(2, len(additionalPkgsInstalled))
+    expected = ["ganglia", "rrd"]
+    for additionalPkg in expected:
+      self.assertTrue(additionalPkg in additionalPkgsInstalled)
+
+    allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
+    self.assertEqual(7, len(allPackages))
+    expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "ganglia", "rrd", "def-def.x86"]
+    for package in expected:
+      self.assertTrue(package in allPackages)

ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py

@@ -18,6 +18,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import re
 import sys
 import logging
 import subprocess
@@ -191,17 +192,13 @@ def getInstalledRepos(hintPackages, allPackages, ignoreRepos, repoList):
   Gets all installed repos by name based on repos that provide any package
   contained in hintPackages
   Repos starting with value in ignoreRepos will not be returned
+  hintPackages must be regexps.
   """
   allRepos = []
   for hintPackage in hintPackages:
     for item in allPackages:
-      if 0 == item[0].find(hintPackage):
-        if not item[2] in allRepos:
-          allRepos.append(item[2])
-      elif hintPackage[0] == '*':
-        if item[0].find(hintPackage[1:]) > 0:
-          if not item[2] in allRepos:
-            allRepos.append(item[2])
+      if re.match(hintPackage, item[0]) and not item[2] in allRepos:
+        allRepos.append(item[2])
 
   for repo in allRepos:
     ignore = False

ambari-server/src/main/resources/custom_actions/scripts/check_host.py

@@ -81,10 +81,10 @@ THP_FILE_UBUNTU = "/sys/kernel/mm/transparent_hugepage/enabled"
 class CheckHost(Script):
   # Package prefixes that are used to find repos (then repos are used to find other packages)
   PACKAGES = [
-    "hadoop", "zookeeper", "webhcat", "oozie", "ambari", "*-manager-server-db",
-    "*-manager-daemons", "mahout", "spark", "falcon", "hbase", "kafka", "knox",
-    "slider", "sqoop", "storm", "flume","hcatalog", "phoenix", "ranger", "accumulo", "hive_*",
-    "pig_", "pig-", "pig." # there's a default 'pigz' package which we should avoid
+    "^hadoop.*$", "^zookeeper.*$", "^webhcat.*$", "^oozie.*$", "^ambari.*$", "^.+-manager-server-db.*$",
+    "^.+-manager-daemons.*$", "^mahout[_\-]\d.*$", "^spark.*$", "^falcon.*$", "^hbase.*$", "^kafka.*$", "^knox.*$",
+    "^slider.*$", "^sqoop.*$", "^storm.*$", "^flume.*$","^hcatalog.*$", "^phoenix.*$", "^ranger.*$", "^accumulo.*$", "^hive_.*$",
+    "^pig[_\-.].*$" # there's a default 'pigz' package which we should avoid
   ]