AMBARI-9106. RMF checks existence of hbase_2_2_* packages incorrectly (dlysnichenko)

ambari-agent/src/test/python/resource_management/TestPackageResource.py

@@ -35,7 +35,7 @@ class TestPackageResource(TestCase):
     with Environment('/') as env:
       Package("some_package",
       )
-    call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^some-package$"),
+    call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package$'"),
                                 call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'}),
                                 call(['/usr/bin/apt-get', 'update', '-qq'], logoutput=False, sudo=True)])
     
@@ -50,12 +50,48 @@ class TestPackageResource(TestCase):
     with Environment('/') as env:
       Package("some_package",
       )
-    call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^some-package$"),
+    call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package$'"),
                                 call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})])
     
     self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
 
+  @patch.object(shell, "call")
+  @patch.object(shell, "checked_call")
+  @patch.object(System, "os_family", new = 'ubuntu')
+  def test_action_install_regex_ubuntu(self, shell_mock, call_mock):
+    call_mock.side_effect = [(0, None),
+                             (0, "some-package1\nsome-package2"),
+                             (0, "Some text.\nStatus: install ok installed\nSome text"),
+                             (0, "Some text.\nStatus: not installed\nSome text"),
+                             (0, None)]
+    with Environment('/') as env:
+      Package("some_package.*",
+      )
+    call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package.*$'"),
+                                call("apt-cache --names-only search '^some-package.*$' | awk '{print $1}'"),
+                                call("dpkg --status 'some-package1'"),
+                                call("dpkg --status 'some-package2'"),
+                                call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package.*'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})])
+    self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
 
+  @patch.object(shell, "call")
+  @patch.object(shell, "checked_call")
+  @patch.object(System, "os_family", new = 'ubuntu')
+  def test_action_install_regex_installed_ubuntu(self, shell_mock, call_mock):
+    call_mock.side_effect = [(0, None),
+                             (0, "some-package1\nsome-package2"),
+                             (0, "Some text.\nStatus: install ok installed\nSome text"),
+                             (0, "Some text.\nStatus: install ok installed\nSome text"),
+                             (0, None)]
+    with Environment('/') as env:
+      Package("some_package.*",
+              )
+    call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package.*$'"),
+                                call("apt-cache --names-only search '^some-package.*$' | awk '{print $1}'"),
+                                call("dpkg --status 'some-package1'"),
+                                call("dpkg --status 'some-package2'")])
+    self.assertEqual(call_mock.call_count, 4, "Package should not be installed")
+    self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
 
   @patch.object(shell, "call")
   @patch.object(shell, "checked_call")
@@ -65,9 +101,33 @@ class TestPackageResource(TestCase):
     with Environment('/') as env:
       Package("some_package",
       )
-    call_mock.assert_called_with('installed_pkgs=`rpm -qa some_package` ; [ ! -z "$installed_pkgs" ]')
+    call_mock.assert_called_with("installed_pkgs=`rpm -qa 'some_package'` ; [ ! -z \"$installed_pkgs\" ]")
     shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package'], logoutput=False, sudo=True)
 
+  @patch.object(shell, "call")
+  @patch.object(shell, "checked_call")
+  @patch.object(System, "os_family", new = 'redhat')
+  def test_action_install_pattern_rhel(self, shell_mock, call_mock):
+    call_mock.side_effect=[(0, None), (1, "Some text")]
+    with Environment('/') as env:
+      Package("some_package*",
+      )
+    call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
+                                call("! yum list available 'some_package*'")])
+    shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package*'], logoutput=False, sudo=True)
+
+  @patch.object(shell, "call")
+  @patch.object(shell, "checked_call")
+  @patch.object(System, "os_family", new = 'redhat')
+  def test_action_install_pattern_installed_rhel(self, shell_mock, call_mock):
+    call_mock.side_effect=[(0, None), (0, "Some text")]
+    with Environment('/') as env:
+      Package("some_package*",
+      )
+    call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
+                                call("! yum list available 'some_package*'")])
+    self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
+
   @patch.object(shell, "call")
   @patch.object(shell, "checked_call")
   @patch.object(System, "os_family", new = 'suse')
@@ -76,9 +136,33 @@ class TestPackageResource(TestCase):
     with Environment('/') as env:
       Package("some_package",
       )
-    call_mock.assert_called_with('installed_pkgs=`rpm -qa some_package` ; [ ! -z "$installed_pkgs" ]')
+    call_mock.assert_called_with("installed_pkgs=`rpm -qa 'some_package'` ; [ ! -z \"$installed_pkgs\" ]")
     shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm', 'some_package'], logoutput=False, sudo=True)
 
+  @patch.object(shell, "call")
+  @patch.object(shell, "checked_call")
+  @patch.object(System, "os_family", new = 'suse')
+  def test_action_install_pattern_suse(self, shell_mock, call_mock):
+    call_mock.side_effect=[(0, None), (0, "Loading repository data...\nReading installed packages...\n\nS | Name\n--+-----\n  | Pack")]
+    with Environment('/') as env:
+      Package("some_package*",
+              )
+    call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
+                                call("zypper --non-interactive search --type package --uninstalled-only --match-exact 'some_package*'")])
+    shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm', 'some_package*'], logoutput=False, sudo=True)
+
+  @patch.object(shell, "call")
+  @patch.object(shell, "checked_call")
+  @patch.object(System, "os_family", new = 'suse')
+  def test_action_install_pattern_suse(self, shell_mock, call_mock):
+    call_mock.side_effect=[(0, None), (0, "Loading repository data...\nReading installed packages...\nNo packages found.\n")]
+    with Environment('/') as env:
+      Package("some_package*",
+              )
+    call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
+                                call("zypper --non-interactive search --type package --uninstalled-only --match-exact 'some_package*'")])
+    self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
+
   @patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
   @patch.object(shell, "checked_call")
   @patch.object(System, "os_family", new = 'redhat')

ambari-common/src/main/python/resource_management/core/providers/package/apt.py

@@ -39,7 +39,11 @@ REMOVE_CMD = {
 }
 REPO_UPDATE_CMD = ['/usr/bin/apt-get', 'update','-qq']
 
-CHECK_CMD = "dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^%s$"
+CHECK_EXISTENCE_CMD = "dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^%s$'"
+GET_PACKAGES_BY_PATTERN_CMD = "apt-cache --names-only search '^%s$' | awk '{print $1}'"
+GET_PACKAGE_STATUS_CMD = "dpkg --status '%s'"
+
+PACKAGE_INSTALLED_STATUS = 'Status: install ok installed'
 
 EMPTY_FILE = "/dev/null"
 APT_SOURCES_LIST_DIR = "/etc/apt/sources.list.d"
@@ -51,6 +55,7 @@ def replace_underscores(function_to_decorate):
     return function_to_decorate(self, name, *args[2:])
   return wrapper
 
+
 class AptProvider(PackageProvider):
 
   @replace_underscores
@@ -113,5 +118,15 @@ class AptProvider(PackageProvider):
 
   @replace_underscores
   def _check_existence(self, name):
-    code, out = shell.call(CHECK_CMD % name)
-    return not bool(code)
+    code, out = shell.call(CHECK_EXISTENCE_CMD % name)
+    if bool(code):
+      return False
+    elif '*' in name or '.' in name:  # Check if all packages matching regexp are installed
+      code1, out1 = shell.call(GET_PACKAGES_BY_PATTERN_CMD % name)
+      for package_name in out1.splitlines():
+        code2, out2 = shell.call(GET_PACKAGE_STATUS_CMD % package_name)
+        if PACKAGE_INSTALLED_STATUS not in out2.splitlines():
+          return False
+      return True
+    else:
+      return True

ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py

@@ -36,7 +36,8 @@ REMOVE_CMD = {
   False: ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'erase'],
 }
 
-CHECK_CMD = "installed_pkgs=`rpm -qa %s` ; [ ! -z \"$installed_pkgs\" ]"
+CHECK_CMD = "installed_pkgs=`rpm -qa '%s'` ; [ ! -z \"$installed_pkgs\" ]"
+CHECK_AVAILABLE_PACKAGES_CMD = "! yum list available '%s'"
 
 class YumProvider(PackageProvider):
   def install_package(self, name, use_repos=[]):
@@ -66,4 +67,10 @@ class YumProvider(PackageProvider):
     if '.' in name:  # To work with names like 'zookeeper_2_2_1_0_2072.noarch'
       name = os.path.splitext(name)[0]
     code, out = shell.call(CHECK_CMD % name)
-    return not bool(code)
+    if bool(code):
+      return False
+    elif '*' in name or '?' in name:  # Check if all packages matching pattern are installed
+      code1, out1 = shell.call(CHECK_AVAILABLE_PACKAGES_CMD % name)
+      return not bool(code1)
+    else:
+      return True

ambari-common/src/main/python/resource_management/core/providers/package/zypper.py

@@ -33,7 +33,11 @@ REMOVE_CMD = {
   True: ['/usr/bin/zypper', 'remove', '--no-confirm'],
   False: ['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm'],
 }
-CHECK_CMD = "installed_pkgs=`rpm -qa %s` ; [ ! -z \"$installed_pkgs\" ]"
+CHECK_CMD = "installed_pkgs=`rpm -qa '%s'` ; [ ! -z \"$installed_pkgs\" ]"
+GET_NOT_INSTALLED_CMD = "zypper --non-interactive search --type package --uninstalled-only --match-exact '%s'"
+
+NO_PACKAGES_FOUND_STATUS = 'No packages found.'
+
 LIST_ACTIVE_REPOS_CMD = ['/usr/bin/zypper', 'repos']
 
 def get_active_base_repos():
@@ -82,4 +86,10 @@ class ZypperProvider(PackageProvider):
 
   def _check_existence(self, name):
     code, out = shell.call(CHECK_CMD % name)
-    return not bool(code)
+    if bool(code):
+      return False
+    elif '*' in name or '?' in name:  # Check if all packages matching pattern are installed
+      code1, out1 = shell.call(GET_NOT_INSTALLED_CMD % name)
+      return NO_PACKAGES_FOUND_STATUS in out1.splitlines()
+    else:
+      return True