AMBARI-24782. Introduce support for Ubuntu 18 LTS (aonishuk)

ambari-agent/src/main/package/dependencies.properties

@@ -29,4 +29,4 @@
 # however should be encouraged manually in pom.xml.
 
 rpm.dependency.list=openssl,\nRequires: rpm-python,\nRequires: zlib,\nRequires: python >= 2.6
-deb.dependency.list=openssl, zlibc, python (>= 2.6)
+deb.dependency.list=openssl, python (>= 2.6)

ambari-agent/src/test/python/resource_management/TestRepositoryResource.py

@@ -188,7 +188,7 @@ class TestRepositoryResource(TestCase):
     @patch.object(OSCheck, "is_suse_family")
     @patch.object(OSCheck, "is_ubuntu_family")
     @patch.object(OSCheck, "is_redhat_family")
-    @patch("resource_management.libraries.providers.repository.checked_call")
+    @patch("resource_management.libraries.providers.repository.call")
     @patch.object(tempfile, "NamedTemporaryFile")
     @patch("resource_management.libraries.providers.repository.Execute")
     @patch("resource_management.libraries.providers.repository.File")
@@ -197,13 +197,13 @@ class TestRepositoryResource(TestCase):
     @patch.object(System, "os_release_name", new='precise')        
     @patch.object(System, "os_family", new='ubuntu')
     def test_create_repo_ubuntu_repo_exists(self, file_mock, execute_mock,
-                                            tempfile_mock, checked_call_mock, is_redhat_family, is_ubuntu_family, is_suse_family):
+                                            tempfile_mock, call_mock, is_redhat_family, is_ubuntu_family, is_suse_family):
       is_redhat_family.return_value = False
       is_ubuntu_family.return_value = True
       is_suse_family.return_value = False
       tempfile_mock.return_value = MagicMock(spec=file)
       tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
-      checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD"
+      call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD"
       
       with Environment('/') as env:
         with patch.object(repository, "__file__", new='/ambari/test/repo/dummy/path/file'):
@@ -228,10 +228,10 @@ class TestRepositoryResource(TestCase):
       #'apt-get update -qq -o Dir::Etc::sourcelist="sources.list.d/HDP.list" -o APT::Get::List-Cleanup="0"')
       execute_command_item = execute_mock.call_args_list[0][0][0]
 
-      self.assertEqual(checked_call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'])
+      self.assertEqual(call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'])
       self.assertEqual(execute_command_item, ('apt-key', 'adv', '--recv-keys', '--keyserver', 'keyserver.ubuntu.com', '123ABCD'))
 
-    @patch("resource_management.libraries.providers.repository.checked_call")
+    @patch("resource_management.libraries.providers.repository.call")
     @patch.object(tempfile, "NamedTemporaryFile")
     @patch("resource_management.libraries.providers.repository.Execute")
     @patch("resource_management.libraries.providers.repository.File")
@@ -240,13 +240,13 @@ class TestRepositoryResource(TestCase):
     @patch.object(System, "os_release_name", new='precise')
     @patch.object(System, "os_family", new='ubuntu')
     def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock, execute_mock,
-                                            tempfile_mock, checked_call_mock):
+                                            tempfile_mock, call_mock):
       """
       Checks that GPG key is extracted from output without \r sign
       """
       tempfile_mock.return_value = MagicMock(spec=file)
       tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
-      checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n"
+      call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n"
 
       with Environment('/') as env:
         with patch.object(repository, "__file__", new='/ambari/test/repo/dummy/path/file'):
@@ -270,7 +270,7 @@ class TestRepositoryResource(TestCase):
       self.assertEqual(copy_item1, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
       execute_command_item = execute_mock.call_args_list[0][0][0]
 
-      self.assertEqual(checked_call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'])
+      self.assertEqual(call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'])
       self.assertEqual(execute_command_item, ('apt-key', 'adv', '--recv-keys', '--keyserver', 'keyserver.ubuntu.com', '123ABCD'))
 
     @patch.object(tempfile, "NamedTemporaryFile")

ambari-common/src/main/python/ambari_commons/resources/os_family.json

@@ -58,7 +58,8 @@
         "versions": [
           12,
           14,
-          16
+          16,
+          18
         ]
       },
       "suse": {

ambari-common/src/main/python/resource_management/libraries/providers/repository.py

@@ -31,9 +31,10 @@ from resource_management.core.source import InlineTemplate
 from resource_management.core.source import StaticFile
 from resource_management.libraries.functions.format import format
 from resource_management.core.environment import Environment
-from resource_management.core.shell import checked_call
+from resource_management.core.shell import checked_call, call
 from resource_management.core import sudo
 from resource_management.core.logger import Logger
+from resource_management.core.exceptions import ExecutionFailed
 import re
 
 REPO_TEMPLATE_FOLDER = 'data'
@@ -119,18 +120,42 @@ class UbuntuRepositoryProvider(Provider):
                content = StaticFile(tmpf.name)
           )
           
-          update_cmd_formatted = [format(x) for x in self.update_cmd]
-          # this is time expensive
-          retcode, out = checked_call(update_cmd_formatted, sudo=True, quiet=False)
-          
-          # add public keys for new repos
-          missing_pkeys = set(re.findall(self.missing_pkey_regex, out))
-          for pkey in missing_pkeys:
-            Execute(self.app_pkey_cmd_prefix + (pkey,),
-                    timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
-                    ignore_failures = True,
-                    sudo = True,
+          try:
+            self.update(repo_file_path)
+          except:
+            # remove created file or else ambari will consider that update was successful and skip repository operations
+            File(repo_file_path,
+                 action = "delete",
             )
+            raise
+
+  def update(self, repo_file_path):
+    repo_file_name = os.path.basename(repo_file_path)
+    update_cmd_formatted = [format(x) for x in self.update_cmd]
+
+    update_failed_exception = None
+
+    try:
+      # this is time expensive
+      retcode, out = call(update_cmd_formatted, sudo=True, quiet=False)
+    except ExecutionFailed as ex:
+      out = ex.out
+      update_failed_exception = ex
+
+    # add public keys for new repos
+    missing_pkeys = set(re.findall(self.missing_pkey_regex, out))
+
+    # failed but NOT due to missing pubkey
+    if update_failed_exception and not missing_pkeys:
+      raise update_failed_exception
+
+    for pkey in missing_pkeys:
+      # add public keys for new repos
+      Execute(self.app_pkey_cmd_prefix + (pkey,),
+              timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
+              ignore_failures = True,
+              sudo = True,
+      )
   
   def action_remove(self):
     repo_file_name = format("{repo_file_name}.list", repo_file_name=self.resource.repo_file_name)

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -358,8 +358,11 @@ class Script(object):
       ex.pre_raise()
       raise
     finally:
-      if self.should_expose_component_version(self.command_name):
-        self.save_component_version_to_structured_out(self.command_name)
+      try:
+        if self.should_expose_component_version(self.command_name):
+          self.save_component_version_to_structured_out(self.command_name)
+      except:
+        Logger.exception("Reporting component version failed")
 
   def get_version(self, env):
     pass

ambari-server/src/main/resources/version_definition.xsd

@@ -60,6 +60,7 @@
       <xs:enumeration value="ubuntu12" />
       <xs:enumeration value="ubuntu14" />
       <xs:enumeration value="ubuntu16" />
+      <xs:enumeration value="ubuntu18" />
       <xs:enumeration value="suse11" />
       <xs:enumeration value="suse12" />
       <xs:enumeration value="amazonlinux2" />