AMBARI-11734 - Missing Falcon conf files after rolling upgrade (jonathanhurley)

ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py

@@ -28,3 +28,29 @@ def archive_dir(output_filename, input_dir):
       tar.add(input_dir, arcname=os.path.basename("."))
     finally:
       tar.close()
+
+
+def archive_directory_dereference(archive, directory):
+  """
+  Creates an archive of the specified directory. This will ensure that
+  symlinks are not included, but instead are followed for recursive inclusion.
+  :param archive:   the name of the archive to create, including path
+  :param directory:   the directory to include
+  :return:  None
+  """
+  tarball = None
+  try:
+    # !!! dereference must be TRUE since the conf is a symlink and we want
+    # its contents instead of the actual symlink
+    tarball = tarfile.open(archive, mode="w", dereference=True)
+
+    # tar the files, chopping off everything in front of directory
+    # /foo/bar/conf/a, /foo/bar/conf/b, /foo/bar/conf/dir/c
+    # becomes
+    # a
+    # b
+    # dir/c
+    tarball.add(directory, arcname=os.path.relpath(directory, start=directory))
+  finally:
+    if tarball:
+      tarball.close()

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py

@@ -23,6 +23,7 @@ import tempfile
 
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions import tar_archive
 
 BACKUP_TEMP_DIR = "falcon-upgrade-backup"
 BACKUP_DATA_ARCHIVE = "falcon-local-backup.tar"
@@ -51,13 +52,8 @@ def post_stop_backup():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
 
 
 def pre_start_restore():

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py

@@ -23,6 +23,7 @@ import tempfile
 
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions import tar_archive
 
 BACKUP_TEMP_DIR = "flume-upgrade-backup"
 BACKUP_CONF_DIR_ARCHIVE = "flume-conf-backup.tar"
@@ -50,13 +51,9 @@ def post_stop_backup():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
+
 
 def pre_start_restore():
   """

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py

@@ -24,6 +24,7 @@ import tempfile
 
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions import tar_archive
 
 BACKUP_TEMP_DIR = "knox-upgrade-backup"
 BACKUP_DATA_ARCHIVE = "knox-data-backup.tar"
@@ -51,15 +52,12 @@ def backup_data():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
+
   return absolute_backup_dir
 
+
 def _get_directory_mappings():
   """
   Gets a dictionary of directory to archive name that represents the

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py

@@ -30,6 +30,7 @@ from resource_management.libraries.functions import Direction
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import compare_versions
 from resource_management.libraries.functions import format_hdp_stack_version
+from resource_management.libraries.functions import tar_archive
 from resource_management.core.resources import File
 from resource_management.core.source import DownloadSource
 
@@ -59,13 +60,8 @@ def backup_configuration():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
 
 
 def restore_configuration():

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -929,6 +929,14 @@ class TestOozieServer(RMFTestCase):
     self.assertTrue(tarfile_open_mock.called)
     self.assertEqual(tarfile_open_mock.call_count,2)
 
+    # check the call args for creation of the tarfile
+    call_args = tarfile_open_mock.call_args_list[0]
+    call_argument_tarfile = call_args[0][0]
+    call_kwargs = call_args[1]
+
+    self.assertTrue("oozie-upgrade-backup/oozie-conf-backup.tar" in call_argument_tarfile)
+    self.assertEquals(True, call_kwargs["dereference"])
+
     self.assertTrue(chmod_mock.called)
     self.assertEqual(chmod_mock.call_count,1)
     chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511)

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -212,6 +212,14 @@ class TestFalconServer(RMFTestCase):
     # 4 calls to tarfile.open (2 directories * read + write)
     self.assertTrue(tarfile_open_mock.called)
     self.assertEqual(tarfile_open_mock.call_count,4)
+
+    # check the call args for creation of the tarfile
+    call_args = tarfile_open_mock.call_args_list[0]
+    call_argument_tarfile = call_args[0][0]
+    call_kwargs = call_args[1]
+
+    self.assertTrue("falcon-upgrade-backup/falcon-conf-backup.tar" in call_argument_tarfile)
+    self.assertEquals(True, call_kwargs["dereference"])
     
   @patch("resource_management.libraries.functions.security_commons.build_expectations")
   @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")