AMBARI-18842: Provide support for removing an mpack on the command line (jluniya)

ambari-server/sbin/ambari-server

@@ -181,13 +181,17 @@ case "${1:-}" in
         echo -e "Installing management pack"
         $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@
         ;;
+  uninstall-mpack)
+        echo -e "Uninstalling management pack"
+        $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@
+        ;;
   upgrade-mpack)
         echo -e "Upgrading management pack"
         $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@
         ;;
   *)
         echo "Usage: $AMBARI_EXECUTABLE
-        {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-cleanup|install-mpack|upgrade-mpack} [options]
+        {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-cleanup|install-mpack|uninstall-mpack|upgrade-mpack} [options]
         Use $AMBARI_PYTHON_EXECUTABLE <action> --help to get details on options available.
         Or, simply invoke ambari-server.py --help to print the options."
         exit 1

ambari-server/src/main/python/ambari-server.py

@@ -39,7 +39,7 @@ from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
 from ambari_server.serverSetup import reset, setup, setup_jce_policy
 from ambari_server.serverUpgrade import upgrade, upgrade_stack, set_current
 from ambari_server.setupHttps import setup_https, setup_truststore
-from ambari_server.setupMpacks import install_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \
+from ambari_server.setupMpacks import install_mpack, uninstall_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \
   SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME
 from ambari_server.setupSso import setup_sso
 from ambari_server.dbCleanup import db_cleanup
@@ -52,7 +52,7 @@ from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SY
   SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \
   UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \
   UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \
-  DB_CLEANUP_ACTION, INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION
+  DB_CLEANUP_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION
 from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam
 from ambari_server.userInput import get_validated_string_input
 
@@ -313,6 +313,35 @@ def restore(args):
 
   BackupRestore_main(restore_command)
 
+_action_option_dependence_map = {
+}
+
+def add_parser_options(*args, **kwargs):
+  required_for_actions = kwargs.pop("required_for_actions", [])
+  optional_for_actions = kwargs.pop("optional_for_actions", [])
+  parser = kwargs.pop("parser")
+  for action in required_for_actions:
+    if not action in _action_option_dependence_map:
+      _action_option_dependence_map[action] = ([], [])
+    _action_option_dependence_map[action][0].append((args[0], kwargs["dest"]))
+  for action in optional_for_actions:
+    if not action in _action_option_dependence_map:
+      _action_option_dependence_map[action] = ([], [])
+    _action_option_dependence_map[action][1].append((args[0], kwargs["dest"]))
+  parser.add_option(*args, **kwargs)
+
+def print_action_arguments_help(action):
+    if action in _action_option_dependence_map:
+      required_options = _action_option_dependence_map[action][0]
+    optional_options = _action_option_dependence_map[action][1]
+    if required_options or optional_options:
+      print "Options used by action {0}:".format(action)
+    if required_options:
+      print "  required:{0}".format(
+          ";".join([print_opt for print_opt, _ in required_options]))
+    if optional_options:
+      print "  optional:{0}".format(
+            ";".join([print_opt for print_opt, _ in optional_options]))
 
 @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
 def init_parser_options(parser):
@@ -358,18 +387,45 @@ def init_parser_options(parser):
                     help="Specifies the path to the JDBC driver JAR file")
   parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation")
   parser.add_option('--skip-database-check', action="store_true", default=False, help="Skip database consistency check", dest="skip_database_check")
-  parser.add_option('--mpack', default=None,
-                    help="Specified the path for management pack to be installed/upgraded",
-                    dest="mpack_path")
-  parser.add_option('--purge', action="store_true", default=False,
-                    help="Purge existing resources specified in purge-list",
-                    dest="purge")
+  add_parser_options('--mpack',
+      default=None,
+      help="Specify the path for management pack to be installed/upgraded",
+      dest="mpack_path",
+      parser=parser,
+      required_for_actions=[INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION]
+  )
+  add_parser_options('--mpack-name',
+      default=None,
+      help="Specify the management pack name to be uninstalled",
+      dest="mpack_name",
+      parser=parser,
+      required_for_actions=[UNINSTALL_MPACK_ACTION]
+  )
+  add_parser_options('--purge',
+      action="store_true",
+      default=False,
+      help="Purge existing resources specified in purge-list",
+      dest="purge",
+      parser=parser,
+      optional_for_actions=[INSTALL_MPACK_ACTION]
+  )
   purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
   default_purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
-  parser.add_option('--purge-list', default=default_purge_resources,
-                    help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources),
-                    dest="purge_list")
-  parser.add_option('--force', action="store_true", default=False, help="Force install management pack", dest="force")
+  add_parser_options('--purge-list',
+      default=default_purge_resources,
+      help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources),
+      dest="purge_list",
+      parser=parser,
+      optional_for_actions=[INSTALL_MPACK_ACTION]
+  )
+  add_parser_options('--force',
+      action="store_true",
+      default=False,
+      help="Force install management pack",
+      dest="force",
+      parser=parser,
+      optional_for_actions=[INSTALL_MPACK_ACTION]
+  )
   # -b and -i the remaining available short options
   # -h reserved for help
 
@@ -438,18 +494,45 @@ def init_parser_options(parser):
   parser.add_option('--stack', dest="stack_name", default=None, type="string",
                     help="Specify stack name for the stack versions that needs to be enabled")
   parser.add_option("-d", "--from-date", dest="cleanup_from_date", default=None, type="string", help="Specify date for the cleanup process in 'yyyy-MM-dd' format")
-  parser.add_option('--mpack', default=None,
-                    help="Specified the path for management pack to be installed/upgraded",
-                    dest="mpack_path")
-  parser.add_option('--purge', action="store_true", default=False,
-                    help="Purge existing resources specified in purge-list",
-                    dest="purge")
+  add_parser_options('--mpack',
+      default=None,
+      help="Specify the path for management pack to be installed/upgraded",
+      dest="mpack_path",
+      parser=parser,
+      required_for_actions=[INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION]
+  )
+  add_parser_options('--mpack-name',
+      default=None,
+      help="Specify the management pack name to be uninstalled",
+      dest="mpack_name",
+      parser=parser,
+      required_for_actions=[UNINSTALL_MPACK_ACTION]
+  )
+  add_parser_options('--purge',
+      action="store_true",
+      default=False,
+      help="Purge existing resources specified in purge-list",
+      dest="purge",
+      parser=parser,
+      optional_for_actions=[INSTALL_MPACK_ACTION]
+  )
   purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
   default_purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
-  parser.add_option('--purge-list', default=default_purge_resources,
-                    help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources),
-                    dest="purge_list")
-  parser.add_option('--force', action="store_true", default=False, help="Force install management pack", dest="force")
+  add_parser_options('--purge-list',
+      default=default_purge_resources,
+      help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources),
+      dest="purge_list",
+      parser=parser,
+      optional_for_actions=[INSTALL_MPACK_ACTION]
+  )
+  add_parser_options('--force',
+      action="store_true",
+      default=False,
+      help="Force install management pack",
+      dest="force",
+      parser=parser,
+      optional_for_actions=[INSTALL_MPACK_ACTION]
+  )
 
   parser.add_option('--ldap-url', default=None, help="Primary url for LDAP", dest="ldap_url")
   parser.add_option('--ldap-secondary-url', default=None, help="Secondary url for LDAP", dest="ldap_secondary_url")
@@ -619,6 +702,7 @@ def create_user_action_map(args, options):
     REFRESH_STACK_HASH_ACTION: UserAction(refresh_stack_hash_action),
     SETUP_SSO_ACTION: UserActionRestart(setup_sso, options),
     INSTALL_MPACK_ACTION: UserAction(install_mpack, options),
+    UNINSTALL_MPACK_ACTION: UserAction(uninstall_mpack, options),
     UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options)
   }
   return action_map
@@ -648,6 +732,7 @@ def create_user_action_map(args, options):
         SETUP_SSO_ACTION: UserActionRestart(setup_sso, options),
         DB_CLEANUP_ACTION: UserAction(db_cleanup, options),
         INSTALL_MPACK_ACTION: UserAction(install_mpack, options),
+        UNINSTALL_MPACK_ACTION: UserAction(uninstall_mpack, options),
         UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options),
         PAM_SETUP_ACTION: UserAction(setup_pam)
       }
@@ -740,6 +825,14 @@ def main(options, args, parser):
   options.exit_code = None
 
   try:
+    if action in _action_option_dependence_map:
+      required, optional = _action_option_dependence_map[action]
+      for opt_str, opt_dest in required:
+        if hasattr(options, opt_dest) and getattr(options, opt_dest) is None:
+          print "Missing option {0} for action {1}".format(opt_str, action)
+          print_action_arguments_help(action)
+          print "Run ambari-server.py --help to see detailed description of each option"
+          raise FatalException(1, "Missing option")
     action_obj.execute()
 
     if action_obj.need_restart:

ambari-server/src/main/python/ambari_server/setupActions.py

@@ -45,5 +45,6 @@ SETUP_JCE_ACTION = "setup-jce"
 ENABLE_STACK_ACTION = "enable-stack"
 DB_CLEANUP_ACTION = "db-cleanup"
 INSTALL_MPACK_ACTION = "install-mpack"
+UNINSTALL_MPACK_ACTION = "uninstall-mpack"
 UPGRADE_MPACK_ACTION = "upgrade-mpack"
 PAM_SETUP_ACTION = "setup-pam"

ambari-server/src/main/python/ambari_server/setupMpacks.py

@@ -34,7 +34,7 @@ from ambari_server.serverConfiguration import get_ambari_properties, get_ambari_
   get_common_services_location, get_mpacks_staging_location, get_server_temp_location, get_extension_location, \
   get_java_exe_path, read_ambari_user, parse_properties_file, JDBC_DATABASE_PROPERTY, get_dashboard_location
 from ambari_server.setupSecurity import ensure_can_start_under_current_user, generate_env
-from ambari_server.setupActions import INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION
+from ambari_server.setupActions import INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION
 from ambari_server.userInput import get_YN_input
 from ambari_server.dbConfiguration import ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST
 
@@ -582,7 +582,7 @@ def search_mpacks(mpack_name, max_mpack_version=None):
           results.append((staged_mpack_name, staged_mpack_version))
   return results
 
-def uninstall_mpacks(mpack_name, max_mpack_version=None):
+def _uninstall_mpacks(mpack_name, max_mpack_version=None):
   """
   Uninstall all "mpack_name" management packs.
   If "max_mpack_version" is specified uninstall only management packs < max_mpack_version
@@ -591,9 +591,9 @@ def uninstall_mpacks(mpack_name, max_mpack_version=None):
   """
   results = search_mpacks(mpack_name, max_mpack_version)
   for result in results:
-    uninstall_mpack(result[0], result[1])
+    _uninstall_mpack(result[0], result[1])
 
-def uninstall_mpack(mpack_name, mpack_version):
+def _uninstall_mpack(mpack_name, mpack_version):
   """
   Uninstall specific management pack
   :param mpack_name: Management pack name
@@ -818,27 +818,43 @@ def get_replay_log_file():
   replay_log_file = os.path.join(mpacks_staging_location, MPACKS_REPLAY_LOG_FILENAME)
   return replay_log_file
 
-def add_replay_log(mpack_command, mpack_archive_path, purge, force, verbose):
+def add_replay_log(mpack_command, mpack_archive_path, purge, purge_list, force, verbose):
   """
   Helper function to add mpack replay log entry
   :param mpack_command: mpack command
   :param mpack_archive_path: mpack archive path (/var/lib/ambari-server/resources/mpacks/mpack.tar.gz)
   :param purge: purge command line option
+  :param purge: purge list command line option
   :param force: force command line option
   :param verbose: verbose command line option
   """
   replay_log_file = get_replay_log_file()
-  log = { 'mpack_command' : mpack_command, 'mpack_path' : mpack_archive_path, 'purge' : purge, 'force' : force, 'verbose' : verbose }
+  log = { 'mpack_command' : mpack_command, 'mpack_path' : mpack_archive_path, 'purge' : purge, 'purge_list': purge_list, 'force' : force, 'verbose' : verbose }
   with open(replay_log_file, "a") as replay_log:
     replay_log.write("{0}\n".format(log))
 
+def remove_replay_logs(mpack_name):
+  replay_log_file = get_replay_log_file()
+  logs = []
+  if os.path.exists(replay_log_file):
+    with open(replay_log_file, "r") as f:
+      for log in f:
+        log = log.strip()
+        options = _named_dict(ast.literal_eval(log))
+        (name, version) = _get_mpack_name_version(options.mpack_path)
+        if mpack_name != name:
+          logs.append(log)
+    with open(replay_log_file, "w") as replay_log:
+      for log in logs:
+        replay_log.write("{0}\n".format(log))
+
 def install_mpack(options, replay_mode=False):
-  logger.info("Install mpack.")
   """
   Install management pack
   :param options: Command line options
   :param replay_mode: Flag to indicate if executing command in replay mode
   """
+  logger.info("Install mpack.")
   # Force install when replaying logs
   if replay_mode:
     options.force = True
@@ -848,15 +864,39 @@ def install_mpack(options, replay_mode=False):
   _execute_hook(mpack_metadata, AFTER_INSTALL_HOOK_NAME, mpack_staging_dir)
 
   if not replay_mode:
-    add_replay_log(INSTALL_MPACK_ACTION, mpack_archive_path, options.purge, options.force, options.verbose)
+    add_replay_log(INSTALL_MPACK_ACTION, mpack_archive_path, options.purge, options.purge_list, options.force, options.verbose)
+
+def uninstall_mpack(options, replay_mode=False):
+  """
+  Uninstall management pack
+  :param options: Command line options
+  :param replay_mode: Flag to indicate if executing command in replay mode
+  """
+  logger.info("Uninstall mpack.")
+  mpack_name = options.mpack_name
+
+  if not mpack_name:
+    print_error_msg("Management pack name not specified!")
+    raise FatalException(-1, 'Management pack name not specified!')
+
+  results = search_mpacks(mpack_name)
+  if not results:
+    print_error_msg("No management packs found that can be uninstalled!")
+    raise FatalException(-1, 'No management packs found that can be uninstalled!')
+
+  _uninstall_mpacks(mpack_name)
+
+  print_info_msg("Management pack {0} successfully uninstalled!".format(mpack_name))
+  if not replay_mode:
+    remove_replay_logs(mpack_name)
 
 def upgrade_mpack(options, replay_mode=False):
-  logger.info("Upgrade mpack.")
   """
   Upgrade management pack
   :param options: command line options
   :param replay_mode: Flag to indicate if executing command in replay mode
   """
+  logger.info("Upgrade mpack.")
   mpack_path = options.mpack_path
   if options.purge:
     print_error_msg("Purge is not supported with upgrade_mpack action!")
@@ -879,14 +919,14 @@ def upgrade_mpack(options, replay_mode=False):
   (mpack_metadata, mpack_name, mpack_version, mpack_staging_dir, mpack_archive_path) = _install_mpack(options, replay_mode, is_upgrade=True)
 
   # Uninstall old management packs
-  uninstall_mpacks(mpack_name, mpack_version)
+  _uninstall_mpacks(mpack_name, mpack_version)
 
   # Execute post upgrade hook
   _execute_hook(mpack_metadata, AFTER_UPGRADE_HOOK_NAME, mpack_staging_dir)
 
   print_info_msg("Management pack {0}-{1} successfully upgraded!".format(mpack_name, mpack_version))
   if not replay_mode:
-    add_replay_log(UPGRADE_MPACK_ACTION, mpack_archive_path, options.purge, options.force, options.verbose)
+    add_replay_log(UPGRADE_MPACK_ACTION, mpack_archive_path, options.purge, options.purge_list, options.force, options.verbose)
 
 def replay_mpack_logs():
   """

ambari-server/src/test/python/TestMpacks.py

@@ -48,7 +48,7 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
                 serverConfiguration.search_file = _search_file
 
 from ambari_server.setupMpacks import install_mpack, upgrade_mpack, replay_mpack_logs, \
-  purge_stacks_and_mpacks, validate_purge, read_mpack_metadata, uninstall_mpack, \
+  purge_stacks_and_mpacks, validate_purge, read_mpack_metadata, _uninstall_mpack, \
   STACK_DEFINITIONS_RESOURCE_NAME, EXTENSION_DEFINITIONS_RESOURCE_NAME, \
   SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME, GRAFANA_DASHBOARDS_DIRNAME, \
   DASHBOARDS_DIRNAME, SERVICE_METRICS_DIRNAME
@@ -546,13 +546,13 @@ class TestMpacks(TestCase):
   @patch("ambari_server.setupMpacks.get_ambari_version")
   @patch("ambari_server.setupMpacks.get_ambari_properties")
   @patch("ambari_server.setupMpacks.add_replay_log")
-  @patch("ambari_server.setupMpacks.uninstall_mpack")
+  @patch("ambari_server.setupMpacks._uninstall_mpack")
   @patch("ambari_server.setupMpacks.purge_stacks_and_mpacks")
   @patch("ambari_server.setupMpacks.expand_mpack")
   @patch("ambari_server.setupMpacks.download_mpack")
   @patch("ambari_server.setupMpacks.run_os_command")
   def test_upgrade_stack_mpack(self, run_os_command_mock, download_mpack_mock, expand_mpack_mock, purge_stacks_and_mpacks_mock,
-                               uninstall_mpack_mock, add_replay_log_mock, get_ambari_properties_mock,
+                               _uninstall_mpack_mock, add_replay_log_mock, get_ambari_properties_mock,
                                get_ambari_version_mock, create_symlink_mock, os_mkdir_mock, shutil_move_mock,
                                os_path_exists_mock, create_symlink_using_path_mock):
     options = self._create_empty_options_mock()
@@ -727,7 +727,7 @@ class TestMpacks(TestCase):
     self.assertEqual(18, create_symlink_mock.call_count)
     create_symlink_using_path_mock.assert_has_calls(create_symlink_using_path_calls)
     self.assertEqual(1, create_symlink_using_path_mock.call_count)
-    uninstall_mpack_mock.assert_has_calls([call("mystack-ambari-mpack", "1.0.0.0")])
+    _uninstall_mpack_mock.assert_has_calls([call("mystack-ambari-mpack", "1.0.0.0")])
     self.assertTrue(add_replay_log_mock.called)
 
 
@@ -786,7 +786,7 @@ class TestMpacks(TestCase):
     common_services_directory = fake_configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
     dashboard_directory = fake_configs[serverConfiguration.DASHBOARD_PATH_PROPERTY]
 
-    uninstall_mpack("mystack-ambari-mpack", "1.0.0.1")
+    _uninstall_mpack("mystack-ambari-mpack", "1.0.0.1")
 
     self.assertEqual(1, sudo_rmtree_mock.call_count)
     self.assertEqual(6, sudo_unlink_mock.call_count)