AMBARI-13644. Stop-and-Start Upgrade: Handle Knox in all upgrade paths and downgrades (alejandro)

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py

@@ -18,14 +18,20 @@ limitations under the License.
 """
 
 from resource_management import *
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_XML
+
 from hdfs_snamenode import snamenode
 from hdfs import hdfs
 from ambari_commons.os_family_impl import OsFamilyImpl
 from ambari_commons import OSConst
 
+from resource_management.core.logger import Logger
+
 class SNameNode(Script):
   def install(self, env):
     import params
@@ -61,7 +67,13 @@ class SNameNodeDefault(SNameNode):
     return {"HDP": "hadoop-hdfs-secondarynamenode"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
-    pass
+    Logger.info("Executing Stack Upgrade pre-restart")
+    import params
+    env.set_params(params)
+
+    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+      conf_select.select(params.stack_name, "hadoop", params.version)
+      hdp_select.select("hadoop-hdfs-secondarynamenode", params.version)
 
   def security_status(self, env):
     import status_params

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py

@@ -48,6 +48,9 @@ from knox_ldap import ldap
 from setup_ranger_knox import setup_ranger_knox
 
 class KnoxGateway(Script):
+  def get_stack_to_component(self):
+    return {"HDP": "knox-server"}
+
   def install(self, env):
     import params
     env.set_params(params)
@@ -104,10 +107,8 @@ class KnoxGatewayWindows(KnoxGateway):
 
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class KnoxGatewayDefault(KnoxGateway):
-  def get_stack_to_component(self):
-    return {"HDP": "knox-server"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
@@ -140,13 +141,13 @@ class KnoxGatewayDefault(KnoxGateway):
                action = "delete",
           )
 
-  def start(self, env, rolling_restart=False):
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     self.configure(env)
     daemon_cmd = format('{knox_bin} start')
     no_op_test = format('ls {knox_pid_file} >/dev/null 2>&1 && ps -p `cat {knox_pid_file}` >/dev/null 2>&1')
-    setup_ranger_knox(rolling_upgrade=rolling_restart)
+    setup_ranger_knox(upgrade_type=upgrade_type)
     # Used to setup symlink, needed to update the knox managed symlink, in case of custom locations
     if os.path.islink(params.knox_managed_pid_symlink):
       Link(params.knox_managed_pid_symlink,
@@ -159,7 +160,7 @@ class KnoxGatewayDefault(KnoxGateway):
             not_if=no_op_test
     )
 
-  def stop(self, env, rolling_restart=False):
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     daemon_cmd = format('{knox_bin} stop')

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py

@@ -19,7 +19,7 @@ limitations under the License.
 """
 from resource_management.core.logger import Logger
 
-def setup_ranger_knox(rolling_upgrade = False):
+def setup_ranger_knox(upgrade_type=None):
   import params
   
   if params.has_ranger_admin:
@@ -30,7 +30,7 @@ def setup_ranger_knox(rolling_upgrade = False):
       from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
     
     hdp_version = None
-    if rolling_upgrade:
+    if upgrade_type is not None:
       hdp_version = params.version
 
     if params.retryAble:

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py

@@ -42,6 +42,8 @@ def backup_data():
   Logger.info('Backing up Knox data directory before upgrade...')
   directoryMappings = _get_directory_mappings_during_upgrade()
 
+  Logger.info("Directory mappings to backup: {0}".format(str(directoryMappings)))
+
   absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
   if not os.path.isdir(absolute_backup_dir):
     os.makedirs(absolute_backup_dir)

ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml

@@ -197,46 +197,46 @@
 
       <!--Changes for stack 2.2-->
       <!--YARN-->
-      <execute-stage service="MAPREDUCE2" component="HISTORYSERVER">
+      <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
         <task xsi:type="configure" id="hdp_2_2_0_0_historyserver_classpath"/>
       </execute-stage>
 
 
       <!--Changes for stack 2.3-->
       <!--HDFS-->
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_modify_hadoop_env"/>
       </execute-stage>
 
 
       <!--YARN-->
-      <execute-stage service="MAPREDUCE2" component="HISTORYSERVER">
+      <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="APP_TIMELINE_SERVER">
+      <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_disable_node_labels"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_clear_default_node_label_expression"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_def_capacity"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
       </execute-stage>
 
 
       <!--HBASE-->
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_set_global_memstore_size"/>
       </execute-stage>
 
@@ -244,27 +244,27 @@
         <task xsi:type="server_action" summary="Calculating HBase Properties" class="org.apache.ambari.server.serveraction.upgrades.HBaseConfigCalculation" />
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_authorization_coprocessors"/>
       </execute-stage>
 
 
       <!--TEZ-->
-      <execute-stage service="TEZ" component="TEZ_CLIENT">
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
       </execute-stage>
 
 
       <!--HIVE-->
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="WEBHCAT_SERVER">
+      <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_env"/>
       </execute-stage>
 
@@ -274,51 +274,35 @@
 
 
       <!--OOZIE-->
-      <execute-stage service="OOZIE" component="OOZIE_SERVER">
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
       </execute-stage>
 
 
       <!--STORM-->
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_update_storm_env"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_add_storm_cluster_logs_content"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_add_storm_worker_logs_content"/>
       </execute-stage>
 
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_update_env_vars"/>
       </execute-stage>
 
     </group>
-
-    <!-- Invoke hdp-select set all after all components have been stopped by 
-    before starting new ones. This step must be done here in order to
-    ensure that components which rely on other components (like ZKFC on hadoop)
-    will start on the correct versions -->
-    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
-      <skippable>true</skippable>
-      <execute-stage title="Update stack to {{version}}">
-        <task xsi:type="execute">
-          <script>scripts/ru_set_all.py</script>
-          <function>actionexecute</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <!-- Now, restart all of the services. -->
-
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -463,6 +447,20 @@
       </service>
     </group>
 
+    <!--
+    Invoke "hdp-select set all" to change any components we may have missed
+    that are installed on the hosts but not known by Ambari.
+    -->
+    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <skippable>true</skippable>
+      <execute-stage title="Update stack to {{version}}">
+        <task xsi:type="execute">
+          <script>scripts/ru_set_all.py</script>
+          <function>actionexecute</function>
+        </task>
+      </execute-stage>
+    </group>
+
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
       <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml

@@ -232,22 +232,7 @@
       </execute-stage>
     </group>
 
-    <!-- Invoke hdp-select set all after all components have been stopped by
-    before starting new ones. This step must be done here in order to
-    ensure that components which rely on other components (like ZKFC on hadoop)
-    will start on the correct versions -->
-    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
-      <skippable>true</skippable>
-      <execute-stage title="Update stack to {{version}}">
-        <task xsi:type="execute">
-          <script>scripts/ru_set_all.py</script>
-          <function>actionexecute</function>      <!-- TODO, parallelize -->
-        </task>
-      </execute-stage>
-    </group>
-
     <!-- Now, restart all of the services. -->
-
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -460,6 +445,20 @@
       </service>
     </group>
 
+    <!--
+    Invoke "hdp-select set all" to change any components we may have missed
+    that are installed on the hosts but not known by Ambari.
+    -->
+    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <skippable>true</skippable>
+      <execute-stage title="Update stack to {{version}}">
+        <task xsi:type="execute">
+          <script>scripts/ru_set_all.py</script>
+          <function>actionexecute</function>
+        </task>
+      </execute-stage>
+    </group>
+
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
       <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml

@@ -296,66 +296,66 @@
       </execute-stage>
 
       <!--HDFS-->
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_modify_hadoop_env"/>
       </execute-stage>
 
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_adjust_ranger_plugin"/>
       </execute-stage>
 
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_policy"/>
       </execute-stage>
 
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_audit"/>
       </execute-stage>
 
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_security"/>
       </execute-stage>
 
-      <execute-stage service="HDFS" component="NAMENODE">
+      <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
         <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_ranger_hdfs_delete_old_properties"/>
       </execute-stage>
 
       <!--YARN-->
-      <execute-stage service="MAPREDUCE2" component="HISTORYSERVER">
+      <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="APP_TIMELINE_SERVER">
+      <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_disable_node_labels"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_clear_default_node_label_expression"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_def_capacity"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
       </execute-stage>
 
 
       <!--HBASE-->
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_rpc_controller_factory"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_set_global_memstore_size"/>
       </execute-stage>
 
@@ -363,141 +363,126 @@
         <task xsi:type="server_action" summary="Calculating HBase Properties" class="org.apache.ambari.server.serveraction.upgrades.HBaseConfigCalculation" />
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_indexed_wal_edit_codec"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_authorization_coprocessors"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_transition_ranger_hbase_policy"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_transition_ranger_hbase_audit"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_copy_ranger_policies"/>
       </execute-stage>
 
-      <execute-stage service="HBASE" component="HBASE_MASTER">
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_delete_old_ranger_properties"/>
       </execute-stage>
 
 
       <!--TEZ-->
-      <execute-stage service="TEZ" component="TEZ_CLIENT">
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
       </execute-stage>
 
 
       <!--HIVE-->
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_policy"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_security"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_audit"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="HIVE_SERVER">
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_remove_deprecated_ranger_properties"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="WEBHCAT_SERVER">
+      <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_env"/>
       </execute-stage>
 
-      <execute-stage service="HIVE" component="WEBHCAT_SERVER">
+      <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_configuration_paths"/>
       </execute-stage>
 
 
       <!--OOZIE-->
-      <execute-stage service="OOZIE" component="OOZIE_SERVER">
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
       </execute-stage>
 
 
       <!--KNOX-->
-      <execute-stage service="KNOX" component="KNOX_GATEWAY">
+      <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
         <task xsi:type="configure" id="hdp_2_3_0_0_knox_configure_ranger_policy"/>
       </execute-stage>
 
-      <execute-stage service="KNOX" component="KNOX_GATEWAY">
+      <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
         <task xsi:type="configure" id="hdp_2_3_0_0_knox_configure_ranger_knox_audit"/>
       </execute-stage>
 
-      <execute-stage service="KNOX" component="KNOX_GATEWAY">
+      <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
         <task xsi:type="configure" id="hdp_2_3_0_0_knox_remove_deprecated_ranger_properties"/>
       </execute-stage>
 
       <!--STORM-->
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_update_storm_env"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_add_storm_cluster_logs_content"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_add_storm_worker_logs_content"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_update_env_vars"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_configure_ranger_policy"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_configure_ranger_audit"/>
       </execute-stage>
 
-      <execute-stage service="STORM" component="NIMBUS">
+      <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
         <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_remove_deprecated_ranger_properties"/>
       </execute-stage>
 
     </group>
 
-    <!-- Invoke hdp-select set all after all components have been stopped but
-    before starting new ones. This step must be done here in order to
-    ensure that components which rely on other components (like ZKFC on hadoop)
-    will start on the correct versions -->
-    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
-      <skippable>true</skippable>
-      <execute-stage title="Update stack to {{version}}">
-        <task xsi:type="execute">
-          <script>scripts/ru_set_all.py</script>
-          <function>actionexecute</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <!-- Now, restart all of the services. -->
-
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -722,6 +707,20 @@
       </service>
     </group>
 
+    <!--
+    Invoke "hdp-select set all" to change any components we may have missed
+    that are installed on the hosts but not known by Ambari.
+    -->
+    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <skippable>true</skippable>
+      <execute-stage title="Update stack to {{version}}">
+        <task xsi:type="execute">
+          <script>scripts/ru_set_all.py</script>
+          <function>actionexecute</function>
+        </task>
+      </execute-stage>
+    </group>
+
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
       <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml

@@ -259,22 +259,7 @@
       </execute-stage>
     </group>
 
-    <!-- Invoke hdp-select set all after all components have been stopped but
-    before starting new ones. This step must be done here in order to
-    ensure that components which rely on other components (like ZKFC on hadoop)
-    will start on the correct versions -->
-    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
-      <skippable>true</skippable>
-      <execute-stage title="Update stack to {{version}}">
-        <task xsi:type="execute">
-          <script>scripts/ru_set_all.py</script>
-          <function>actionexecute</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <!-- Now, restart all of the services. -->
-
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -514,6 +499,20 @@
       </service>
     </group>
 
+    <!--
+    Invoke "hdp-select set all" to change any components we may have missed
+    that are installed on the hosts but not known by Ambari.
+    -->
+    <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <skippable>true</skippable>
+      <execute-stage title="Update stack to {{version}}">
+        <task xsi:type="execute">
+          <script>scripts/ru_set_all.py</script>
+          <function>actionexecute</function>
+        </task>
+      </execute-stage>
+    </group>
+
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
       <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py

@@ -105,7 +105,7 @@ class TestFalconClient(RMFTestCase):
 
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -122,7 +122,7 @@ class TestFalconClient(RMFTestCase):
     self.assertNoMoreResources()
 
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock):
+  def test_pre_upgrade_restart_23(self, call_mock):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -489,7 +489,7 @@ class TestFalconServer(RMFTestCase):
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
 
   @patch('os.path.isfile', new=MagicMock(return_value=True))
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json"
 
     with open(config_file, "r") as f:
@@ -511,7 +511,7 @@ class TestFalconServer(RMFTestCase):
   @patch.object(tarfile, 'open')
   @patch.object(shutil, 'rmtree')
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock):
+  def test_pre_upgrade_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock):
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json"
 
     with open(config_file, "r") as f:

ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py

@@ -226,7 +226,7 @@ class TestKnoxGateway(RMFTestCase):
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
 
   @patch("os.path.isdir")
-  def test_pre_rolling_restart(self, isdir_mock):
+  def test_pre_upgrade_restart(self, isdir_mock):
     isdir_mock.return_value = True
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
     with open(config_file, "r") as f:
@@ -236,7 +236,7 @@ class TestKnoxGateway(RMFTestCase):
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
                        classname = "KnoxGateway",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -262,7 +262,7 @@ class TestKnoxGateway(RMFTestCase):
   @patch("os.path.exists")
   @patch("os.path.isdir")
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_to_hdp_2300(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+  def test_pre_upgrade_restart_to_hdp_2300(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
     """
     In HDP 2.3.0.0, Knox was using a data dir of /var/lib/knox/data
     """
@@ -278,7 +278,7 @@ class TestKnoxGateway(RMFTestCase):
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
                        classname = "KnoxGateway",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -330,7 +330,7 @@ class TestKnoxGateway(RMFTestCase):
   @patch("os.path.exists")
   @patch("os.path.isdir")
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+  def test_pre_upgrade_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
     """
     In RU from HDP 2.3.0.0 to 2.3.2.0, should backup the data dir used by the source version, which
     is /var/lib/knox/data
@@ -350,7 +350,7 @@ class TestKnoxGateway(RMFTestCase):
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
                        classname = "KnoxGateway",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -402,7 +402,7 @@ class TestKnoxGateway(RMFTestCase):
   @patch("os.path.exists")
   @patch("os.path.isdir")
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_from_hdp_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+  def test_pre_upgrade_restart_from_hdp_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
     """
     In RU from HDP 2.3.2 to anything higher, should backup the data dir used by the source version, which
     is /var/lib/knox/data_${source_version}
@@ -422,7 +422,7 @@ class TestKnoxGateway(RMFTestCase):
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
                        classname = "KnoxGateway",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py

@@ -62,7 +62,7 @@ class TestMahoutClient(RMFTestCase):
 
     self.assertNoMoreResources()
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder() + "/test/python/stacks/2.2/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -79,7 +79,7 @@ class TestMahoutClient(RMFTestCase):
     self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'mahout-client', '2.2.1.0-3242'), sudo=True)
     self.assertNoMoreResources()
 
-  def test_pre_rolling_restart_23(self):
+  def test_pre_upgrade_restart_23(self):
     config_file = self.get_src_folder() + "/test/python/stacks/2.2/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)