AMBARI-13828. Express Upgrade: Misc fixes for NameNode in Kerberized cluster, Storm PreCheck, Knox message (alejandro)

ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java

@@ -79,7 +79,7 @@ public enum CheckDescription {
       }}),
 
   STORM_REST_API_MUST_BE_DELETED(PrereqCheckType.SERVICE,
-      "The STORM_REST_API component will no longer be available and must be deleted from the cluster before upgrading. The same functionality is now provided by STORM_UI_SERVER. First, stop the component. Next, delete it using the API, e.g., curl -u $user:$password -X DELETE -H 'X-Requested-By:admin' http://$server:8080/api/v1/clusters/$name/services/STORM/components/STORM_REST_API ",
+      "The STORM_REST_API component will no longer be available and must be deleted from the cluster before upgrading. The same functionality is now provided by STORM_UI_SERVER. First, stop the entire Storm service. Next, delete STORM_REST_API using the API, e.g., curl -u $user:$password -X DELETE -H 'X-Requested-By:admin' http://$server:8080/api/v1/clusters/$name/services/STORM/components/STORM_REST_API . Finally, start Storm service.",
       new HashMap<String, String>() {{
         put(AbstractCheckDescriptor.DEFAULT, "The following component must be deleted from the cluster: {{fails}}.");
       }}),

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py

@@ -96,14 +96,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, e
       Logger.info(format("Previous file system image dir present is {is_previous_image_dir}"))
 
       if params.upgrade_direction == Direction.UPGRADE:
-        if params.dfs_ha_enabled:
-          if params.desired_namenode_role is None:
-            raise Fail("Did not receive parameter \"desired_namenode_role\" to indicate the role that this NameNode should have.")
-
-          # Both Active and Standby can use the same command
-          options = "-rollingUpgrade started"
-        else:
-          options = "-rollingUpgrade started"
+        options = "-rollingUpgrade started"
       elif params.upgrade_direction == Direction.DOWNGRADE:
         options = "-rollingUpgrade downgrade"
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py

@@ -157,8 +157,8 @@ class NameNodeDefault(NameNode):
     Logger.info("Preparing the NameNodes for a NonRolling (aka Express) Upgrade.")
 
     if params.security_enabled:
-      Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
-              user=params.hdfs_user)
+      kinit_command = format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}")
+      Execute(kinit_command, user=params.hdfs_user, logoutput=True)
 
     hdfs_binary = self.get_hdfs_binary()
     namenode_upgrade.prepare_upgrade_check_for_previous_dir()
@@ -182,6 +182,11 @@ class NameNodeDefault(NameNode):
     import params
 
     Logger.info("Wait to leafe safemode since must transition from ON to OFF.")
+
+    if params.security_enabled:
+      kinit_command = format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}")
+      Execute(kinit_command, user=params.hdfs_user, logoutput=True)
+
     try:
       hdfs_binary = self.get_hdfs_binary()
       # Note, this fails if namenode_address isn't prefixed with "params."

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml

@@ -129,7 +129,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/ ~/knox_backup" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -206,7 +206,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /var/lib/knox/data/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml

@@ -56,28 +56,6 @@
       </execute-stage>
     </group>
 
-    <!--
-    On an HDP 2.3 to 2.2 downgrade, the /etc/component/conf must be turned back from a symlink
-    into a normal directory. This will avoid the circular symbolic link that will happen after
-    hdp-select sets /usr/hdp/current/component back to 2.2:
-
-    # new symlink created for 2.3
-    /etc/flume/conf -> /usr/hdp/current/flume-server/conf
-
-    # circular reference from 2.2 (current)
-    /usr/hdp/2.2.0.0-1234/flume/conf -> /etc/flume/conf
-    /usr/hdp/current/flume-server/conf -> /etc/flume/conf
-    -->
-    <group xsi:type="cluster" name="RESTORE_CONFIG_DIRS" title="Restore Configuration Directories">
-      <direction>DOWNGRADE</direction>
-      <execute-stage title="Restore configuration directories and remove HDP 2.3 symlinks">
-        <task xsi:type="execute">
-          <script>scripts/ru_set_all.py</script>
-          <function>unlink_all_configs</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="stop" name="STOP_HIGH_LEVEL_SERVICE_COMPONENTS" title="Stop Components for High-Level Services">
       <direction>UPGRADE</direction>
       <skippable>true</skippable>
@@ -157,10 +135,9 @@
         </task>
       </execute-stage>
 
-      <!--TODO(dlysnichenko): This step is missing in 2.3->2.3+ RU pack -->
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/ ~/knox_backup" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -236,7 +213,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /var/lib/knox/data/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -247,6 +224,28 @@
       </execute-stage>
     </group>
 
+    <!--
+    On an HDP 2.3 to 2.2 downgrade, the /etc/component/conf must be turned back from a symlink
+    into a normal directory. This will avoid the circular symbolic link that will happen after
+    hdp-select sets /usr/hdp/current/component back to 2.2:
+
+    # new symlink created for 2.3
+    /etc/flume/conf -> /usr/hdp/current/flume-server/conf
+
+    # circular reference from 2.2 (current)
+    /usr/hdp/2.2.0.0-1234/flume/conf -> /etc/flume/conf
+    /usr/hdp/current/flume-server/conf -> /etc/flume/conf
+    -->
+    <group xsi:type="cluster" name="RESTORE_CONFIG_DIRS" title="Restore Configuration Directories">
+      <direction>DOWNGRADE</direction>
+      <execute-stage title="Restore configuration directories and remove HDP 2.3 symlinks">
+        <task xsi:type="execute">
+          <script>scripts/ru_set_all.py</script>
+          <function>unlink_all_configs</function>
+        </task>
+      </execute-stage>
+    </group>
+
     <!-- After processing this group, will change the effective Stack of the UpgradeContext object. -->
     <group xsi:type="update-stack" name="UPDATE_DESIRED_STACK_ID" title="Update Target Stack">
       <execute-stage title="Update Target Stack" service="" component="">

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml

@@ -142,7 +142,7 @@
       <!--TODO(dlysnichenko): This step is missing in 2.3->2.3+ RU pack -->
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/ ~/knox_backup" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -229,7 +229,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /usr/hdp/$(old_version)/knox/data/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>