AMBARI-14570. Add 2.2->2.4 EU pack, fix Mahout/STS upgrade, other fixes (dlysnichenko)

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml

@@ -68,6 +68,11 @@
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
       <service-check>false</service-check>
       <parallel-scheduler/>
+      <service-check>false</service-check>
+
+      <service name="ATLAS">
+        <component>ATLAS_SERVER</component>
+      </service>
 
       <service name="FLUME">
         <component>FLUME_HANDLER</component>
@@ -79,7 +84,7 @@
         <component>ACCUMULO_TSERVER</component>
         <component>ACCUMULO_MONITOR</component>
         <component>ACCUMULO_MASTER</component>
-      </service>      
+      </service>
 
       <service name="STORM">
         <component>DRPC_SERVER</component>
@@ -106,6 +111,7 @@
 
       <service name="SPARK">
         <component>SPARK_JOBHISTORYSERVER</component>
+        <component>SPARK_THRIFTSERVER</component>
       </service>
 
       <service name="HIVE">
@@ -132,7 +138,7 @@
 
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database">
         <task xsi:type="manual">
-          <message>Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}.</message>
+          <message>Before continuing, please backup the Oozie Server database on {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -144,7 +150,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -167,6 +173,12 @@
           <message>Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Backup Ranger KMS Database">
+        <task xsi:type="manual">
+          <message>Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}.</message>
+        </task>
+      </execute-stage>
     </group>
 
     <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services">
@@ -179,6 +191,7 @@
       <service name="HBASE">
         <component>HBASE_REGIONSERVER</component>
         <component>HBASE_MASTER</component>
+        <component>PHOENIX_QUERY_SERVER</component>
       </service>
 
       <service name="HDFS">
@@ -187,6 +200,7 @@
         <component>SECONDARY_NAMENODE</component>
         <component>ZKFC</component>
         <component>JOURNALNODE</component>
+        <component>NFS_GATEWAY</component>
       </service>
 
       <service name="RANGER">
@@ -194,6 +208,10 @@
         <component>RANGER_ADMIN</component>
       </service>
 
+      <service name="RANGER_KMS">
+        <component>RANGER_KMS_SERVER</component>
+      </service>
+
       <service name="ZOOKEEPER">
         <component>ZOOKEEPER_SERVER</component>
       </service>
@@ -220,7 +238,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /var/lib/knox/data/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /usr/hdp/$(old_version)/knox/data/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -229,6 +247,12 @@
           <message>Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Restore Ranger KMS Database">
+        <task xsi:type="manual">
+          <message>Before continuing, please restore Ranger KMS database</message>
+        </task>
+      </execute-stage>
     </group>
 
     <!--
@@ -262,8 +286,8 @@
     </group>
 
     <group xsi:type="cluster" name="Upgrade service configs" title="Upgrade service configs">
-      <direction>UPGRADE</direction>    <!--  prevent config changes on downgrade -->
-      <skippable>true</skippable>       <!-- May fix configuration problems manually -->
+      <direction>UPGRADE</direction>   <!--  prevent config changes on downgrade -->
+      <skippable>true</skippable>  <!-- May fix configuration problems manually -->
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
 
       <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger">
@@ -410,7 +434,6 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
       </execute-stage>
 
-      <!-- Apply HDP 2.4 since same as that in HDP 2.3 -->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
@@ -453,7 +476,7 @@
       </execute-stage>
 
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
-        <task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_configuration_paths"/>
+        <task xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths"/>
       </execute-stage>
 
       <!--OOZIE-->
@@ -504,8 +527,9 @@
       </execute-stage>
 
       <!-- KAFKA  -->
+
       <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka">
-        <task xsi:type="configure" id ="hdp_2_3_0_0_kafka_broker_listeners"/>
+        <task xsi:type="configure" id ="hdp_2_4_0_0_kafka_broker_listeners"/>
       </execute-stage>
 
     </group>
@@ -534,6 +558,16 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="RANGER_KMS" title="Ranger KMS">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <parallel-scheduler/>
+      <service name="RANGER_KMS">
+        <component>RANGER_KMS_SERVER</component>
+      </service>
+    </group>
+
     <group xsi:type="restart" name="HDFS" title="HDFS">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -544,6 +578,7 @@
         <component>ZKFC</component>
         <component>NAMENODE</component>
         <component>SECONDARY_NAMENODE</component>
+        <component>NFS_GATEWAY</component>
         <component>HDFS_CLIENT</component>
       </service>
     </group>
@@ -607,6 +642,7 @@
         <component>HBASE_MASTER</component>
         <component>HBASE_REGIONSERVER</component>
         <component>HBASE_CLIENT</component>
+        <component>PHOENIX_QUERY_SERVER</component>
       </service>
     </group>
 
@@ -618,6 +654,10 @@
         <component>TEZ_CLIENT</component>
       </service>
 
+      <service name="MAHOUT">
+        <component>MAHOUT</component>
+      </service>
+
       <service name="PIG">
         <component>PIG</component>
       </service>
@@ -659,6 +699,7 @@
       <parallel-scheduler/>
       <service name="SPARK">
         <component>SPARK_JOBHISTORYSERVER</component>
+        <component>SPARK_THRIFTSERVER</component>
       </service>
     </group>
 
@@ -671,6 +712,16 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="ATLAS" title="Atlas">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <parallel-scheduler/>
+      <service name="ATLAS">
+        <component>ATLAS_SERVER</component>
+      </service>
+    </group>
+
     <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
     <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
       <direction>UPGRADE</direction>
@@ -787,7 +838,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-
+    
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -880,6 +931,14 @@
       </component>
     </service>
 
+    <service name="RANGER_KMS">
+      <component name="RANGER_KMS_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="HDFS">
       <component name="NAMENODE">
         <upgrade>
@@ -910,6 +969,12 @@
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
+
+      <component name="NFS_GATEWAY">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
     </service>
 
     <service name="MAPREDUCE2">
@@ -980,6 +1045,14 @@
       </component>
     </service>
 
+    <service name="MAHOUT">
+      <component name="MAHOUT">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="PIG">
       <component name="PIG">
         <upgrade>
@@ -1034,6 +1107,11 @@
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
+      <component name="SPARK_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
       <component name="SPARK_CLIENT">
         <upgrade>
           <task xsi:type="restart-task"/>
@@ -1190,6 +1268,44 @@
       </component>
     </service>
 
+    <service name="ACCUMULO">
+      <component name="ACCUMULO_MASTER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_TSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_MONITOR">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_GC">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_TRACER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="SLIDER">
       <component name="SLIDER">
         <upgrade>
@@ -1205,5 +1321,14 @@
         </upgrade>
       </component>
     </service>
+
+    <service name="ATLAS">
+      <component name="ATLAS_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+    </service>
   </processing>
 </upgrade>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml

@@ -175,7 +175,7 @@
 
       <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Backup Ranger KMS Database">
         <task xsi:type="manual">
-          <message>Before continuing, please backup Ranger KMS database</message>
+          <message>Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
     </group>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml

@@ -66,8 +66,12 @@
       <direction>UPGRADE</direction>
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <service-check>false</service-check>
       <parallel-scheduler/>
+      <service-check>false</service-check>
+
+      <service name="ATLAS">
+        <component>ATLAS_SERVER</component>
+      </service>
 
       <service name="FLUME">
         <component>FLUME_HANDLER</component>
@@ -79,7 +83,7 @@
         <component>ACCUMULO_TSERVER</component>
         <component>ACCUMULO_MONITOR</component>
         <component>ACCUMULO_MASTER</component>
-      </service>      
+      </service>
 
       <service name="STORM">
         <component>DRPC_SERVER</component>
@@ -106,6 +110,7 @@
 
       <service name="SPARK">
         <component>SPARK_JOBHISTORYSERVER</component>
+        <component>SPARK_THRIFTSERVER</component>
       </service>
 
       <service name="HIVE">
@@ -144,7 +149,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -167,6 +172,12 @@
           <message>Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Backup Ranger KMS Database">
+        <task xsi:type="manual">
+          <message>Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}.</message>
+        </task>
+      </execute-stage>
     </group>
 
     <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services">
@@ -179,6 +190,7 @@
       <service name="HBASE">
         <component>HBASE_REGIONSERVER</component>
         <component>HBASE_MASTER</component>
+        <component>PHOENIX_QUERY_SERVER</component>
       </service>
 
       <service name="HDFS">
@@ -187,6 +199,7 @@
         <component>SECONDARY_NAMENODE</component>
         <component>ZKFC</component>
         <component>JOURNALNODE</component>
+        <component>NFS_GATEWAY</component>
       </service>
 
       <service name="RANGER">
@@ -194,6 +207,10 @@
         <component>RANGER_ADMIN</component>
       </service>
 
+      <service name="RANGER_KMS">
+        <component>RANGER_KMS_SERVER</component>
+      </service>
+
       <service name="ZOOKEEPER">
         <component>ZOOKEEPER_SERVER</component>
       </service>
@@ -220,7 +237,7 @@
 
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data">
         <task xsi:type="manual">
-          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /var/lib/knox/data/" on the following host(s): {{hosts.all}}.</message>
+          <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /usr/hdp/$(old_version)/knox/data/" on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
 
@@ -229,6 +246,13 @@
           <message>Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Restore Ranger KMS Database">
+        <task xsi:type="manual">
+          <message>Before continuing, please restore Ranger KMS database</message>
+        </task>
+      </execute-stage>
+
     </group>
 
     <!-- After processing this group, will change the effective Stack of the UpgradeContext object. -->
@@ -288,6 +312,16 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="RANGER_KMS" title="Ranger KMS">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <parallel-scheduler/>
+      <service name="RANGER_KMS">
+        <component>RANGER_KMS_SERVER</component>
+      </service>
+    </group>
+
     <group xsi:type="restart" name="HDFS" title="HDFS">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -298,6 +332,7 @@
         <component>ZKFC</component>
         <component>NAMENODE</component>
         <component>SECONDARY_NAMENODE</component>
+        <component>NFS_GATEWAY</component>
         <component>HDFS_CLIENT</component>
       </service>
     </group>
@@ -361,6 +396,7 @@
         <component>HBASE_MASTER</component>
         <component>HBASE_REGIONSERVER</component>
         <component>HBASE_CLIENT</component>
+        <component>PHOENIX_QUERY_SERVER</component>
       </service>
     </group>
 
@@ -372,6 +408,10 @@
         <component>TEZ_CLIENT</component>
       </service>
 
+      <service name="MAHOUT">
+        <component>MAHOUT</component>
+      </service>
+
       <service name="PIG">
         <component>PIG</component>
       </service>
@@ -413,6 +453,7 @@
       <parallel-scheduler/>
       <service name="SPARK">
         <component>SPARK_JOBHISTORYSERVER</component>
+        <component>SPARK_THRIFTSERVER</component>
       </service>
     </group>
 
@@ -425,6 +466,16 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="ATLAS" title="Atlas">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <parallel-scheduler/>
+      <service name="ATLAS">
+        <component>ATLAS_SERVER</component>
+      </service>
+    </group>
+
     <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
     <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
       <direction>UPGRADE</direction>
@@ -541,7 +592,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-
+    
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -634,6 +685,14 @@
       </component>
     </service>
 
+    <service name="RANGER_KMS">
+      <component name="RANGER_KMS_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="HDFS">
       <component name="NAMENODE">
         <upgrade>
@@ -670,6 +729,12 @@
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
+
+      <component name="NFS_GATEWAY">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
     </service>
 
     <service name="MAPREDUCE2">
@@ -740,6 +805,14 @@
       </component>
     </service>
 
+    <service name="MAHOUT">
+      <component name="MAHOUT">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="PIG">
       <component name="PIG">
         <upgrade>
@@ -794,6 +867,11 @@
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
+      <component name="SPARK_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
       <component name="SPARK_CLIENT">
         <upgrade>
           <task xsi:type="restart-task"/>
@@ -950,6 +1028,44 @@
       </component>
     </service>
 
+    <service name="ACCUMULO">
+      <component name="ACCUMULO_MASTER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_TSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_MONITOR">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_GC">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_TRACER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+      <component name="ACCUMULO_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="SLIDER">
       <component name="SLIDER">
         <upgrade>
@@ -965,5 +1081,14 @@
         </upgrade>
       </component>
     </service>
+
+    <service name="ATLAS">
+      <component name="ATLAS_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+
+    </service>
   </processing>
 </upgrade>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml

@@ -113,7 +113,7 @@
 
       <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Pre Upgrade Ranger KMS Backup">
         <task xsi:type="manual">
-          <message>Before continuing, please backup Ranger KMS database</message>
+          <message>Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
     </group>

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml

@@ -111,6 +111,7 @@
 
       <service name="SPARK">
         <component>SPARK_JOBHISTORYSERVER</component>
+        <component>SPARK_THRIFTSERVER</component>
       </service>
 
       <service name="HIVE">
@@ -175,7 +176,7 @@
 
       <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Backup Ranger KMS Database">
         <task xsi:type="manual">
-          <message>Before continuing, please backup Ranger KMS database</message>
+          <message>Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
     </group>
@@ -852,6 +853,11 @@
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
+      <component name="SPARK_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
       <component name="SPARK_CLIENT">
         <upgrade>
           <task xsi:type="restart-task"/>

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml

@@ -111,7 +111,7 @@
 
       <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Pre Upgrade Ranger KMS Backup">
         <task xsi:type="manual">
-          <message>Before continuing, please backup Ranger KMS database</message>
+          <message>Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}.</message>
         </task>
       </execute-stage>
     </group>
@@ -672,6 +672,11 @@
           <task xsi:type="restart-task" />
         </upgrade>
       </component>
+      <component name="SPARK_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
       <component name="SPARK_CLIENT">
         <upgrade>
           <task xsi:type="restart-task" />