AMBARI-20109. HDP stack - enable shuffle connection keep-alive. (Siddharth Seth via Swapan Shridhar).

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml

@@ -170,6 +170,14 @@
         </definition>
       </changes>
     </component>
+    <component name="NODEMANAGER">
+      <changes>
+        <definition xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive">
+          <type>mapred-site</type>
+          <set key="mapreduce.shuffle.connection-keep-alive.enable" value="true"/>
+        </definition>
+      </changes>
+    </component>
   </service>
 
   <service name="MAPREDUCE2">
@@ -347,6 +355,17 @@
     </component>
     </service>
 
+    <service name="TEZ">
+      <component name="TEZ_CLIENT">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive">
+            <type>tez-site</type>
+            <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/>
+          </definition>
+        </changes>
+      </component>
+    </service>
+
     <service name="PIG">
       <component name="PIG">
         <changes>
@@ -426,6 +445,11 @@
             <set key="llap_headroom_space" value="12288"/>
           </definition>
 
+          <definition xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive">
+            <type>tez-interactive-site</type>
+            <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/>
+          </definition>
+
         </changes>
 
       </component>

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml

@@ -287,6 +287,13 @@
         </task>
       </execute-stage>
 
+      <!--YARN-->
+      <execute-stage service="YARN" component="NODEMANAGER" title="Enable ShuffleHandler keep-alive">
+        <task xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive">
+          <summary>Enabling connection keep alive for the MapReduce Shuffle Handler</summary>
+        </task>
+      </execute-stage>
+
       <!--Yarn-->
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager">
         <task xsi:type="configure" id="yarn_log4j_parameterize">
@@ -334,6 +341,13 @@
         </task>
       </execute-stage>
 
+      <!--TEZ-->
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Enable Tez Client Shuffle Connectio Keep-Alive">
+        <task xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive">
+          <summary>Enabling connection keep alive for Tez shuffle client</summary>
+        </task>
+      </execute-stage>
+
       <!--OOZIE-->
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
@@ -523,6 +537,12 @@
           <summary>Updating the Hive Log4J2 properties to include parameterizations</summary>
         </task>
       </execute-stage>
+
+      <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Enable Tez-HiveInteractive Client Shuffle Connectio Keep-Alive">
+        <task xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive">
+          <summary>Enabling connection keep alive for Tez-HiveInteractive shuffle client</summary>
+        </task>
+      </execute-stage>
       
       <!-- SPARK -->
       <execute-stage service="SPARK" component="LIVY_SERVER" title="Apply config changes for Livy Server">

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml

@@ -683,6 +683,10 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive"/>
+        </pre-upgrade>
+        <pre-downgrade />
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -802,6 +806,7 @@
           <task xsi:type="configure" id="hive_log4j2_parameterize"/>
           <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/>
           <task xsi:type="configure" id="llap_update_headroom"/>
+          <task xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive"/>
         </pre-upgrade>
         
         <pre-downgrade />
@@ -898,6 +903,7 @@
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
+          <task xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive"/>
         </pre-upgrade>
 
         <pre-downgrade>

ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml

@@ -74,5 +74,11 @@
     </description>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>tez.runtime.shuffle.keep-alive.enabled</name>
+    <value>true</value>
+    <description>Enables/Disables connection keep from the client side</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
 
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml

@@ -25,4 +25,11 @@
     </description>
     <on-ambari-upgrade add="false"/>
   </property>
+  <property>
+    <name>tez.runtime.shuffle.keep-alive.enabled</name>
+    <value>true</value>
+    <description>Enables/Disables connection keep from the client side</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
+  <property>
+    <name>mapreduce.shuffle.connection-keep-alive.enable</name>
+    <value>true</value>
+    <description>Enable/Disable connection keep-alive for the MapReduce shuffle handler</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>