AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java

@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
@@ -123,7 +124,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
    */
   @Override
   protected void executePreDMLUpdates() throws AmbariException, SQLException {
-
+    updateHiveLlapConfigs();
   }
 
   /**
@@ -201,6 +202,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     return content;
   }
 
+  protected void updateHiveLlapConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Set<String> installedServices = cluster.getServices().keySet();
+
+          if (installedServices.contains("HIVE")) {
+            Config hiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+            if (hiveSite != null) {
+              Map<String, String> hiveSiteProperties = hiveSite.getProperties();
+              String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+              if (schedulerDelay != null) {
+                // Property exists. Change to new default if set to -1.
+                if (schedulerDelay.length() != 0) {
+                  try {
+                    int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+                    if (schedulerDelayInt == -1) {
+                      // Old default. Set to new default.
+                      updateConfigurationProperties("hive-interactive-site", Collections
+                              .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+                          false);
+                    }
+                  } catch (NumberFormatException e) {
+                    // Invalid existing value. Set to new default.
+                    updateConfigurationProperties("hive-interactive-site", Collections
+                            .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+                        false);
+                  }
+                }
+              }
+              updateConfigurationProperties("hive-interactive-site",
+                  Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
+                  false);
+              updateConfigurationProperties("tez-interactive-site",
+                  Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), true,
+                  false);
+              // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since it's related to security,
+              // and we don't know if the value is set by the user or as a result of the previous default.
+            }
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Creates the servicecomponent_version table
    *

ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml

@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration supports_final="true">
+  <!--
+  Note: This file includes only those configs which are supposed to have different value from the parent hive/hive-site.
+       It inherits the other required configs from hive/hive-site.
+       The inheritance logic in order to get hive2/hive-site goes like this :
+
+       1. We read the hive/hive-site which includes the stack defaults and Stack Advisor recommended values.
+       2. We take the read hive/hive-site (step 1), and on top of it apply the hive-interactive-site to get
+          hive2/hive-site.
+
+       Therefore, any config defined here will override the config value read from hive2/hive-site (step 1).
+  -->
+  <property>
+    <name>hive.llap.task.scheduler.locality.delay</name>
+    <value>8000</value>
+    <description>
+      Amount of time to wait (in ms) before allocating a request which contains location information,
+      to a location other than the ones requested. Set to -1 for an infinite delay, 0
+      for no delay.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+  <property>
+    <name>hive.mapjoin.hybridgrace.hashtable</name>
+    <value>true</value>
+    <description>Whether to use hybrid grace hash join as the join method for mapjoin. Tez only.</description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+  <property>
+    <name>hive.llap.enable.grace.join.in.llap</name>
+    <value>false</value>
+    <description>Override if grace join should be allowed to run in llap for regular map joins.
+      Dynamic partitioned joins will honor the hive.mapjoin.hybridgrace.hashtable property in LLAP
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration supports_final="true">
+  <!--
+  Note: This file includes only those configs which are supposed to have different value from the parent tez/tez-site.
+        It inherits the other required configs from tez/tez-site.
+        The inheritance logic in order to get tez_hive2/tez-site goes like this :
+
+        1. We read the tez/tez-site which includes the stack defaults and Stack Advisor recommended values.
+        2. We take the read tez/tez-site (step 1), and on top of it apply the tez-interactive-site to get tez_hive2/tez-site.
+
+        Therefore, any config defined here will override the config value read from tez/tez-site (step 1).
+  -->
+  <property>
+    <name>tez.task.heartbeat.timeout.check-ms</name>
+    <value>11000</value>
+    <description>The time interval, in milliseconds, at which the AM will check
+      for timed out tasks
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.task.timeout-ms</name>
+    <value>20000</value>
+    <description>Amount of time the Tez AM waits before marking a task which
+      has not sent in a heartbeat, as timed out
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.am.client.heartbeat.timeout.secs</name>
+    <value>60</value>
+    <description>The time interval, after which an AM will kill itself, if it
+      does not receive a heartbeat from the client.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.am.client.heartbeat.poll.interval.millis</name>
+    <value>6000</value>
+    <description>The interval at which the AM checks for a client heartbeat
+      timeout
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.am.node-blacklisting.enabled</name>
+    <value>false</value>
+    <description>Whether to enable blacklisting in Tez AMs. Disable for LLAP
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.session.am.dag.submit.timeout.secs</name>
+    <value>1209600</value>
+    <description>The amoung of time an AM will wait, before killing itself,
+      if not DAG is submitted.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+</configuration>