Pārlūkot izejas kodu

AMBARI-7739. hive command line fails to start after setting hive.execution.engine to tez in HDP 2.2.1 (alejandro)

Alejandro Fernandez 11 gadi atpakaļ
vecāks
revīzija
a0d37dcc32

+ 1 - 0
ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py

@@ -30,6 +30,7 @@ def tez():
     recursive = True
   )
 
+  # TODO, tez-site.xml needs to use versioned tez.tar.gz file to allow currently running jobs to finish during a rolling upgrade.
   XmlConfig( "tez-site.xml",
             conf_dir = params.config_dir,
             configurations = params.config['configurations']['tez-site'],

+ 7 - 6
ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml

@@ -165,24 +165,25 @@ for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`
 do
   JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
 done
-#Add libraries required by oracle connector
+# Add libraries required by oracle connector
 for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`
 do
   JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
 done
-#Add libraries required by nodemanager
+# Add libraries required by nodemanager
 MAPREDUCE_LIBS={{mapreduce_libs_path}}
 export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
 
-if [ -d "/usr/lib/tez" ]; then
-  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf
+if [[ -d "/usr/hdp/current/tez-client" &&  -d "/etc/tez/conf/" ]]; then
+  # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.
+  export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/
 fi
 
 # Setting path to hdfs command line
 export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
 
-#Mostly required for hadoop 2.0
-export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64
+# Mostly required for hadoop 2.0
+export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64
 
 #Hadoop logging options
 export HADOOP_ROOT_LOGGER={{hadoop_root_logger}}

+ 64 - 0
ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <!-- hive-env.sh -->
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for hive-env.sh file</description>
+    <value>
+ if [ "$SERVICE" = "cli" ]; then
+   if [ -z "$DEBUG" ]; then
+     export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit"
+   else
+     export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
+   fi
+ fi
+
+# The heap size of the jvm stared by hive shell script can be controlled via:
+
+export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
+
+# Larger heap size may be required when running queries over large number of files or partitions.
+# By default hive shell scripts use a heap size of 256 (MB).  Larger heap size would also be
+# appropriate for hive server (hwi etc).
+
+
+# Set HADOOP_HOME to point to a specific hadoop install directory
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+# Hive Configuration Directory can be controlled by:
+export HIVE_CONF_DIR={{hive_config_dir}}
+
+# Folder containing extra libraries required for hive compilation/execution can be controlled by:
+if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
+  export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}
+elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then
+  export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog
+fi
+
+export METASTORE_PORT={{hive_metastore_port}}
+    </value>
+  </property>
+  
+</configuration>

+ 0 - 6
ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml

@@ -26,12 +26,6 @@ limitations under the License.
     The user defined authorization class should implement interface org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider.  </description>
   </property>
 
-  <property>
-    <name>hive.aux.jars.path</name>
-    <value>file:///usr/hdp/current/hive-hcatalog/share/hcatalog</value>
-    <description>Comma-separated list of the location of the plugin jars that contain implementations of user defined functions and SerDes.</description>
-  </property>
-
   <property>
     <name>hive.cluster.delegation.token.store.class</name>
     <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value>

+ 5 - 2
ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml

@@ -21,8 +21,11 @@
 
   <property>
     <name>tez.lib.uris</name>
-    <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/,hdfs:///apps/tez/tez.tar.gz</value>
-    <description>The location of the Tez libraries which will be localized for DAGs</description>
+    <value>hdfs:///apps/tez/tez.tar.gz</value>  <!-- TODO, this needs to be a versioned file to allow currently running jobs to finish during a rolling upgrade. -->
+    <description>Comma-delimited list of the location of the Tez libraries which will be localized for DAGs.
+      Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after . and .*).
+      If multiple files are specified - files are localized as regular files, contents of directories are localized as regular files (non-recursive).
+    </description>
   </property>
 
 </configuration>