AMBARI-8074. Provide validated Oozie configs for Ambari deployments. (swagle)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py

@@ -31,12 +31,12 @@ def oozie(is_server=False # TODO: see if see can remove this
                          owner=params.oozie_user,
                          mode=params.oozie_hdfs_user_mode
     )
-  Directory( params.conf_dir,
+  Directory(params.conf_dir,
              recursive = True,
              owner = params.oozie_user,
              group = params.user_group
   )
-  XmlConfig( "oozie-site.xml",
+  XmlConfig("oozie-site.xml",
     conf_dir = params.conf_dir,
     configurations = params.config['configurations']['oozie-site'],
     configuration_attributes=params.config['configuration_attributes']['oozie-site'],
@@ -90,14 +90,14 @@ def oozie(is_server=False # TODO: see if see can remove this
       not_if  = format("[ -f {check_db_connection_jar} ]"),
       environment=environment
     )
-    
-  oozie_ownership( )
+  pass
+
+  oozie_ownership()
   
   if is_server:      
-    oozie_server_specific( )
+    oozie_server_specific()
   
-def oozie_ownership(
-):
+def oozie_ownership():
   import params
   
   File ( format("{conf_dir}/hadoop-config.xml"),
@@ -120,8 +120,7 @@ def oozie_ownership(
     group = params.user_group
   )
   
-def oozie_server_specific(
-):
+def oozie_server_specific():
   import params
   
   File(params.pid_file,
@@ -157,4 +156,30 @@ def oozie_server_specific(
     user = params.oozie_user,
     not_if  = no_op_test
   )
+
+  if params.stack_is_hdp22_or_further:
+    # Create hive-site and tez-site configs for oozie
+    Directory(params.hive_conf_dir,
+        recursive = True,
+        owner = params.oozie_user,
+        group = params.user_group
+    )
+    if params.config['configurations']['hive-site']:
+      XmlConfig("hive-site.xml",
+        conf_dir=params.hive_conf_dir,
+        configurations=params.config['configurations']['hive-site'],
+        configuration_attributes=params.config['configuration_attributes']['hive-site'],
+        owner=params.oozie_user,
+        group=params.user_group,
+        mode=0644)
+    if params.config['configurations']['tez-site']:
+      XmlConfig( "tez-site.xml",
+        conf_dir = params.hive_conf_dir,
+        configurations = params.config['configurations']['tez-site'],
+        configuration_attributes=params.config['configuration_attributes']['tez-site'],
+        owner = params.oozie_user,
+        group = params.user_group,
+        mode = 0664
+  )
+  pass
   

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py

@@ -63,6 +63,7 @@ execute_path = oozie_bin_dir + os.pathsep + hadoop_bin_dir
 
 hadoop_conf_dir = "/etc/hadoop/conf"
 conf_dir = "/etc/oozie/conf"
+hive_conf_dir = "/etc/oozie/conf/action-conf/hive"
 oozie_user = config['configurations']['oozie-env']['oozie_user']
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 user_group = config['configurations']['cluster-env']['user_group']

ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/configuration/oozie-env.xml

@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <!-- oozie-env.sh -->
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for oozie-env.sh file</description>
+    <value>
+#!/bin/bash
+
+if [ -d "/usr/lib/bigtop-tomcat" ]; then
+  export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}
+  export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}
+  export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}
+  export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat
+fi
+
+#Set JAVA HOME
+export JAVA_HOME={{java_home}}
+
+export JRE_HOME=${JAVA_HOME}
+
+# Set Oozie specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs Oozie
+# Java System properties for Oozie should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# Oozie configuration file to load from Oozie configuration directory
+#
+# export OOZIE_CONFIG_FILE=oozie-site.xml
+
+# Oozie logs directory
+#
+export OOZIE_LOG={{oozie_log_dir}}
+
+# Oozie pid directory
+#
+export CATALINA_PID={{pid_file}}
+
+#Location of the data for oozie
+export OOZIE_DATA={{oozie_data_dir}}
+
+# Oozie Log4J configuration file to load from Oozie configuration directory
+#
+# export OOZIE_LOG4J_FILE=oozie-log4j.properties
+
+# Reload interval of the Log4J configuration file, in seconds
+#
+# export OOZIE_LOG4J_RELOAD=10
+
+# The port Oozie server runs
+#
+export OOZIE_HTTP_PORT={{oozie_server_port}}
+
+# The admin port Oozie server runs
+#
+export OOZIE_ADMIN_PORT={{oozie_server_admin_port}}
+
+# The host name Oozie server runs on
+#
+# export OOZIE_HTTP_HOSTNAME=`hostname -f`
+
+# The base URL for callback URLs to Oozie
+#
+# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"
+export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64
+
+# At least 1 minute of retry time to account for server downtime during
+# upgrade/downgrade
+export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 "
+
+# This is needed so that Oozie does not run into OOM or GC Overhead limit
+# exceeded exceptions. If the oozie server is handling large number of
+# workflows/coordinator jobs, the memory settings may need to be revised
+export CATALINA_OPTS="${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m "
+    </value>
+  </property>
+
+</configuration>

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py

@@ -126,6 +126,7 @@ class TestOozieClient(RMFTestCase):
                               )
     self.assertNoMoreResources()
 
+
   def test_configure_default_hdp22(self):
     config_file = "stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -343,3 +343,40 @@ class TestOozieServer(RMFTestCase):
                               user = 'oozie',
                               )
 
+    def test_configure_default_hdp22(self):
+      config_file = "stacks/2.0.6/configs/default.json"
+      with open(config_file, "r") as f:
+        default_json = json.load(f)
+
+      default_json['hostLevelParams']['stack_version']= '2.2'
+      self.executeScript("2.0.6/services/OOZIE/package/scripts/oozie_server.py",
+                       classname = "OozieServer",
+                       command = "configure",
+                       config_file="default.json"
+      )
+      self.assert_configure_default()
+      self.assertResourceCalled('Directory', '/etc/oozie/conf/action-conf/hive',
+                              owner = 'oozie',
+                              group = 'hadoop',
+                              recursive = True
+                              )
+      self.assertResourceCalled('XmlConfig', 'hive-site',
+                                owner = 'oozie',
+                                group = 'hadoop',
+                                mode = 0664,
+                                conf_dir = '/etc/oozie/conf/action-conf/hive',
+                                configurations = self.getConfig()['configurations']['hive-site'],
+                                configuration_attributes = self.getConfig()['configuration_attributes']['hive-site']
+      )
+      self.assertResourceCalled('XmlConfig', 'tez-site',
+                                owner = 'oozie',
+                                group = 'hadoop',
+                                mode = 0664,
+                                conf_dir = '/etc/oozie/conf/action-conf/hive',
+                                configurations = self.getConfig()['configurations']['tez-site'],
+                                configuration_attributes = self.getConfig()['configuration_attributes']['tez-site']
+      )
+      self.assertNoMoreResources()
+
+
+