AMBARI-7524 Unable to start oozie server on HDP2.2 (dsen)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/configuration/oozie-env.xml

@@ -67,7 +67,7 @@
 
 if [ -d "/usr/lib/bigtop-tomcat" ]; then
   export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}
-  export CATALINA_BASE=${CATALINA_BASE:-/var/lib/oozie/oozie-server}
+  export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}
   export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}
   export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat
 fi

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/files/oozieSmoke2.sh

@@ -35,7 +35,7 @@ function checkOozieJobStatus {
   num_of_tries=${num_of_tries:-10}
   local i=0
   local rc=1
-  local cmd="source ${oozie_conf_dir}/oozie-env.sh ; /usr/bin/oozie job -oozie ${OOZIE_SERVER} -info $job_id"
+  local cmd="source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie job -oozie ${OOZIE_SERVER} -info $job_id"
   su - ${smoke_test_user} -c "$cmd"
   while [ $i -lt $num_of_tries ] ; do
     cmd_output=`su - ${smoke_test_user} -c "$cmd"`
@@ -58,11 +58,13 @@ function checkOozieJobStatus {
 }
 
 export oozie_conf_dir=$1
-export hadoop_conf_dir=$2
-export smoke_test_user=$3
-export security_enabled=$4
-export smoke_user_keytab=$5
-export kinit_path_local=$6
+export oozie_bin_dir=$2
+export hadoop_conf_dir=$3
+export hadoop_bin_dir=$4
+export smoke_test_user=$5
+export security_enabled=$6
+export smoke_user_keytab=$7
+export kinit_path_local=$8
 
 export OOZIE_EXIT_CODE=0
 export JOBTRACKER=`getValueFromField ${hadoop_conf_dir}/yarn-site.xml yarn.resourcemanager.address`
@@ -77,6 +79,9 @@ fi
   
 
 export OOZIE_EXAMPLES_DIR=`$LIST_PACKAGE_FILES_CMD oozie-client | grep 'oozie-examples.tar.gz$' | xargs dirname`
+if [[ -z "$OOZIE_EXAMPLES_DIR" ]] ; then
+  export OOZIE_EXAMPLES_DIR='/usr/hdp/current/oozie/doc/'
+fi
 cd $OOZIE_EXAMPLES_DIR
 
 tar -zxf oozie-examples.tar.gz
@@ -93,12 +98,12 @@ else
   kinitcmd=""
 fi
 
-su - ${smoke_test_user} -c "hdfs --config ${hadoop_conf_dir} dfs -rm -r examples"
-su - ${smoke_test_user} -c "hdfs --config ${hadoop_conf_dir} dfs -rm -r input-data"
-su - ${smoke_test_user} -c "hdfs --config ${hadoop_conf_dir} dfs -copyFromLocal $OOZIE_EXAMPLES_DIR/examples examples"
-su - ${smoke_test_user} -c "hdfs --config ${hadoop_conf_dir} dfs -copyFromLocal $OOZIE_EXAMPLES_DIR/examples/input-data input-data"
+su - ${smoke_test_user} -c "${hadoop_bin_dir}/hdfs --config ${hadoop_conf_dir} dfs -rm -r examples"
+su - ${smoke_test_user} -c "${hadoop_bin_dir}/hdfs --config ${hadoop_conf_dir} dfs -rm -r input-data"
+su - ${smoke_test_user} -c "${hadoop_bin_dir}/hdfs --config ${hadoop_conf_dir} dfs -copyFromLocal $OOZIE_EXAMPLES_DIR/examples examples"
+su - ${smoke_test_user} -c "${hadoop_bin_dir}/hdfs --config ${hadoop_conf_dir} dfs -copyFromLocal $OOZIE_EXAMPLES_DIR/examples/input-data input-data"
 
-cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; /usr/bin/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $OOZIE_EXAMPLES_DIR/examples/apps/map-reduce/job.properties  -run"
+cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $OOZIE_EXAMPLES_DIR/examples/apps/map-reduce/job.properties  -run"
 echo $cmd
 job_info=`su - ${smoke_test_user} -c "$cmd" | grep "job:"`
 job_id="`echo $job_info | cut -d':' -f2`"

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py

@@ -121,18 +121,18 @@ def oozie_server_specific(
     not_if="ls {pid_file} >/dev/null 2>&1 && !(ps `cat {pid_file}` >/dev/null 2>&1)"
   )
   
-  oozie_server_directorties = [params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir]            
+  oozie_server_directorties = [params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir]
   Directory( oozie_server_directorties,
     owner = params.oozie_user,
     mode = 0755,
     recursive = True
   )
 
-  cmd1 = "cd /usr/lib/oozie && tar -xvf oozie-sharelib.tar.gz"
-  cmd2 =  format("cd /usr/lib/oozie && mkdir -p {oozie_tmp_dir}")
+  cmd1 = format("cd {oozie_home} && tar -xvf oozie-sharelib.tar.gz")
+  cmd2 = format("cd {oozie_home} && mkdir -p {oozie_tmp_dir}")
   
   # this is different for HDP1
-  cmd3 = format("cd /usr/lib/oozie && chown {oozie_user}:{user_group} {oozie_tmp_dir} && mkdir -p {oozie_libext_dir} && cp {ext_js_path} {oozie_libext_dir}")
+  cmd3 = format("cd {oozie_home} && chown {oozie_user}:{user_group} {oozie_tmp_dir} && mkdir -p {oozie_libext_dir} && cp {ext_js_path} {oozie_libext_dir}")
   if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
     cmd3 += format(" && cp {jdbc_driver_jar} {oozie_libext_dir}")
   #falcon el extension

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie_service.py

@@ -27,7 +27,7 @@ def oozie_service(action = 'start'): # 'start' or 'stop'
   no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
   
   if action == 'start':
-    start_cmd = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-start.sh")
+    start_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-start.sh")
     
     if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
        params.jdbc_driver_name == "org.postgresql.Driver" or \
@@ -36,15 +36,15 @@ def oozie_service(action = 'start'): # 'start' or 'stop'
     else:
       db_connection_check_command = None
       
-    cmd1 =  format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/ooziedb.sh create -sqlfile oozie.sql -run")
+    cmd1 =  format("cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run")
     cmd2 =  format("{kinit_if_needed} {put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share")
 
     if not os.path.isfile(params.jdbc_driver_jar) and params.jdbc_driver_name == "org.postgresql.Driver":
       print "ERROR: jdbc file " + params.jdbc_driver_jar + " is unavailable. Please, follow next steps:\n" \
-        "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p /usr/lib/oozie/libserver/\n" \
+        "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \
         "3) Copy postgresql-9.0-801.jdbc4.jar to newly created dir: cp /path/to/jdbc/postgresql-9.0-801.jdbc4.jar " \
-        "/usr/lib/oozie/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \
-        "/path/to/jdbc/postgresql-9.0-801.jdbc4.jar /usr/lib/oozie/libext/\n"
+        "{oozie_home}/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \
+        "/path/to/jdbc/postgresql-9.0-801.jdbc4.jar {oozie_home}/libext/\n"
       exit(1)
 
     if db_connection_check_command:
@@ -57,8 +57,9 @@ def oozie_service(action = 'start'): # 'start' or 'stop'
     ) 
     
     Execute( cmd2,
-      user = params.oozie_user,       
-      not_if = format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'")
+      user = params.oozie_user,
+      not_if = format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'"),
+      path = params.execute_path
     )
     
     Execute( start_cmd,
@@ -66,7 +67,7 @@ def oozie_service(action = 'start'): # 'start' or 'stop'
       not_if  = no_op_test,
     )
   elif action == 'stop':
-    stop_cmd  = format("su - {oozie_user} -c  'cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-stop.sh' && rm -f {pid_file}")
+    stop_cmd  = format("su - {oozie_user} -c  'cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-stop.sh' && rm -f {pid_file}")
     Execute( stop_cmd,
       only_if  = no_op_test
     )

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 from resource_management import *
 import status_params
+import os
 
 # server configurations
 config = Script.get_config()
@@ -33,14 +34,35 @@ if rpm_version is not None:
   hadoop_bin_dir = "/usr/hdp/current/hadoop/bin"
   hadoop_lib_home = "/usr/hdp/current/hadoop/lib"
   mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce/*"
+  oozie_lib_dir = "/usr/hdp/current/oozie/"
+  oozie_setup_sh = "/usr/hdp/current/oozie/bin/oozie-setup.sh"
+  oozie_webapps_dir = "/usr/hdp/current/oozie/oozie-server/webapps"
+  oozie_webapps_conf_dir = "/usr/hdp/current/oozie/oozie-server/conf"
+  oozie_libext_dir = "/usr/hdp/current/oozie/libext"
+  oozie_server_dir = "/usr/hdp/current/oozie/oozie-server"
+  oozie_shared_lib = "/usr/hdp/current/oozie/share"
+  oozie_home = "/usr/hdp/current/oozie"
+  oozie_bin_dir = "/usr/hdp/current/oozie/bin"
+  falcon_home = '/usr/hdp/current/falcon'
 else:
   hadoop_bin_dir = "/usr/bin"
   hadoop_lib_home = "/usr/lib/hadoop/lib"
   mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+  oozie_lib_dir = "/var/lib/oozie/"
+  oozie_setup_sh = "/usr/lib/oozie/bin/oozie-setup.sh"
+  oozie_webapps_dir = "/var/lib/oozie/oozie-server/webapps/"
+  oozie_webapps_conf_dir = "/var/lib/oozie/oozie-server/conf"
+  oozie_libext_dir = "/usr/lib/oozie/libext"
+  oozie_server_dir = "/var/lib/oozie/oozie-server"
+  oozie_shared_lib = "/usr/lib/oozie/share"
+  oozie_home = "/usr/lib/oozie"
+  oozie_bin_dir = "/usr/bin"
+  falcon_home = '/usr/lib/falcon'
+
+execute_path = oozie_bin_dir + os.pathsep + hadoop_bin_dir
 
 hadoop_conf_dir = "/etc/hadoop/conf"
 conf_dir = "/etc/oozie/conf"
-
 oozie_user = config['configurations']['oozie-env']['oozie_user']
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 user_group = config['configurations']['cluster-env']['user_group']
@@ -55,7 +77,6 @@ hadoop_jar_location = "/usr/lib/hadoop/"
 hdp_stack_version = config['hostLevelParams']['stack_version']
 # for HDP1 it's "/usr/share/HDP-oozie/ext.zip"
 ext_js_path = "/usr/share/HDP-oozie/ext-2.2.zip"
-oozie_libext_dir = "/usr/lib/oozie/libext"
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 
 kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
@@ -76,10 +97,6 @@ oozie_data_dir = config['configurations']['oozie-env']['oozie_data_dir']
 oozie_server_port = get_port_from_url(config['configurations']['oozie-site']['oozie.base.url'])
 oozie_server_admin_port = config['configurations']['oozie-env']['oozie_admin_port']
 oozie_env_sh_template = config['configurations']['oozie-env']['content']
-oozie_lib_dir = "/var/lib/oozie/"
-oozie_webapps_dir = "/var/lib/oozie/oozie-server/webapps/"
-oozie_setup_sh = "/usr/lib/oozie/bin/oozie-setup.sh"
-oozie_shared_lib = "/usr/lib/oozie/share"
 fs_root = config['configurations']['core-site']['fs.defaultFS']
 
 if str(hdp_stack_version).startswith('2.0') or str(hdp_stack_version).startswith('2.1'):
@@ -103,7 +120,6 @@ hostname = config["hostname"]
 ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 falcon_host = default("/clusterHostInfo/falcon_server_hosts", [])
 has_falcon_host = not len(falcon_host)  == 0
-falcon_home = '/usr/lib/falcon'
 
 #oozie-log4j.properties
 if (('oozie-log4j' in config['configurations']) and ('content' in config['configurations']['oozie-log4j'])):

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/service_check.py

@@ -43,12 +43,13 @@ def oozie_smoke_shell_file(
   os_family = System.get_instance().os_family
   
   if params.security_enabled:
-    sh_cmd = format("{tmp_dir}/{file_name} {os_family} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local}")
+    sh_cmd = format("{tmp_dir}/{file_name} {os_family} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local}")
   else:
-    sh_cmd = format("{tmp_dir}/{file_name} {os_family} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled}")
+    sh_cmd = format("{tmp_dir}/{file_name} {os_family} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled}")
 
   Execute( format("{tmp_dir}/{file_name}"),
     command   = sh_cmd,
+    path      = params.execute_path,
     tries     = 3,
     try_sleep = 5,
     logoutput = True

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -48,6 +48,7 @@ class TestOozieServer(RMFTestCase):
     self.assertResourceCalled('Execute', ' hadoop --config /etc/hadoop/conf dfs -put /usr/lib/oozie/share /user/oozie ; hadoop --config /etc/hadoop/conf dfs -chmod -R 755 /user/oozie/share',
         not_if = " hadoop --config /etc/hadoop/conf dfs -ls /user/oozie/share | awk 'BEGIN {count=0;} /share/ {count++} END {if (count > 0) {exit 0} else {exit 1}}'",
         user = 'oozie',
+        path = ['/usr/bin:/usr/bin'],
         )
     self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh',
         not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
@@ -94,6 +95,7 @@ class TestOozieServer(RMFTestCase):
     self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/oozie.service.keytab oozie/c6402.ambari.apache.org@EXAMPLE.COM; hadoop --config /etc/hadoop/conf dfs -put /usr/lib/oozie/share /user/oozie ; hadoop --config /etc/hadoop/conf dfs -chmod -R 755 /user/oozie/share',
                               not_if = "/usr/bin/kinit -kt /etc/security/keytabs/oozie.service.keytab oozie/c6402.ambari.apache.org@EXAMPLE.COM; hadoop --config /etc/hadoop/conf dfs -ls /user/oozie/share | awk 'BEGIN {count=0;} /share/ {count++} END {if (count > 0) {exit 0} else {exit 1}}'",
                               user = 'oozie',
+                              path = ['/usr/bin:/usr/bin'],
                               )
     self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh',
                               not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
@@ -202,6 +204,11 @@ class TestOozieServer(RMFTestCase):
                               recursive = True,
                               mode = 0755,
                               )
+    self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server/conf',
+                              owner = 'oozie',
+                              recursive = True,
+                              mode = 0755,
+                              )
     self.assertResourceCalled('Execute', 'cd /usr/lib/oozie && tar -xvf oozie-sharelib.tar.gz',
                               not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
                               )
@@ -307,6 +314,11 @@ class TestOozieServer(RMFTestCase):
                               recursive = True,
                               mode = 0755,
                               )
+    self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server/conf',
+                              owner = 'oozie',
+                              recursive = True,
+                              mode = 0755,
+                              )
     self.assertResourceCalled('Execute', 'cd /usr/lib/oozie && tar -xvf oozie-sharelib.tar.gz',
                               not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
                               )

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py

@@ -51,6 +51,7 @@ class TestServiceCheck(RMFTestCase):
     self.assertResourceCalled('Execute', '/tmp/oozieSmoke2.sh',
         logoutput = True,
         tries = 3,
-        command = '/tmp/oozieSmoke2.sh suse /etc/oozie/conf /etc/hadoop/conf ambari-qa False',
+        command = '/tmp/oozieSmoke2.sh suse /etc/oozie/conf /usr/bin /etc/hadoop/conf /usr/bin ambari-qa False',
+        path = ['/usr/bin:/usr/bin'],
         try_sleep = 5,
     )