AMBARI-9811. Clean up spark stack (Gautam Borad via smohanty)

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py

@@ -43,15 +43,12 @@ class JobHistoryServer(Script):
   def install(self, env):
     self.install_packages(env)
     import params
-
     env.set_params(params)
-    self.configure(env)
 
   def stop(self, env, rolling_restart=False):
     import params
 
     env.set_params(params)
-    self.configure(env)
     daemon_cmd = format('{spark_history_server_stop}')
     Execute(daemon_cmd,
             user=params.spark_user,
@@ -65,18 +62,12 @@ class JobHistoryServer(Script):
     import params
 
     env.set_params(params)
-    self.configure(env)
+    setup_spark(env, 'server', action = 'start')
 
     if params.security_enabled:
       spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
       Execute(spark_kinit_cmd, user=params.spark_user)
 
-    # FIXME! TODO! remove this after soft link bug is fixed:
-    #if not os.path.islink('/usr/hdp/current/spark'):
-    #  hdp_version = get_hdp_version()
-    #  cmd = 'ln -s /usr/hdp/' + hdp_version + '/spark /usr/hdp/current/spark'
-    #  Execute(cmd)
-
     daemon_cmd = format('{spark_history_server_start}')
     no_op_test = format(
       'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
@@ -94,12 +85,12 @@ class JobHistoryServer(Script):
     # Recursively check all existing gmetad pid files
     check_process_status(pid_file)
 
-
+  # Note: This function is not called from start()/install()
   def configure(self, env):
     import params
 
     env.set_params(params)
-    setup_spark(env)
+    setup_spark(env, 'server', action = 'config')
 
 if __name__ == "__main__":
   JobHistoryServer().execute()

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py

@@ -28,7 +28,7 @@ from resource_management.core.logger import Logger
 from resource_management.core import shell
 
 
-def setup_spark(env):
+def setup_spark(env, type, action = None):
   import params
 
   env.set_params(params)
@@ -38,11 +38,13 @@ def setup_spark(env):
             group=params.user_group,
             recursive=True
   )
-  #params.HdfsDirectory(params.spark_hdfs_user_dir,
-  #                     action="create",
-  #                     owner=params.spark_user,
-  #                     mode=0775
-  #)
+  if type == 'server':
+    if action == 'start' or action == 'config':
+      params.HdfsDirectory(params.spark_hdfs_user_dir,
+                         action="create",
+                         owner=params.spark_user,
+                         mode=0775
+      )
 
   file_path = params.spark_conf + '/spark-defaults.conf'
   create_file(file_path)

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py

@@ -46,7 +46,7 @@ class SparkClient(Script):
     import params
 
     env.set_params(params)
-    setup_spark(env)
+    setup_spark(env, 'client', action = 'config')
 
   def status(self, env):
     raise ClientComponentHasNoStatus()

ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json

@@ -6,6 +6,7 @@
     "RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_ADMIN-START"],
     "RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_USERSYNC-START"],
     "SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"], 
+    "SPARK_JOBHISTORYSERVER-START" : ["NAMENODE-START"],
     "FALCON_SERVER-START": ["NAMENODE-START", "DATANODE-START", "OOZIE_SERVER-START"],
     "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"],
     "FLUME_SERVICE_CHECK-SERVICE_CHECK": ["FLUME_HANDLER-START"],