AMBARI-9811. Spark: spark-defaults.conf file does not contains added Custom properties from UI (Gautam Borad via alejandro)

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml

@@ -86,7 +86,16 @@ SPARK_NICENESS=0
 export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
 export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}
 
-    </value>
+# The java implementation to use.
+export JAVA_HOME={{java_home}}
+
+if [ -d "/etc/tez/conf/" ]; then
+  export TEZ_CONF_DIR=/etc/tez/conf
+else
+  export TEZ_CONF_DIR=
+fi
+
+</value>
   </property>
 
 </configuration>

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml

@@ -127,6 +127,7 @@
         <config-type>spark-env</config-type>
         <config-type>spark-log4j-properties</config-type>
         <config-type>spark-metrics-properties</config-type>
+        <config-type>spark-javaopts-properties</config-type>
       </configuration-dependencies>
 
       <commandScript>

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py

@@ -45,6 +45,7 @@ stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_sta
 
 if stack_is_hdp22_or_further:
   hadoop_home = "/usr/hdp/current/hadoop-client"
+  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   spark_conf = '/etc/spark/conf'
   spark_log_dir = config['configurations']['spark-env']['spark_log_dir']
   spark_pid_dir = status_params.spark_pid_dir
@@ -63,10 +64,14 @@ else:
 
 java_home = config['hostLevelParams']['java_home']
 hadoop_conf_dir = "/etc/hadoop/conf"
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 
 spark_user = status_params.spark_user
 spark_group = status_params.spark_group
 user_group = status_params.user_group
+spark_hdfs_user_dir = format("/user/{spark_user}")
 spark_history_server_pid_file = status_params.spark_history_server_pid_file
 
 spark_history_server_start = format("{spark_home}/sbin/start-history-server.sh")
@@ -86,6 +91,7 @@ else:
 
 # spark-defaults params
 spark_yarn_historyServer_address = default(spark_history_server_host, "localhost")
+
 spark_yarn_applicationMaster_waitTries = default(
   "/configurations/spark-defaults/spark.yarn.applicationMaster.waitTries", '10')
 spark_yarn_submit_file_replication = default("/configurations/spark-defaults/spark.yarn.submit.file.replication", '3')
@@ -130,3 +136,18 @@ spark_kerberos_keytab =  config['configurations']['spark-defaults']['spark.histo
 spark_kerberos_principal =  config['configurations']['spark-defaults']['spark.history.kerberos.principal']
 if security_enabled:
   spark_principal = spark_kerberos_principal.replace('_HOST',spark_history_server_host.lower())
+
+
+
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+  HdfsDirectory,
+  conf_dir=hadoop_conf_dir,
+  hdfs_user=hdfs_principal_name if security_enabled else hdfs_user,
+  security_enabled = security_enabled,
+  keytab = hdfs_user_keytab,
+  kinit_path_local = kinit_path_local,
+  bin_dir = hadoop_bin_dir
+)

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py

@@ -38,6 +38,11 @@ def setup_spark(env):
             group=params.user_group,
             recursive=True
   )
+  params.HdfsDirectory(params.spark_hdfs_user_dir,
+                       action="create",
+                       owner=params.spark_user,
+                       mode=0775
+  )
 
   file_path = params.spark_conf + '/spark-defaults.conf'
   create_file(file_path)
@@ -101,6 +106,10 @@ def get_hive_config():
 def spark_properties(params):
   spark_dict = dict()
 
+  all_spark_config  = params.config['configurations']['spark-defaults']
+  #Add all configs unfiltered first to handle Custom case.
+  spark_dict = all_spark_config.copy()
+
   spark_dict['spark.yarn.executor.memoryOverhead'] = params.spark_yarn_executor_memoryOverhead
   spark_dict['spark.yarn.driver.memoryOverhead'] = params.spark_yarn_driver_memoryOverhead
   spark_dict['spark.yarn.applicationMaster.waitTries'] = params.spark_yarn_applicationMaster_waitTries
@@ -121,6 +130,7 @@ def spark_properties(params):
   spark_dict['spark.driver.extraJavaOptions'] = params.spark_driver_extraJavaOptions
   spark_dict['spark.yarn.am.extraJavaOptions'] = params.spark_yarn_am_extraJavaOptions
 
+
   return spark_dict
 
 
@@ -184,6 +194,7 @@ def get_hdp_version():
       'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
 
   hdp_version = re.sub('hadoop-client - ', '', hdp_output)
+  hdp_version = hdp_version.rstrip()
   match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
   if match is None: