AMBARI-9583. Add kerberos support for spark (Zhan Zhang via rlevas)

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml

@@ -86,4 +86,14 @@
     <value></value>
   </property>
 
+  <property>
+    <name>spark.history.kerberos.principal</name>
+    <value>none</value>
+  </property>
+
+  <property>
+    <name>spark.history.kerberos.keytab</name>
+    <value>none</value>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/kerberos.json

@@ -0,0 +1,40 @@
+{
+  "services": [
+    {
+      "name": "SPARK",
+      "identities": [
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "components": [
+        {
+          "name": "SPARK_JOBHISTORYSERVER",
+          "identities": [
+            {
+              "name": "spark_historyserver",
+              "principal": {
+                "value": "spark/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "spark-defaults/spark.history.kerberos.principal",
+                "local_username" : "${spark-env/spark_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/spark.service.keytab",
+                "owner": {
+                  "name": "${spark-env/spark_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "spark-defaults/spark.history.kerberos.keytab"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py

@@ -66,6 +66,10 @@ class JobHistoryServer(Script):
     env.set_params(params)
     self.configure(env)
 
+    if params.security_enabled:
+      spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
+      Execute(spark_kinit_cmd, user=params.spark_user)
+
     # FIXME! TODO! remove this after soft link bug is fixed:
     if not os.path.islink('/usr/hdp/current/spark'):
       hdp_version = get_hdp_version()

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py

@@ -116,3 +116,10 @@ if spark_driver_extraJavaOptions.find('-Dhdp.version') == -1:
 spark_yarn_am_extraJavaOptions = str(config['configurations']['spark-defaults']['spark.yarn.am.extraJavaOptions'])
 if spark_yarn_am_extraJavaOptions.find('-Dhdp.version') == -1:
   spark_yarn_am_extraJavaOptions = spark_yarn_am_extraJavaOptions + ' -Dhdp.version=' + str(hdp_full_version)
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+spark_kerberos_keytab =  config['configurations']['spark-defaults']['spark.history.kerberos.keytab']
+spark_kerberos_principal =  config['configurations']['spark-defaults']['spark.history.kerberos.principal']
+if security_enabled:
+  spark_principal = spark_kerberos_principal.replace('_HOST',spark_history_server_host.lower())

ambari-server/src/test/java/org/apache/ambari/server/stack/KerberosDescriptorTest.java

@@ -141,6 +141,14 @@ public class KerberosDescriptorTest {
     Assert.notNull(descriptor.getService("ZOOKEEPER"));
   }
 
+  @Test
+  public void testCommonSparkServiceDescriptor() throws IOException {
+    KerberosDescriptor descriptor = getKerberosDescriptor(commonServicesDirectory, "SPARK", "1.2.0.2.2");
+    Assert.notNull(descriptor);
+    Assert.notNull(descriptor.getServices());
+    Assert.notNull(descriptor.getService("SPARK"));
+  }
+
   private KerberosDescriptor getKerberosDescriptor(File baseDirectory, String service, String version) throws IOException {
     File serviceDirectory = new File(baseDirectory, service);
     File serviceVersionDirectory = new File(serviceDirectory, version);