AMBARI-7799. Add Knox kerberos setup to the existing Ambari security capabilities. (jaimin)

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/package/files/validateKnoxStatus.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import optparse
+import socket
+
+#
+# Main.
+#
+def main():
+  parser = optparse.OptionParser(usage="usage: %prog [options]")
+  parser.add_option("-p", "--port", dest="port", help="Port for Knox process")
+  parser.add_option("-n", "--hostname", dest="hostname", help="Hostname of Knox Gateway component")
+
+  (options, args) = parser.parse_args()
+  timeout_seconds = 5
+  try:
+    s = socket.create_connection((options.hostname, int(options.port)),timeout=timeout_seconds)
+    print "Successfully connected to %s on port %s" % (options.hostname, options.port)
+    s.close()
+  except socket.error, e:
+    print "Connection to %s on port %s failed: %s" % (options.hostname, options.port, e)
+    exit(1)
+
+if __name__ == "__main__":
+  main()
+

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/package/scripts/knox.py

@@ -50,8 +50,13 @@ def knox():
          owner=params.knox_user,
          content=InlineTemplate(params.topology_template)
     )
+    if params.security_enabled:
+      TemplateConfig( format("{knox_conf_dir}/krb5JAASLogin.conf"),
+                      owner = params.knox_user,
+                      template_tag = None
+      )
 
-    cmd = format('{knox_client_bin} create-master --master {knox_master_secret}')
+    cmd = format('{knox_client_bin} create-master --master {knox_master_secret!p}')
     Execute(cmd,
             user=params.knox_user,
             environment={'JAVA_HOME': params.java_home},

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/package/scripts/params.py

@@ -23,6 +23,7 @@ from resource_management import *
 import status_params
 
 config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
 stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
@@ -120,4 +121,13 @@ gateway_log4j = config['configurations']['gateway-log4j']['content']
 ldap_log4j = config['configurations']['ldap-log4j']['content']
 users_ldif = config['configurations']['users-ldif']['content']
 java_home = config['hostLevelParams']['java_home']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+if security_enabled:
+  knox_keytab_path = config['configurations']['knox-env']['knox_keytab_path']
+  _hostname_lowercase = config['hostname'].lower()
+  knox_principal_name = config['configurations']['knox-env']['knox_principal_name'].replace('_HOST',_hostname_lowercase)
+
 

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/package/scripts/service_check.py

@@ -19,7 +19,6 @@ limitations under the License.
 """
 
 from resource_management import *
-import socket
 import sys
 
 class KnoxServiceCheck(Script):
@@ -28,18 +27,32 @@ class KnoxServiceCheck(Script):
         import params
         env.set_params(params)
 
-        address = format("{knox_host_name}")
-        port = int(format("{knox_host_port}"))
-        s = socket.socket()
+        validateKnoxFileName = "validateKnoxStatus.py"
+        validateKnoxFilePath = format("{tmp_dir}/{validateKnoxFileName}")
+        python_executable = sys.executable
+        validateStatusCmd = format("{python_executable} {validateKnoxFilePath} -p {knox_host_port} -n {knox_host_name}")
+        if params.security_enabled:
+          kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
+          smoke_cmd = format("{kinit_cmd} {validateStatusCmd}")
+        else:
+          smoke_cmd = validateStatusCmd
+
         print "Test connectivity to knox server"
-        try:
-            s.connect((address, port))
-            print "Successfully connected to %s on port %s" % (address, port)
-            s.close()
-        except socket.error, e:
-            print "Connection to %s on port %s failed: %s" % (address, port, e)
-            sys.exit(1)
 
 
+        File(validateKnoxFilePath,
+          content=StaticFile(validateKnoxFileName),
+          mode=0755
+          )
+
+        Execute(smoke_cmd,
+          tries=3,
+          try_sleep=5,
+          path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+          user=params.smokeuser,
+          timeout=5,
+          logoutput=True
+        )
+
 if __name__ == "__main__":
     KnoxServiceCheck().execute()

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/package/templates/krb5JAASLogin.conf.j2

@@ -0,0 +1,30 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+com.sun.security.jgss.initiate {
+com.sun.security.auth.module.Krb5LoginModule required
+renewTGT=true
+doNotPrompt=true
+useKeyTab=true
+keyTab="{{knox_keytab_path}}"
+principal="{{knox_principal_name}}"
+isInitiator=true
+storeKey=true
+useTicketCache=true
+client=true;
+};
+

ambari-web/app/app.js

@@ -65,7 +65,7 @@ module.exports = Em.Application.create({
       return (!!ATS && !!ATS.get('minToInstall'));
     }
     return false;
-  }.property('App.router.clusterController.isLoaded'),
+  }.property('router.clusterController.isLoaded'),
 
   clusterName: null,
   clockDistance: null, // server clock - client clock

ambari-web/app/assets/test/tests.js

@@ -37,6 +37,7 @@ require('utils/ajax/ajax_queue');
 
 var files = ['test/init_model_test',
   'test/app_test',
+  'test/data/secure_mapping_test',
   'test/data/HDP2/secure_mapping_test',
   'test/controllers/global/background_operations_test',
   'test/controllers/global/cluster_controller_test',

ambari-web/app/controllers/main/admin/security.js

@@ -92,7 +92,8 @@ App.MainAdminSecurityController = Em.Controller.extend({
     'nagios_user': {defaultValue: 'nagios', siteName: 'nagios-env', serviceName: 'NAGIOS'},
     'user_group': {defaultValue: 'hadoop', siteName: 'hadoop-env', serviceName: 'HDFS'},
     'storm_user': {defaultValue: 'storm', siteName: 'storm-env', serviceName: 'STORM'},
-    'falcon_user': {defaultValue: 'falcon', siteName: 'falcon-env', serviceName: 'FALCON'}
+    'falcon_user': {defaultValue: 'falcon', siteName: 'falcon-env', serviceName: 'FALCON'},
+    'knox_user': {defaultValue: 'knox', siteName: 'knox-env', serviceName: 'KNOX'}
   },
 
   loadStep: function () {

ambari-web/app/controllers/main/admin/security/add/step2.js

@@ -145,6 +145,11 @@ App.MainAdminSecurityAddStep2Controller = Em.Controller.extend({
       serviceName: 'STORM',
       configName: 'nimbus_host',
       components: ['NIMBUS']
+    },
+    {
+      serviceName: 'KNOX',
+      configName: 'knox_gateway_hosts',
+      components: ['KNOX_GATEWAY']
     }
   ],
 

ambari-web/app/controllers/main/admin/security/add/step3.js

@@ -45,7 +45,8 @@ App.MainAdminSecurityAddStep3Controller = Em.Controller.extend({
       'SUPERVISOR': 'storm_user',
       'NIMBUS': 'storm_user',
       'STORM_UI_SERVER': 'storm_user',
-      'FALCON_SERVER': 'falcon_user'
+      'FALCON_SERVER': 'falcon_user',
+      'KNOX_GATEWAY': 'knox_user'
     };
     if (App.get('isHadoop22Stack')) {
       map['DRPC_SERVER'] = 'storm_user'
@@ -350,7 +351,7 @@ App.MainAdminSecurityAddStep3Controller = Em.Controller.extend({
   setHostComponentsSecureValue: function (result, host, addedPrincipalsHost, securityUsers, hadoopGroupId) {
     var componentsToDisplay = ['NAMENODE', 'SECONDARY_NAMENODE', 'DATANODE', 'JOBTRACKER', 'ZOOKEEPER_SERVER', 'HIVE_SERVER', 'TASKTRACKER',
       'OOZIE_SERVER', 'NAGIOS_SERVER', 'HBASE_MASTER', 'HBASE_REGIONSERVER', 'HISTORYSERVER', 'RESOURCEMANAGER', 'NODEMANAGER', 'JOURNALNODE',
-      'SUPERVISOR', 'NIMBUS', 'STORM_UI_SERVER', 'FALCON_SERVER'];
+      'SUPERVISOR', 'NIMBUS', 'STORM_UI_SERVER', 'FALCON_SERVER', 'KNOX_GATEWAY'];
     if (App.get('isHadoop22Stack')) {
       componentsToDisplay.push('DRPC_SERVER');
     }

ambari-web/app/data/HDP2/secure_configs.js

@@ -149,6 +149,16 @@ var configs = [
     ],
     sites: ['falcon-startup.properties'],
     configs: configProperties.filterProperty('serviceName', 'FALCON')
+  },
+  {
+    serviceName: 'KNOX',
+    displayName: 'Knox',
+    filename: 'gateway-site.xml',
+    configCategories: [
+      App.ServiceConfigCategory.create({ name: 'Knox Gateway', displayName:  'Knox Gateway'})
+    ],
+    sites: ['gateway-site','knox-env'],
+    configs: configProperties.filterProperty('serviceName', 'KNOX')
   }
 ];
 

ambari-web/app/data/HDP2/secure_mapping.js

@@ -67,7 +67,8 @@ module.exports = [
     "templateName": ["snamenode_principal_name", "kerberos_domain"],
     "foreignKey": null,
     "value": "<templateName[0]>@<templateName[1]>",
-    "filename": "hdfs-site.xml"
+    "filename": "hdfs-site.xml",
+    "serviceName": "HDFS"
   },
   {
     "name": "dfs.secondary.namenode.keytab.file",
@@ -587,7 +588,8 @@ module.exports = [
     "templateName": [],
     "foreignKey": null,
     "value": "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.hadoop.hbase.security.access.AccessController",
-    "filename": "hbase-site.xml"
+    "filename": "hbase-site.xml",
+    "serviceName": "HBASE"
   },
   {
     "name": "hbase.bulkload.staging.dir",
@@ -666,7 +668,66 @@ module.exports = [
     "filename": "falcon-startup.properties.xml",
     "serviceName": "FALCON"
   },
-  /******************************************************************************************/
+
+  /***************************************KNOX***********************************************/
+  {
+    "name": "gateway.hadoop.kerberos.secured",
+    "templateName": [],
+    "foreignKey": null,
+    "value": "true",
+    "nonSecureValue": "false",
+    "filename": "gateway-site.xml",
+    "serviceName": "KNOX"
+  },
+  {
+    "name": "hadoop.proxyuser.<foreignKey[0]>.groups",
+    "templateName": ["proxyuser_group"],
+    "foreignKey": ["knox_primary_name"],
+    "value": "<templateName[0]>",
+    "filename": "core-site.xml",
+    "serviceName": "KNOX"
+  },
+  {
+    "name": "hadoop.proxyuser.<foreignKey[0]>.hosts",
+    "templateName": ["knox_gateway_hosts"],
+    "foreignKey": ["knox_primary_name"],
+    "value": "<templateName[0]>",
+    "filename": "core-site.xml",
+    "serviceName": "KNOX"
+  },
+  {
+    "name": "webhcat.proxyuser.<foreignKey[0]>.groups",
+    "templateName": ["proxyuser_group"],
+    "foreignKey": ["knox_primary_name"],
+    "value": "<templateName[0]>",
+    "filename": "webhcat-site.xml",
+    "serviceName": "KNOX"
+  },
+  {
+    "name": "webhcat.proxyuser.<foreignKey[0]>.hosts",
+    "templateName": ["knox_gateway_hosts"],
+    "foreignKey": ["knox_primary_name"],
+    "value": "<templateName[0]>",
+    "filename": "webhcat-site.xml",
+    "serviceName": "KNOX"
+  },
+  {
+    "name": "hadoop.proxyuser.<foreignKey[0]>.groups",
+    "templateName": ["proxyuser_group"],
+    "foreignKey": ["knox_primary_name"],
+    "value": "<templateName[0]>",
+    "filename": "oozie-site.xml",
+    "serviceName": "KNOX"
+  },
+  {
+    "name": "hadoop.proxyuser.<foreignKey[0]>.hosts",
+    "templateName": ["knox_gateway_hosts"],
+    "foreignKey": ["knox_primary_name"],
+    "value": "<templateName[0]>",
+    "filename": "oozie-site.xml",
+    "serviceName": "KNOX"
+  },
+/***************************************core-site***************************************************/
   {
     "name": "hadoop.proxyuser.<foreignKey[0]>.groups",
     "templateName": ["proxyuser_group"],

ambari-web/app/data/HDP2/secure_properties.js

@@ -1095,6 +1095,51 @@ var props = {
       "isOverridable": false,
       "serviceName": "FALCON",
       "category": "Falcon Server"
+    },
+
+  /**********************************************Knox***************************************/
+    {
+      "id": "puppet var",
+      "name": "knox_gateway_hosts",
+      "displayName": "Knox Gateway hosts",
+      "value": "",
+      "defaultValue": "",
+      "description": "The hosts that has been assigned to run Knox Gateway",
+      "displayType": "masterHosts",
+      "isOverridable": false,
+      "isVisible": true,
+      "serviceName": "KNOX",
+      "category": "Knox Gateway"
+    },
+    {
+      "id": "puppet var",
+      "name": "knox_principal_name",
+      "displayName": "Principal name",
+      "value": "",
+      "defaultValue": "knox/_HOST",
+      "description": "This is the principal name for Knox Gateway",
+      "displayType": "principal",
+      "isVisible": true,
+      "isOverridable": false,
+      "serviceName": "KNOX",
+      "filename": "knox-env.xml",
+      "category": "Knox Gateway",
+      "component": "KNOX_GATEWAY"
+    },
+    {
+      "id": "puppet var",
+      "name": "knox_keytab_path",
+      "displayName": "Path to keytab file",
+      "value": "",
+      "defaultValue": "/etc/security/keytabs/knox.service.keytab",
+      "description": "This is the keytab file for Knox Gateway",
+      "displayType": "directory",
+      "isVisible": true,
+      "isOverridable": false,
+      "serviceName": "KNOX",
+      "filename": "knox-env.xml",
+      "category": "Knox Gateway",
+      "component": "KNOX_GATEWAY"
     }
   ]
 };

ambari-web/app/data/secure_mapping.js

@@ -66,7 +66,8 @@ module.exports = [
     "templateName": ["snamenode_principal_name", "kerberos_domain"],
     "foreignKey": null,
     "value": "<templateName[0]>@<templateName[1]>",
-    "filename": "hdfs-site.xml"
+    "filename": "hdfs-site.xml",
+    "serviceName": "HDFS"
   },
   {
     "name": "dfs.secondary.namenode.keytab.file",
@@ -446,7 +447,8 @@ module.exports = [
     "templateName": [],
     "foreignKey": null,
     "value": "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.hadoop.hbase.security.access.AccessController",
-    "filename": "hbase-site.xml"
+    "filename": "hbase-site.xml",
+    "serviceName": "HBASE"
   },
   {
     "name": "hbase.bulkload.staging.dir",

ambari-web/app/messages.js

@@ -1095,6 +1095,7 @@ Em.I18n.translations = {
   'admin.addSecurity.falcon.user.httpUser': 'Falcon SPNEGO User',
   'admin.addSecurity.user.yarn.atsUser': 'YARN ATS User',
   'admin.addSecurity.user.yarn.atsHTTPUser': 'YARN ATS HTTP User',
+  'admin.addSecurity.knox.user': 'Knox Gateway',
   'admin.addSecurity.enable.onClose': 'You are in the process of enabling security on your cluster. ' +
     'Are you sure you want to quit? If you quit, ' +
     'you may have to re-run the security wizard from the beginning to enable security.',

ambari-web/app/mixins/wizard/addSecurityConfigs.js

@@ -92,6 +92,10 @@ App.AddSecurityConfigs = Em.Mixin.create({
         name: 'zookeeper_principal_name',
         serviceName: 'ZOOKEEPER'
       },
+      {
+        name: 'knox_principal_name',
+        serviceName: 'KNOX'
+      },
       {
         name: 'storm_principal_name',
         serviceName: 'STORM'
@@ -242,7 +246,9 @@ App.AddSecurityConfigs = Em.Mixin.create({
    */
   loadUiSideSecureConfigs: function () {
     var uiConfig = [];
-    var configs = this.get('secureMapping').filterProperty('foreignKey', null);
+    var configs = this.get('secureMapping').filterProperty('foreignKey', null).filter(function(_configProperty){
+      return (App.Service.find().mapProperty('serviceName').contains(_configProperty.serviceName));
+    },this);
     configs.forEach(function (_config) {
       var value = _config.value;
       if (_config.hasOwnProperty('dependedServiceName')) {

ambari-web/test/controllers/main/admin/security/add/addSecurity_controller_test.js

@@ -86,7 +86,8 @@ describe('App.AddSecurityController', function () {
         "OOZIE",
         "NAGIOS",
         "STORM",
-        "FALCON"
+        "FALCON",
+        "KNOX"
       ];
       sinon.stub(App, 'get', function () {
         return true;

ambari-web/test/data/HDP2/secure_mapping_test.js

@@ -18,14 +18,20 @@
 
 var App = require('app');
 require('utils/helper');
-var mappedProperties = require('data/HDP2/secure_mapping');
+var mappedHdp2Properties = require('data/HDP2/secure_mapping');
 
 describe('hdp2SiteMapping', function () {
 
   // All mapped properties should have value of string type
-  mappedProperties.forEach(function(mappedProperty){
+  mappedHdp2Properties.forEach(function(mappedProperty){
     it('Value of "' + mappedProperty.name  + '"' + ' should be string', function () {
       expect(mappedProperty.value).to.be.a('string');
     });
   });
+  mappedHdp2Properties.forEach(function(mappedProperty){
+    it('Value of "' + mappedProperty.name  + '"' + ' should have serviceName and filename attribute', function () {
+      expect(mappedProperty).to.have.property('serviceName');
+      expect(mappedProperty).to.have.property('filename');
+    });
+  });
 });

ambari-web/test/data/secure_mapping_test.js

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('utils/helper');
+var mappedProperties = require('data/secure_mapping');
+
+describe('hdp1SiteMapping', function () {
+
+  // All mapped properties should have value of string type
+  mappedProperties.forEach(function(mappedProperty){
+    it('Value of "' + mappedProperty.name  + '"' + ' should be string', function () {
+      expect(mappedProperty.value).to.be.a('string');
+    });
+  });
+  mappedProperties.forEach(function(mappedProperty){
+    it('Value of "' + mappedProperty.name  + '"' + ' should have serviceName and filename attribute', function () {
+      expect(mappedProperty).to.have.property('serviceName');
+      expect(mappedProperty).to.have.property('filename');
+    });
+  });
+});