AMBARI-2800. YARN and MRv2 should work in kerberos secure environment. (jaimin)

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp

@@ -140,6 +140,12 @@ define hdp-hadoop::namenode::create_app_directories($service_state)
         owner         => $hdp::params::hbase_user,
         service_state => $service_state
       }
+      $hbase_staging_dir = $hdp::params::hbase_staging_dir
+        hdp-hadoop::hdfs::directory { $hbase_staging_dir:
+          owner         => $hdp::params::hbase_user,
+          service_state => $service_state,
+          mode             => '711',
+        }
     }
 
     if ($hdp::params::hive_server_host != "") {

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp

@@ -83,6 +83,13 @@ class hdp-hadoop::params(
 
   $mapred_pid_dir_prefix = hdp_default("mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
 
+  ### JSVC_HOME path is correct for AMD64 only, but can be changed through API
+  if ($hdp::params::hdp_os_type == "suse") {
+    $jsvc_path = hdp_default("jsvc_path","/usr/lib/hadoop/sbin/Linux-amd64-64/")
+  } else {
+    $jsvc_path = hdp_default("jsvc_path","/usr/libexec/bigtop-utils")
+  }
+
   ### compression related
   if (($hdp::params::lzo_enabled == true) and ($hdp::params::snappy_enabled == true)) {
     $mapred_compress_map_output = true

ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb

@@ -31,6 +31,9 @@ export HADOOP_HOME_WARN_SUPPRESS=1
 #TODO: if env var set that can cause problems
 export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-<%=scope.function_hdp_template_var("conf_dir")%>}
 
+# Path to jsvc required by secure HDP 2.0 datanode
+export JSVC_HOME=<%=scope.function_hdp_template_var("jsvc_path")%>
+
 # The maximum amount of heap to use, in MB. Default is 1000.
 export HADOOP_HEAPSIZE="<%=scope.function_hdp_template_var("hadoop_heapsize")%>"
 
@@ -106,8 +109,8 @@ do
   JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
 done
 #Add libraries required by nodemanager
-MAPREDUCE_LIBS=<%=scope.function_hdp_template_var("mapreduce_libs_path")%> 
+MAPREDUCE_LIBS=<%=scope.function_hdp_template_var("mapreduce_libs_path")%>
 export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
 
 # Setting path to hdfs command line
-export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%>
+export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%>

ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp

@@ -97,35 +97,15 @@ class hdp-hbase(
     hdp-hbase::configfile { 'regionservers':}
 
     if ($security_enabled == true) {
-      if ($type == 'master') {
+      if ($type == 'master' and $service_state == 'running') {
         hdp-hbase::configfile { 'hbase_master_jaas.conf' : }
       } elsif ($type == 'regionserver' and $service_state == 'running') {
-
-        $hbase_grant_premissions_file = '/tmp/hbase_grant_permissions.sh'
-
-        file { $hbase_grant_premissions_file:
-          owner   => $hbase_user,
-          group   => $hdp::params::user_group,
-          mode => '0644',
-          content => template('hdp-hbase/hbase_grant_permissions.erb')
-        }
-
-        hdp::exec { '${smokeuser}_grant_privileges' :
-          command => "su - ${smoke_test_user} -c 'hbase --config $conf_dir shell ${hbase_grant_premissions_file}'",
-          require => File[$hbase_grant_premissions_file]
-        }
-
-        Hdp-hbase::Configfile<||> -> File[$hbase_grant_premissions_file] ->
-        Hdp::Exec['${smokeuser}_grant_privileges'] -> Anchor['hdp-hbase::end']
-
-      } elsif ($type == 'regionserver') {
         hdp-hbase::configfile { 'hbase_regionserver_jaas.conf' : }
-      } else {
+      } elsif ($type == 'client') {
         hdp-hbase::configfile { 'hbase_client_jaas.conf' : }
       }
     }
-
-    Anchor['hdp-hbase::begin'] -> Hdp::Package['hbase'] -> Hdp::Directory[$config_dir] -> 
+    Anchor['hdp-hbase::begin'] -> Hdp::Package['hbase'] -> Hdp::Directory[$config_dir] ->
     Hdp-hbase::Configfile<||> ->  Anchor['hdp-hbase::end']
   }
 }

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp

@@ -101,4 +101,19 @@ define hdp-yarn::generate_common_configs() {
     owner          => $hdp-yarn::params::yarn_user,
     mode           => 755
   }
+  if ($hdp::params::security_enabled == true) {
+    $container_executor = "${hdp::params::yarn_container_bin}/container-executor"
+    file { $container_executor:
+      ensure => present,
+      group => 'hadoop',
+      mode => 6050
+    }
+
+    hdp::configfile { "${yarn_config_dir}/container-executor.cfg" :
+      component => 'yarn',
+      owner => 'root',
+      group   => $hdp::params::user_group,
+      mode  => '0644'
+    }
+  }
 }

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp

@@ -22,9 +22,14 @@ class hdp-yarn::params(
 ) inherits hdp-hadoop::params 
 {
 
-  $conf_dir = $hdp::params::yarn_conf_dir 
+  $conf_dir = $hdp::params::yarn_conf_dir
   $stack_version = $hdp::params::stack_version
-    
+  $smoke_test_user = $hdp::params::smokeuser
+  ## security params
+  $security_enabled = $hdp::params::security_enabled
+  $smoke_user_keytab = $hdp::params::smokeuser_keytab
+  $kinit_cmd = "${hdp::params::kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user};"
+
   ## yarn-env 
   $hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec")
   

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp

@@ -39,12 +39,23 @@ define hdp-yarn::smoketest(
     hdp_fail("Unsupported component name: $component_name")
   }
 
+  $security_enabled = $hdp::params::security_enabled
+  $smoke_user_keytab = $hdp::params::smokeuser_keytab
   $smoke_test_user = $hdp::params::smokeuser
+  $kinit_cmd = "${hdp::params::kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user};"
+
 
   $validateStatusFileName = "validateYarnComponentStatus.py"
   $validateStatusFilePath = "/tmp/$validateStatusFileName"
 
-  $validateStatusCmd = "su - ${smoke_test_user} -c 'python $validateStatusFilePath $component_type -p $component_address'"
+  $validateStatusCmd = "python $validateStatusFilePath $component_type -p $component_address"
+
+    if ($security_enabled == true) {
+         $smoke_cmd = "${kinit_cmd}  $validateStatusCmd"
+        } else {
+          $smoke_cmd = $validateStatusCmd
+        }
+
 
   file { $validateStatusFilePath:
     ensure => present,
@@ -53,11 +64,12 @@ define hdp-yarn::smoketest(
   }
 
   exec { $validateStatusFilePath:
-    command   => $validateStatusCmd,
+    command   => $smoke_cmd,
     tries     => 3,
     try_sleep => 5,
     path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-    logoutput => "true"
+    logoutput => "true",
+    user     =>  $smoke_test_user
 }
   anchor{"hdp-yarn::smoketest::begin":} -> File[$validateStatusFilePath] -> Exec[$validateStatusFilePath] -> anchor{"hdp-yarn::smoketest::end":}
 }

ambari-agent/src/main/puppet/modules/hdp-yarn/templates/container-executor.cfg.erb

@@ -0,0 +1,22 @@
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+yarn.nodemanager.local-dirs=<%=scope.function_hdp_default(["yarn-site/yarn.nodemanager.local-dirs","/hadoop/yarn"])%>
+yarn.nodemanager.log-dirs=<%=scope.function_hdp_default(["yarn-site/yarn.nodemanager.log-dirs","/var/log/hadoop/yarn"])%>
+yarn.nodemanager.linux-container-executor.group=<%=scope.function_hdp_default(["yarn-site/yarn.nodemanager.linux-container-executor.group","hadoop"])%>
+banned.users = hfds,yarn,mapred,bin
+min.user.id=1000

ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp

@@ -231,6 +231,7 @@ class hdp::params()
   $hive_apps_whs_dir = hdp_default("hive_apps_whs_dir", "/apps/hive/warehouse")
   $webhcat_apps_dir = hdp_default("webhcat_apps_dir", "/apps/webhcat")
   $hbase_hdfs_root_dir = hdp_default("hbase-site/hbase.hdfs.root.dir","/apps/hbase/data")
+  $hbase_staging_dir = hdp_default("hbase-site/hbase.bulkload.staging.dir","/apps/hbase/staging")
 
   $yarn_nm_app_log_dir = hdp_default("yarn-site/yarn.nodemanager.remote-app-log-dir","/app-logs")
 
@@ -353,6 +354,7 @@ class hdp::params()
       $hadoop_deps = ['hadoop','hadoop-libhdfs','hadoop-native','hadoop-pipes','hadoop-sbin','hadoop-lzo', 'hadoop-lzo-native']
     }
     $yarn_bin = "/usr/lib/hadoop-yarn/sbin"
+    $yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
     $mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
     $hadoop_conf_dir = "/etc/hadoop/conf"
     $yarn_conf_dir = "/etc/hadoop/conf"

ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/core-site.xml

@@ -160,6 +160,7 @@
     <name>hadoop.security.auth_to_local</name>
     <value>
         RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
+        RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
         RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
         RULE:[2:$1@$0](hm@.*)s/.*/hbase/
         RULE:[2:$1@$0](rs@.*)s/.*/hbase/