AMBARI-4824: A utility script for Kerberos environment setup on Ambari cluster. (jaimin)

ambari-server/src/main/resources/scripts/distribute_keytabs.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from optparse import OptionParser
-import sys
-import subprocess
-import re
-import os
-import glob
-
-
-def sendTarfileToHosts(hostnames,identity_file,krb5_conf):
-  user = "root"
-  for i in range(len(hostnames)):
-    remotehost, localfile = hostnames[i]
-    os.system('scp -i "%s" -oStrictHostKeyChecking=no "%s" "%s"@"%s":/' % (identity_file, localfile, user, remotehost))
-    if krb5_conf:
-      os.system('scp -i "%s" -oStrictHostKeyChecking=no "%s" "%s"@"%s":%s' % (identity_file, krb5_conf, user, remotehost, krb5_conf))
-    sshCommand = "tar xvf /" + os.path.basename(localfile) + " -C /"
-    ssh = subprocess.Popen(["ssh", "-i", "%s" % identity_file, "-oStrictHostKeyChecking=no","%s" % remotehost, sshCommand],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
-    result = ssh.stdout.readlines()
-    if result == []:
-      error = ssh.stderr.readlines()
-      print >>sys.stderr, "ERROR: %s" % error
-    else:
-      print result
-
-
-def getHostnames(filenames,regex):
-  return [(hostname.group(1),tarfile) for tarfile in filenames for hostname in (regex(tarfile),) if hostname]
-
-
-def main():
-  parser = OptionParser()
-  parser.add_option('-d','--directory', help='Path to the Directory containing tar files',dest='dirPath', default='.')
-  parser.add_option('-i','--identity-file', help='Path to the identity file',dest='identity_file', default='/tmp/ec2-keypair')
-  parser.add_option('-k','--krb5-conf', help='Path to the krb5_conf file',dest='krb5_conf')
-  (options, args) = parser.parse_args()
-  pattern = options.dirPath + "/*.tar"
-  tarfiles = glob.glob(pattern)
-  hostnames_regex = re.compile("(?<=keytabs_)(.*)(?=.tar)").search
-  hostnames = getHostnames(tarfiles,hostnames_regex)
-  sendTarfileToHosts(hostnames,options.identity_file,options.krb5_conf)
-
-
-if __name__ == '__main__':
-  main()
-
-

ambari-server/src/main/resources/scripts/kerberos-setup.sh

@@ -0,0 +1,234 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+############################
+## NOTE:
+##      1) This script should be executed on NameNode host as that host is guaranteed to have all the users needed while creating keytab file
+##      2) The script has been verified to work in gce environment and 
+##         vagrant environment documented at ambari wiki: https://cwiki.apache.org/confluence/display/AMBARI/Quick+Start+Guide 
+###########################
+
+usage () {
+echo "Usage: keytabs.sh <HOST_PRINCIPAL_KEYTABLE.csv> <SSH_LOGIN_KEY_PATH>";
+echo "  <HOST_PRINCIPAL_KEYTABLE.csv>: CSV file generated by 'Enable Security Wizard' of Ambari";
+echo "  <SSH_LOGIN_KEY_PATH>: File path to the ssh login key for root user";
+exit 1;
+}
+
+###################
+## processCSVFile()
+###################
+processCSVFile () {
+    csvFile=$1;
+    csvFile=$(printf '%q' "$csvFile")
+    touch generate_keytabs.sh;
+    chmod 755 generate_keytabs.sh;
+
+    echo "#!/bin/bash" > generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "## " >> generate_keytabs.sh;
+    echo "## Ambari Security Script Generator" >> generate_keytabs.sh;
+    echo "## "  >> generate_keytabs.sh;
+    echo "## Ambari security script is generated which should be run on the" >> generate_keytabs.sh;
+    echo "## Kerberos server machine." >> generate_keytabs.sh;
+    echo "## " >> generate_keytabs.sh;
+    echo "## Running the generated script will create host specific keytabs folders." >> generate_keytabs.sh;
+    echo "## Each of those folders will contain service specific keytab files with " >> generate_keytabs.sh;
+    echo "## appropriate permissions. There folders should be copied as the appropriate" >> generate_keytabs.sh;
+    echo "## host's '/etc/security/keytabs' folder" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    
+    rm -f commands.mkdir;
+    rm -f commands.chmod;
+    rm -f commands.addprinc;
+    rm -f commands.xst
+    rm -f commands.xst.cp
+    rm -f commands.chown.1
+    rm -f commands.chmod.1
+    rm -f commands.chmod.2
+    rm -f commands.tar
+    
+    seenHosts="";
+    seenPrincipals="";
+    
+    echo "mkdir -p ./tmp_keytabs" >> commands.mkdir;
+    cat $csvFile | while read line; do
+        hostName=`echo $line|cut -d , -f 1`;
+        service=`echo $line|cut -d , -f 2`;
+        principal=`echo $line|cut -d , -f 3`;
+        keytabFile=`echo $line|cut -d , -f 4`;
+        keytabFilePath=`echo $line|cut -d , -f 5`;
+        owner=`echo $line|cut -d , -f 6`;
+        group=`echo $line|cut -d , -f 7`;
+        acl=`echo $line|cut -d , -f 8`;
+        
+        if [[ $seenHosts != *$hostName* ]]; then
+              echo "mkdir -p ./keytabs_$hostName" >> commands.mkdir;
+              echo "chmod 755 ./keytabs_$hostName" >> commands.chmod;
+              echo "chown -R root:$group `pwd`/keytabs_$hostName" >> commands.chown.1
+              echo "tar -cvf keytabs_$hostName.tar -C keytabs_$hostName ." >> commands.tar
+              seenHosts="$seenHosts$hostName";
+        fi
+        
+        if [[ $seenPrincipals != *$principal* ]]; then
+          echo -e "kadmin.local -q \"addprinc -randkey $principal\"" >> commands.addprinc;
+          seenPrincipals="$seenPrincipals$principal"
+        fi
+        tmpKeytabFile="`pwd`/tmp_keytabs/$keytabFile";
+	    newKeytabPath="`pwd`/keytabs_$hostName$keytabFilePath";
+	    newKeytabFile="$newKeytabPath/$keytabFile";
+        if [ ! -f $tmpKeytabFile ]; then
+          echo "kadmin.local -q \"xst -k $tmpKeytabFile $principal\"" >> commands.xst;          
+        fi
+        if [ ! -d $newKeytabPath ]; then
+            echo "mkdir -p $newKeytabPath" >> commands.mkdir;
+        fi
+        echo "cp $tmpKeytabFile $newKeytabFile" >> commands.xst.cp
+        echo "chmod $acl $newKeytabFile" >> commands.chmod.2
+        echo "chown $owner:$group $newKeytabFile" >> commands.chown.1
+    done;
+    
+    
+    echo "" >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Making host specific keytab folders" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    cat commands.mkdir >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Changing permissions for host specific keytab folders" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    cat commands.chmod >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Creating Kerberos Principals" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    cat commands.addprinc >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Creating Kerberos Principal keytabs in host specific keytab folders" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    cat commands.xst >> generate_keytabs.sh;
+    cat commands.xst.cp >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Changing ownerships of host specific keytab files" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    cat commands.chown.1 >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Changing access permissions of host specific keytab files" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    #cat commands.chmod.1
+    cat commands.chmod.2 >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Packaging keytab folders" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    cat commands.tar >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "# Cleanup" >> generate_keytabs.sh;
+    echo "###########################################################################" >> generate_keytabs.sh;
+    echo "rm -rf ./tmp_keytabs" >> generate_keytabs.sh;
+    echo "" >> generate_keytabs.sh;
+    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
+    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
+    echo "echo \"** Copy and extract 'keytabs_[hostname].tar' files onto respective hosts. **\"" >> generate_keytabs.sh;
+    echo "echo \"**                                                                        **\"" >> generate_keytabs.sh;
+    echo "echo \"** Generated keytab files are preserved in the 'tmp_keytabs' folder.      **\"" >> generate_keytabs.sh;
+    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
+    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
+    
+    rm -f commands.mkdir >> generate_keytabs.sh;
+    rm -f commands.chmod >> generate_keytabs.sh;
+    rm -f commands.addprinc >> generate_keytabs.sh;
+    rm -f commands.xst >> generate_keytabs.sh;
+    rm -f commands.xst.cp >> generate_keytabs.sh;
+    rm -f commands.chown.1 >> generate_keytabs.sh;
+    rm -f commands.chmod.1 >> generate_keytabs.sh;
+    rm -f commands.chmod.2 >> generate_keytabs.sh;
+    rm -f commands.tar >> generate_keytabs.sh;
+    # generate keytabs
+    sh ./generate_keytabs.sh
+}
+
+installKDC () {
+  csvFile=$1;
+  sshLoginKey=$2;
+  krb5_conf="/etc/krb5.conf"
+  # Configure /etc/krb5.conf
+  sed -c -i "/FILE/!s/\(kdc *= *\).*/\1$HOSTNAME/" $krb5_conf
+  sed -c -i "/FILE/!s/\(admin_server *= *\).*/\1$HOSTNAME/" $krb5_conf
+  # Install kdc server on this host
+  yum install krb5-server krb5-libs krb5-auth-dialog  krb5-workstation -y; 
+  # Install rng tools
+  yum install rng-tools -y
+  sed -c -i "s/\(EXTRAOPTIONS *= *\).*/\1\"-r \/dev\/urandom\"/" "/etc/sysconfig/rngd"
+  # start rngd
+  /etc/init.d/rngd start
+  (echo; echo;) | kdb5_util create -s
+  /sbin/service krb5kdc start
+  /sbin/service kadmin start
+  # Install pdsh on this host
+  yum install pdsh -y; 
+  chown root:root -R /usr;
+  eval `ssh-agent`
+  ssh-add $sshLoginKey
+  hostNames='';
+  while read line; do
+    hostName=`echo $line|cut -d , -f 1`;
+    if [ -z "$hostNames" ]; then
+      hostNames=$hostName;
+      continue;
+    fi
+    if [[ $hostNames != *$hostName* ]]; then
+      hostNames=$hostNames,$hostName;
+    fi
+  done < $csvFile
+  pdsh -w $hostNames yum install krb5-workstation -y
+  pdsh -w $hostNames yum install pdsh -y
+  pdsh -w $hostNames chown root:root -R /usr
+  pdcp -w $hostNames $krb5_conf $krb5_conf
+}
+
+distributeKeytabs () {
+  shopt -s nullglob  
+  filearray=(keytabs_*tar)
+  for i in ${filearray[@]}; do
+    derivedname=${i%.*}
+    derivedname=${derivedname##keytabs_}
+    echo $derivedname
+    scp $i root@$derivedname:/
+    ssh root@$derivedname "cd /;tar xvf $i"
+  done
+}
+
+if (($# != 2)); then
+    usage
+fi
+
+installKDC $@
+processCSVFile $@
+distributeKeytabs $@
+