HADOOP-6526. Need mapping from long principal names to local OS user names

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@953388 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -47,6 +47,9 @@ Trunk (unreleased changes)
     HADOOP-6813. Add a new newInstance method in FileSystem that takes 
     a "user" as argument (ddas via boryas)
 
+    HADOOP-6526. Need mapping from long principal names to local OS 
+    user names. (boryas)
+
   BUG FIXES
     HADOOP-6638. try to relogin in a case of failed RPC connection (expired tgt) 
     only in case the subject is loginUser or proxyUgi.realUser. (boryas)

build.xml

@@ -627,6 +627,7 @@
     <attribute name="test.dir" />
     <attribute name="fileset.dir" />
     <attribute name="hadoop.conf.dir.deployed" default="" />
+    <attribute name="test.krb5.conf.filename" default="" />
     <sequential>
       <delete file="${test.build.dir}/testsfailed"/>
       <delete dir="@{test.dir}/data" />
@@ -656,6 +657,7 @@
         <sysproperty key="hadoop.log.dir" value="${test.log.dir}" />
         <sysproperty key="test.src.dir" value="${test.src.dir}" />
         <sysproperty key="test.build.extraconf" value="@{test.dir}/extraconf" />
+         <sysproperty key="java.security.krb5.conf" value="@{test.krb5.conf.filename}"/>
         <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml" />
         <sysproperty key="java.library.path"
           value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
@@ -711,6 +713,7 @@
                        classpath="${test.classpath.id}"
                        test.dir="${test.build.dir}"
                        fileset.dir="${test.src.dir}"
+                       test.krb5.conf.filename="${test.src.dir}/krb5.conf"
                        >
     </macro-test-runner>
   </target>   

src/java/org/apache/hadoop/security/KerberosName.java

@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+
+import sun.security.krb5.Config;
+import sun.security.krb5.KrbException;
+
+/**
+ * This class implements parsing and handling of Kerberos principal names. In 
+ * particular, it splits them apart and translates them down into local
+ * operating system names.
+ */
+@SuppressWarnings("all")
+public class KerberosName {
+  /** The first component of the name */
+  private final String serviceName;
+  /** The second component of the name. It may be null. */
+  private final String hostName;
+  /** The realm of the name. */
+  private final String realm;
+
+  /**
+   * A pattern that matches a Kerberos name with at most 2 components.
+   */
+  private static final Pattern nameParser = 
+    Pattern.compile("([^/@]*)(/([^/@]*))?@([^/@]*)");
+
+  /** 
+   * A pattern that matches a string with out '$' and then a single
+   * parameter with $n.
+   */
+  private static Pattern parameterPattern = 
+    Pattern.compile("([^$]*)(\\$(\\d*))?");
+
+  /**
+   * A pattern for parsing a auth_to_local rule.
+   */
+  private static final Pattern ruleParser =
+    Pattern.compile("\\s*((DEFAULT)|(RULE:\\[(\\d*):([^\\]]*)](\\(([^)]*)\\))?"+
+                    "(s/([^/]*)/([^/]*)/(g)?)?))");
+  
+  /**
+   * A pattern that recognizes simple/non-simple names.
+   */
+  private static final Pattern nonSimplePattern = Pattern.compile("[/@]");
+  
+  /**
+   * The list of translation rules.
+   */
+  private static List<Rule> rules;
+
+  private static String defaultRealm;
+  private static Config kerbConf;
+  
+  static {
+    try {
+      kerbConf = Config.getInstance();
+      defaultRealm = kerbConf.getDefaultRealm();
+    } catch (KrbException ke) {
+      if(UserGroupInformation.isSecurityEnabled())
+        throw new IllegalArgumentException("Can't get Kerberos configuration",ke);
+      else 
+        defaultRealm="";
+    }
+  }
+
+  /**
+   * Create a name from the full Kerberos principal name.
+   * @param name
+   */
+  public KerberosName(String name) {
+    Matcher match = nameParser.matcher(name);
+    if (!match.matches()) {
+      if (name.contains("@")) {
+        throw new IllegalArgumentException("Malformed Kerberos name: " + name);
+      } else {
+        serviceName = name;
+        hostName = null;
+        realm = null;
+      }
+    } else {
+      serviceName = match.group(1);
+      hostName = match.group(3);
+      realm = match.group(4);
+    }
+  }
+
+  /**
+   * Get the configured default realm.
+   * @return the default realm from the krb5.conf
+   */
+  public String getDefaultRealm() {
+    return defaultRealm;
+  }
+
+  /**
+   * Put the name back together from the parts.
+   */
+  @Override
+  public String toString() {
+    StringBuilder result = new StringBuilder();
+    result.append(serviceName);
+    if (hostName != null) {
+      result.append('/');
+      result.append(hostName);
+    }
+    if (realm != null) {
+      result.append('@');
+      result.append(realm);
+    }
+    return result.toString();
+  }
+
+  /**
+   * Get the first component of the name.
+   * @return the first section of the Kerberos principal name
+   */
+  public String getServiceName() {
+    return serviceName;
+  }
+
+  /**
+   * Get the second component of the name.
+   * @return the second section of the Kerberos principal name, and may be null
+   */
+  public String getHostName() {
+    return hostName;
+  }
+  
+  /**
+   * Get the realm of the name.
+   * @return the realm of the name, may be null
+   */
+  public String getRealm() {
+    return realm;
+  }
+  
+  /**
+   * An encoding of a rule for translating kerberos names.
+   */
+  private static class Rule {
+    private final boolean isDefault;
+    private final int numOfComponents;
+    private final String format;
+    private final Pattern match;
+    private final Pattern fromPattern;
+    private final String toPattern;
+    private final boolean repeat;
+
+    Rule() {
+      isDefault = true;
+      numOfComponents = 0;
+      format = null;
+      match = null;
+      fromPattern = null;
+      toPattern = null;
+      repeat = false;
+    }
+
+    Rule(int numOfComponents, String format, String match, String fromPattern,
+         String toPattern, boolean repeat) {
+      isDefault = false;
+      this.numOfComponents = numOfComponents;
+      this.format = format;
+      this.match = match == null ? null : Pattern.compile(match);
+      this.fromPattern = 
+        fromPattern == null ? null : Pattern.compile(fromPattern);
+      this.toPattern = toPattern;
+      this.repeat = repeat;
+    }
+    
+    @Override
+    public String toString() {
+      StringBuilder buf = new StringBuilder();
+      if (isDefault) {
+        buf.append("DEFAULT");
+      } else {
+        buf.append("RULE:[");
+        buf.append(numOfComponents);
+        buf.append(':');
+        buf.append(format);
+        buf.append(']');
+        if (match != null) {
+          buf.append('(');
+          buf.append(match);
+          buf.append(')');
+        }
+        if (fromPattern != null) {
+          buf.append("s/");
+          buf.append(fromPattern);
+          buf.append('/');
+          buf.append(toPattern);
+          buf.append('/');
+          if (repeat) {
+            buf.append('g');
+          }
+        }
+      }
+      return buf.toString();
+    }
+    
+    /**
+     * Replace the numbered parameters of the form $n where n is from 1 to 
+     * the length of params. Normal text is copied directly and $n is replaced
+     * by the corresponding parameter.
+     * @param format the string to replace parameters again
+     * @param params the list of parameters
+     * @return the generated string with the parameter references replaced.
+     * @throws BadFormatString
+     */
+    static String replaceParameters(String format, 
+                                    String[] params) throws BadFormatString {
+      Matcher match = parameterPattern.matcher(format);
+      int start = 0;
+      StringBuilder result = new StringBuilder();
+      while (start < format.length() && match.find(start)) {
+        result.append(match.group(1));
+        String paramNum = match.group(3);
+        if (paramNum != null) {
+          try {
+            int num = Integer.parseInt(paramNum);
+            if (num < 0 || num > params.length) {
+              throw new BadFormatString("index " + num + " from " + format +
+                                        " is outside of the valid range 0 to " +
+                                        (params.length - 1));
+            }
+            result.append(params[num]);
+          } catch (NumberFormatException nfe) {
+            throw new BadFormatString("bad format in username mapping in " + 
+                                      paramNum, nfe);
+          }
+          
+        }
+        start = match.end();
+      }
+      return result.toString();
+    }
+
+    /**
+     * Replace the matches of the from pattern in the base string with the value
+     * of the to string.
+     * @param base the string to transform
+     * @param from the pattern to look for in the base string
+     * @param to the string to replace matches of the pattern with
+     * @param repeat whether the substitution should be repeated
+     * @return
+     */
+    static String replaceSubstitution(String base, Pattern from, String to, 
+                                      boolean repeat) {
+      Matcher match = from.matcher(base);
+      if (repeat) {
+        return match.replaceAll(to);
+      } else {
+        return match.replaceFirst(to);
+      }
+    }
+
+    /**
+     * Try to apply this rule to the given name represented as a parameter
+     * array.
+     * @param params first element is the realm, second and later elements are
+     *        are the components of the name "a/b@FOO" -> {"FOO", "a", "b"}
+     * @return the short name if this rule applies or null
+     * @throws IOException throws if something is wrong with the rules
+     */
+    String apply(String[] params) throws IOException {
+      String result = null;
+      if (isDefault) {
+        if (defaultRealm.equals(params[0])) {
+          result = params[1];
+        }
+      } else if (params.length - 1 == numOfComponents) {
+        String base = replaceParameters(format, params);
+        if (match == null || match.matcher(base).matches()) {
+          if (fromPattern == null) {
+            result = base;
+          } else {
+            result = replaceSubstitution(base, fromPattern, toPattern,  repeat);
+          }
+        }
+      }
+      if (result != null && nonSimplePattern.matcher(result).find()) {
+        throw new NoMatchingRule("Non-simple name " + result +
+                                 " after auth_to_local rule " + this);
+      }
+      return result;
+    }
+  }
+
+  static List<Rule> parseRules(String rules) {
+    List<Rule> result = new ArrayList<Rule>();
+    String remaining = rules.trim();
+    while (remaining.length() > 0) {
+      Matcher matcher = ruleParser.matcher(remaining);
+      if (!matcher.lookingAt()) {
+        throw new IllegalArgumentException("Invalid rule: " + remaining);
+      }
+      if (matcher.group(2) != null) {
+        result.add(new Rule());
+      } else {
+        result.add(new Rule(Integer.parseInt(matcher.group(4)),
+                            matcher.group(5),
+                            matcher.group(7),
+                            matcher.group(9),
+                            matcher.group(10),
+                            "g".equals(matcher.group(11))));
+      }
+      remaining = remaining.substring(matcher.end());
+    }
+    return result;
+  }
+
+  /**
+   * Set the static configuration to get the rules.
+   * @param conf the new configuration
+   * @throws IOException
+   */
+  public static void setConfiguration(Configuration conf) throws IOException {
+    String ruleString = conf.get("hadoop.security.auth_to_local", "DEFAULT");
+    rules = parseRules(ruleString);
+  }
+
+  @SuppressWarnings("serial")
+  public static class BadFormatString extends IOException {
+    BadFormatString(String msg) {
+      super(msg);
+    }
+    BadFormatString(String msg, Throwable err) {
+      super(msg, err);
+    }
+  }
+
+  @SuppressWarnings("serial")
+  public static class NoMatchingRule extends IOException {
+    NoMatchingRule(String msg) {
+      super(msg);
+    }
+  }
+
+  /**
+   * Get the translation of the principal name into an operating system
+   * user name.
+   * @return the short name
+   * @throws IOException
+   */
+  public String getShortName() throws IOException {
+    String[] params;
+    if (hostName == null) {
+      // if it is already simple, just return it
+      if (realm == null) {
+        return serviceName;
+      }
+      params = new String[]{realm, serviceName};
+    } else {
+      params = new String[]{realm, serviceName, hostName};
+    }
+    for(Rule r: rules) {
+      String result = r.apply(params);
+      if (result != null) {
+        return result;
+      }
+    }
+    throw new NoMatchingRule("No rules applied to " + toString());
+  }
+
+  static void printRules() throws IOException {
+    int i = 0;
+    for(Rule r: rules) {
+      System.out.println(++i + " " + r);
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    for(String arg: args) {
+      KerberosName name = new KerberosName(arg);
+      System.out.println("Name: " + name + " to " + name.getShortName());
+    }
+  }
+}

src/java/org/apache/hadoop/security/User.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.security;
 
+import java.io.IOException;
 import java.security.Principal;
 
 import javax.security.auth.login.LoginContext;
@@ -38,18 +39,13 @@ class User implements Principal {
   }
   
   public User(String name, AuthenticationMethod authMethod, LoginContext login) {
-    fullName = name;
-    int atIdx = name.indexOf('@');
-    if (atIdx == -1) {
-      shortName = name;
-    } else {
-      int slashIdx = name.indexOf('/');
-      if (slashIdx == -1 || atIdx < slashIdx) {
-        shortName = name.substring(0, atIdx);
-      } else {
-        shortName = name.substring(0, slashIdx);
-      }
+    try {
+      shortName = new KerberosName(name).getShortName();
+    } catch (IOException ioe) {
+      throw new IllegalArgumentException("Illegal principal name " + name, ioe);
     }
+    fullName = name;
+
     this.authMethod = authMethod;
     this.login = login;
   }

src/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -177,6 +177,14 @@ public class UserGroupInformation {
     // circular dependence.
     javax.security.auth.login.Configuration.setConfiguration
         (new HadoopConfiguration());
+    
+    // give the configuration on how to translate Kerberos names
+    try {
+      KerberosName.setConfiguration(conf);
+    } catch (IOException ioe) {
+      throw new RuntimeException("Problem with Kerberos auth_to_local name " +
+          "configuration", ioe);
+    }
     isInitialized = true;
   }
 

src/test/core/org/apache/hadoop/security/TestDoAsEffectiveUser.java

@@ -57,10 +57,21 @@ public class TestDoAsEffectiveUser {
       GROUP2_NAME };
   private static final String ADDRESS = "0.0.0.0";
   private TestProtocol proxy;
+  private static Configuration masterConf = new Configuration();
+  
   
   public static final Log LOG = LogFactory
       .getLog(TestDoAsEffectiveUser.class);
   
+  
+  static {
+    masterConf.set("hadoop.security.auth_to_local",
+        "RULE:[2:$1@$0](.*@HADOOP.APACHE.ORG)s/@.*//" +
+        "RULE:[1:$1@$0](.*@HADOOP.APACHE.ORG)s/@.*//"
+        + "DEFAULT");
+    UserGroupInformation.setConfiguration(masterConf);
+  }
+
   private void configureSuperUserIPAddresses(Configuration conf,
       String superUserShortName) throws IOException {
     ArrayList<String> ipList = new ArrayList<String>();
@@ -380,7 +391,7 @@ public class TestDoAsEffectiveUser {
    */
   @Test
   public void testProxyWithToken() throws Exception {
-    final Configuration conf = new Configuration();
+    final Configuration conf = new Configuration(masterConf);
     TestTokenSecretManager sm = new TestTokenSecretManager();
     conf
         .set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
@@ -437,7 +448,7 @@ public class TestDoAsEffectiveUser {
   @Test
   public void testTokenBySuperUser() throws Exception {
     TestTokenSecretManager sm = new TestTokenSecretManager();
-    final Configuration newConf = new Configuration();
+    final Configuration newConf = new Configuration(masterConf);
     newConf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
     UserGroupInformation.setConfiguration(newConf);

src/test/core/org/apache/hadoop/security/TestKerberosName.java

@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestKerberosName {
+
+  @Before
+  public void setUp() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set("hadoop.security.auth_to_local", 
+             ("RULE:[1:$1@$0](.*@YAHOO\\.COM)s/@.*//\n" +
+              "RULE:[2:$1](johndoe)s/^.*$/guest/\n" +
+              "RULE:[2:$1;$2](^.*;admin$)s/;admin$//\n" +
+              "RULE:[2:$2](root)\n" +
+              "DEFAULT"));
+    conf.set("hadoop.security.authentication", "kerberos");
+    KerberosName.setConfiguration(conf);
+    KerberosName.printRules();
+  }
+
+  private void checkTranslation(String from, String to) throws Exception {
+    System.out.println("Translate " + from);
+    KerberosName nm = new KerberosName(from);
+    String simple = nm.getShortName();
+    System.out.println("to " + simple);
+    assertEquals("short name incorrect", to, simple);
+  }
+
+  @Test
+  public void testRules() throws Exception {
+    checkTranslation("omalley@APACHE.ORG", "omalley");
+    checkTranslation("hdfs/10.0.0.1@APACHE.ORG", "hdfs");
+    checkTranslation("oom@YAHOO.COM", "oom");
+    checkTranslation("johndoe/zoo@FOO.COM", "guest");
+    checkTranslation("joe/admin@FOO.COM", "joe");
+    checkTranslation("joe/root@FOO.COM", "root");
+  }
+  
+  private void checkBadName(String name) {
+    System.out.println("Checking " + name + " to ensure it is bad.");
+    try {
+      new KerberosName(name);
+      fail("didn't get exception for " + name);
+    } catch (IllegalArgumentException iae) {
+      // PASS
+    }
+  }
+  
+  private void checkBadTranslation(String from) {
+    System.out.println("Checking bad translation for " + from);
+    KerberosName nm = new KerberosName(from);
+    try {
+      nm.getShortName();
+      fail("didn't get exception for " + from);
+    } catch (IOException ie) {
+      // PASS
+    }
+  }
+  
+  @Test
+  public void testAntiPatterns() throws Exception {
+    checkBadName("owen/owen/owen@FOO.COM");
+    checkBadName("owen@foo/bar.com");
+    checkBadTranslation("foo@ACME.COM");
+    checkBadTranslation("root/joe@FOO.COM");
+  }
+}

src/test/core/org/apache/hadoop/security/TestUserGroupInformation.java

@@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-
 import static org.mockito.Mockito.mock;
 
 import java.io.BufferedReader;
@@ -36,6 +35,7 @@ import javax.security.auth.login.LoginContext;
 
 import junit.framework.Assert;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -48,6 +48,15 @@ public class TestUserGroupInformation {
   final private static String GROUP3_NAME = "group3";
   final private static String[] GROUP_NAMES = 
     new String[]{GROUP1_NAME, GROUP2_NAME, GROUP3_NAME};
+  
+  static {
+    Configuration conf = new Configuration();
+    conf.set("hadoop.security.auth_to_local",
+        "RULE:[2:$1@$0](.*@HADOOP.APACHE.ORG)s/@.*//" +
+        "RULE:[1:$1@$0](.*@HADOOP.APACHE.ORG)s/@.*//"
+        + "DEFAULT");
+    UserGroupInformation.setConfiguration(conf);
+  }
 
   /**
    * given user name - get all the groups.

src/test/krb5.conf

@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+[libdefaults]
+	default_realm = APACHE.ORG
+	udp_preference_limit = 1
+	extra_addresses = 127.0.0.1
+[realms]
+	APACHE.ORG = {
+		admin_server = localhost:88
+		kdc = localhost:88
+	}
+[domain_realm]
+	localhost = APACHE.ORG