Merge branch 'trunk' into HDFS-6581

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -768,6 +768,9 @@ Release 2.6.0 - UNRELEASED
     HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
     determine if in proxyuser mode or not. (tucu)
 
+    HADOOP-11073. Credential Provider related Unit Tests Failure on Windows.
+    (Xiaoyu Yao via cnauroth)
+
 Release 2.5.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java

@@ -55,18 +55,18 @@ public class TestKeyProviderFactory {
   @Test
   public void testFactory() throws Exception {
     Configuration conf = new Configuration();
+    final String userUri = UserProvider.SCHEME_NAME + ":///";
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
+    final String jksUri = JavaKeyStoreProvider.SCHEME_NAME +
+        "://file" + jksPath.toUri().toString();
     conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
-        UserProvider.SCHEME_NAME + ":///," +
-            JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+        userUri + "," + jksUri);
     List<KeyProvider> providers = KeyProviderFactory.getProviders(conf);
     assertEquals(2, providers.size());
     assertEquals(UserProvider.class, providers.get(0).getClass());
     assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
-    assertEquals(UserProvider.SCHEME_NAME +
-        ":///", providers.get(0).toString());
-    assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
-        "://file" + tmpDir + "/test.jks",
-        providers.get(1).toString());
+    assertEquals(userUri, providers.get(0).toString());
+    assertEquals(jksUri, providers.get(1).toString());
   }
 
   @Test
@@ -207,8 +207,9 @@ public class TestKeyProviderFactory {
   @Test
   public void testJksProvider() throws Exception {
     Configuration conf = new Configuration();
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
     final String ourUrl =
-        JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
+        JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
 
     File file = new File(tmpDir, "test.jks");
     file.delete();
@@ -317,8 +318,9 @@ public class TestKeyProviderFactory {
   @Test
   public void testJksProviderPasswordViaConfig() throws Exception {
     Configuration conf = new Configuration();
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
     final String ourUrl =
-        JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
+        JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
     File file = new File(tmpDir, "test.jks");
     file.delete();
     try {
@@ -360,8 +362,8 @@ public class TestKeyProviderFactory {
   @Test
   public void testGetProviderViaURI() throws Exception {
     Configuration conf = new Configuration(false);
-    URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir +
-        "/test.jks");
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
+    URI uri = new URI(JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
     KeyProvider kp = KeyProviderFactory.get(uri, conf);
     Assert.assertNotNull(kp);
     Assert.assertEquals(JavaKeyStoreProvider.class, kp.getClass());

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java

@@ -40,6 +40,7 @@ import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@@ -165,8 +166,9 @@ public class TestLdapGroupsMapping {
     File testDir = new File(System.getProperty("test.build.data",
                                                "target/test-dir"));
     Configuration conf = new Configuration();
+    final Path jksPath = new Path(testDir.toString(), "test.jks");
     final String ourUrl =
-        JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+        JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
 
     File file = new File(testDir, "test.jks");
     file.delete();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java

@@ -52,19 +52,19 @@ public class TestCredentialProviderFactory {
   @Test
   public void testFactory() throws Exception {
     Configuration conf = new Configuration();
+    final String userUri = UserProvider.SCHEME_NAME + ":///";
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
+    final String jksUri = JavaKeyStoreProvider.SCHEME_NAME +
+        "://file" + jksPath.toUri();
     conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
-        UserProvider.SCHEME_NAME + ":///," +
-            JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+        userUri + "," + jksUri);
     List<CredentialProvider> providers = 
         CredentialProviderFactory.getProviders(conf);
     assertEquals(2, providers.size());
     assertEquals(UserProvider.class, providers.get(0).getClass());
     assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
-    assertEquals(UserProvider.SCHEME_NAME +
-        ":///", providers.get(0).toString());
-    assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
-        "://file" + tmpDir + "/test.jks",
-        providers.get(1).toString());
+    assertEquals(userUri, providers.get(0).toString());
+    assertEquals(jksUri, providers.get(1).toString());
   }
 
   @Test
@@ -188,8 +188,9 @@ public class TestCredentialProviderFactory {
   @Test
   public void testJksProvider() throws Exception {
     Configuration conf = new Configuration();
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
     final String ourUrl =
-        JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
+        JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
 
     File file = new File(tmpDir, "test.jks");
     file.delete();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.security.ssl;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@@ -392,8 +393,9 @@ public class KeyStoreTestUtil {
         "target/test-dir"));
 
     Configuration conf = new Configuration();
+    final Path jksPath = new Path(testDir.toString(), "test.jks");
     final String ourUrl =
-    JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+    JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
 
     File file = new File(testDir, "test.jks");
     file.delete();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java

@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
@@ -305,8 +306,9 @@ public class TestSSLFactory {
       if (useCredProvider) {
         File testDir = new File(System.getProperty("test.build.data",
             "target/test-dir"));
+        final Path jksPath = new Path(testDir.toString(), "test.jks");
         final String ourUrl =
-            JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+            JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
         sslConf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
       }
     } else {

hadoop-yarn-project/CHANGES.txt

@@ -184,6 +184,15 @@ Release 2.6.0 - UNRELEASED
     YARN-2508. Cross Origin configuration parameters prefix are not honored
     (Mit Desai via jeagles)
 
+    YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
+    (Jonathan Eagles via zjshen)
+
+    YARN-2507. Documented CrossOriginFilter configurations for the timeline
+    server. (Jonathan Eagles via zjshen)
+
+    YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
+    (Tsuyoshi OZAWA via jianhe)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java

@@ -18,8 +18,10 @@
 
 package org.apache.hadoop.yarn.api.records;
 
-import java.text.NumberFormat;
+import com.google.common.base.Splitter;
 
+import java.text.NumberFormat;
+import java.util.Iterator;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records;
 @Public
 @Stable
 public abstract class ContainerId implements Comparable<ContainerId>{
+  private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
+  private static final String CONTAINER_PREFIX = "container";
 
   @Private
   @Unstable
@@ -163,5 +167,38 @@ public abstract class ContainerId implements Comparable<ContainerId>{
     return sb.toString();
   }
 
+  @Public
+  @Unstable
+  public static ContainerId fromString(String containerIdStr) {
+    Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
+    if (!it.next().equals(CONTAINER_PREFIX)) {
+      throw new IllegalArgumentException("Invalid ContainerId prefix: "
+          + containerIdStr);
+    }
+    try {
+      ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
+      int id = Integer.parseInt(it.next());
+      int epoch = 0;
+      if (it.hasNext()) {
+        epoch = Integer.parseInt(it.next());
+      }
+      int cid = (epoch << 22) | id;
+      ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
+      return containerId;
+    } catch (NumberFormatException n) {
+      throw new IllegalArgumentException("Invalid ContainerId: "
+          + containerIdStr, n);
+    }
+  }
+
+  private static ApplicationAttemptId toApplicationAttemptId(
+      Iterator<String> it) throws NumberFormatException {
+    ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+        Integer.parseInt(it.next()));
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
+    return appAttemptId;
+  }
+
   protected abstract void build();
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java

@@ -168,20 +168,7 @@ public class ConverterUtils {
   }
 
   public static ContainerId toContainerId(String containerIdStr) {
-    Iterator<String> it = _split(containerIdStr).iterator();
-    if (!it.next().equals(CONTAINER_PREFIX)) {
-      throw new IllegalArgumentException("Invalid ContainerId prefix: "
-          + containerIdStr);
-    }
-    try {
-      ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
-      ContainerId containerId =
-          ContainerId.newInstance(appAttemptID, Integer.parseInt(it.next()));
-      return containerId;
-    } catch (NumberFormatException n) {
-      throw new IllegalArgumentException("Invalid ContainerId: "
-          + containerIdStr, n);
-    }
+    return ContainerId.fromString(containerIdStr);
   }
 
   public static ApplicationAttemptId toApplicationAttemptId(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java

@@ -54,10 +54,14 @@ public class TestContainerId {
     long ts = System.currentTimeMillis();
     ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811);
     Assert.assertEquals("container_10_0001_01_000001", c1.toString());
+    Assert.assertEquals(c1,
+        ContainerId.fromString("container_10_0001_01_000001"));
     Assert.assertEquals(479987, 0x003fffff & c6.getId());
     Assert.assertEquals(6, c6.getId() >> 22);
     Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06",
         c6.toString());
+    Assert.assertEquals(c6,
+        ContainerId.fromString("container_" + ts + "_36473_4365472_479987_06"));
   }
 
   public static ContainerId newContainerId(int appId, int appAttemptId,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java

@@ -55,6 +55,15 @@ public class TestConverterUtils {
     assertEquals(gen, id);
   }
 
+  @Test
+  public void testContainerIdWithEpoch() throws URISyntaxException {
+    ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
+    String cid = ConverterUtils.toString(id);
+    assertEquals("container_0_0000_00_479987_06", cid);
+    ContainerId gen = ConverterUtils.toContainerId(cid);
+    assertEquals(gen.toString(), id.toString());
+  }
+
   @Test
   public void testContainerIdNull() throws URISyntaxException {
     assertNull(ConverterUtils.toString((ContainerId)null));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java

@@ -24,6 +24,8 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
@@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
 
   @VisibleForTesting
   boolean isOriginAllowed(String origin) {
-    return allowAllOrigins || allowedOrigins.contains(origin);
+    if (allowAllOrigins) {
+      return true;
+    }
+
+    for (String allowedOrigin : allowedOrigins) {
+      if (allowedOrigin.contains("*")) {
+        String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
+        Pattern p = Pattern.compile(regex);
+        Matcher m = p.matcher(origin);
+        if (m.matches()) {
+          return true;
+        }
+      } else if (allowedOrigin.equals(origin)) {
+        return true;
+      }
+    }
+    return false;
   }
 
   private boolean areHeadersAllowed(String accessControlRequestHeaders) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java

@@ -77,7 +77,26 @@ public class TestCrossOriginFilter {
     // Object under test
     CrossOriginFilter filter = new CrossOriginFilter();
     filter.init(filterConfig);
-    Assert.assertTrue(filter.isOriginAllowed("example.org"));
+    Assert.assertTrue(filter.isOriginAllowed("example.com"));
+  }
+
+  @Test
+  public void testPatternMatchingOrigins() throws ServletException, IOException {
+
+    // Setup the configuration settings of the server
+    Map<String, String> conf = new HashMap<String, String>();
+    conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "*.example.com");
+    FilterConfig filterConfig = new FilterConfigTest(conf);
+
+    // Object under test
+    CrossOriginFilter filter = new CrossOriginFilter();
+    filter.init(filterConfig);
+
+    // match multiple sub-domains
+    Assert.assertFalse(filter.isOriginAllowed("example.com"));
+    Assert.assertFalse(filter.isOriginAllowed("foo:example.com"));
+    Assert.assertTrue(filter.isOriginAllowed("foo.example.com"));
+    Assert.assertTrue(filter.isOriginAllowed("foo.bar.example.com"));
   }
 
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm

@@ -102,6 +102,43 @@ YARN Timeline Server
   <name>yarn.timeline-service.handler-thread-count</name>
   <value>10</value>
 </property>
+
+<property>
+  <description>Enables cross-origin support (CORS) for web services where
+  cross-origin web response headers are needed. For example, javascript making
+  a web services request to the timeline server.</description>
+  <name>yarn.timeline-service.http-cross-origin.enabled</name>
+  <value>false</value>
+</property>
+
+<property>
+  <description>Comma separated list of origins that are allowed for web
+  services needing cross-origin (CORS) support. Wildcards (*) and patterns
+  allowed</description>
+  <name>yarn.timeline-service.http-cross-origin.allowed-origins</name>
+  <value>*</value>
+</property>
+
+<property>
+  <description>Comma separated list of methods that are allowed for web
+  services needing cross-origin (CORS) support.</description>
+  <name>yarn.timeline-service.http-cross-origin.allowed-methods</name>
+  <value>GET,POST,HEAD</value>
+</property>
+
+<property>
+  <description>Comma separated list of headers that are allowed for web
+  services needing cross-origin (CORS) support.</description>
+  <name>yarn.timeline-service.http-cross-origin.allowed-headers</name>
+  <value>X-Requested-With,Content-Type,Accept,Origin</value>
+</property>
+
+<property>
+  <description>The number of seconds a pre-flighted request can be cached
+  for web services needing cross-origin (CORS) support.</description>
+  <name>yarn.timeline-service.http-cross-origin.max-age</name>
+  <value>1800</value>
+</property>
 +---+
 
 * Generic-data related Configuration