YARN-1157. Fixed ResourceManager UI to behave correctly when apps like distributed-shell do not set tracking urls. Contributed by Xuan Gong.

svn merge --ignore-ancestry -c 1526371 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.1-beta@1526374 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -37,6 +37,9 @@ Release 2.1.2 - UNRELEASED
     YARN-49. Improve distributed shell application to work on a secure cluster.
     (Vinod Kumar Vavilapalli via hitesh)
 
+    YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
+    distributed-shell do not set tracking urls. (Xuan Gong via vinodkv)
+
 Release 2.1.1-beta - 2013-09-23
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java

@@ -100,11 +100,22 @@ public abstract class FinishApplicationMasterRequest {
   public abstract String getTrackingUrl();
 
   /**
-   * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
-   * This url if contains scheme then that will be used by resource manager
-   * web application proxy otherwise it will default to http.
-   * @param url <em>tracking URL</em>for the
-   *                   <code>ApplicationMaster</code>
+   * Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+   * This is the web-URL to which ResourceManager or web-application proxy will
+   * redirect client/users once the application is finished and the
+   * <code>ApplicationMaster</code> is gone.
+   * <p>
+   * If the passed url has a scheme then that will be used by the
+   * ResourceManager and web-application proxy, otherwise the scheme will
+   * default to http.
+   * </p>
+   * <p>
+   * Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+   * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+   * <p>
+   *
+   * @param url
+   *          <em>tracking URL</em>for the <code>ApplicationMaster</code>
    */
   @Public
   @Stable

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java

@@ -112,11 +112,22 @@ public abstract class RegisterApplicationMasterRequest {
   public abstract String getTrackingUrl();
   
   /**
-   * Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
-   * This url if contains scheme then that will be used by resource manager
-   * web application proxy otherwise it will default to http.
-   * @param trackingUrl <em>tracking URL</em> for the 
-   *                    <code>ApplicationMaster</code>
+   * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+   * it is running. This is the web-URL to which ResourceManager or
+   * web-application proxy will redirect client/users while the application and
+   * the <code>ApplicationMaster</code> are still running.
+   * <p>
+   * If the passed url has a scheme then that will be used by the
+   * ResourceManager and web-application proxy, otherwise the scheme will
+   * default to http.
+   * </p>
+   * <p>
+   * Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+   * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+   * <p>
+   *
+   * @param trackingUrl
+   *          <em>tracking URL</em>for the <code>ApplicationMaster</code>
    */
   @Public
   @Stable

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java

@@ -994,7 +994,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     }
   }
 
-  static final class AMRegisteredTransition extends BaseTransition {
+  private static final class AMRegisteredTransition extends BaseTransition {
     @Override
     public void transition(RMAppAttemptImpl appAttempt,
         RMAppAttemptEvent event) {
@@ -1003,7 +1003,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
           = (RMAppAttemptRegistrationEvent) event;
       appAttempt.host = registrationEvent.getHost();
       appAttempt.rpcPort = registrationEvent.getRpcport();
-      appAttempt.origTrackingUrl = registrationEvent.getTrackingurl();
+      appAttempt.origTrackingUrl =
+          sanitizeTrackingUrl(registrationEvent.getTrackingurl());
       appAttempt.proxiedTrackingUrl = 
         appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
 
@@ -1138,7 +1139,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
       RMAppAttemptUnregistrationEvent unregisterEvent
         = (RMAppAttemptUnregistrationEvent) event;
       appAttempt.diagnostics.append(unregisterEvent.getDiagnostics());
-      appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl();
+      appAttempt.origTrackingUrl =
+          sanitizeTrackingUrl(unregisterEvent.getTrackingUrl());
       appAttempt.proxiedTrackingUrl = 
         appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
       appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
@@ -1292,4 +1294,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     appAttempt.rmContext.getAMRMTokenSecretManager()
       .applicationMasterFinished(appAttempt.getAppAttemptId());
   }
+
+  private static String sanitizeTrackingUrl(String url) {
+    return (url == null || url.trim().isEmpty()) ? "N/A" : url;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java

@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
-
-import junit.framework.Assert;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
-
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-public class TestRMAppAttemptImpl {
-  
-  private void testTrackingUrl(String url, boolean unmanaged) {
-    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance
-        (ApplicationId.newInstance(1, 2), 1);
-    EventHandler handler = Mockito.mock(EventHandler.class);    
-    Dispatcher dispatcher = Mockito.mock(Dispatcher.class);
-    Mockito.when(dispatcher.getEventHandler()).thenReturn(handler);
-    RMContext rmContext = Mockito.mock(RMContext.class);
-    Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher);
-    
-    ApplicationSubmissionContext appContext = 
-        Mockito.mock(ApplicationSubmissionContext.class);
-    Mockito.when(appContext.getUnmanagedAM()).thenReturn(unmanaged);
-    
-    RMAppAttemptImpl attempt = new RMAppAttemptImpl(attemptId, rmContext, null, 
-        null, appContext, new YarnConfiguration(), null);
-    RMAppAttemptRegistrationEvent event = 
-        Mockito.mock(RMAppAttemptRegistrationEvent.class);
-    Mockito.when(event.getHost()).thenReturn("h");
-    Mockito.when(event.getRpcport()).thenReturn(0);
-    Mockito.when(event.getTrackingurl()).thenReturn(url);
-    new RMAppAttemptImpl.AMRegisteredTransition().transition(attempt, event);
-    if (unmanaged) {
-      Assert.assertEquals(url, attempt.getTrackingUrl());
-    } else {
-      Assert.assertNotSame(url, attempt.getTrackingUrl());
-      Assert.assertTrue(attempt.getTrackingUrl().contains(
-          ProxyUriUtils.PROXY_SERVLET_NAME));
-      Assert.assertTrue(attempt.getTrackingUrl().contains(
-          attemptId.getApplicationId().toString()));
-    }
-  }
-
-  @Test
-  public void testTrackingUrlUnmanagedAM() {
-    testTrackingUrl("http://foo:8000/x", true);
-  }
-
-  @Test
-  public void testTrackingUrlManagedAM() {
-    testTrackingUrl("bar:8000/x", false);
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java

@@ -30,14 +30,18 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -85,8 +89,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -261,8 +267,22 @@ public class TestRMAppAttemptTransitions {
   
 
   private String getProxyUrl(RMAppAttempt appAttempt) {
-    return pjoin(RM_WEBAPP_ADDR, "proxy",
-        appAttempt.getAppAttemptId().getApplicationId(), "");
+    String url = null;
+    try {
+      URI trackingUri =
+          StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
+              ProxyUriUtils
+                  .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
+      String proxy = WebAppUtils.getProxyHostAndPort(conf);
+      URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
+      URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+          appAttempt.getAppAttemptId().getApplicationId());
+      url = result.toASCIIString().substring(
+          HttpConfig.getSchemePrefix().length());
+    } catch (URISyntaxException ex) {
+      Assert.fail();
+    }
+    return url;
   }
 
   /**
@@ -448,9 +468,9 @@ public class TestRMAppAttemptTransitions {
     assertEquals(container, applicationAttempt.getMasterContainer());
     assertEquals(host, applicationAttempt.getHost());
     assertEquals(rpcPort, applicationAttempt.getRpcPort());
-    assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+    verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
     if (unmanagedAM) {
-      assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl());
+      verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
     } else {
       assertEquals(getProxyUrl(applicationAttempt), 
           applicationAttempt.getTrackingUrl());
@@ -468,7 +488,7 @@ public class TestRMAppAttemptTransitions {
     assertEquals(RMAppAttemptState.FINISHING,
         applicationAttempt.getAppAttemptState());
     assertEquals(diagnostics, applicationAttempt.getDiagnostics());
-    assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+    verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
     assertEquals(getProxyUrl(applicationAttempt),
         applicationAttempt.getTrackingUrl());
     assertEquals(container, applicationAttempt.getMasterContainer());
@@ -487,9 +507,9 @@ public class TestRMAppAttemptTransitions {
     assertEquals(RMAppAttemptState.FINISHED, 
         applicationAttempt.getAppAttemptState());
     assertEquals(diagnostics, applicationAttempt.getDiagnostics());
-    assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+    verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
     if (unmanagedAM) {
-      assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl());
+      verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
       
     } else {
       assertEquals(getProxyUrl(applicationAttempt),
@@ -603,9 +623,7 @@ public class TestRMAppAttemptTransitions {
         trackingUrl, diagnostics);
   }
 
-  
-  @Test
-  public void testUnmanagedAMSuccess() {
+  private void testUnmanagedAMSuccess(String url) {
     unmanagedAM = true;
     when(submissionContext.getUnmanagedAM()).thenReturn(true);
     // submit AM and check it goes to LAUNCHED state
@@ -615,7 +633,7 @@ public class TestRMAppAttemptTransitions {
         applicationAttempt.getAppAttemptId());
 
     // launch AM
-    runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true);
+    runApplicationAttempt(null, "host", 8042, url, true);
 
     // complete a container
     applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
@@ -623,13 +641,12 @@ public class TestRMAppAttemptTransitions {
     applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
         applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
     // complete AM
-    String trackingUrl = "mytrackingurl";
     String diagnostics = "Successful";
     FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
     applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
-        applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
+        applicationAttempt.getAppAttemptId(), url, finalStatus,
         diagnostics));
-    testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1,
+    testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
         true);
   }
   
@@ -824,12 +841,42 @@ public class TestRMAppAttemptTransitions {
         "Killed by user");
   }
 
+  @Test
+  public void testTrackingUrlUnmanagedAM() {
+    testUnmanagedAMSuccess("oldTrackingUrl");
+  }
 
   @Test
-  public void testNoTrackingUrl() {
+  public void testEmptyTrackingUrlUnmanagedAM() {
+    testUnmanagedAMSuccess("");
+  }
+
+  @Test
+  public void testNullTrackingUrlUnmanagedAM() {
+    testUnmanagedAMSuccess(null);
+  }
+
+  @Test
+  public void testManagedAMWithTrackingUrl() {
+    testTrackingUrlManagedAM("theTrackingUrl");
+  }
+
+  @Test
+  public void testManagedAMWithEmptyTrackingUrl() {
+    testTrackingUrlManagedAM("");
+  }
+
+  @Test
+  public void testManagedAMWithNullTrackingUrl() {
+    testTrackingUrlManagedAM(null);
+  }
+
+  private void testTrackingUrlManagedAM(String url) {
     Container amContainer = allocateApplicationAttempt();
     launchApplicationAttempt(amContainer);
-    runApplicationAttempt(amContainer, "host", 8042, "", false);
+    runApplicationAttempt(amContainer, "host", 8042, url, false);
+    unregisterApplicationAttempt(amContainer,
+        FinalApplicationStatus.SUCCEEDED, url, "Successful");
   }
 
   @Test
@@ -927,4 +974,12 @@ public class TestRMAppAttemptTransitions {
       }
     }
   }
+
+  private void verifyUrl(String url1, String url2) {
+    if (url1 == null || url1.trim().isEmpty()) {
+      assertEquals("N/A", url2);
+    } else {
+      assertEquals(url1, url2);
+    }
+  }
 }