YARN-8769. [Submarine] Allow user to specify customized quicklink(s) when submit Submarine job. Contributed by Wangda Tan.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java

@@ -49,6 +49,7 @@ public class CliConstants {
   public static final String WAIT_JOB_FINISH = "wait_job_finish";
   public static final String PS_DOCKER_IMAGE = "ps_docker_image";
   public static final String WORKER_DOCKER_IMAGE = "worker_docker_image";
+  public static final String QUICKLINK = "quicklink";
   public static final String TENSORBOARD_DOCKER_IMAGE =
       "tensorboard_docker_image";
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java

@@ -117,6 +117,14 @@ public class RunJobCli extends AbstractCli {
     options.addOption(CliConstants.WORKER_DOCKER_IMAGE, true,
         "Specify docker image for WORKER, when this is not specified, WORKER "
             + "uses --" + CliConstants.DOCKER_IMAGE + " as default.");
+    options.addOption(CliConstants.QUICKLINK, true, "Specify quicklink so YARN"
+        + "web UI shows link to given role instance and port. When "
+        + "--tensorboard is speciied, quicklink to tensorboard instance will "
+        + "be added automatically. The format of quick link is: "
+        + "Quick_link_label=http(or https)://role-name:port. For example, "
+        + "if want to link to first worker's 7070 port, and text of quicklink "
+        + "is Notebook_UI, user need to specify --quicklink "
+        + "Notebook_UI=https://master-0:7070");
     options.addOption("h", "help", false, "Print help");
     return options;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/Quicklink.java

@@ -0,0 +1,71 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.yarn.submarine.client.cli.param;
+
+import org.apache.commons.cli.ParseException;
+
+/**
+ * A class represents quick links to a web page.
+ */
+public class Quicklink {
+  private String label;
+  private String componentInstanceName;
+  private String protocol;
+  private int port;
+
+  public void parse(String quicklinkStr) throws ParseException {
+    if (!quicklinkStr.contains("=")) {
+      throw new ParseException("Should be <label>=<link> format for quicklink");
+    }
+
+    int index = quicklinkStr.indexOf("=");
+    label = quicklinkStr.substring(0, index);
+    quicklinkStr = quicklinkStr.substring(index + 1);
+
+    if (quicklinkStr.startsWith("http://")) {
+      protocol = "http://";
+    } else if (quicklinkStr.startsWith("https://")) {
+      protocol = "https://";
+    } else {
+      throw new ParseException("Quicklink should start with http or https");
+    }
+
+    quicklinkStr = quicklinkStr.substring(protocol.length());
+    index = quicklinkStr.indexOf(":");
+
+    if (index == -1) {
+      throw new ParseException("Quicklink should be componet-id:port form");
+    }
+
+    componentInstanceName = quicklinkStr.substring(0, index);
+    port = Integer.parseInt(quicklinkStr.substring(index + 1));
+  }
+
+  public String getLabel() {
+    return label;
+  }
+
+  public String getComponentInstanceName() {
+    return componentInstanceName;
+  }
+
+  public String getProtocol() {
+    return protocol;
+  }
+
+  public int getPort() {
+    return port;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java

@@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.submarine.client.cli.CliUtils;
 import org.apache.hadoop.yarn.submarine.common.ClientContext;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Parameters used to run a job
@@ -41,6 +43,7 @@ public class RunJobParameters extends RunParameters {
   private String tensorboardDockerImage;
   private String workerLaunchCmd;
   private String psLaunchCmd;
+  private List<Quicklink> quicklinks = new ArrayList<>();
 
   private String psDockerImage = null;
   private String workerDockerImage = null;
@@ -119,6 +122,17 @@ public class RunJobParameters extends RunParameters {
       this.waitJobFinish = true;
     }
 
+    // Quicklinks
+    String[] quicklinkStrs = parsedCommandLine.getOptionValues(
+        CliConstants.QUICKLINK);
+    if (quicklinkStrs != null) {
+      for (String ql : quicklinkStrs) {
+        Quicklink quicklink = new Quicklink();
+        quicklink.parse(ql);
+        quicklinks.add(quicklink);
+      }
+    }
+
     psDockerImage = parsedCommandLine.getOptionValue(
         CliConstants.PS_DOCKER_IMAGE);
     workerDockerImage = parsedCommandLine.getOptionValue(
@@ -247,4 +261,8 @@ public class RunJobParameters extends RunParameters {
   public String getTensorboardDockerImage() {
     return tensorboardDockerImage;
   }
+
+  public List<Quicklink> getQuicklinks() {
+    return quicklinks;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java

@@ -15,7 +15,6 @@
 package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -29,6 +28,7 @@ import org.apache.hadoop.yarn.service.api.records.Resource;
 import org.apache.hadoop.yarn.service.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.submarine.client.cli.param.Quicklink;
 import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
 import org.apache.hadoop.yarn.submarine.common.ClientContext;
 import org.apache.hadoop.yarn.submarine.common.Envs;
@@ -40,10 +40,14 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileNotFoundException;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.StringTokenizer;
@@ -54,6 +58,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
  * Submit a job to cluster
  */
 public class YarnServiceJobSubmitter implements JobSubmitter {
+  public static final String TENSORBOARD_QUICKLINK_LABEL = "Tensorboard";
   private static final Logger LOG =
       LoggerFactory.getLogger(YarnServiceJobSubmitter.class);
   ClientContext clientContext;
@@ -98,7 +103,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
   }
 
   private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
-      FileWriter fw, Component comp) throws IOException {
+      PrintWriter fw, Component comp) throws IOException {
     // Find envs to use HDFS
     String hdfsHome = null;
     String javaHome = null;
@@ -191,7 +196,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
     envs.put(Envs.TASK_TYPE_ENV, taskType.name());
   }
 
-  private String getUserName() {
+  @VisibleForTesting
+  protected String getUserName() {
     return System.getProperty("user.name");
   }
 
@@ -205,18 +211,19 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
   private String generateCommandLaunchScript(RunJobParameters parameters,
       TaskType taskType, Component comp) throws IOException {
     File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
-    FileWriter fw = new FileWriter(file);
+    Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+    PrintWriter pw = new PrintWriter(w);
 
     try {
-      fw.append("#!/bin/bash\n");
+      pw.append("#!/bin/bash\n");
 
-      addHdfsClassPathIfNeeded(parameters, fw, comp);
+      addHdfsClassPathIfNeeded(parameters, pw, comp);
 
       if (taskType.equals(TaskType.TENSORBOARD)) {
         String tbCommand =
             "export LC_ALL=C && tensorboard --logdir=" + parameters
                 .getCheckpointPath();
-        fw.append(tbCommand + "\n");
+        pw.append(tbCommand + "\n");
         LOG.info("Tensorboard command=" + tbCommand);
       } else{
         // When distributed training is required
@@ -226,20 +233,20 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
               taskType.getComponentName(), parameters.getNumWorkers(),
               parameters.getNumPS(), parameters.getName(), getUserName(),
               getDNSDomain());
-          fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
+          pw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
         }
 
         // Print launch command
         if (taskType.equals(TaskType.WORKER) || taskType.equals(
             TaskType.PRIMARY_WORKER)) {
-          fw.append(parameters.getWorkerLaunchCmd() + '\n');
+          pw.append(parameters.getWorkerLaunchCmd() + '\n');
 
           if (SubmarineLogs.isVerbose()) {
             LOG.info(
                 "Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
           }
         } else if (taskType.equals(TaskType.PS)) {
-          fw.append(parameters.getPSLaunchCmd() + '\n');
+          pw.append(parameters.getPSLaunchCmd() + '\n');
 
           if (SubmarineLogs.isVerbose()) {
             LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
@@ -247,7 +254,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
         }
       }
     } finally {
-      fw.close();
+      pw.close();
     }
     return file.getAbsolutePath();
   }
@@ -421,18 +428,51 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
     return new Artifact().type(Artifact.TypeEnum.DOCKER).id(dockerImageName);
   }
 
+  private void handleQuicklinks(RunJobParameters runJobParameters)
+      throws IOException {
+    List<Quicklink> quicklinks = runJobParameters.getQuicklinks();
+    if (null != quicklinks && !quicklinks.isEmpty()) {
+      for (Quicklink ql : quicklinks) {
+        // Make sure it is a valid instance name
+        String instanceName = ql.getComponentInstanceName();
+        boolean found = false;
+
+        for (Component comp : serviceSpec.getComponents()) {
+          for (int i = 0; i < comp.getNumberOfContainers(); i++) {
+            String possibleInstanceName = comp.getName() + "-" + i;
+            if (possibleInstanceName.equals(instanceName)) {
+              found = true;
+              break;
+            }
+          }
+        }
+
+        if (!found) {
+          throw new IOException(
+              "Couldn't find a component instance = " + instanceName
+                  + " while adding quicklink");
+        }
+
+        String link = ql.getProtocol() + YarnServiceUtils.getDNSName(
+            serviceSpec.getName(), instanceName, getUserName(), getDNSDomain(),
+            ql.getPort());
+        YarnServiceUtils.addQuicklink(serviceSpec, ql.getLabel(), link);
+      }
+    }
+  }
+
   private Service createServiceByParameters(RunJobParameters parameters)
       throws IOException {
     componentToLocalLaunchScriptPath.clear();
-    Service service = new Service();
-    service.setName(parameters.getName());
-    service.setVersion(String.valueOf(System.currentTimeMillis()));
-    service.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
+    serviceSpec = new Service();
+    serviceSpec.setName(parameters.getName());
+    serviceSpec.setVersion(String.valueOf(System.currentTimeMillis()));
+    serviceSpec.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
 
-    handleServiceEnvs(service, parameters);
+    handleServiceEnvs(serviceSpec, parameters);
 
     if (parameters.getNumWorkers() > 0) {
-      addWorkerComponents(service, parameters);
+      addWorkerComponents(serviceSpec, parameters);
     }
 
     if (parameters.getNumPS() > 0) {
@@ -450,7 +490,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
             getDockerArtifact(parameters.getPsDockerImage()));
       }
       handleLaunchCommand(parameters, TaskType.PS, psComponent);
-      service.addComponent(psComponent);
+      serviceSpec.addComponent(psComponent);
     }
 
     if (parameters.isTensorboardEnabled()) {
@@ -470,14 +510,20 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
 
       // Add tensorboard to quicklink
       String tensorboardLink = "http://" + YarnServiceUtils.getDNSName(
-          parameters.getName(), TaskType.TENSORBOARD.getComponentName(), 0,
-          getUserName(), getDNSDomain(), 6006);
+          parameters.getName(),
+          TaskType.TENSORBOARD.getComponentName() + "-" + 0, getUserName(),
+          getDNSDomain(), 6006);
       LOG.info("Link to tensorboard:" + tensorboardLink);
-      service.addComponent(tbComponent);
-      service.setQuicklinks(ImmutableMap.of("Tensorboard", tensorboardLink));
+      serviceSpec.addComponent(tbComponent);
+
+      YarnServiceUtils.addQuicklink(serviceSpec, TENSORBOARD_QUICKLINK_LABEL,
+          tensorboardLink);
     }
 
-    return service;
+    // After all components added, handle quicklinks
+    handleQuicklinks(parameters);
+
+    return serviceSpec;
   }
 
   /**
@@ -486,12 +532,11 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
   @Override
   public ApplicationId submitJob(RunJobParameters parameters)
       throws IOException, YarnException {
-    Service service = createServiceByParameters(parameters);
+    createServiceByParameters(parameters);
     ServiceClient serviceClient = YarnServiceUtils.createServiceClient(
         clientContext.getYarnConfig());
-    ApplicationId appid = serviceClient.actionCreate(service);
+    ApplicationId appid = serviceClient.actionCreate(serviceSpec);
     serviceClient.stop();
-    this.serviceSpec = service;
     return appid;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java

@@ -16,10 +16,20 @@ package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.client.ServiceClient;
 import org.apache.hadoop.yarn.submarine.common.Envs;
+import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
 
 public class YarnServiceUtils {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(YarnServiceUtils.class);
+
   // This will be true only in UT.
   private static ServiceClient stubServiceClient = null;
 
@@ -40,10 +50,10 @@ public class YarnServiceUtils {
     YarnServiceUtils.stubServiceClient = stubServiceClient;
   }
 
-  public static String getDNSName(String serviceName, String componentName,
-      int index, String userName, String domain, int port) {
-    return componentName + "-" + index + getDNSNameCommonSuffix(serviceName,
-        userName, domain, port);
+  public static String getDNSName(String serviceName,
+      String componentInstanceName, String userName, String domain, int port) {
+    return componentInstanceName + getDNSNameCommonSuffix(serviceName, userName,
+        domain, port);
   }
 
   private static String getDNSNameCommonSuffix(String serviceName,
@@ -66,12 +76,18 @@ public class YarnServiceUtils {
         commonEndpointSuffix) + ",";
     String ps = getComponentArrayJson("ps", nPs, commonEndpointSuffix) + "},";
 
-    String task =
-        "\\\"task\\\":{" + " \\\"type\\\":\\\"" + curCommponentName + "\\\","
-            + " \\\"index\\\":" + '$' + Envs.TASK_INDEX_ENV + "},";
+    StringBuilder sb = new StringBuilder();
+    sb.append("\\\"task\\\":{");
+    sb.append(" \\\"type\\\":\\\"");
+    sb.append(curCommponentName);
+    sb.append("\\\",");
+    sb.append(" \\\"index\\\":");
+    sb.append('$');
+    sb.append(Envs.TASK_INDEX_ENV + "},");
+    String task = sb.toString();
     String environment = "\\\"environment\\\":\\\"cloud\\\"}";
 
-    StringBuilder sb = new StringBuilder();
+    sb = new StringBuilder();
     sb.append(json);
     sb.append(master);
     sb.append(worker);
@@ -81,6 +97,21 @@ public class YarnServiceUtils {
     return sb.toString();
   }
 
+  public static void addQuicklink(Service serviceSpec, String label,
+      String link) {
+    Map<String, String> quicklinks = serviceSpec.getQuicklinks();
+    if (null == quicklinks) {
+      quicklinks = new HashMap<>();
+      serviceSpec.setQuicklinks(quicklinks);
+    }
+
+    if (SubmarineLogs.isVerbose()) {
+      LOG.info("Added quicklink, " + label + "=" + link);
+    }
+
+    quicklinks.put(label, link);
+  }
+
   private static String getComponentArrayJson(String componentName, int count,
       String endpointSuffix) {
     String component = "\\\"" + componentName + "\\\":";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.submarine.client.cli.yarnservice;
 
+import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -100,6 +101,32 @@ public class TestYarnServiceRunJobCli {
     Assert.assertTrue(SubmarineLogs.isVerbose());
   }
 
+  private void verifyQuicklink(Service serviceSpec,
+      Map<String, String> expectedQuicklinks) {
+    Map<String, String> actualQuicklinks = serviceSpec.getQuicklinks();
+    if (actualQuicklinks == null || actualQuicklinks.isEmpty()) {
+      Assert.assertTrue(
+          expectedQuicklinks == null || expectedQuicklinks.isEmpty());
+      return;
+    }
+
+    Assert.assertEquals(expectedQuicklinks.size(), actualQuicklinks.size());
+    for (Map.Entry<String, String> expectedEntry : expectedQuicklinks
+        .entrySet()) {
+      Assert.assertTrue(actualQuicklinks.containsKey(expectedEntry.getKey()));
+
+      // $USER could be changed in different environment. so replace $USER by
+      // "user"
+      String expectedValue = expectedEntry.getValue();
+      String actualValue = actualQuicklinks.get(expectedEntry.getKey());
+
+      String userName = System.getProperty("user.name");
+      actualValue = actualValue.replaceAll(userName, "username");
+
+      Assert.assertEquals(expectedValue, actualValue);
+    }
+  }
+
   @Test
   public void testBasicRunJobForDistributedTraining() throws Exception {
     MockClientContext mockClientContext =
@@ -120,6 +147,8 @@ public class TestYarnServiceRunJobCli {
     Assert.assertEquals(3, serviceSpec.getComponents().size());
 
     commonVerifyDistributedTrainingSpec(serviceSpec);
+
+    verifyQuicklink(serviceSpec, null);
   }
 
   @Test
@@ -147,6 +176,10 @@ public class TestYarnServiceRunJobCli {
 
     verifyTensorboardComponent(runJobCli, serviceSpec,
         Resources.createResource(4096, 1));
+
+    verifyQuicklink(serviceSpec, ImmutableMap
+        .of(YarnServiceJobSubmitter.TENSORBOARD_QUICKLINK_LABEL,
+            "http://tensorboard-0.my-job.username.null:6006"));
   }
 
   @Test
@@ -232,6 +265,9 @@ public class TestYarnServiceRunJobCli {
 
     verifyTensorboardComponent(runJobCli, serviceSpec,
         Resources.createResource(2048, 2));
+    verifyQuicklink(serviceSpec, ImmutableMap
+        .of(YarnServiceJobSubmitter.TENSORBOARD_QUICKLINK_LABEL,
+            "http://tensorboard-0.my-job.username.null:6006"));
   }
 
   private void commonTestSingleNodeTraining(Service serviceSpec)
@@ -372,4 +408,62 @@ public class TestYarnServiceRunJobCli {
     Assert.assertEquals(jobInfo.get(StorageKeyConstants.INPUT_PATH),
         "s3://input");
   }
+
+  @Test
+  public void testAddQuicklinksWithoutTensorboard() throws Exception {
+    MockClientContext mockClientContext =
+        YarnServiceCliTestUtils.getMockClientContext();
+    RunJobCli runJobCli = new RunJobCli(mockClientContext);
+    Assert.assertFalse(SubmarineLogs.isVerbose());
+
+    runJobCli.run(
+        new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+            "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+            "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+            "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+            "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+            "ps.image", "--worker_docker_image", "worker.image",
+            "--ps_launch_cmd", "python run-ps.py", "--verbose", "--quicklink",
+            "AAA=http://master-0:8321", "--quicklink",
+            "BBB=http://worker-0:1234" });
+    Service serviceSpec = getServiceSpecFromJobSubmitter(
+        runJobCli.getJobSubmitter());
+    Assert.assertEquals(3, serviceSpec.getComponents().size());
+
+    commonVerifyDistributedTrainingSpec(serviceSpec);
+
+    verifyQuicklink(serviceSpec, ImmutableMap
+        .of("AAA", "http://master-0.my-job.username.null:8321", "BBB",
+            "http://worker-0.my-job.username.null:1234"));
+  }
+
+  @Test
+  public void testAddQuicklinksWithTensorboard() throws Exception {
+    MockClientContext mockClientContext =
+        YarnServiceCliTestUtils.getMockClientContext();
+    RunJobCli runJobCli = new RunJobCli(mockClientContext);
+    Assert.assertFalse(SubmarineLogs.isVerbose());
+
+    runJobCli.run(
+        new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+            "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+            "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+            "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+            "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+            "ps.image", "--worker_docker_image", "worker.image",
+            "--ps_launch_cmd", "python run-ps.py", "--verbose", "--quicklink",
+            "AAA=http://master-0:8321", "--quicklink",
+            "BBB=http://worker-0:1234", "--tensorboard" });
+    Service serviceSpec = getServiceSpecFromJobSubmitter(
+        runJobCli.getJobSubmitter());
+    Assert.assertEquals(4, serviceSpec.getComponents().size());
+
+    commonVerifyDistributedTrainingSpec(serviceSpec);
+
+    verifyQuicklink(serviceSpec, ImmutableMap
+        .of("AAA", "http://master-0.my-job.username.null:8321", "BBB",
+            "http://worker-0.my-job.username.null:1234",
+            YarnServiceJobSubmitter.TENSORBOARD_QUICKLINK_LABEL,
+            "http://tensorboard-0.my-job.username.null:6006"));
+  }
 }