YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java

@@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService {
         attemptId.getApplicationId().toString());
     serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
         PersistencePolicies.APPLICATION);
-    serviceRecord.description = "Yarn Service Master";
-
-    serviceRecord.addExternalEndpoint(RegistryTypeUtils
-        .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
-            context.clientAMService.getBindAddress()));
+    serviceRecord.description = "YarnServiceMaster";
 
     // set any provided attributes
     setUserProvidedServiceRecordAttributes(service.getConfiguration(),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java

@@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService
     if (!StringUtils.isEmpty(args.getServiceName())) {
       service.setName(args.getServiceName());
     }
+    if (!StringUtils.isEmpty(args.queue)) {
+      service.setQueue(args.queue);
+    }
     return service;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java

@@ -37,7 +37,7 @@ public abstract class AbstractClusterBuildingActionArgs
   }
 
   @Parameter(names = {
-      ARG_QUEUE }, description = "Queue to submit the service")
+      ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
   public String queue;
 
   @Parameter(names = {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java

@@ -77,6 +77,7 @@ public interface Arguments {
   String ARG_PATH = "--path";
   String ARG_PRINCIPAL = "--principal";
   String ARG_QUEUE = "--queue";
+  String ARG_SHORT_QUEUE = "-q";
   String ARG_LIFETIME = "--lifetime";
   String ARG_RESOURCE = "--resource";
   String ARG_RESOURCE_MANAGER = "--rm";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java

@@ -59,6 +59,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
+import static org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes.*;
 import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
 import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
 import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
@@ -356,12 +357,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
       YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
     ServiceRecord record = new ServiceRecord();
     String containerId = status.getContainerId().toString();
-    record.set(YarnRegistryAttributes.YARN_ID, containerId);
+    record.set(YARN_ID, containerId);
     record.description = getCompInstanceName();
-    record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
-        PersistencePolicies.CONTAINER);
-    record.set("yarn:ip", status.getIPs());
-    record.set("yarn:hostname", status.getHost());
+    record.set(YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
+    record.set(YARN_IP, status.getIPs().get(0));
+    record.set(YARN_HOSTNAME, status.getHost());
     try {
       yarnRegistry
           .putComponent(RegistryPathUtils.encodeYarnID(containerId), record);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.yarn.service.provider;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.service.api.records.Service;
@@ -91,13 +92,16 @@ public abstract class AbstractProviderService implements ProviderService,
         component, tokensForSubstitution, instance, context);
 
     // substitute launch command
-    String launchCommand = ProviderUtils
-        .substituteStrWithTokens(component.getLaunchCommand(),
-            tokensForSubstitution);
-    CommandLineBuilder operation = new CommandLineBuilder();
-    operation.add(launchCommand);
-    operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
-    launcher.addCommand(operation.build());
+    String launchCommand = component.getLaunchCommand();
+    // docker container may have empty commands
+    if (!StringUtils.isEmpty(launchCommand)) {
+      launchCommand = ProviderUtils
+          .substituteStrWithTokens(launchCommand, tokensForSubstitution);
+      CommandLineBuilder operation = new CommandLineBuilder();
+      operation.add(launchCommand);
+      operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+      launcher.addCommand(operation.build());
+    }
 
     // By default retry forever every 30 seconds
     launcher.setRetryContext(YarnServiceConf

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java

@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import static org.apache.hadoop.registry.client.types.AddressTypes.*;
 import org.apache.hadoop.registry.client.types.Endpoint;
 import org.apache.hadoop.registry.client.types.ProtocolTypes;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
@@ -36,6 +35,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.hadoop.registry.client.types.AddressTypes.*;
+
 /**
  * Static methods to work with registry types —primarily endpoints and the
  * list representation of addresses.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java

@@ -18,6 +18,8 @@ package org.apache.hadoop.registry.server.dns;
 
 import org.apache.hadoop.registry.client.types.Endpoint;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.xbill.DNS.Name;
 import org.xbill.DNS.Type;
 
@@ -32,7 +34,8 @@ import java.util.List;
  */
 public class ApplicationServiceRecordProcessor extends
     BaseServiceRecordProcessor {
-
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ApplicationServiceRecordProcessor.class);
   /**
    * Create an application service record processor.
    *
@@ -57,6 +60,10 @@ public class ApplicationServiceRecordProcessor extends
    */
   @Override public void initTypeToInfoMapping(ServiceRecord serviceRecord)
       throws Exception {
+    if (serviceRecord.external.isEmpty()) {
+      LOG.info(serviceRecord.description + ": No external endpoints defined.");
+      return;
+    }
     for (int type : getRecordTypes()) {
       switch (type) {
       case Type.A:
@@ -309,6 +316,9 @@ public class ApplicationServiceRecordProcessor extends
         throws Exception {
       this.setNames(new Name[] {getServiceName()});
       List<Endpoint> endpoints = serviceRecord.external;
+      if (endpoints.isEmpty()) {
+        return;
+      }
       // TODO:  do we need a "hostname" attribute for an application record or
       // can we rely on the first endpoint record.
       this.setTarget(InetAddress.getByName(
@@ -342,6 +352,9 @@ public class ApplicationServiceRecordProcessor extends
     @Override protected void init(ServiceRecord serviceRecord)
         throws Exception {
       super.init(serviceRecord);
+      if (getTarget() == null) {
+        return;
+      }
       try {
         this.setTarget(getIpv6Address(getTarget()));
       } catch (UnknownHostException e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java

@@ -52,8 +52,8 @@ public abstract class BaseServiceRecordProcessor
   private String domain;
 
   private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
-  private static final String SLIDER_API_PREFIX =
-      "classpath:org.apache.slider.";
+  private static final String YARN_SERVICE_API_PREFIX =
+      "classpath:org.apache.hadoop.yarn.service.";
   private static final String HTTP_API_TYPE = "http://";
 
   /**
@@ -425,8 +425,8 @@ public abstract class BaseServiceRecordProcessor
      */
     protected String getDNSApiFragment(String api) {
       String dnsApi = null;
-      if (api.startsWith(SLIDER_API_PREFIX)) {
-        dnsApi = api.substring(SLIDER_API_PREFIX.length());
+      if (api.startsWith(YARN_SERVICE_API_PREFIX)) {
+        dnsApi = api.substring(YARN_SERVICE_API_PREFIX.length());
       } else if (api.startsWith(HTTP_API_TYPE)) {
         dnsApi = "http";
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java

@@ -69,7 +69,8 @@ public class TestRegistryDNS extends Assert {
       + "  \"type\" : \"JSONServiceRecord\",\n"
       + "  \"description\" : \"Slider Application Master\",\n"
       + "  \"external\" : [ {\n"
-      + "    \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+      + "    \"api\" : \"classpath:org.apache.hadoop.yarn.service.appmaster.ipc"
+      + "\",\n"
       + "    \"addressType\" : \"host/port\",\n"
       + "    \"protocolType\" : \"hadoop/IPC\",\n"
       + "    \"addresses\" : [ {\n"
@@ -84,7 +85,8 @@ public class TestRegistryDNS extends Assert {
       + "      \"uri\" : \"http://192.168.1.5:1027\"\n"
       + "    } ]\n"
       + "  }, {\n"
-      + "    \"api\" : \"classpath:org.apache.slider.management\",\n"
+      + "    \"api\" : \"classpath:org.apache.hadoop.yarn.service.management\""
+      + ",\n"
       + "    \"addressType\" : \"uri\",\n"
       + "    \"protocolType\" : \"REST\",\n"
       + "    \"addresses\" : [ {\n"
@@ -92,14 +94,16 @@ public class TestRegistryDNS extends Assert {
       + "    } ]\n"
       + "  } ],\n"
       + "  \"internal\" : [ {\n"
-      + "    \"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+      + "    \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.secure"
+      + "\",\n"
       + "    \"addressType\" : \"uri\",\n"
       + "    \"protocolType\" : \"REST\",\n"
       + "    \"addresses\" : [ {\n"
       + "      \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n"
       + "    } ]\n"
       + "  }, {\n"
-      + "    \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+      + "    \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.oneway"
+      + "\",\n"
       + "    \"addressType\" : \"uri\",\n"
       + "    \"protocolType\" : \"REST\",\n"
       + "    \"addresses\" : [ {\n"

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -812,13 +812,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
           .executePrivilegedOperation(null, privOp, null,
               null, true, false);
       LOG.info("Docker inspect output for " + containerId + ": " + output);
+      // strip off quotes if any
+      output = output.replaceAll("['\"]", "");
       int index = output.lastIndexOf(',');
       if (index == -1) {
         LOG.error("Incorrect format for ip and host");
         return null;
       }
-      // strip off quotes if any
-      output = output.replaceAll("['\"]", "");
       String ips = output.substring(0, index).trim();
       String host = output.substring(index+1).trim();
       String[] ipAndHost = new String[2];

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md

@@ -92,6 +92,7 @@ Usage `yarn service [sub-command] [service-name] [options]`
 
    Options:
     --file,-f       The local path to the service definition file.
+    --queue,-q      The queue to which the service is submitted.
     --example,-e    The name of the example service such as:
                     Sleeper      A simple service that launches a few non-docker sleep containers on YARN.
    ```