YARN-7186. Add examples in yarn-service. Contributed by Jian He

hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

@@ -97,6 +97,13 @@
       <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
       <outputDirectory>etc/hadoop</outputDirectory>
     </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
     <fileSet>
       <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json

@@ -0,0 +1,15 @@
+{
+  "name": "sleeper-service",
+  "components" :
+  [
+    {
+      "name": "sleeper",
+      "number_of_containers": 2,
+      "launch_command": "sleep 900000",
+      "resource": {
+        "cpus": 1,
+        "memory": "256"
+      }
+    }
+  ]
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java

@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
@@ -160,12 +161,15 @@ public class ServiceClient extends CompositeService
       AbstractClusterBuildingActionArgs args) throws IOException {
     File file = args.getFile();
     Path filePath = new Path(file.getAbsolutePath());
-    LOG.info("Loading app json from: " + filePath);
+    LOG.info("Loading service definition from: " + filePath);
     Service service = jsonSerDeser
         .load(FileSystem.getLocal(getConfig()), filePath);
     if (args.lifetime > 0) {
       service.setLifetime(args.lifetime);
     }
+    if (!StringUtils.isEmpty(args.getServiceName())) {
+      service.setName(args.getServiceName());
+    }
     return service;
   }
 
@@ -182,9 +186,23 @@ public class ServiceClient extends CompositeService
     return EXIT_SUCCESS;
   }
 
-  public int actionCreate(AbstractClusterBuildingActionArgs args)
+  public int actionCreate(ActionCreateArgs args)
       throws IOException, YarnException {
-    actionCreate(loadAppJsonFromLocalFS(args));
+    Service serviceDef;
+    if (args.file != null) {
+      serviceDef = loadAppJsonFromLocalFS(args);
+    } else if (!StringUtils.isEmpty(args.example)) {
+      // create an example service
+      String yarnHome = System
+          .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+      args.file = new File(MessageFormat
+          .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
+              yarnHome, args.example, args.example));
+      serviceDef = loadAppJsonFromLocalFS(args);
+    } else {
+      throw new YarnException("No service definition provided!");
+    }
+    actionCreate(serviceDef);
     return EXIT_SUCCESS;
   }
 
@@ -213,7 +231,7 @@ public class ServiceClient extends CompositeService
     Map<String, Long> componentCounts =
         new HashMap<>(flexArgs.getComponentMap().size());
     Service persistedService =
-        ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+        ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
     if (!StringUtils.isEmpty(persistedService.getId())) {
       cachedAppIds.put(persistedService.getName(),
           ApplicationId.fromString(persistedService.getId()));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java

@@ -50,7 +50,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
    * get the name: relies on arg 1 being the cluster name in all operations 
    * @return the name argument, null if there is none
    */
-  public String getClusterName() {
+  public String getServiceName() {
     return (parameters.isEmpty()) ? null : parameters.get(0);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java

@@ -28,7 +28,7 @@ import java.io.File;
  */
 public abstract class AbstractClusterBuildingActionArgs
     extends AbstractActionArgs {
-  @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },  required = true,
+  @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
       description = "The path to the service definition file in JSON format.")
   public File file;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.service.client.params;
 
 import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 
 @Parameters(commandNames = { SliderActions.ACTION_BUILD},
             commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
@@ -30,7 +31,9 @@ public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
   }
 
   @Override
-  public int getMinParams() {
-    return 0;
+  public void validate() throws BadCommandArgumentsException {
+    if (file == null) {
+      throw new BadCommandArgumentsException("No service definition provided.");
+    }
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java

@@ -18,21 +18,29 @@
 
 package org.apache.hadoop.yarn.service.client.params;
 
+import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 
 @Parameters(commandNames = { SliderActions.ACTION_CREATE},
             commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
 
 public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
 
+  @Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
+      description = "The name of the example service such as sleeper")
+  public String example;
+
   @Override
   public String getActionName() {
     return SliderActions.ACTION_CREATE;
   }
 
   @Override
-  public int getMinParams() {
-    return 0;
+  public void validate() throws BadCommandArgumentsException {
+    if (file == null && example == null) {
+      throw new BadCommandArgumentsException("No service definition provided.");
+    }
   }
 }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java

@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES},
-    commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES)
-public class ActionExamples {
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java

@@ -42,8 +42,8 @@ public interface Arguments {
   String ARG_DELETE = "--delete";
   String ARG_DEST = "--dest";
   String ARG_DESTDIR = "--destdir";
-  String ARG_FILESYSTEM = "--fs";
-  String ARG_FILESYSTEM_LONG = "--filesystem";
+  String ARG_EXAMPLE = "--example";
+  String ARG_EXAMPLE_SHORT = "-e";
   String ARG_FOLDER = "--folder";
   String ARG_FORCE = "--force";
   String ARG_FORMAT = "--format";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java

@@ -19,8 +19,6 @@
 package org.apache.hadoop.yarn.service.client.params;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -104,51 +102,10 @@ public class ClientArgs extends CommonArgs {
     return actionDependencyArgs;
   }
 
-  public ActionDestroyArgs getActionDestroyArgs() {
-    return actionDestroyArgs;
-  }
-
-  public ActionExistsArgs getActionExistsArgs() {
-    return actionExistsArgs;
-  }
-
   public ActionFlexArgs getActionFlexArgs() {
     return actionFlexArgs;
   }
 
-  public ActionFreezeArgs getActionFreezeArgs() {
-    return actionFreezeArgs;
-  }
-
-  public ActionListArgs getActionListArgs() {
-    return actionListArgs;
-  }
-
-
-  public ActionRegistryArgs getActionRegistryArgs() {
-    return actionRegistryArgs;
-  }
-
-  public ActionResolveArgs getActionResolveArgs() {
-    return actionResolveArgs;
-  }
-
-  public ActionResourceArgs getActionResourceArgs() {
-    return actionResourceArgs;
-  }
-
-  public ActionStatusArgs getActionStatusArgs() {
-    return actionStatusArgs;
-  }
-
-  public ActionThawArgs getActionThawArgs() {
-    return actionThawArgs;
-  }
-
-  public ActionTokensArgs getActionTokenArgs() {
-    return actionTokenArgs;
-  }
-
   /**
    * Look at the chosen action and bind it as the core action for the operation.
    * @throws SliderException bad argument or similar
@@ -227,7 +184,6 @@ public class ClientArgs extends CommonArgs {
       case ACTION_UPDATE:
         bindCoreAction(actionUpdateArgs);
         break;
-
       default:
         throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
         + " " + action);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java

@@ -24,7 +24,6 @@ import com.beust.jcommander.ParameterDescription;
 import com.beust.jcommander.ParameterException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -88,7 +87,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
    * @return the name argument, null if there is none
    */
   public String getClusterName() {
-    return coreAction.getClusterName();
+    return coreAction.getServiceName();
   }
 
   protected CommonArgs(String[] args) {
@@ -142,10 +141,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
     return result;
   }
 
-  public static String usage(CommonArgs serviceArgs) {
-    return usage(serviceArgs, null);
-  }
-
   /**
    * Parse routine -includes registering the action-specific argument classes
    * and postprocess it
@@ -164,14 +159,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
     postProcess();
   }
 
-  /**
-   * Add a command
-   * @param name action
-   * @param arg value
-   */
-  protected void addAction(String name, Object arg) {
-    commander.addCommand(name, arg);
-  }
 
   protected void addActions(Object... actions) {
     for (Object action : actions) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java

@@ -30,7 +30,6 @@ public interface SliderActions {
   String ACTION_UPGRADE = "upgrade";
   String ACTION_DESTROY = "destroy";
   String ACTION_EXISTS = "exists";
-  String ACTION_EXAMPLES = "examples";
   String ACTION_FLEX = "flex";
   String ACTION_STOP = "stop";
   String ACTION_HELP = "help";
@@ -59,7 +58,6 @@ public interface SliderActions {
         "Destroy a stopped service, service must be stopped first before destroying.";
   String DESCRIBE_ACTION_EXISTS =
             "Probe for a service running";
-  String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
   String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
   String DESCRIBE_ACTION_FREEZE =
               "Stop a running service";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md

@@ -76,28 +76,32 @@ Usage `yarn service [sub-command] [service-name] [options]`
 
 * `build`:  Build a service with its specifications, but do not start it.
     ```
-    Usage: yarn service build --file [file]
-    ```
+    Usage: yarn service build [service-name] --file [file]
+    Fields:
+    service-name     Optional. If specified, it will override the name in the service definition.
 
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --file or -f | The local path to the service definition file |
+    Options:
+      --file,-f      The local path to the service definition file
+    ```
 
 * `create`:  create a service, it's equivalent to first invoke build and then start.
    ```
-   Usage: yarn service create --file [file]
+   Usage: yarn service create [service-name] --file [file]
+   Fields:
+    service-name    Optional. If specified, it will override the name in the service definition.
+
+   Options:
+    --file,-f       The local path to the service definition file.
+    --example,-e    The name of the example service such as:
+                    Sleeper      A simple service that launches a few non-docker sleep containers on YARN.
    ```
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --file or -f | The local path to the service definition file |
 
 * `dependency`:  Yarn service framework dependency (libraries) management.
   ```
   Usage: yarn service dependency [options]
+  Option:
+     --upload      Pre-upload the dependency jars onto HDFS to expediate service launch process.
   ```
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
 
 * `destroy`:  Destroy a stopped service, service must be stopped first before destroying.
   ```
@@ -106,10 +110,10 @@ Usage `yarn service [sub-command] [service-name] [options]`
 * `flex`:   Flex a service's component by increasing or decreasing the number of containers.
   ```
   Usage: yarn service flex [service-name] --component [component-name] [count]
+  Options:
+    --component [component-name] [count]
+            Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
   ```
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
 * `status`:  Get the status of a service.
   ```
   Usage: yarn service status [service-name]

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md

@@ -43,7 +43,14 @@ Below is a simple service definition that launches sleep containers on YARN by w
     ]
 }
 ```
-
+User can simply run a pre-built example service on YARN using below command:
+```
+yarn service create [service-name] --example [example-name]
+```
+e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
+```
+yarn service create my-sleeper --example sleeper
+```
 For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
 
 ## Manage services on YARN via CLI