YARN-5770. Performance improvement of native-services REST API service. Contributed by Gour Saha

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java

@@ -20,8 +20,7 @@ package org.apache.hadoop.yarn.services.api.impl;
 import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
 import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
 
-import java.io.File;
-import java.io.FileReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.security.PrivilegedExceptionAction;
@@ -36,7 +35,6 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.regex.Pattern;
 
-import javax.inject.Singleton;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
@@ -52,6 +50,7 @@ import javax.ws.rs.core.Response.Status;
 
 import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -78,7 +77,6 @@ import org.apache.slider.common.params.ActionFlexArgs;
 import org.apache.slider.common.params.ActionFreezeArgs;
 import org.apache.slider.common.params.ActionListArgs;
 import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
 import org.apache.slider.common.params.ActionThawArgs;
 import org.apache.slider.common.params.ComponentArgsDelegate;
 import org.apache.slider.common.tools.SliderUtils;
@@ -98,6 +96,7 @@ import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
+import com.google.inject.Singleton;
 
 @Singleton
 @Path(APPLICATIONS_API_RESOURCE_PATH)
@@ -109,6 +108,11 @@ public class ApplicationApiService implements ApplicationApi {
   private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG;
   private static UserGroupInformation SLIDER_USER;
   private static SliderClient SLIDER_CLIENT;
+  private static Response SLIDER_VERSION;
+  private static final JsonParser JSON_PARSER = new JsonParser();
+  private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject();
+  private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs();
+  private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs();
 
   static {
     init();
@@ -119,24 +123,27 @@ public class ApplicationApiService implements ApplicationApi {
     SLIDER_CONFIG = getSliderClientConfiguration();
     SLIDER_USER = getSliderUser();
     SLIDER_CLIENT = createSliderClient();
+    SLIDER_VERSION = initSliderVersion();
   }
 
   @GET
-  @Path("/slider-version")
+  @Path("/versions/slider-version")
   @Consumes({ MediaType.APPLICATION_JSON })
   @Produces({ MediaType.APPLICATION_JSON })
   public Response getSliderVersion() {
     logger.info("GET: getSliderVersion");
+    return SLIDER_VERSION;
+  }
 
+  private static Response initSliderVersion() {
     Map<String, Object> metadata = new HashMap<>();
     BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services");
     String sliderVersion = metadata.toString();
     logger.info("Slider version = {}", sliderVersion);
     String hadoopVersion = SliderVersionInfo.getHadoopVersionString();
     logger.info("Hadoop version = {}", hadoopVersion);
-    return Response.ok(
-        "{ \"slider_version\": \"" + sliderVersion
-            + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
+    return Response.ok("{ \"slider_version\": \"" + sliderVersion
+        + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
   }
 
   @POST
@@ -196,7 +203,8 @@ public class ApplicationApiService implements ApplicationApi {
     }
 
     // If the application has no components do top-level checks
-    if (application.getComponents() == null) {
+    if (application.getComponents() == null
+        || application.getComponents().size() == 0) {
       // artifact
       if (application.getArtifact() == null) {
         throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID);
@@ -222,6 +230,9 @@ public class ApplicationApiService implements ApplicationApi {
       if (application.getNumberOfContainers() == null) {
         throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID);
       }
+
+      // Since it is a simple app with no components, create a default component
+      application.setComponents(getDefaultComponentAsList(application));
     } else {
       // If the application has components, then run checks for each component.
       // Let global values take effect if component level values are not
@@ -274,11 +285,6 @@ public class ApplicationApiService implements ApplicationApi {
       }
     }
 
-    // If it is a simple app with no components, then create a default component
-    if (application.getComponents() == null) {
-      application.setComponents(getDefaultComponentAsList(application));
-    }
-
     // Application lifetime if not specified, is set to unlimited lifetime
     if (application.getLifetime() == null) {
       application.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
@@ -853,15 +859,12 @@ public class ApplicationApiService implements ApplicationApi {
     // TODO: add status
     app.setState(ApplicationState.ACCEPTED);
     JsonObject appStatus = null;
-    JsonObject appRegistryDocker = null;
     JsonObject appRegistryQuicklinks = null;
     try {
       appStatus = getSliderApplicationStatus(appName);
-      appRegistryDocker = getSliderApplicationRegistry(appName, "docker");
       appRegistryQuicklinks = getSliderApplicationRegistry(appName,
           "quicklinks");
-      return populateAppData(app, appStatus, appRegistryDocker,
-          appRegistryQuicklinks);
+      return populateAppData(app, appStatus, appRegistryQuicklinks);
     } catch (BadClusterStateException | NotFoundException e) {
       logger.error(
           "Get application failed, application not in running state yet", e);
@@ -881,7 +884,7 @@ public class ApplicationApiService implements ApplicationApi {
   }
 
   private Response populateAppData(Application app, JsonObject appStatus,
-      JsonObject appRegistryDocker, JsonObject appRegistryQuicklinks) {
+      JsonObject appRegistryQuicklinks) {
     String appName = jsonGetAsString(appStatus, "name");
     Long totalNumberOfRunningContainers = 0L;
     Long totalExpectedNumberOfRunningContainers = 0L;
@@ -944,7 +947,7 @@ public class ApplicationApiService implements ApplicationApi {
     JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics");
     if (applicationRoles == null) {
       // initialize to empty object to avoid too many null checks
-      applicationRoles = new JsonObject();
+      applicationRoles = EMPTY_JSON_OBJECT;
     }
     if (applicationStatus != null) {
       JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live");
@@ -954,8 +957,9 @@ public class ApplicationApiService implements ApplicationApi {
             continue;
           }
           componentNames.add(entry.getKey());
-          JsonObject componentRole = applicationRoles.get(entry.getKey()) == null ? new JsonObject()
-              : applicationRoles.get(entry.getKey()).getAsJsonObject();
+          JsonObject componentRole = applicationRoles
+              .get(entry.getKey()) == null ? EMPTY_JSON_OBJECT
+                  : applicationRoles.get(entry.getKey()).getAsJsonObject();
           JsonObject liveContainers = entry.getValue().getAsJsonObject();
           if (liveContainers != null) {
             for (Map.Entry<String, JsonElement> liveContainerEntry : liveContainers
@@ -1052,67 +1056,57 @@ public class ApplicationApiService implements ApplicationApi {
 
   private JsonObject getSliderApplicationStatus(final String appName)
       throws IOException, YarnException, InterruptedException {
-    final File appStatusOutputFile = File.createTempFile("status_", ".json");
-    final ActionStatusArgs statusArgs = new ActionStatusArgs();
-    statusArgs.output = appStatusOutputFile.getAbsolutePath();
 
-    return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
-      @Override
-      public JsonObject run(SliderClient sliderClient) throws YarnException,
-          IOException, InterruptedException {
-        sliderClient.actionStatus(appName, statusArgs);
-        JsonParser parser = new JsonParser();
-        FileReader reader = null;
-        JsonElement statusElement = null;
-        try {
-          reader = new FileReader(appStatusOutputFile);
-          statusElement = parser.parse(reader);
-        } finally {
-          if (reader != null) {
-            reader.close();
+    return invokeSliderClientRunnable(
+        new SliderClientContextRunnable<JsonObject>() {
+          @Override
+          public JsonObject run(SliderClient sliderClient)
+              throws YarnException, IOException, InterruptedException {
+            String status = null;
+            try {
+              status = sliderClient.actionStatus(appName);
+            } catch (Exception e) {
+              logger.error("Exception calling slider.actionStatus", e);
+              return EMPTY_JSON_OBJECT;
+            }
+            JsonElement statusElement = JSON_PARSER.parse(status);
+            return (statusElement == null || statusElement instanceof JsonNull)
+                ? EMPTY_JSON_OBJECT : (JsonObject) statusElement;
           }
-          appStatusOutputFile.delete();
-        }
-        return (statusElement == null || statusElement instanceof JsonNull) ?
-            new JsonObject() : (JsonObject) statusElement;
-      }
-    });
+        });
   }
 
   private JsonObject getSliderApplicationRegistry(final String appName,
-      final String registryName) throws IOException, YarnException,
-      InterruptedException {
-    final File appRegistryOutputFile = File
-        .createTempFile("registry_", ".json");
+      final String registryName)
+      throws IOException, YarnException, InterruptedException {
     final ActionRegistryArgs registryArgs = new ActionRegistryArgs();
-    registryArgs.out = appRegistryOutputFile;
     registryArgs.name = appName;
     registryArgs.getConf = registryName;
     registryArgs.format = ConfigFormat.JSON.toString();
 
-    return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
-      @Override
-      public JsonObject run(SliderClient sliderClient) throws YarnException,
-          IOException, InterruptedException {
-        sliderClient.actionRegistry(registryArgs);
-        JsonParser parser = new JsonParser();
-        FileReader reader = null;
-        JsonElement registryElement = null;
-        try {
-          reader = new FileReader(appRegistryOutputFile);
-          registryElement = parser.parse(reader);
-        } catch (Throwable t) {
-          logger.error("Error reading file {}", appRegistryOutputFile);
-        } finally {
-          if (reader != null) {
-            reader.close();
+    return invokeSliderClientRunnable(
+        new SliderClientContextRunnable<JsonObject>() {
+          @Override
+          public JsonObject run(SliderClient sliderClient)
+              throws YarnException, IOException, InterruptedException {
+            String registry = null;
+            try {
+              registry = sliderClient.actionRegistryGetConfig(registryArgs)
+                .asJson();
+            } catch (FileNotFoundException | PathNotFoundException e) {
+              // ignore and return empty object
+              return EMPTY_JSON_OBJECT;
+            } catch (Exception e) {
+              logger.error("Exception calling slider.actionRegistryGetConfig",
+                  e);
+              return EMPTY_JSON_OBJECT;
+            }
+            JsonElement registryElement = JSON_PARSER.parse(registry);
+            return (registryElement == null
+                || registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT
+                    : (JsonObject) registryElement;
           }
-          appRegistryOutputFile.delete();
-        }
-        return (registryElement == null || registryElement instanceof JsonNull) ?
-            new JsonObject() : (JsonObject) registryElement;
-      }
-    });
+        });
   }
 
   private Integer getSliderList(final String appName)
@@ -1130,8 +1124,7 @@ public class ApplicationApiService implements ApplicationApi {
         if (liveOnly) {
           status = sliderClient.actionList(appName);
         } else {
-          ActionListArgs listArgs = new ActionListArgs();
-          status = sliderClient.actionList(appName, listArgs);
+          status = sliderClient.actionList(appName, ACTION_LIST_ARGS);
         }
         return status;
       }
@@ -1228,8 +1221,7 @@ public class ApplicationApiService implements ApplicationApi {
       @Override
       public Response run(SliderClient sliderClient) throws YarnException,
           IOException, InterruptedException {
-        ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
-        int returnCode = sliderClient.actionFreeze(appName, freezeArgs);
+        int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS);
         if (returnCode == 0) {
           logger.info("Successfully stopped application {}", appName);
           return Response.status(Status.NO_CONTENT).build();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java

@@ -44,7 +44,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
 @XmlRootElement
 @JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, containers " })
+@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
+    "lifetime", "containers" })
 public class Application extends BaseResource {
   private static final long serialVersionUID = -4491694636566094885L;
 
@@ -174,8 +175,8 @@ public class Application extends BaseResource {
 
   @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
   @JsonProperty("launch_time")
-  public String getLaunchTime() {
-    return launchTime == null ? null : launchTime.toString();
+  public Date getLaunchTime() {
+    return launchTime == null ? null : (Date) launchTime.clone();
   }
 
   @XmlElement(name = "launch_time")

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java

@@ -79,8 +79,8 @@ public class Container extends BaseResource {
 
   @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
   @JsonProperty("launch_time")
-  public String getLaunchTime() {
-    return launchTime == null ? null : launchTime.toString();
+  public Date getLaunchTime() {
+    return launchTime == null ? null : (Date) launchTime.clone();
   }
 
   @XmlElement(name = "launch_time")

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java

@@ -27,6 +27,7 @@ import java.util.Arrays;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
 import org.mortbay.jetty.webapp.Configuration;
@@ -95,13 +96,10 @@ public class ApplicationApiWebApp extends AbstractService {
         .setName("services-rest-api")
         .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
 
-    String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP
-        + "org.apache.hadoop.yarn.services.api.impl" + SEP
-        + "org.apache.hadoop.yarn.services.resource" + SEP
-        + "org.apache.hadoop.yarn.services.utils" + SEP
-        + "org.apache.hadoop.yarn.services.webapp" + SEP
-        + GenericExceptionHandler.class.getPackage().getName() + SEP
-        + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+    String apiPackages =
+        ApplicationApiService.class.getPackage().getName() + SEP
+            + GenericExceptionHandler.class.getPackage().getName() + SEP
+            + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
     applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
         + "/*");
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java

@@ -3137,22 +3137,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 
   @Override
   @VisibleForTesting
-  public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
-                                              YarnException,
-                                              IOException {
-    verifyBindingsDefined();
-    validateClusterName(clustername);
+  public int actionStatus(String clustername, ActionStatusArgs statusArgs)
+      throws YarnException, IOException {
+    ClusterDescription status = verifyAndGetClusterDescription(clustername);
     String outfile = statusArgs.getOutput();
-    ClusterDescription status = getClusterDescription(clustername);
-    String text = status.toJsonString();
     if (outfile == null) {
-      log.info(text);
+      log.info(status.toJsonString());
     } else {
       status.save(new File(outfile).getAbsoluteFile());
     }
     return EXIT_SUCCESS;
   }
 
+  @Override
+  public String actionStatus(String clustername)
+      throws YarnException, IOException {
+    return verifyAndGetClusterDescription(clustername).toJsonString();
+  }
+
+  private ClusterDescription verifyAndGetClusterDescription(String clustername)
+      throws YarnException, IOException {
+    verifyBindingsDefined();
+    validateClusterName(clustername);
+    return getClusterDescription(clustername);
+  }
+
   @Override
   public int actionVersion() {
     SliderVersionInfo.loadAndPrintVersionInfo(log);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java

@@ -279,6 +279,17 @@ public interface SliderClientAPI extends Service {
   int actionStatus(String clustername, ActionStatusArgs statusArgs)
       throws YarnException, IOException;
 
+  /**
+   * Status operation which returns the status object as a string instead of
+   * printing it to the console or file.
+   *
+   * @param clustername cluster name
+   * @return cluster status details
+   * @throws YarnException
+   * @throws IOException
+   */
+  String actionStatus(String clustername) throws YarnException, IOException;
+
   /**
    * Version Details
    * @return exit code