/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.ambari.view.slider;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;

import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.slider.ViewStatus.Validation;
import org.apache.ambari.view.slider.clients.AmbariCluster;
import org.apache.ambari.view.slider.clients.AmbariClusterInfo;
import org.apache.ambari.view.slider.clients.AmbariHostComponent;
import org.apache.ambari.view.slider.clients.AmbariService;
import org.apache.ambari.view.slider.clients.AmbariServiceInfo;
import org.apache.ambari.view.slider.rest.client.AmbariHttpClient;
import org.apache.ambari.view.slider.rest.client.Metric;
import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient;
import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient.SliderAppMasterData;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.log4j.Logger;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.client.SliderClient;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionInstallKeytabArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;

import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.google.inject.Inject;
import com.google.inject.Singleton;

@Singleton
public class SliderAppsViewControllerImpl implements SliderAppsViewController {
  private static final Logger logger = Logger
      .getLogger(SliderAppsViewControllerImpl.class);
  private static String METRICS_PREFIX = "metrics/";
  @Inject
  private ViewContext viewContext;
  private List<SliderAppType> appTypes;
  private Integer createAppCounter = -1;
  @Inject
  private SliderAppsAlerts sliderAlerts;
  private Map<String, MetricsHolder> appMetrics = new HashMap<String, MetricsHolder>();

  private String getAppsFolderPath() {
    return viewContext.getAmbariProperty("resources.dir") + "/apps";
  }

  private String getAppsCreateFolderPath() {
    return getAppsFolderPath() + "/create";
  }

  @Override
  public ViewStatus getViewStatus() {
    ViewStatus status = new ViewStatus();
    Map<String, String> newHadoopConfigs = new HashMap<String, String>();
    status.setVersion(SliderAppsConfiguration.INSTANCE.getVersion());
    String ambariCluster = getViewParameterValue(PARAM_AMBARI_CLUSTER_API);
    String ambariUsername = getViewParameterValue(PARAM_AMBARI_USERNAME);
    String ambariPassword = getViewParameterValue(PARAM_AMBARI_PASSWORD);
    if (ambariCluster != null && ambariUsername != null
        && ambariPassword != null && ambariCluster.trim().length() > 0
        && ambariUsername.trim().length() > 0
        && ambariPassword.trim().length() > 0) {
      String APIPREFIX = "/api/v1/clusters/";
      int index = ambariCluster.indexOf(APIPREFIX);
      if (index > 0) {
        String ambariUrl = ambariCluster.substring(0, index);
        String clusterName = ambariCluster
            .substring(index + APIPREFIX.length());
        if (clusterName.endsWith("/")) {
          clusterName = clusterName.substring(0, clusterName.length() - 1);
        }
        AmbariHttpClient ambariClient = new AmbariHttpClient(ambariUrl,
            ambariUsername, ambariPassword);
        try {
          AmbariClusterInfo clusterInfo = ambariClient.getClusterInfo();
          if (clusterInfo!=null && clusterName.equals(clusterInfo.getName())) {
            AmbariCluster cluster = ambariClient.getCluster(clusterInfo);
            AmbariServiceInfo hdfsServiceInfo = null;
            AmbariServiceInfo yarnServiceInfo = null;
            // Validate stack-version
            Validation validateStackVersion = validateStackVersion(clusterInfo.getVersion());
            if (validateStackVersion != null) {
              status.getValidations().add(validateStackVersion);
            }
            for (AmbariServiceInfo svc : cluster.getServices()) {
              if ("HDFS".equals(svc.getId())) {
                hdfsServiceInfo = svc;
              } else if ("YARN".equals(svc.getId())) {
                yarnServiceInfo = svc;
              }
            }
            // HDFS
            if (hdfsServiceInfo != null) {
              if (!hdfsServiceInfo.isStarted()) {
                status.getValidations().add(
                    new ViewStatus.Validation("HDFS service is not started"));
              }
            } else {
              status.getValidations().add(
                  new ViewStatus.Validation("HDFS service is not installed"));
            }
            // YARN
            if (yarnServiceInfo != null) {
              if (!yarnServiceInfo.isStarted()) {
                status.getValidations().add(
                    new ViewStatus.Validation("YARN service is not started"));
              }
            } else {
              status.getValidations().add(
                  new ViewStatus.Validation("YARN service is not installed"));
            }
            // JAVA_HOME
            Map<String, String> ambariServerConfigs = ambariClient.getAmbariServerConfigs();
            if (ambariServerConfigs.containsKey("java.home")) {
              newHadoopConfigs.put("java.home", ambariServerConfigs.get("java.home"));
              status.getParameters().put(PROPERTY_JAVA_HOME, ambariServerConfigs.get("java.home"));
            }
            // Configs
            if (cluster.getDesiredConfigs().containsKey("core-site")) {
              Map<String, String> coreSiteConfigs = ambariClient
                  .getConfiguration(cluster, "core-site", cluster
                      .getDesiredConfigs().get("core-site"));
              newHadoopConfigs.putAll(coreSiteConfigs);
            }
            if (cluster.getDesiredConfigs().containsKey("cluster-env")) {
              Map<String, String> clusterEnvConfigs = ambariClient
                  .getConfiguration(cluster, "cluster-env", cluster
                      .getDesiredConfigs().get("cluster-env"));
              newHadoopConfigs.put("security_enabled",
                  clusterEnvConfigs.get("security_enabled"));
            }
            if (cluster.getDesiredConfigs().containsKey("hdfs-site")) {
              Map<String, String> hdfsSiteConfigs = ambariClient
                  .getConfiguration(cluster, "hdfs-site", cluster
                      .getDesiredConfigs().get("hdfs-site"));
              newHadoopConfigs.putAll(hdfsSiteConfigs);
            }
            if (cluster.getDesiredConfigs().containsKey("yarn-site")) {
              Map<String, String> yarnSiteConfigs = ambariClient
                  .getConfiguration(cluster, "yarn-site", cluster
                      .getDesiredConfigs().get("yarn-site"));
              newHadoopConfigs.putAll(yarnSiteConfigs);
              status.getParameters().put(PROPERTY_YARN_RM_WEBAPP_URL,
                  newHadoopConfigs.get("yarn.resourcemanager.webapp.address"));
            }
            if (cluster.getDesiredConfigs().containsKey("yarn-env")) {
              Map<String, String> yarnEnvConfigs = ambariClient.getConfiguration(cluster, "yarn-env", cluster
                  .getDesiredConfigs().get("yarn-env"));
              String yarnUser = yarnEnvConfigs.get("yarn_user");
              if (yarnUser == null || yarnUser.trim().length() < 1) {
                yarnUser = "yarn";
              }
              newHadoopConfigs.put("yarn_user", yarnUser); // YARN service user
            }
            newHadoopConfigs.put("slider.user", getUserToRunAs(newHadoopConfigs)); // Slider user
            status.getParameters().put(PROPERTY_SLIDER_USER, newHadoopConfigs.get("slider.user"));
            if (newHadoopConfigs.containsKey("security_enabled")) {
              boolean securityEnabled = Boolean.valueOf(newHadoopConfigs.get("security_enabled"));
              if (securityEnabled) {
                String yarnUser = newHadoopConfigs.get("yarn_user");
                if (yarnUser != null && yarnUser.equals(newHadoopConfigs.get("slider.user"))) {
                  status.getValidations().add(
                      new ViewStatus.Validation("Slider view does not support accessing secured YARN cluster as YARN superuser (" + yarnUser + ")"));
                }
              }
            }
            if (cluster.getDesiredConfigs().containsKey("zoo.cfg")) {
              Map<String, String> zkEnvConfigs = ambariClient.getConfiguration(
                  cluster, "zoo.cfg",
                  cluster.getDesiredConfigs().get("zoo.cfg"));
              StringBuilder zkQuorumBuilder = new StringBuilder();
              String port = zkEnvConfigs.get("clientPort");
              AmbariService zkService = ambariClient.getService(cluster,
                  "ZOOKEEPER");
              if (zkService != null) {
                List<AmbariHostComponent> hostsList = zkService
                    .getComponentsToHostComponentsMap().get("ZOOKEEPER_SERVER");
                int count = 1;
                for (AmbariHostComponent host : hostsList) {
                  zkQuorumBuilder.append(host.getHostName() + ":" + port);
                  if (count++ < hostsList.size()) {
                    zkQuorumBuilder.append(",");
                  }
                }
                newHadoopConfigs.put(PROPERTY_SLIDER_ZK_QUORUM,
                    zkQuorumBuilder.toString());
              } else {
                status.getValidations().add(
                    new ViewStatus.Validation(
                        "ZooKeeper service is not installed"));
              }
            } else {
              status.getValidations()
              .add(
                  new ViewStatus.Validation(
                      "ZooKeeper service is not installed"));
            }
            if (cluster.getDesiredConfigs().containsKey("ams-site")) {
              Map<String, String> amsConfigs = ambariClient.getConfiguration(cluster, "ams-site", cluster.getDesiredConfigs().get("ams-site"));
              AmbariService amsService = ambariClient.getService(cluster, "AMBARI_METRICS");
              List<AmbariHostComponent> hostsList = amsService.getComponentsToHostComponentsMap().get("METRICS_COLLECTOR");
              if (hostsList != null && hostsList.size() > 0) {
                String collectorHostName = hostsList.get(0).getHostName();
                newHadoopConfigs.put(PROPERTY_METRICS_SERVER_HOSTNAME, collectorHostName);
                status.getParameters().put(PROPERTY_METRICS_SERVER_HOSTNAME, collectorHostName);
              }
              if (amsConfigs != null && amsConfigs.containsKey("timeline.metrics.service.webapp.address")) {
                String portString = amsConfigs.get("timeline.metrics.service.webapp.address");
                int sepIndex = portString.indexOf(':');
                if (sepIndex > -1) {
                  portString = portString.substring(sepIndex + 1);
                }
                newHadoopConfigs.put(PROPERTY_METRICS_SERVER_PORT, portString);
                status.getParameters().put(PROPERTY_METRICS_SERVER_PORT, portString);
              }
              newHadoopConfigs.put(PROPERTY_METRICS_LIBRARY_PATH, "file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar");
              status.getParameters().put(PROPERTY_METRICS_LIBRARY_PATH, "file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar");
            }
            Validation validateHDFSAccess = validateHDFSAccess(newHadoopConfigs, hdfsServiceInfo);
            if (validateHDFSAccess != null) {
              status.getValidations().add(validateHDFSAccess);
            }
          } else {
            status.getValidations().add(
                new ViewStatus.Validation("Ambari cluster with ID ["
                    + clusterName + "] was not found on Ambari server"));
          }
        } catch (Throwable e) {
          logger.warn("Exception determining view status", e);
          String message = e.getClass().getName() + ": " + e.getMessage();
          if (e instanceof RuntimeException && e.getCause() != null) {
            message = e.getCause().getClass().getName() + ": " + e.getMessage();
          }
          message = String.format("Unable to initialize Slider view: %s", message);
          status.getValidations().add(new ViewStatus.Validation(message));
        }
      } else {
        status
            .getValidations()
            .add(
                new ViewStatus.Validation(
                    "Ambari server cluster API URL should include cluster name, for example http://ambari.server:8080/api/v1/clusters/c1"));
      }
    } else {
      status.getValidations().add(
          new ViewStatus.Validation(
              "View parameters specifying Ambari details required"));
    }
    synchronized (viewContext) {
      if (!newHadoopConfigs.equals(viewContext.getInstanceData())) {
        Set<String> removeKeys = new HashSet<String>(viewContext.getInstanceData().keySet());
        for (Entry<String, String> e : newHadoopConfigs.entrySet()) {
          viewContext.putInstanceData(e.getKey(), e.getValue());
          removeKeys.remove(e.getKey());
        }
        for (String key : removeKeys) {
          viewContext.removeInstanceData(key);
        }
      }
    }
    return status;
  }

  /**
   * Slider-view supports only some stack-versions. This method validates that the targeted cluster is supported.
   *
   * @param clusterVersion
   * @return
   */
  private Validation validateStackVersion(String clusterVersion) {
    // Assuming cluster versions are of the format "X-a.b.c.d"
    String stackName = clusterVersion;
    String stackVersion = clusterVersion;
    int dashIndex = clusterVersion.indexOf('-');
    if (dashIndex > -1 && dashIndex < clusterVersion.length() - 1) {
      stackName = stackName.substring(0, dashIndex);
      stackVersion = stackVersion.substring(dashIndex + 1);
    }
    String[] versionSplits = stackVersion.split("\\.");
    if (!"HDP".equals(stackName) || versionSplits.length < 2) {
      return new Validation("Stack version (" + clusterVersion + ") used by cluster is not supported");
    }
    try {
      int majorVersion = Integer.parseInt(versionSplits[0]);
      int minorVersion = Integer.parseInt(versionSplits[1]);
      if (!(majorVersion >= 2 && minorVersion >= 2)) {
        return new Validation("Stack version (" + clusterVersion + ") used by cluster is not supported");
      }
    } catch (NumberFormatException e) {
      return new Validation("Stack version (" + clusterVersion + ") used by cluster is not supported");
    }
    return null;
  }

  private Validation validateHDFSAccess(final Map<String, String> hadoopConfigs, AmbariServiceInfo hdfsServiceInfo) {
    if (hdfsServiceInfo != null && hdfsServiceInfo.isStarted()) {
      if (hadoopConfigs.containsKey("fs.defaultFS")) {
        try {
          invokeHDFSClientRunnable(new HDFSClientRunnable<Boolean>() {
            @Override
            public Boolean run(FileSystem fs) throws IOException, InterruptedException {
              Path homePath = fs.getHomeDirectory();
              fs.listFiles(homePath, false);
              return Boolean.TRUE;
            }
          }, hadoopConfigs);
        } catch (IOException e) {
          String message = "Slider View requires access to user's home directory in HDFS to proceed. Contact your administrator to create the home directory. ("
              + e.getMessage() + ")";
          logger.warn(message, e);
          return new Validation(message);
        } catch (InterruptedException e) {
          String message = "Slider View requires access to user's home directory in HDFS to proceed. Contact your administrator to create the home directory. ("
              + e.getMessage() + ")";
          logger.warn(message, e);
          return new Validation(message);
        }
      } else {
        return new Validation("Location of HDFS filesystem is unknown for verification. Please check the 'fs.defaultFS' config in core-site.xml");
      }
    }
    return null;
  }

  private String getApplicationIdString(ApplicationId appId) {
    return Long.toString(appId.getClusterTimestamp()) + "_"
        + Integer.toString(appId.getId());
  }

  private ApplicationId getApplicationId(String appIdString) {
    if (appIdString != null) {
      int index = appIdString.indexOf('_');
      if (index > -1 && index < appIdString.length() - 1) {
        ApplicationId appId = ApplicationId.newInstance(
            Long.parseLong(appIdString.substring(0, index)),
            Integer.parseInt(appIdString.substring(index + 1)));
        return appId;
      }
    }
    return null;
  }

  private static interface SliderClientContextRunnable<T> {
    public T run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException;
  }

  private static interface HDFSClientRunnable<T> {
    public T run(FileSystem fs) throws IOException, InterruptedException;
  }

  private String getUserToRunAs() {
    return getUserToRunAs(getHadoopConfigs());
  }

  private String getUserToRunAs(Map<String, String> hadoopConfigs) {
    String user = getViewParameterValue(PARAM_SLIDER_USER);
    if (user == null || user.trim().length() < 1) {
      if (hadoopConfigs.containsKey("yarn_user")) {
        return hadoopConfigs.get("yarn_user");
      }
      return "yarn";
    } else if ("${username}".equals(user)) {
      return viewContext.getUsername();
    } else {
      return user;
    }
  }

  private <T> T invokeHDFSClientRunnable(final HDFSClientRunnable<T> runnable, final Map<String, String> hadoopConfigs) throws IOException,
      InterruptedException {
    ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {
      boolean securityEnabled = Boolean.valueOf(hadoopConfigs.get("security_enabled"));
      final HdfsConfiguration hdfsConfiguration = new HdfsConfiguration();
      for (Entry<String, String> entry : hadoopConfigs.entrySet()) {
        hdfsConfiguration.set(entry.getKey(), entry.getValue());
      }
      UserGroupInformation.setConfiguration(hdfsConfiguration);
      UserGroupInformation sliderUser;
      String loggedInUser = getUserToRunAs(hadoopConfigs);
      if (securityEnabled) {
        String viewPrincipal = getViewParameterValue(PARAM_VIEW_PRINCIPAL);
        String viewPrincipalKeytab = getViewParameterValue(PARAM_VIEW_PRINCIPAL_KEYTAB);
        UserGroupInformation ambariUser = UserGroupInformation.loginUserFromKeytabAndReturnUGI(viewPrincipal, viewPrincipalKeytab);
        if (loggedInUser.equals(ambariUser.getShortUserName())) {
          // HDFS throws exception when caller tries to impresonate themselves.
          // User: admin@EXAMPLE.COM is not allowed to impersonate admin
          sliderUser = ambariUser;
        } else {
          sliderUser = UserGroupInformation.createProxyUser(loggedInUser, ambariUser);
        }
      } else {
        sliderUser = UserGroupInformation.getBestUGI(null, loggedInUser);
      }
      try {
        T value = sliderUser.doAs(new PrivilegedExceptionAction<T>() {
          @Override
          public T run() throws Exception {
            String fsPath = hadoopConfigs.get("fs.defaultFS");
            FileSystem fs = FileSystem.get(URI.create(fsPath), hdfsConfiguration);
            try {
              return runnable.run(fs);
            } finally {
              fs.close();
            }
          }
        });
        return value;
      } catch (UndeclaredThrowableException e) {
        throw e;
      }
    } finally {
      Thread.currentThread().setContextClassLoader(currentClassLoader);
    }
  }

  private <T> T invokeSliderClientRunnable(final SliderClientContextRunnable<T> runnable) throws IOException, InterruptedException, YarnException {
    ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {
      boolean securityEnabled = Boolean.valueOf(getHadoopConfigs().get("security_enabled"));
      UserGroupInformation.setConfiguration(getSliderClientConfiguration());
      UserGroupInformation sliderUser;
      String loggedInUser = getUserToRunAs();
      if (securityEnabled) {
        String viewPrincipal = getViewParameterValue(PARAM_VIEW_PRINCIPAL);
        String viewPrincipalKeytab = getViewParameterValue(PARAM_VIEW_PRINCIPAL_KEYTAB);
        UserGroupInformation ambariUser = UserGroupInformation.loginUserFromKeytabAndReturnUGI(viewPrincipal, viewPrincipalKeytab);
        if (loggedInUser.equals(ambariUser.getShortUserName())) {
          // HDFS throws exception when caller tries to impresonate themselves.
          // User: admin@EXAMPLE.COM is not allowed to impersonate admin
          sliderUser = ambariUser;
        } else {
          sliderUser = UserGroupInformation.createProxyUser(loggedInUser, ambariUser);
        }
      } else {
        sliderUser = UserGroupInformation.getBestUGI(null, loggedInUser);
      }
      try {
        T value = sliderUser.doAs(new PrivilegedExceptionAction<T>() {
          @Override
          public T run() throws Exception {
            final SliderClient sliderClient = createSliderClient();
            try {
              return runnable.run(sliderClient);
            } finally {
              destroySliderClient(sliderClient);
            }
          }
        });
        return value;
      } catch (UndeclaredThrowableException e) {
        Throwable cause = e.getCause();
        if (cause instanceof YarnException) {
          YarnException ye = (YarnException) cause;
          throw ye;
        }
        throw e;
      }
    } finally {
      Thread.currentThread().setContextClassLoader(currentClassLoader);
    }
  }

  @Override
  public boolean appExists(final String appName) throws IOException, InterruptedException, YarnException {
    return invokeSliderClientRunnable(new SliderClientContextRunnable<Boolean>() {
      @Override
      public Boolean run(SliderClient sliderClient) throws YarnException, IOException {
        if (appName != null) {
          try {
            return sliderClient.actionExists(appName, false) == SliderClient.EXIT_SUCCESS;
          } catch (UnknownApplicationInstanceException e) {
            return Boolean.FALSE;
          }
        }
        return Boolean.FALSE;
      }
    });
  }

  @Override
  public SliderApp getSliderApp(final String applicationId, final Set<String> properties)
     throws YarnException, IOException, InterruptedException {
    return invokeSliderClientRunnable(new SliderClientContextRunnable<SliderApp>() {
      @Override
      public SliderApp run(SliderClient sliderClient) throws YarnException, IOException {
        if (applicationId!=null) {
          ApplicationId appId = getApplicationId(applicationId);
          if (appId != null) {
            ApplicationReport yarnApp = sliderClient.getApplicationReport(appId);
            return createSliderAppObject(yarnApp, properties, sliderClient);
          }
        }
        return null;
      }
    });
  }

  private SliderApp createSliderAppObject(ApplicationReport yarnApp,
      Set<String> properties, SliderClient sliderClient) {
    if (yarnApp == null) {
      return null;
    }

    SliderApp app = new SliderApp();
    app.setState(yarnApp.getYarnApplicationState().name());

    // Valid Slider App?
    // We want all Slider apps except the ones which properly finished.
    if (YarnApplicationState.FINISHED.equals(yarnApp.getYarnApplicationState())) {
      try {
        if (sliderClient.actionExists(yarnApp.getName(), false) == LauncherExitCodes.EXIT_SUCCESS) {
          app.setState(SliderApp.STATE_FROZEN);
        }
      } catch (UnknownApplicationInstanceException e) {
        return null; // Application not in HDFS - means it is not frozen
      } catch (YarnException e) {
        logger.warn(
            "Unable to determine frozen state for " + yarnApp.getName(), e);
        return null;
      } catch (IOException e) {
        logger.warn(
            "Unable to determine frozen state for " + yarnApp.getName(), e);
        return null;
      }
    }
    if (YarnApplicationState.KILLED.equals(yarnApp.getYarnApplicationState()) || YarnApplicationState.FAILED.equals(yarnApp.getYarnApplicationState())) {
      try {
        if (sliderClient.actionExists(yarnApp.getName(), false) != LauncherExitCodes.EXIT_SUCCESS) {
          // YARN application is killed or failed, and no HDFS content - Application has been destroyed.
          return null;
        }
      } catch (UnknownApplicationInstanceException e) {
        return null; // Application not in HDFS - means it is not frozen
      } catch (YarnException e) {
        logger.warn("Unable to determine status of killed app " + yarnApp.getName(), e);
        return null;
      } catch (IOException e) {
        logger.warn("Unable to determine status of killed app " + yarnApp.getName(), e);
        return null;
      }
    }

    app.setId(getApplicationIdString(yarnApp.getApplicationId()));
    app.setName(yarnApp.getName());
    app.setUser(yarnApp.getUser());
    app.setDiagnostics(yarnApp.getDiagnostics());
    app.setYarnId(yarnApp.getApplicationId().toString());
    app.setStartTime(yarnApp.getStartTime());
    app.setEndTime(yarnApp.getFinishTime());
    Set<String> applicationTags = yarnApp.getApplicationTags();
    if (applicationTags != null && applicationTags.size() > 0) {
      for (String tag : applicationTags) {
        int index = tag.indexOf(':');
        if (index > 0 && index < tag.length() - 1) {
          String key = tag.substring(0, index).trim();
          String value = tag.substring(index + 1).trim();
          if ("name".equals(key)) {
            app.setType(value);
          } else if ("version".equals(key)) {
            app.setAppVersion(value);
          } else if ("description".equals(key)) {
            app.setDescription(value);
          }
        }
      }
    }
    if (properties != null && !properties.isEmpty()) {
      SliderAppType matchedAppType = null;
      List<SliderAppType> matchingAppTypes = getSliderAppTypes(null);
      if (matchingAppTypes != null && matchingAppTypes.size() > 0) {
        for (SliderAppType appType : matchingAppTypes) {
          if ((appType.getTypeName() != null && appType.getTypeName()
              .equalsIgnoreCase(app.getType()))
              && (appType.getTypeVersion() != null && appType.getTypeVersion()
                  .equalsIgnoreCase(app.getAppVersion()))) {
            matchedAppType = appType;
            break;
          }
        }
      }

      SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null
          : new SliderAppMasterClient(yarnApp.getTrackingUrl());
      SliderAppMasterData appMasterData = null;
      Map<String, String> quickLinks = new HashMap<String, String>();
      Set<String> metrics = new HashSet<String>();
      for (String property : properties) {
        if ("RUNNING".equals(app.getState())) {
          if (sliderAppClient != null) {
            if (appMasterData == null) {
              appMasterData = sliderAppClient.getAppMasterData();
            }
            if (appMasterData!=null && "urls".equals(property.toLowerCase())) {
              if (quickLinks.isEmpty()) {
                quickLinks = sliderAppClient
                    .getQuickLinks(appMasterData.publisherUrl);
              }
              app.setUrls(quickLinks);
            } else if (appMasterData!=null && "configs".equals(property.toLowerCase())) {
              Map<String, Map<String, String>> configs = sliderAppClient
                  .getConfigs(appMasterData.publisherUrl);
              app.setConfigs(configs);
            } else if (appMasterData!=null && "jmx".equals(property.toLowerCase())) {
              if (quickLinks.isEmpty()) {
                quickLinks = sliderAppClient
                    .getQuickLinks(appMasterData.publisherUrl);
              }
              if (quickLinks != null && quickLinks.containsKey("JMX")) {
                String jmxUrl = quickLinks.get("JMX");
                if (matchedAppType != null) {
                  MetricsHolder metricsHolder = appMetrics.get(matchedAppType
                      .uniqueName());
                  app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext,
                      matchedAppType, metricsHolder));
                }
              }
              Map<String, Map<String, String>> configs = sliderAppClient
                  .getConfigs(appMasterData.publisherUrl);
              app.setConfigs(configs);
            } else if ("components".equals(property.toLowerCase())) {
              try {
                ClusterDescription description = sliderClient
                    .getClusterDescription(yarnApp.getName());
                if (description != null && description.status != null
                    && !description.status.isEmpty()) {
                  Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>();
                  for (Entry<String, Object> e : description.status.entrySet()) {
                    @SuppressWarnings("unchecked")
                    Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e
                        .getValue();
                    boolean isLive = "live".equals(e.getKey());
                    for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj
                        .entrySet()) {
                      SliderAppComponent appComponent = componentTypeMap
                          .get(componentEntry.getKey());
                      if (appComponent == null) {
                        appComponent = new SliderAppComponent();
                        appComponent.setComponentName(componentEntry.getKey());
                        appComponent
                            .setActiveContainers(new HashMap<String, Map<String, String>>());
                        appComponent
                            .setCompletedContainers(new HashMap<String, Map<String, String>>());
                        componentTypeMap.put(componentEntry.getKey(),
                            appComponent);
                      }
                      for (Entry<String, Map<String, Object>> containerEntry : componentEntry
                          .getValue().entrySet()) {
                        Map<String, String> containerDataMap = new HashMap<String, String>();
                        String containerId = containerEntry.getKey();
                        Map<String, Object> containerValues = containerEntry
                            .getValue();
                        for (String containerProperty : containerValues
                            .keySet()) {
                          Object containerPropertyValue = containerValues
                              .get(containerProperty);
                          containerDataMap.put(containerProperty,
                              containerPropertyValue.toString());
                        }
                        if (isLive) {
                          appComponent.getActiveContainers().put(containerId,
                              containerDataMap);
                        } else {
                          appComponent.getCompletedContainers().put(
                              containerId, containerDataMap);
                        }
                      }
                      // Set total instances count from statistics
                      appComponent.setInstanceCount(appComponent
                          .getActiveContainers().size()
                          + appComponent.getCompletedContainers().size());
                      if (description.statistics != null
                          && description.statistics.containsKey(componentEntry.getKey())) {
                        Map<String, Integer> statisticsMap = description.statistics.get(componentEntry.getKey());
                        if (statisticsMap.containsKey("containers.desired")) {
                          appComponent.setInstanceCount(statisticsMap.get("containers.desired"));
                        }
                      }
                    }
                  }
                  app.setAlerts(sliderAlerts.generateComponentsAlerts(componentTypeMap, app.getType()));
                  app.setComponents(componentTypeMap);
                }
              } catch (UnknownApplicationInstanceException e) {
                logger.warn(
                    "Unable to determine app components for "
                        + yarnApp.getName(), e);
              } catch (YarnException e) {
                logger.warn(
                    "Unable to determine app components for "
                        + yarnApp.getName(), e);
                throw new RuntimeException(e.getMessage(), e);
              } catch (IOException e) {
                logger.warn(
                    "Unable to determine app components for "
                        + yarnApp.getName(), e);
                throw new RuntimeException(e.getMessage(), e);
              }
            } else if (property.startsWith(METRICS_PREFIX)) {
              metrics.add(property.substring(METRICS_PREFIX.length()));
            } else if ("supportedMetrics".equals(property)) {
              if (matchedAppType != null) {
                app.setSupportedMetrics(matchedAppType.getSupportedMetrics());
              }
            }
          }
        }
      }
      if (metrics.size() > 0) {
        if (quickLinks.isEmpty()) {
          quickLinks = sliderAppClient
              .getQuickLinks(appMasterData.publisherUrl);
        }
        if (quickLinks != null && quickLinks.containsKey(METRICS_API_NAME)) {
          String metricsUrl = quickLinks.get(METRICS_API_NAME);
          MetricsHolder metricsHolder = appMetrics.get(matchedAppType
              .uniqueName());
          app.setMetrics(sliderAppClient.getMetrics(yarnApp.getName(),
              metricsUrl, metrics, null, viewContext, matchedAppType,
              metricsHolder));
        }
      }
    }
    return app;
  }

  /**
   * Creates a new {@link SliderClient} initialized with appropriate
   * configuration and started. This slider client can be used to invoke
   * individual API.
   * 
   * When work with this client is done,
   * {@link #destroySliderClient(SliderClient)} must be called.
   * 
   * @return created {@link SliderClient}
   * @see #destroySliderClient(SliderClient)
   * @see #runSliderCommand(String...)
   */
  protected SliderClient createSliderClient() {
    Configuration sliderClientConfiguration = getSliderClientConfiguration();
    SliderClient client = new SliderClient() {
      @Override
      public void init(Configuration conf) {
        super.init(conf);
        try {
          initHadoopBinding();
        } catch (SliderException e) {
          throw new RuntimeException("Unable to automatically init Hadoop binding", e);
        } catch (IOException e) {
          throw new RuntimeException("Unable to automatically init Hadoop binding", e);
        }
      }
    };
    try {
      if (logger.isDebugEnabled()) {
        logger.debug("Slider Client configuration: " + sliderClientConfiguration.toString());
      }
      sliderClientConfiguration = client.bindArgs(sliderClientConfiguration,
          new String[] { "help" });
      client.init(sliderClientConfiguration);
      client.start();
    } catch (Exception e) {
      logger.warn("Unable to create SliderClient", e);
      throw new RuntimeException(e.getMessage(), e);
    } catch (Throwable e) {
      logger.warn("Unable to create SliderClient", e);
      throw new RuntimeException(e.getMessage(), e);
    }
    return client;
  }

  protected void destroySliderClient(SliderClient sliderClient) {
    sliderClient.stop();
    sliderClient = null;
  }

  private String getViewParameterValue(String parameterName) {
    String value = viewContext.getProperties().get(parameterName);
    if ("null".equals(value)) {
      return null;
    }
    return value;
  }

  protected Map<String, String> getHadoopConfigs() {
    return viewContext.getInstanceData();
  }

  /**
   * Dynamically determines Slider client configuration. If unable to determine,
   * <code>null</code> is returned.
   * 
   * @return
   */
  private Configuration getSliderClientConfiguration() {
    HdfsConfiguration hdfsConfig = new HdfsConfiguration();
    YarnConfiguration yarnConfig = new YarnConfiguration(hdfsConfig);

    Map<String, String> hadoopConfigs = getHadoopConfigs();
    for(Entry<String, String> entry: hadoopConfigs.entrySet()) {
      String entryValue = entry.getValue();
      if (entryValue == null) {
        entryValue = "";
      }
      yarnConfig.set(entry.getKey(), entryValue);
    }
    yarnConfig.set(PROPERTY_SLIDER_SECURITY_ENABLED, hadoopConfigs.get("security_enabled"));
    if (hadoopConfigs.containsKey(PROPERTY_SLIDER_ZK_QUORUM)) {
      yarnConfig.set(PROPERTY_SLIDER_ZK_QUORUM, hadoopConfigs.get(PROPERTY_SLIDER_ZK_QUORUM));
    }
    return yarnConfig;
  }

  private boolean areViewParametersSet() {
    Map<String, String> hadoopConfigs = getHadoopConfigs();
    return hadoopConfigs.containsKey("fs.defaultFS")
        && hadoopConfigs.containsKey("yarn.resourcemanager.address")
        && hadoopConfigs.containsKey("yarn.resourcemanager.webapp.address")
        && hadoopConfigs.containsKey(PROPERTY_SLIDER_ZK_QUORUM);
  }

  @Override
  public List<SliderApp> getSliderApps(final Set<String> properties)
      throws YarnException, IOException, InterruptedException {
    if (!areViewParametersSet()) {
      return Collections.emptyList();
    }
    return invokeSliderClientRunnable(new SliderClientContextRunnable<List<SliderApp>>() {
      @Override
      public List<SliderApp> run(SliderClient sliderClient)
          throws YarnException, IOException {
        List<SliderApp> sliderApps = new ArrayList<SliderApp>();
        Map<String, SliderApp> sliderAppsMap = new HashMap<String, SliderApp>();
        List<ApplicationReport> yarnApps = sliderClient.listSliderInstances(null);
        for (ApplicationReport yarnApp : yarnApps) {
          SliderApp sliderAppObject = createSliderAppObject(yarnApp, properties,
              sliderClient);
          if (sliderAppObject != null) {
            if (sliderAppsMap.containsKey(sliderAppObject.getName())) {
              if (sliderAppsMap.get(sliderAppObject.getName()).getId()
                  .compareTo(sliderAppObject.getId()) < 0) {
                sliderAppsMap.put(sliderAppObject.getName(), sliderAppObject);
              }
            } else {
              sliderAppsMap.put(sliderAppObject.getName(), sliderAppObject);
            }
          }
        }
        if (sliderAppsMap.size() > 0)
          sliderApps.addAll(sliderAppsMap.values());
        return sliderApps;
      }
    });
  }

  @Override
  public void deleteSliderApp(final String applicationId) throws YarnException,
      IOException, InterruptedException {
    Integer code = invokeSliderClientRunnable(new SliderClientContextRunnable<Integer>() {
      @Override
      public Integer run(SliderClient sliderClient) throws YarnException,
          IOException, InterruptedException {
        Set<String> properties = new HashSet<String>();
        properties.add("id");
        properties.add("name");
        SliderApp sliderApp = getSliderApp(applicationId, properties);
        if (sliderApp == null) {
          throw new ApplicationNotFoundException(applicationId);
        }
        return sliderClient.actionDestroy(sliderApp.getName());
      }
    });
    logger.info("Deleted Slider App [" + applicationId + "] with exit code " + code);
  }

  @Override
  public SliderAppType getSliderAppType(String appTypeId, Set<String> properties) {
    List<SliderAppType> appTypes = getSliderAppTypes(properties);
    if (appTypeId != null && appTypes != null) {
      for (SliderAppType appType : appTypes) {
        if (appTypeId != null && appTypeId.equals(appType.getId())) {
          return appType;
        }
      }
    }
    return null;
  }

  @Override
  public List<SliderAppType> getSliderAppTypes(Set<String> properties) {
    try {
      // Need to determine security enablement before loading app types
      getViewStatus();
    } catch (Throwable t) {
      logger.warn("Unable to determine if cluster is secured when loading app-types", t);
    }
    Map<String, String> hadoopConfigs = getHadoopConfigs();
    final boolean securityEnabled = Boolean.valueOf(hadoopConfigs.get("security_enabled"));
    if (appTypes == null) {
      appTypes = loadAppTypes();
    }
    if (appTypes != null) {
      for (SliderAppType appType : appTypes) {
        Map<String, String> configs = appType.typeConfigsUnsecured;
        JsonObject resourcesObj = appType.resourcesUnsecured;
        if (securityEnabled) {
          configs = appType.typeConfigsSecured;
          if (configs == null || configs.isEmpty()) {
            configs = appType.typeConfigsUnsecured;
          }
        }
        if (securityEnabled) {
          resourcesObj = appType.resourcesSecured;
          if (resourcesObj == null) {
            resourcesObj = appType.resourcesUnsecured;
          }
        }
        Map<String, String> appTypeConfigs = new HashMap<String, String>();
        for (Entry<String, String> e : configs.entrySet()) {
          String valueString = e.getValue();
          if (valueString != null && valueString.contains("${USER_NAME}")) {
            valueString = valueString.replace("${USER_NAME}", getUserToRunAs(hadoopConfigs));
          }
          appTypeConfigs.put(e.getKey(), valueString);
        }
        appType.setTypeConfigs(appTypeConfigs);

        if (resourcesObj != null) {
          for (SliderAppTypeComponent component : appType.getTypeComponents()) {
            JsonElement componentJson = resourcesObj.get(component.getName());
            if (componentJson != null && componentJson.isJsonObject()) {
              JsonObject componentObj = componentJson.getAsJsonObject();
              if (componentObj.has("yarn.role.priority")) {
                component.setPriority(Integer.parseInt(componentObj.get("yarn.role.priority").getAsString()));
              }
              if (componentObj.has("yarn.memory")) {
                component.setYarnMemory(Integer.parseInt(componentObj.get("yarn.memory").getAsString()));
              }
            }
          }
        }
      }
    }
    return appTypes;
  }

  private List<SliderAppType> loadAppTypes() {
    List<SliderAppType> appTypes = null;
    String appsFolderPath = getAppsFolderPath();
    File appsFolder = new File(appsFolderPath);
    if (appsFolder.exists()) {
      File[] appZips = appsFolder
          .listFiles((FilenameFilter) new RegexFileFilter("^.*\\.zip$"));
      if (appZips != null) {
        appTypes = new ArrayList<SliderAppType>();
        for (File appZip : appZips) {
          try {
            ZipFile zipFile = new ZipFile(appZip);
            Metainfo metainfo = new MetainfoParser().parse(zipFile
                .getInputStream(zipFile.getEntry("metainfo.xml")));
            // Create app type object
            if (metainfo.getApplication() != null) {
              Application application = metainfo.getApplication();
              ZipArchiveEntry appConfigZipEntry = zipFile.getEntry("appConfig-default.json");
              ZipArchiveEntry appConfigSecuredZipEntry = zipFile.getEntry("appConfig-secured-default.json");
              if (appConfigZipEntry == null) {
                throw new IllegalStateException("Slider App package '" + appZip.getName() + "' does not contain 'appConfig-default.json' file");
              }
              ZipArchiveEntry resourcesZipEntry = zipFile.getEntry("resources-default.json");
              ZipArchiveEntry resourcesSecuredZipEntry = zipFile.getEntry("resources-secured-default.json");
              if (resourcesZipEntry == null) {
                throw new IllegalStateException("Slider App package '" + appZip.getName() + "' does not contain 'resources-default.json' file");
              }
              SliderAppType appType = new SliderAppType();
              appType.setId(application.getName());
              appType.setTypeName(application.getName());
              appType.setTypeDescription(application.getComment());
              appType.setTypeVersion(application.getVersion());
              appType.setTypePackageFileName(appZip.getName());
              // Configs
              appType.typeConfigsUnsecured = parseAppTypeConfigs(zipFile, appConfigZipEntry, appZip.getName(), application.getName());
              if (appConfigSecuredZipEntry != null) {
                appType.typeConfigsSecured = parseAppTypeConfigs(zipFile, appConfigSecuredZipEntry, appZip.getName(), application.getName());
              }
              // Resources
              appType.resourcesUnsecured = parseAppTypeResources(zipFile, resourcesZipEntry);
              if (resourcesSecuredZipEntry != null) {
                appType.resourcesSecured = parseAppTypeResources(zipFile, resourcesSecuredZipEntry);
              }
              // Components
              ArrayList<SliderAppTypeComponent> appTypeComponentList = new ArrayList<SliderAppTypeComponent>();
              for (Component component : application.getComponents()) {
                if ("CLIENT".equals(component.getCategory())) {
                  continue;
                }
                SliderAppTypeComponent appTypeComponent = new SliderAppTypeComponent();
                appTypeComponent.setDisplayName(component.getName());
                appTypeComponent.setId(component.getName());
                appTypeComponent.setName(component.getName());
                appTypeComponent.setYarnMemory(1024);
                appTypeComponent.setYarnCpuCores(1);
                // Updated below if present in resources.json
                appTypeComponent.setInstanceCount(1);
                // appTypeComponent.setPriority(component.);
                if (component.getMinInstanceCount() != null) {
                  appTypeComponent.setInstanceCount(Integer.parseInt(component
                      .getMinInstanceCount()));
                }
                if (component.getMaxInstanceCount() != null) {
                  appTypeComponent.setMaxInstanceCount(Integer
                      .parseInt(component.getMaxInstanceCount()));
                }
                appTypeComponent.setCategory(component.getCategory());
                appTypeComponentList.add(appTypeComponent);
              }

              MetricsHolder metricsHolder = new MetricsHolder();
              metricsHolder.setJmxMetrics(readMetrics(zipFile,
                  "jmx_metrics.json"));
              metricsHolder.setTimelineMetrics(readMetrics(zipFile,
                  "timeline_metrics.json"));
              appType.setSupportedMetrics(getSupportedMetrics(metricsHolder
                  .getTimelineMetrics()));
              appMetrics.put(appType.uniqueName(), metricsHolder);

              appType.setTypeComponents(appTypeComponentList);
              appTypes.add(appType);
            }
          } catch (ZipException e) {
            logger.warn("Unable to parse Slider App package " + appZip.getAbsolutePath(), e);
          } catch (IOException e) {
            logger.warn("Unable to parse Slider App package " + appZip.getAbsolutePath(), e);
          } catch (Throwable e) {
            logger.warn("Unable to parse Slider App package " + appZip.getAbsolutePath(), e);
          }
        }
      }
    }
    return appTypes;
  }

  private JsonObject parseAppTypeResources(ZipFile zipFile, ZipArchiveEntry resourcesZipEntry) throws ZipException, IOException {
    String resourcesJsonString = IOUtils.toString(zipFile.getInputStream(resourcesZipEntry), "UTF-8");
    JsonElement resourcesJson = new JsonParser().parse(resourcesJsonString);
    return resourcesJson.getAsJsonObject().get("components").getAsJsonObject();
  }

  private Map<String, String> parseAppTypeConfigs(ZipFile zipFile, ZipArchiveEntry appConfigZipEntry, String zipFileName, String appName) throws IOException,
      ZipException {
    String appConfigJsonString = IOUtils.toString(zipFile.getInputStream(appConfigZipEntry), "UTF-8");
    JsonElement appConfigJson = new JsonParser().parse(appConfigJsonString);
    Map<String, String> configsMap = new HashMap<String, String>();
    JsonObject appTypeGlobalJson = appConfigJson.getAsJsonObject().get("global").getAsJsonObject();
    for (Entry<String, JsonElement> e : appTypeGlobalJson.entrySet()) {
      String key = e.getKey();
      String valueString = e.getValue().getAsString();
      if ("application.def".equals(key)) {
        valueString = String.format(".slider/package/%s/%s", appName, zipFileName);
      }
      configsMap.put(key, valueString);
    }
    return configsMap;
  }

  private List<String> getSupportedMetrics(
      Map<String, Map<String, Map<String, Metric>>> metrics) {
    Set<String> supportedMetrics = new HashSet<String>();
    if (metrics != null && metrics.size() > 0) {
      for (Map<String, Map<String, Metric>> compMetrics : metrics
          .values()) {
        for (Map<String, Metric> compMetric : compMetrics.values()) {
          supportedMetrics.addAll(compMetric.keySet());
        }
      }
    }
    return new ArrayList<String>(supportedMetrics);
  }

  Map<String, Map<String, Map<String, Metric>>> readMetrics(ZipFile zipFile,
      String fileName) {
    Map<String, Map<String, Map<String, Metric>>> metrics = null;
    try {
      InputStream inputStream = zipFile.getInputStream(zipFile
          .getEntry(fileName));
      ObjectMapper mapper = new ObjectMapper();

      metrics = mapper.readValue(inputStream,
          new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() {
          });
    } catch (IOException e) {
      logger.info("Error reading metrics for file " + fileName + ". " + e.getMessage());
    }

    return metrics;
  }

  @Override
  public String createSliderApp(JsonObject json) throws IOException,
      YarnException, InterruptedException {
    if (json.has("name") && json.has("typeConfigs")
        && json.has("resources") && json.has("typeName")) {
      final String appType = json.get("typeName").getAsString();
      final String appName = json.get("name").getAsString();
      final String queueName = json.has("queue") ? json.get("queue").getAsString() : null;
      final boolean securityEnabled = Boolean.valueOf(getHadoopConfigs().get("security_enabled"));
      JsonObject configs = json.get("typeConfigs").getAsJsonObject();
      JsonObject resourcesObj = json.get("resources").getAsJsonObject();
      JsonArray componentsArray = resourcesObj.get("components").getAsJsonArray();
      String appsCreateFolderPath = getAppsCreateFolderPath();
      File appsCreateFolder = new File(appsCreateFolderPath);
      if (!appsCreateFolder.exists()) {
        appsCreateFolder.mkdirs();
      }
      int appCount;
      synchronized (createAppCounter) {
        if (createAppCounter < 0) {
          // Not initialized
          createAppCounter = 0;
          String[] apps = appsCreateFolder.list();
          for (String app : apps) {
            try {
              int count = Integer.parseInt(app);
              if (count > createAppCounter) {
                createAppCounter = count;
              }
            } catch (NumberFormatException e) {
            }
          }
        }
        appCount = ++createAppCounter;
      }
      File appCreateFolder = new File(appsCreateFolder,
          Integer.toString(appCount));
      appCreateFolder.mkdirs();
      File appConfigJsonFile = new File(appCreateFolder, "appConfig.json");
      File resourcesJsonFile = new File(appCreateFolder, "resources.json");
      saveAppConfigs(configs, componentsArray, appName, appType, securityEnabled, appConfigJsonFile);
      saveAppResources(resourcesObj, resourcesJsonFile);

      final ActionCreateArgs createArgs = new ActionCreateArgs();
      createArgs.template = appConfigJsonFile;
      createArgs.resources = resourcesJsonFile;
      if (queueName != null && queueName.trim().length() > 0) {
        createArgs.queue = queueName;
      }

      final ActionInstallPackageArgs installArgs = new ActionInstallPackageArgs();
      SliderAppType sliderAppType = getSliderAppType(appType, null);
      String localAppPackageFileName = sliderAppType.getTypePackageFileName();
      installArgs.name = appType;
      installArgs.packageURI = getAppsFolderPath() + "/" + localAppPackageFileName;
      installArgs.replacePkg = true;

      final List<ActionInstallKeytabArgs> installKeytabActions = new ArrayList<ActionInstallKeytabArgs>();
      if (securityEnabled) {
        for (String keytab : getUserToRunAsKeytabs(appType)) {
          ActionInstallKeytabArgs keytabArgs = new ActionInstallKeytabArgs();
          keytabArgs.keytabUri = keytab;
          keytabArgs.folder = appName;
          keytabArgs.overwrite = true;
          installKeytabActions.add(keytabArgs);
        }
      }

      return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() {
        @Override
        public String run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException {
          try {
            File sliderJarFile = SliderUtils.findContainingJar(SliderClient.class);
            if (sliderJarFile != null) {
              if (logger.isDebugEnabled()) {
                logger.debug("slider.libdir=" + sliderJarFile.getParentFile().getAbsolutePath());
              }
              System.setProperty("slider.libdir", sliderJarFile.getParentFile().getAbsolutePath());
            }
          } catch (Throwable t) {
            logger.warn("Unable to determine 'slider.libdir' path", t);
          }
          if (securityEnabled) {
            for (ActionInstallKeytabArgs keytabArgs : installKeytabActions) {
              if (logger.isDebugEnabled()) {
                logger.debug("Installing keytab " + keytabArgs.keytabUri);
              }
              sliderClient.actionInstallKeytab(keytabArgs);
            }
          }
          sliderClient.actionInstallPkg(installArgs);
          sliderClient.actionCreate(appName, createArgs);
          ApplicationId applicationId = sliderClient.applicationId;
          if (applicationId != null) {
            return getApplicationIdString(applicationId);
          }
          return null;
        }
      });
    }
    return null;
  }

  private void saveAppResources(JsonObject clientResourcesObj,
      File resourcesJsonFile) throws IOException {
    JsonObject resourcesObj = new JsonObject();
    JsonArray clientComponentsArray = clientResourcesObj.get("components").getAsJsonArray();
    resourcesObj.addProperty("schema",
        "http://example.org/specification/v2.0.0");
    resourcesObj.add("metadata", new JsonObject());
    resourcesObj.add("global",
        clientResourcesObj.has("global") ? clientResourcesObj.get("global")
            .getAsJsonObject() : new JsonObject());
    JsonObject componentsObj = new JsonObject();
    if (clientComponentsArray != null) {
      for (int i = 0; i < clientComponentsArray.size(); i++) {
        JsonObject inputComponent = clientComponentsArray.get(i).getAsJsonObject();
        if (inputComponent.has("id")) {
          JsonObject componentValue = new JsonObject();
          if (inputComponent.has("priority")) {
            componentValue.addProperty("yarn.role.priority", inputComponent
                .get("priority").getAsString());
          }
          if (inputComponent.has("instanceCount")) {
            componentValue.addProperty("yarn.component.instances",
                inputComponent.get("instanceCount").getAsString());
          }
          if (inputComponent.has("yarnMemory")) {
            componentValue.addProperty("yarn.memory",
                inputComponent.get("yarnMemory").getAsString());
          }
          if (inputComponent.has("yarnCpuCores")) {
            componentValue.addProperty("yarn.vcores",
                inputComponent.get("yarnCpuCores").getAsString());
          }
          if (inputComponent.has("yarnLabel")) {
            componentValue.addProperty("yarn.label.expression", inputComponent
                .get("yarnLabel").getAsString());
          }
          componentsObj.add(inputComponent.get("id").getAsString(),
              componentValue);
        }
      }
    }
    resourcesObj.add("components", componentsObj);
    String jsonString = new GsonBuilder().setPrettyPrinting().create().toJson(resourcesObj);
    FileOutputStream fos = null;
    try {
      fos = new FileOutputStream(resourcesJsonFile);
      IOUtils.write(jsonString, fos);
    } finally {
      if (fos != null) {
        fos.close();
      }
    }
  }

  /*
   * When security is enabled, the AppMaster itself needs the keytab identifying the calling user.
   * The user's keytab should be at the same location as the view's keytab, and should be
   * named as ${username}.headless.keytab.
   * 
   * This method returns the list of keytabs where the first keytab is always the AppMaster's 
   * keytab. Additional keys will be provided, only if found at the location of the view's keytab.
   * Additional keytabs should be of the format ${username}.<APP_TYPE>.*.keytab
   */
  private List<String> getUserToRunAsKeytabs(String appType) {
    List<String> keytabsList = new ArrayList<String>();
    String viewKeytab = viewContext.getProperties().get(PARAM_VIEW_PRINCIPAL_KEYTAB);
    String folderPath = "";
    int index = viewKeytab.lastIndexOf('/');
    if (index > -1) {
      folderPath = viewKeytab.substring(0, index);
    }
    String username = getUserToRunAs();
    String userKeytab = folderPath + "/" + username + ".headless.keytab";
    File folder = new File(folderPath);
    if (folder.exists()) {
      final Pattern userKeytabPattern = Pattern.compile("^" + username + "\\." + appType + "\\..*\\.keytab");
      String[] keytabNames = folder.list(new FilenameFilter() {
        @Override
        public boolean accept(File dir, String name) {
          return userKeytabPattern.matcher(name).matches();
        }
      });
      if (keytabNames != null) {
        for (String keytabName : keytabNames) {
          keytabsList.add(folderPath + "/" + keytabName);
        }
      }
    }
    keytabsList.add(0, userKeytab);
    if (logger.isDebugEnabled()) {
      logger.debug(username + " keytabs: " + keytabsList);
    }
    return keytabsList;
  }

  private void saveAppConfigs(JsonObject configs, JsonArray componentsArray,
      String appName, String appType, boolean securityEnabled, File appConfigJsonFile) throws IOException {
    JsonObject appConfigs = new JsonObject();
    appConfigs.addProperty("schema", "http://example.org/specification/v2.0.0");
    appConfigs.add("metadata", new JsonObject());
    appConfigs.add("global", configs);
    JsonObject componentsObj = new JsonObject();
    if (componentsArray != null) {
      for (int i = 0; i < componentsArray.size(); i++) {
        JsonObject inputComponent = componentsArray.get(i).getAsJsonObject();
        if (inputComponent.has("id")) {
          componentsObj.add(inputComponent.get("id").getAsString(),
              new JsonObject());
        }
      }
    }
    if (securityEnabled) {
      JsonObject appMasterComponent = new JsonObject();
      String userToRunAsKeytab = getUserToRunAsKeytabs(appType).get(0);
      String fileName = userToRunAsKeytab.substring(userToRunAsKeytab.lastIndexOf('/') + 1);
      String userName = fileName.substring(0, fileName.indexOf('.'));
      String viewPrincipalName = getViewParameterValue(PARAM_VIEW_PRINCIPAL);
      int atIndex = viewPrincipalName.lastIndexOf('@');
      String viewPrincipalDomain = atIndex > -1 ? viewPrincipalName.substring(atIndex+1) : "";
      appMasterComponent.add("slider.keytab.principal.name", new JsonPrimitive(userName + "@" + viewPrincipalDomain));
      appMasterComponent.add("slider.am.login.keytab.name", new JsonPrimitive(fileName));
      appMasterComponent.add("slider.hdfs.keytab.dir", new JsonPrimitive(".slider/keytabs/" + appName));
      componentsObj.add("slider-appmaster", appMasterComponent);
    }
   appConfigs.add("components", componentsObj);
    String jsonString = new GsonBuilder().setPrettyPrinting().create().toJson(appConfigs);
    FileOutputStream fos = null;
    try {
      fos = new FileOutputStream(appConfigJsonFile);
      IOUtils.write(jsonString, fos);
    } finally {
      if (fos != null) {
        fos.close();
      }
    }
    if (logger.isDebugEnabled()) {
      logger.debug("Saved appConfigs.json at " + appConfigJsonFile.getAbsolutePath());
    }
  }

  @Override
  public void freezeApp(final String appId) throws YarnException, IOException,
      InterruptedException {
    ApplicationId applicationId = invokeSliderClientRunnable(new SliderClientContextRunnable<ApplicationId>() {
      @Override
      public ApplicationId run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException {
        Set<String> properties = new HashSet<String>();
        properties.add("id");
        properties.add("name");
        final SliderApp sliderApp = getSliderApp(appId, properties);
        if (sliderApp == null)
          throw new ApplicationNotFoundException(appId);
        ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
        sliderClient.actionFreeze(sliderApp.getName(), freezeArgs);
        return sliderClient.applicationId;
      }
    });
    logger.info("Frozen Slider App [" + appId + "] with response: " + applicationId.toString());
  }

  @Override
  public void thawApp(final String appId) throws YarnException, IOException, InterruptedException {
    ApplicationId applicationId = invokeSliderClientRunnable(new SliderClientContextRunnable<ApplicationId>() {
      @Override
      public ApplicationId run(SliderClient sliderClient) throws YarnException,
          IOException, InterruptedException {
        Set<String> properties = new HashSet<String>();
        properties.add("id");
        properties.add("name");
        final SliderApp sliderApp = getSliderApp(appId, properties);
        if (sliderApp == null)
          throw new ApplicationNotFoundException(appId);
        ActionThawArgs thawArgs = new ActionThawArgs();
        sliderClient.actionThaw(sliderApp.getName(), thawArgs);
        return sliderClient.applicationId;
      }
    });
    logger.info("Thawed Slider App [" + appId + "] with response: " + applicationId.toString());
  }

  @Override
  public void flexApp(final String appId, final Map<String, Integer> componentsMap)
      throws YarnException, IOException, InterruptedException {
    ApplicationId applicationId = invokeSliderClientRunnable(new SliderClientContextRunnable<ApplicationId>() {
      @Override
      public ApplicationId run(SliderClient sliderClient) throws YarnException,
          IOException, InterruptedException {
        Set<String> properties = new HashSet<String>();
        properties.add("id");
        properties.add("name");
        final SliderApp sliderApp = getSliderApp(appId, properties);
        if (sliderApp == null) {
          throw new ApplicationNotFoundException(appId);
        }
        ActionFlexArgs flexArgs = new ActionFlexArgs();
        flexArgs.parameters.add(sliderApp.getName());
        for (Entry<String, Integer> e : componentsMap.entrySet()) {
          flexArgs.componentDelegate.componentTuples.add(e.getKey());
          flexArgs.componentDelegate.componentTuples.add(e.getValue()
              .toString());
        }
        sliderClient.actionFlex(sliderApp.getName(), flexArgs);
        return sliderClient.applicationId;
      }
    });
    logger.info("Flexed Slider App [" + appId + "] with response: " + applicationId);
  }

}
