YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

hadoop-project/pom.xml

@@ -965,11 +965,6 @@
         <artifactId>jackson-module-jaxb-annotations</artifactId>
         <version>${jackson2.version}</version>
       </dependency>
-      <dependency>
-        <groupId>com.fasterxml.jackson.jaxrs</groupId>
-        <artifactId>jackson-jaxrs-json-provider</artifactId>
-        <version>${jackson2.version}</version>
-      </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.dataformat</groupId>
         <artifactId>jackson-dataformat-cbor</artifactId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java

@@ -30,7 +30,7 @@ import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.mortbay.jetty.webapp.Configuration;
+import org.eclipse.jetty.webapp.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml

@@ -225,7 +225,7 @@
 
     <dependency>
       <groupId>javax.servlet</groupId>
-      <artifactId>servlet-api</artifactId>
+      <artifactId>javax.servlet-api</artifactId>
     </dependency>
 
     <dependency>
@@ -288,21 +288,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>org.mortbay.jetty</groupId>
-      <artifactId>jetty</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.mortbay.jetty</groupId>
-      <artifactId>jetty-util</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.mortbay.jetty</groupId>
-      <artifactId>jetty-sslengine</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>javax.servlet.jsp</groupId>
       <artifactId>jsp-api</artifactId>
@@ -314,12 +299,6 @@
       <artifactId>jettison</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.mortbay.jetty</groupId>
-      <artifactId>jetty-sslengine</artifactId>
-      <version>6.1.26</version>
-    </dependency>
-
     <dependency>
       <groupId>org.yaml</groupId>
       <artifactId>snakeyaml</artifactId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java

@@ -70,7 +70,6 @@ import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.launch.ClasspathConstructor;
 import org.apache.slider.core.main.LauncherExitCodes;
 import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.providers.agent.application.metadata.Component;
 import org.apache.slider.server.services.utility.PatternValidator;
 import org.apache.slider.server.services.workflow.ForkedProcessService;
 import org.apache.zookeeper.server.util.KerberosUtil;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java

@@ -19,17 +19,17 @@
 package org.apache.slider.providers;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.types.AddressTypes;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.types.AddressTypes;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.tools.ConfigHelper;
@@ -44,7 +44,6 @@ import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
 import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
 import org.apache.slider.server.services.workflow.ForkedProcessService;
 import org.apache.slider.server.services.workflow.ServiceParent;
 import org.apache.slider.server.services.workflow.WorkflowSequenceService;
@@ -77,7 +76,6 @@ public abstract class AbstractProviderService
   private static final Logger log =
     LoggerFactory.getLogger(AbstractProviderService.class);
   protected StateAccessForProviders amState;
-  protected AgentRestOperations restOps;
   protected URL amWebAPI;
   protected YarnRegistryViewForProviders yarnRegistry;
   protected QueueAccess queueAccess;
@@ -126,19 +124,10 @@ public abstract class AbstractProviderService
     return yarnRegistry;
   }
 
-  @Override
-  public AgentRestOperations getAgentRestOperations() {
-    return restOps;
-  }
-
   @Override
   public void notifyContainerCompleted(ContainerId containerId) {
   }
 
-  public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
-    this.restOps = agentRestOperations;
-  }
-
   /**
    * Load default Configuration
    * @param confDir configuration directory

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java

@@ -20,10 +20,10 @@ package org.apache.slider.providers;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.common.tools.SliderFileSystem;
@@ -37,7 +37,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
 import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
 import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
 
 import java.io.File;
@@ -175,12 +174,6 @@ public interface ProviderService extends ProviderCore,
    */
   void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
 
-  /**
-   * Returns the agent rest operations interface.
-   * @return  the interface if available, null otherwise.
-   */
-  AgentRestOperations getAgentRestOperations();
-
   /**
    * Build up the endpoint details for this service
    * @param details

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java

@@ -1,713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.io.Files;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.ClientUtils;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.providers.agent.application.metadata.Package;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-import static org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
-
-/** This class implements  the client-side aspects of the agent deployer */
-public class AgentClientProvider extends AbstractClientProvider
-    implements AgentKeys, SliderKeys {
-
-
-  protected static final Logger log =
-      LoggerFactory.getLogger(AgentClientProvider.class);
-  protected static final String NAME = "agent";
-  private static final ProviderUtils providerUtils = new ProviderUtils(log);
-  public static final String E_COULD_NOT_READ_METAINFO
-      = "Not a valid app package. Could not read metainfo.";
-
-  protected Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
-
-  protected AgentClientProvider(Configuration conf) {
-    super(conf);
-  }
-
-  @Override
-  public String getName() {
-    return NAME;
-  }
-
-  @Override
-  public List<ProviderRole> getRoles() {
-    return AgentRoles.getRoles();
-  }
-
-  @Override //Client
-  public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
-                                                    String clustername,
-                                                    Configuration configuration,
-                                                    AggregateConf instanceDefinition,
-                                                    Path clusterDirPath,
-                                                    Path generatedConfDirPath,
-                                                    boolean secure) throws
-      SliderException,
-      IOException {
-    super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
-                                                configuration,
-                                                instanceDefinition,
-                                                clusterDirPath,
-                                                generatedConfDirPath, secure);
-
-    String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
-        .getAppConfOperations());
-    Path appDefPath = new Path(appDef);
-    sliderFileSystem.verifyFileExists(appDefPath);
-
-    String agentConf = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AGENT_CONF, "");
-    if (StringUtils.isNotEmpty(agentConf)) {
-      sliderFileSystem.verifyFileExists(new Path(agentConf));
-    }
-
-    String appHome = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().get(PACKAGE_PATH);
-    if (SliderUtils.isUnset(appHome)) {
-      String agentImage = instanceDefinition.getInternalOperations().
-          get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
-      sliderFileSystem.verifyFileExists(new Path(agentImage));
-    }
-  }
-
-  @Override
-  public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
-      SliderException {
-    super.validateInstanceDefinition(instanceDefinition, fs);
-    log.debug("Validating conf {}", instanceDefinition);
-    ConfTreeOperations resources =
-        instanceDefinition.getResourceOperations();
-
-    providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
-                                    0, -1);
-
-    String appDef = null;
-    try {
-      // Validate the app definition
-      appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
-          .getAppConfOperations());
-    } catch (BadConfigException bce) {
-      throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
-    }
-
-    log.info("Validating app definition {}", appDef);
-    String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
-    if (!"zip".equals(extension.toLowerCase(Locale.ENGLISH))) {
-      throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
-    }
-
-    Set<String> names = resources.getComponentNames();
-    names.remove(COMPONENT_AM);
-    Map<Integer, String> priorityMap = new HashMap<Integer, String>();
-
-    for (String name : names) {
-      try {
-        // Validate the app definition
-        appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
-            .getAppConfOperations(), name);
-      } catch (BadConfigException bce) {
-        throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
-      }
-      Metainfo metaInfo = getMetainfo(fs, appDef);
-
-      MapOperations component = resources.getMandatoryComponent(name);
-
-      if (metaInfo != null) {
-        Component componentDef = metaInfo.getApplicationComponent(
-            AgentUtils.getMetainfoComponentName(name,
-                instanceDefinition.getAppConfOperations()));
-        if (componentDef == null) {
-          throw new BadConfigException(
-              "Component %s is not a member of application.", name);
-        }
-      }
-
-      int priority =
-          component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
-      if (priority <= 0) {
-        throw new BadConfigException("Component %s %s value out of range %d",
-                                     name,
-                                     ResourceKeys.COMPONENT_PRIORITY,
-                                     priority);
-      }
-
-      String existing = priorityMap.get(priority);
-      if (existing != null) {
-        throw new BadConfigException(
-            "Component %s has a %s value %d which duplicates that of %s",
-            name,
-            ResourceKeys.COMPONENT_PRIORITY,
-            priority,
-            existing);
-      }
-      priorityMap.put(priority, name);
-
-      // fileSystem may be null for tests
-      if (metaInfo != null) {
-        Component componentDef = metaInfo.getApplicationComponent(
-            AgentUtils.getMetainfoComponentName(name,
-                instanceDefinition.getAppConfOperations()));
-
-        // ensure that intance count is 0 for client components
-        if ("CLIENT".equals(componentDef.getCategory())) {
-          MapOperations componentConfig = resources.getMandatoryComponent(name);
-          int count =
-              componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
-          if (count > 0) {
-            throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated."
-                                         + " Use \"slider client install ...\" command instead.",
-                                         name);
-          }
-        } else {
-          MapOperations componentConfig = resources.getMandatoryComponent(name);
-          int count =
-              componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
-          int definedMinCount = componentDef.getMinInstanceCountInt();
-          int definedMaxCount = componentDef.getMaxInstanceCountInt();
-          if (count < definedMinCount || count > definedMaxCount) {
-            throw new BadConfigException("Component %s, %s value %d out of range. "
-                                         + "Expected minimum is %d and maximum is %d",
-                                         name,
-                                         ResourceKeys.COMPONENT_INSTANCES,
-                                         count,
-                                         definedMinCount,
-                                         definedMaxCount);
-          }
-        }
-      }
-    }
-  }
-
-
-  @Override
-  public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
-                                          Configuration serviceConf,
-                                          AbstractLauncher launcher,
-                                          AggregateConf instanceDefinition,
-                                          Path snapshotConfDirPath,
-                                          Path generatedConfDirPath,
-                                          Configuration clientConfExtras,
-                                          String libdir,
-                                          Path tempPath,
-                                          boolean miniClusterTestRun) throws
-      IOException,
-      SliderException {
-    String agentImage = instanceDefinition.getInternalOperations().
-        get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
-    if (SliderUtils.isUnset(agentImage)) {
-      Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
-      log.info("Automatically uploading the agent tarball at {}", agentPath);
-      fileSystem.getFileSystem().mkdirs(agentPath);
-      if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
-        instanceDefinition.getInternalOperations().set(
-            InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH,
-            new Path(agentPath, AGENT_TAR).toUri());
-      }
-    }
-  }
-
-  @Override
-  public Set<String> getApplicationTags(SliderFileSystem fileSystem,
-      ConfTreeOperations appConf) throws SliderException {
-    return getApplicationTags(fileSystem,
-        getApplicationDefinitionPath(appConf));
-  }
-
-  public Set<String> getApplicationTags(SliderFileSystem fileSystem,
-                                        String appDef) throws SliderException {
-    Set<String> tags;
-    Metainfo metaInfo = getMetainfo(fileSystem, appDef);
-
-    if (metaInfo == null) {
-      log.error("Error retrieving metainfo from {}", appDef);
-      throw new SliderException("Error parsing metainfo file, possibly bad structure.");
-    }
-
-    Application application = metaInfo.getApplication();
-    tags = new HashSet<String>();
-    tags.add("Name: " + application.getName());
-    tags.add("Version: " + application.getVersion());
-    tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
-
-    return tags;
-  }
-
-  @Override
-  public void processClientOperation(SliderFileSystem fileSystem,
-                                     RegistryOperations rops,
-                                     Configuration configuration,
-                                     String operation,
-                                     File clientInstallPath,
-                                     File appPackage,
-                                     JSONObject config,
-                                     String name) throws SliderException {
-    // create temp folder
-    // create sub-folders app_pkg, agent_pkg, command
-    File tmpDir = Files.createTempDir();
-    log.info("Command is being executed at {}", tmpDir.getAbsolutePath());
-    File appPkgDir = new File(tmpDir, "app_pkg");
-    appPkgDir.mkdir();
-
-    File agentPkgDir = new File(tmpDir, "agent_pkg");
-    agentPkgDir.mkdir();
-
-    File cmdDir = new File(tmpDir, "command");
-    cmdDir.mkdir();
-
-    Metainfo metaInfo = null;
-    JSONObject defaultConfig = null;
-    try {
-      // expand app package into /app_pkg
-      ZipInputStream zipInputStream = null;
-      try {
-        zipInputStream = new ZipInputStream(new FileInputStream(appPackage));
-        {
-          ZipEntry zipEntry = zipInputStream.getNextEntry();
-          while (zipEntry != null) {
-            log.info("Processing {}", zipEntry.getName());
-            String filePath = appPkgDir + File.separator + zipEntry.getName();
-            if (!zipEntry.isDirectory()) {
-              log.info("Extracting file {}", filePath);
-              extractFile(zipInputStream, filePath);
-
-              if ("metainfo.xml".equals(zipEntry.getName())) {
-                FileInputStream input = null;
-                try {
-                  input = new FileInputStream(filePath);
-                  metaInfo = new MetainfoParser().fromXmlStream(input);
-                } finally {
-                  IOUtils.closeStream(input);
-                }
-              } else if ("metainfo.json".equals(zipEntry.getName())) {
-                FileInputStream input = null;
-                try {
-                  input = new FileInputStream(filePath);
-                  metaInfo = new MetainfoParser().fromJsonStream(input);
-                } finally {
-                  IOUtils.closeStream(input);
-                }
-              } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
-                try {
-                  defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
-                } catch (JSONException jex) {
-                  throw new SliderException("Unable to read default client config.", jex);
-                }
-              }
-            } else {
-              log.info("Creating dir {}", filePath);
-              File dir = new File(filePath);
-              dir.mkdir();
-            }
-            zipInputStream.closeEntry();
-            zipEntry = zipInputStream.getNextEntry();
-          }
-        }
-      } finally {
-        zipInputStream.close();
-      }
-
-      if (metaInfo == null) {
-        throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
-      }
-
-      String clientScript = null;
-      String clientComponent = null;
-      for (Component component : metaInfo.getApplication().getComponents()) {
-        if (component.getCategory().equals("CLIENT")) {
-          clientComponent = component.getName();
-          if (component.getCommandScript() != null) {
-            clientScript = component.getCommandScript().getScript();
-          }
-          break;
-        }
-      }
-
-      if (SliderUtils.isUnset(clientScript)) {
-        log.info("Installing CLIENT without script");
-        List<Package> packages = metaInfo.getApplication().getPackages();
-        if (packages.size() > 0) {
-          // retrieve package resources from HDFS and extract
-          for (Package pkg : packages) {
-            Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
-            if (!fileSystem.isFile(pkgPath) && name != null) {
-              pkgPath = fileSystem.buildResourcePath(name, pkg.getName());
-            }
-            if (!fileSystem.isFile(pkgPath)) {
-              throw new IOException("Package doesn't exist as a resource: " +
-                  pkg.getName());
-            }
-            if ("archive".equals(pkg.getType())) {
-              File pkgFile = new File(tmpDir, pkg.getName());
-              fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
-              expandTar(pkgFile, clientInstallPath);
-            } else {
-              File pkgFile = new File(clientInstallPath, pkg.getName());
-              fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
-            }
-          }
-        } else {
-          // extract tarball from app def
-          for (OSSpecific osSpecific : metaInfo.getApplication()
-              .getOSSpecifics()) {
-            for (OSPackage pkg : osSpecific.getPackages()) {
-              if ("tarball".equals(pkg.getType())) {
-                File pkgFile = new File(appPkgDir, pkg.getName());
-                expandTar(pkgFile, clientInstallPath);
-              }
-            }
-          }
-        }
-        if (name == null) {
-          log.warn("Conf files not being generated because no app name was " +
-              "provided");
-          return;
-        }
-        File confInstallDir;
-        String clientRoot = null;
-        if (config != null) {
-          try {
-            clientRoot = config.getJSONObject("global")
-                .getString(APP_CLIENT_ROOT);
-          } catch (JSONException e) {
-            log.info("Couldn't read {} from provided client config, falling " +
-                "back on default", APP_CLIENT_ROOT);
-          }
-        }
-        if (clientRoot == null && defaultConfig != null) {
-          try {
-            clientRoot = defaultConfig.getJSONObject("global")
-                .getString(APP_CLIENT_ROOT);
-          } catch (JSONException e) {
-            log.info("Couldn't read {} from default client config, using {}",
-                APP_CLIENT_ROOT, clientInstallPath);
-          }
-        }
-        if (clientRoot == null) {
-          confInstallDir = clientInstallPath;
-        } else {
-          confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf");
-          if (!confInstallDir.exists()) {
-            confInstallDir.mkdirs();
-          }
-        }
-        String user = RegistryUtils.currentUser();
-        for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) {
-          retrieveConfigFile(rops, configuration, configFile, name, user,
-              confInstallDir);
-        }
-      } else {
-        log.info("Installing CLIENT using script {}", clientScript);
-        expandAgentTar(agentPkgDir);
-
-        JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
-        FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
-        try {
-          file.write(commandJson.toString());
-
-        } catch (IOException e) {
-          log.error("Couldn't write command.json to file");
-        } finally {
-          file.flush();
-          file.close();
-        }
-
-        runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript);
-      }
-
-    } catch (IOException ioex) {
-      log.warn("Error while executing INSTALL command {}", ioex.getMessage());
-      throw new SliderException("INSTALL client failed.");
-    }
-  }
-
-  protected void runCommand(
-      File appPkgDir,
-      File agentPkgDir,
-      File cmdDir,
-      String clientScript) throws SliderException {
-    int exitCode = 0;
-    Exception exp = null;
-    try {
-      String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" +
-                                File.separator + clientScript;
-      List<String> command = Arrays.asList(PYTHON_EXE,
-               "-S",
-               clientScriptPath,
-               "INSTALL",
-               cmdDir.getAbsolutePath() + File.separator + "command.json",
-               appPkgDir.getAbsolutePath() + File.separator + "package",
-               cmdDir.getAbsolutePath() + File.separator + "command-out.json",
-               "DEBUG");
-      ProcessBuilder pb = new ProcessBuilder(command);
-      log.info("Command: " + StringUtils.join(pb.command(), " "));
-      pb.environment().put(PYTHONPATH,
-                           agentPkgDir.getAbsolutePath()
-                           + File.separator + "slider-agent" + File.pathSeparator
-                           + agentPkgDir.getAbsolutePath()
-                           + File.separator + "slider-agent/jinja2");
-      log.info("{}={}", PYTHONPATH, pb.environment().get(PYTHONPATH));
-
-      Process proc = pb.start();
-      InputStream stderr = proc.getErrorStream();
-      InputStream stdout = proc.getInputStream();
-      BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout));
-      BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr));
-
-      proc.waitFor();
-
-      String line;
-      while ((line = stdOutReader.readLine()) != null) {
-        log.info("Stdout: " + line);
-      }
-      while ((line = stdErrReader.readLine()) != null) {
-        log.info("Stderr: " + line);
-      }
-
-      exitCode = proc.exitValue();
-      log.info("Exit value is {}", exitCode);
-    } catch (IOException e) {
-      exp = e;
-    } catch (InterruptedException e) {
-      exp = e;
-    }
-
-    if (exitCode != 0) {
-      throw new SliderException("INSTALL client failed with exit code " + exitCode);
-    }
-
-    if (exp != null) {
-      log.error("Error while executing INSTALL command {}. Stack trace {}",
-                exp.getMessage(),
-                ExceptionUtils.getStackTrace(exp));
-      throw new SliderException("INSTALL client failed.", exp);
-    }
-  }
-
-  private void expandAgentTar(File agentPkgDir) throws IOException {
-    String libDirProp = SliderUtils.getLibDir();
-    File tarFile = new File(libDirProp, AGENT_TAR);
-    expandTar(tarFile, agentPkgDir);
-  }
-
-  private void expandTar(File tarFile, File destDir) throws IOException {
-    log.info("Expanding tar {} to {}", tarFile, destDir);
-    TarArchiveInputStream tarIn = new TarArchiveInputStream(
-        new GzipCompressorInputStream(
-            new BufferedInputStream(
-                new FileInputStream(tarFile)
-            )
-        )
-    );
-    try {
-      TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
-      while (tarEntry != null) {
-        File destPath = new File(destDir, tarEntry.getName());
-        File parent = destPath.getParentFile();
-        if (!parent.exists()) {
-          parent.mkdirs();
-        }
-        if (tarEntry.isDirectory()) {
-          destPath.mkdirs();
-        } else {
-          byte[] byteToRead = new byte[1024];
-          BufferedOutputStream buffOut =
-              new BufferedOutputStream(new FileOutputStream(destPath));
-          try {
-            int len;
-            while ((len = tarIn.read(byteToRead)) != -1) {
-              buffOut.write(byteToRead, 0, len);
-            }
-          } finally {
-            buffOut.close();
-          }
-        }
-        if ((tarEntry.getMode() & 0100) != 0) {
-          destPath.setExecutable(true);
-        }
-        tarEntry = tarIn.getNextTarEntry();
-      }
-    } finally {
-      tarIn.close();
-    }
-  }
-
-  private void retrieveConfigFile(RegistryOperations rops,
-      Configuration configuration, ConfigFile configFile, String name,
-      String user, File destDir) throws IOException, SliderException {
-    log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
-        destDir);
-    PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
-        configuration, configFile.getDictionaryName(), name, user, true);
-    ClientUtils.saveOrReturnConfig(published, configFile.getType(),
-        destDir, configFile.getFileName());
-  }
-
-  protected JSONObject getCommandJson(JSONObject defaultConfig,
-                                      JSONObject inputConfig,
-                                      Metainfo metainfo,
-                                      File clientInstallPath,
-                                      String name) throws SliderException {
-    try {
-      JSONObject pkgList = new JSONObject();
-      pkgList.put(PACKAGE_LIST,
-                  AgentProviderService.getPackageListFromApplication(metainfo.getApplication()));
-      JSONObject obj = new JSONObject();
-      obj.put("hostLevelParams", pkgList);
-
-      String user = RegistryUtils.currentUser();
-      JSONObject configuration = new JSONObject();
-      JSONObject global = new JSONObject();
-      global.put("app_install_dir", clientInstallPath.getAbsolutePath());
-      global.put("app_user", user);
-      if (name != null) {
-        global.put("app_name", name);
-      }
-
-      if (defaultConfig != null) {
-        readConfigEntries(defaultConfig, clientInstallPath, global, name, user);
-      }
-      if (inputConfig != null) {
-        readConfigEntries(inputConfig, clientInstallPath, global, name, user);
-      }
-
-      configuration.put("global", global);
-      obj.put("configurations", configuration);
-      return obj;
-    } catch (JSONException jex) {
-      log.warn("Error while executing INSTALL command {}", jex.getMessage());
-      throw new SliderException("INSTALL client failed.");
-    }
-  }
-
-  private void readConfigEntries(JSONObject inpConfig,
-                                 File clientInstallPath,
-                                 JSONObject globalConfig,
-                                 String name, String user)
-      throws JSONException {
-    JSONObject globalSection = inpConfig.getJSONObject("global");
-    Iterator it = globalSection.keys();
-    while (it.hasNext()) {
-      String key = (String) it.next();
-      String value = globalSection.getString(key);
-      if (SliderUtils.isSet(value)) {
-        value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath());
-        value = value.replace("{app_user}", user);
-        if (name != null) {
-          value = value.replace("{app_name}", name);
-        }
-      }
-      if (globalConfig.has(key)) {
-        // last one wins
-        globalConfig.remove(key);
-      }
-      globalConfig.put(key, value);
-    }
-  }
-
-  private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException {
-    BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath));
-    try {
-      byte[] bytesRead = new byte[4096];
-      int read = 0;
-      while ((read = zipInputStream.read(bytesRead)) != -1) {
-        output.write(bytesRead, 0, read);
-      }
-    } finally {
-      output.close();
-    }
-  }
-
-  private Metainfo getMetainfo(SliderFileSystem fs, String appDef) {
-    Metainfo metaInfo = metaInfoMap.get(appDef);
-    if (fs != null && metaInfo == null) {
-      try {
-        metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef, false);
-        metaInfoMap.put(appDef, metaInfo);
-      } catch (IOException ioe) {
-        // Ignore missing metainfo file for now
-        log.info("Missing metainfo {}", ioe.getMessage());
-      } catch (BadConfigException bce) {
-        log.info("Bad Configuration {}", bce.getMessage());
-      }
-    }
-    return metaInfo;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java

@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-class AgentLaunchParameter {
-  public static final Logger log =
-      LoggerFactory.getLogger(AgentLaunchParameter.class);
-  private static final String DEFAULT_PARAMETER = "";
-  private static final String ANY_COMPONENT = "ANY";
-  private static final String NONE_VALUE = "NONE";
-  private final Map<String, CommandTracker> launchParameterTracker;
-
-  public AgentLaunchParameter(String parameters) {
-    launchParameterTracker = parseExpectedLaunchParameters(parameters);
-  }
-
-  /**
-   * Get command for the component type
-   *
-   * @param componentGroup
-   *
-   * @return
-   */
-  public String getNextLaunchParameter(String componentGroup) {
-    if (launchParameterTracker != null) {
-      if (launchParameterTracker.containsKey(componentGroup)
-          || launchParameterTracker.containsKey(ANY_COMPONENT)) {
-        synchronized (this) {
-          CommandTracker indexTracker = null;
-          if (launchParameterTracker.containsKey(componentGroup)) {
-            indexTracker = launchParameterTracker.get(componentGroup);
-          } else {
-            indexTracker = launchParameterTracker.get(ANY_COMPONENT);
-          }
-
-          return indexTracker.getNextCommand();
-        }
-      }
-    }
-
-    return DEFAULT_PARAMETER;
-  }
-
-  /**
-   * Parse launch parameters of the form ANY:PARAM_FOR_FIRST:PARAM_FOR_SECOND:...:PARAM_FOR_REST|HBASE_MASTER:...
-   *
-   * E.g. ANY:DO_NOT_REGISTER:DO_NOT_HEARTBEAT:NONE For any container, first one gets DO_NOT_REGISTER second one gets
-   * DO_NOT_HEARTBEAT, then all of the rest get nothing
-   *
-   * E.g. HBASE_MASTER:FAIL_AFTER_START:NONE For HBASE_MASTER, first one gets FAIL_AFTER_START then "" for all
-   *
-   * @param launchParameters
-   *
-   * @return
-   */
-  Map<String, CommandTracker> parseExpectedLaunchParameters(String launchParameters) {
-    Map<String, CommandTracker> trackers = null;
-    if (launchParameters != null && launchParameters.length() > 0) {
-      String[] componentSpecificParameters = launchParameters.split(Pattern.quote("|"));
-      for (String componentSpecificParameter : componentSpecificParameters) {
-        if (componentSpecificParameter.length() != 0) {
-          String[] parameters = componentSpecificParameter.split(Pattern.quote(":"));
-
-          if (parameters.length > 1 && parameters[0].length() > 0) {
-
-            for (int index = 1; index < parameters.length; index++) {
-              if (parameters[index].equals(NONE_VALUE)) {
-                parameters[index] = DEFAULT_PARAMETER;
-              }
-            }
-
-            if (trackers == null) {
-              trackers = new HashMap<String, CommandTracker>(10);
-            }
-            String componentName = parameters[0];
-            CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));
-            trackers.put(componentName, tracker);
-          }
-        }
-      }
-    }
-
-    return trackers;
-  }
-
-  class CommandTracker {
-    private final int maxIndex;
-    private final String[] launchCommands;
-    private int currentIndex;
-
-    CommandTracker(String[] launchCommands) {
-      this.currentIndex = 0;
-      this.maxIndex = launchCommands.length - 1;
-      this.launchCommands = launchCommands;
-    }
-
-    String getNextCommand() {
-      String retVal = launchCommands[currentIndex];
-      if (currentIndex != maxIndex) {
-        currentIndex++;
-      }
-
-      return retVal;
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java

@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-
-public class AgentProviderFactory extends SliderProviderFactory {
-
-  public static final String CLASSNAME =
-      "org.apache.slider.providers.agent.AgentProviderFactory";
-
-  public AgentProviderFactory() {
-  }
-
-  public AgentProviderFactory(Configuration conf) {
-    super(conf);
-  }
-
-  @Override
-  public AbstractClientProvider createClientProvider() {
-    return new AgentClientProvider(getConf());
-  }
-
-  @Override
-  public ProviderService createServerProvider() {
-    return new AgentProviderService();
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java

@@ -1,2850 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.MonitorDetail;
-import org.apache.slider.providers.ProviderCore;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.agent.application.metadata.AbstractComponent;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.apache.slider.providers.agent.application.metadata.CommandScript;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
-import org.apache.slider.providers.agent.application.metadata.ComponentExport;
-import org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import org.apache.slider.providers.agent.application.metadata.DockerContainer;
-import org.apache.slider.providers.agent.application.metadata.Export;
-import org.apache.slider.providers.agent.application.metadata.ExportGroup;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.providers.agent.application.metadata.Package;
-import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
-import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
-import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
-import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
-import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
-import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
-import org.apache.slider.server.appmaster.web.rest.agent.Register;
-import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
-import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
-import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
-
-/**
- * This class implements the server-side logic for application deployment through Slider application package
- */
-public class AgentProviderService extends AbstractProviderService implements
-    ProviderCore,
-    AgentKeys,
-    SliderKeys, AgentRestOperations {
-
-
-  protected static final Logger log =
-      LoggerFactory.getLogger(AgentProviderService.class);
-  private static final ProviderUtils providerUtils = new ProviderUtils(log);
-  private static final String LABEL_MAKER = "___";
-  private static final String CONTAINER_ID = "container_id";
-  private static final String GLOBAL_CONFIG_TAG = "global";
-  private static final String COMPONENT_TAG = "component";
-  private static final String APPLICATION_TAG = "application";
-  private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
-  private static final String SHARED_PORT_TAG = "SHARED";
-  private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}";
-  private static final int MAX_LOG_ENTRIES = 40;
-  private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
-
-  private final Object syncLock = new Object();
-  private final ComponentTagProvider tags = new ComponentTagProvider();
-  private int heartbeatMonitorInterval = 0;
-  private AgentClientProvider clientProvider;
-  private AtomicInteger taskId = new AtomicInteger(0);
-  private volatile Map<String, MetainfoHolder> metaInfoMap = new HashMap<>();
-  private SliderFileSystem fileSystem = null;
-  private Map<String, DefaultConfig> defaultConfigs = null;
-  private ComponentCommandOrder commandOrder = new ComponentCommandOrder();
-  private HeartbeatMonitor monitor;
-  private Boolean canAnyMasterPublish = null;
-  private AgentLaunchParameter agentLaunchParameter = null;
-  private String clusterName = null;
-  private boolean isInUpgradeMode;
-  private Set<String> upgradeContainers = new HashSet<String>();
-  private boolean appStopInitiated;
-
-  private final Map<String, ComponentInstanceState> componentStatuses =
-      new ConcurrentHashMap<String, ComponentInstanceState>();
-  private final Map<String, Map<String, String>> componentInstanceData =
-      new ConcurrentHashMap<String, Map<String, String>>();
-  private final Map<String, Map<String, List<ExportEntry>>> exportGroups =
-      new ConcurrentHashMap<String, Map<String, List<ExportEntry>>>();
-  private final Map<String, Map<String, String>> allocatedPorts =
-      new ConcurrentHashMap<String, Map<String, String>>();
-  private final Map<String, Metainfo> packageMetainfo = 
-      new ConcurrentHashMap<String, Metainfo>();
-
-  private final Map<String, ExportEntry> logFolderExports =
-      Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
-        protected boolean removeEldestEntry(Map.Entry eldest) {
-          return size() > MAX_LOG_ENTRIES;
-        }
-      });
-  private final Map<String, ExportEntry> workFolderExports =
-      Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
-        protected boolean removeEldestEntry(Map.Entry eldest) {
-          return size() > MAX_LOG_ENTRIES;
-        }
-      });
-  private final Map<String, Set<String>> containerExportsMap =
-      new HashMap<String, Set<String>>();
-
-  private static class MetainfoHolder {
-    Metainfo metaInfo;
-    private Map<String, DefaultConfig> defaultConfigs = null;
-
-    public MetainfoHolder(Metainfo metaInfo,
-        Map<String, DefaultConfig> defaultConfigs) {
-      this.metaInfo = metaInfo;
-      this.defaultConfigs = defaultConfigs;
-    }
-  }
-
-  /**
-   * Create an instance of AgentProviderService
-   */
-  public AgentProviderService() {
-    super("AgentProviderService");
-    setAgentRestOperations(this);
-    setHeartbeatMonitorInterval(DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
-  }
-
-  @Override
-  public String getHumanName() {
-    return "Slider Agent";
-  }
-
-  @Override
-  public List<ProviderRole> getRoles() {
-    return AgentRoles.getRoles();
-  }
-
-  @Override
-  protected void serviceInit(Configuration conf) throws Exception {
-    super.serviceInit(conf);
-    clientProvider = new AgentClientProvider(conf);
-  }
-
-  @Override
-  public void validateInstanceDefinition(AggregateConf instanceDefinition)
-      throws
-      SliderException {
-    clientProvider.validateInstanceDefinition(instanceDefinition, null);
-
-    ConfTreeOperations resources =
-        instanceDefinition.getResourceOperations();
-
-    Set<String> names = resources.getComponentNames();
-    names.remove(COMPONENT_AM);
-    for (String name : names) {
-      Component componentDef = getApplicationComponent(name);
-      if (componentDef == null) {
-        // component member is validated elsewhere, so we don't need to throw
-        // an exception here
-        continue;
-      }
-
-      MapOperations componentConfig = resources.getMandatoryComponent(name);
-      int count =
-          componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
-      int definedMinCount = componentDef.getMinInstanceCountInt();
-      int definedMaxCount = componentDef.getMaxInstanceCountInt();
-      if (count < definedMinCount || count > definedMaxCount) {
-        throw new BadConfigException("Component %s, %s value %d out of range. "
-                                     + "Expected minimum is %d and maximum is %d",
-                                     name,
-                                     ResourceKeys.COMPONENT_INSTANCES,
-                                     count,
-                                     definedMinCount,
-                                     definedMaxCount);
-      }
-    }
-  }
-
-  // Reads the metainfo.xml in the application package and loads it
-  private void buildMetainfo(AggregateConf instanceDefinition,
-                             SliderFileSystem fileSystem,
-                             String roleGroup)
-      throws IOException, SliderException {
-    String mapKey = instanceDefinition.getAppConfOperations()
-        .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY);
-    String appDef = SliderUtils.getApplicationDefinitionPath(
-        instanceDefinition.getAppConfOperations(), roleGroup);
-    MapOperations component = null;
-    if (roleGroup != null) {
-      component = instanceDefinition.getAppConfOperations().getComponent(roleGroup);
-    }
-
-    MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey);
-    if (metaInfoHolder == null) {
-      synchronized (syncLock) {
-        if (this.fileSystem == null) {
-          this.fileSystem = fileSystem;
-        }
-        metaInfoHolder = metaInfoMap.get(mapKey);
-        if (metaInfoHolder == null) {
-          readAndSetHeartbeatMonitoringInterval(instanceDefinition);
-          initializeAgentDebugCommands(instanceDefinition);
-
-          Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false);
-          log.info("Master package metainfo: {}", metaInfo.toString());
-          if (metaInfo == null || metaInfo.getApplication() == null) {
-            log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
-            throw new SliderException(
-                "metainfo.xml is required in app package.");
-          }
-          List<CommandOrder> commandOrders = metaInfo.getApplication()
-              .getCommandOrders();
-          if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
-            for (Component comp : metaInfo.getApplication().getComponents()) {
-              comp.setName(mapKey + comp.getName());
-              log.info("Modifying external metainfo component name to {}",
-                  comp.getName());
-            }
-            for (CommandOrder co : commandOrders) {
-              log.info("Adding prefix {} to command order {}",
-                  mapKey, co);
-              co.setCommand(mapKey + co.getCommand());
-              co.setRequires(mapKey + co.getRequires());
-            }
-          }
-          log.debug("Merging command orders {} for {}", commandOrders,
-              roleGroup);
-          commandOrder.mergeCommandOrders(commandOrders,
-              instanceDefinition.getResourceOperations());
-          Map<String, DefaultConfig> defaultConfigs =
-              initializeDefaultConfigs(fileSystem, appDef, metaInfo);
-          metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs));
-          monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
-          monitor.start();
-
-          // build a map from component to metainfo
-          String addonAppDefString = instanceDefinition.getAppConfOperations()
-              .getGlobalOptions().getOption(ADDONS, null);
-          if (component != null) {
-            addonAppDefString = component.getOption(ADDONS, addonAppDefString);
-          }
-          log.debug("All addon appdefs: {}", addonAppDefString);
-          if (addonAppDefString != null) {
-            Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
-            while (scanner.hasNext()) {
-              String addonAppDef = scanner.next();
-              String addonAppDefPath = instanceDefinition
-                  .getAppConfOperations().getGlobalOptions().get(addonAppDef);
-              if (component != null) {
-                addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath);
-              }
-              log.debug("Addon package {} is stored at: {}", addonAppDef
-                  + addonAppDefPath);
-              Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem,
-                  addonAppDefPath, true);
-              addonMetaInfo.validate();
-              packageMetainfo.put(addonMetaInfo.getApplicationPackage()
-                  .getName(), addonMetaInfo);
-            }
-            log.info("Metainfo map for master and addon: {}",
-                packageMetainfo.toString());
-          }
-        }
-      }
-    }
-  }
-
-  @Override
-  public void initializeApplicationConfiguration(
-      AggregateConf instanceDefinition, SliderFileSystem fileSystem,
-      String roleGroup)
-      throws IOException, SliderException {
-    buildMetainfo(instanceDefinition, fileSystem, roleGroup);
-  }
-
-  @Override
-  public void buildContainerLaunchContext(ContainerLauncher launcher,
-                                          AggregateConf instanceDefinition,
-                                          Container container,
-                                          ProviderRole providerRole,
-                                          SliderFileSystem fileSystem,
-                                          Path generatedConfPath,
-                                          MapOperations resourceComponent,
-                                          MapOperations appComponent,
-                                          Path containerTmpDirPath) throws
-      IOException,
-      SliderException {
-    
-    String roleName = providerRole.name;
-    String roleGroup = providerRole.group;
-    String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
-        .getAppConfOperations(), roleGroup);
-
-    initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup);
-
-    log.info("Build launch context for Agent");
-    log.debug(instanceDefinition.toString());
-    
-    //if we are launching docker based app on yarn, then we need to pass docker image
-    if (isYarnDockerContainer(roleGroup)) {
-      launcher.setYarnDockerMode(true);
-      launcher.setDockerImage(getConfigFromMetaInfo(roleGroup, "image"));
-      launcher.setDockerNetwork(getConfigFromMetaInfo(roleGroup, "network"));
-      launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(roleGroup, "runPriviledgedContainer"));
-      launcher
-          .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding(
-              roleGroup, "yarn.container.mount.points"));
-    }
-
-    // Set the environment
-    launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
-        providerUtils.getStandardTokenMap(getAmState().getAppConfSnapshot(),
-            getAmState().getInternalsSnapshot(), roleName, roleGroup,
-            getClusterName())));
-
-    String workDir = ApplicationConstants.Environment.PWD.$();
-    launcher.setEnv("AGENT_WORK_ROOT", workDir);
-    log.info("AGENT_WORK_ROOT set to {}", workDir);
-    String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
-    launcher.setEnv("AGENT_LOG_ROOT", logDir);
-    log.info("AGENT_LOG_ROOT set to {}", logDir);
-    if (System.getenv(HADOOP_USER_NAME) != null) {
-      launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
-    }
-    // for 2-Way SSL
-    launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase());
-    //add english env
-    launcher.setEnv("LANG", "en_US.UTF-8");
-    launcher.setEnv("LC_ALL", "en_US.UTF-8");
-    launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
-    //local resources
-
-    // TODO: Should agent need to support App Home
-    String scriptPath = new File(AGENT_MAIN_SCRIPT_ROOT, AGENT_MAIN_SCRIPT).getPath();
-    String appHome = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().get(PACKAGE_PATH);
-    if (SliderUtils.isSet(appHome)) {
-      scriptPath = new File(appHome, AGENT_MAIN_SCRIPT).getPath();
-    }
-
-    // set PYTHONPATH
-    List<String> pythonPaths = new ArrayList<String>();
-    pythonPaths.add(AGENT_MAIN_SCRIPT_ROOT);
-    pythonPaths.add(AGENT_JINJA2_ROOT);
-    String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
-    launcher.setEnv(PYTHONPATH, pythonPath);
-    log.info("PYTHONPATH set to {}", pythonPath);
-
-    Path agentImagePath = null;
-    String agentImage = instanceDefinition.getInternalOperations().
-        get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
-    if (SliderUtils.isUnset(agentImage)) {
-      agentImagePath =
-          new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
-                                     container.getId().getApplicationAttemptId().getApplicationId().toString()),
-                            PROVIDER_AGENT),
-                   AGENT_TAR);
-    } else {
-       agentImagePath = new Path(agentImage);
-    }
-
-    if (fileSystem.getFileSystem().exists(agentImagePath)) {
-      LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
-      launcher.addLocalResource(AGENT_INSTALL_DIR, agentImageRes);
-    } else {
-      String msg =
-          String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
-      MapOperations compOps = appComponent;
-      boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
-          getOptionBool(TEST_RELAX_VERIFICATION, false)) : false;
-      log.error(msg);
-
-      if (!relaxVerificationForTest) {
-        throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg);
-      }
-    }
-
-    log.info("Using {} for agent.", scriptPath);
-    LocalResource appDefRes = fileSystem.createAmResource(
-        fileSystem.getFileSystem().resolvePath(new Path(appDef)),
-        LocalResourceType.ARCHIVE);
-    launcher.addLocalResource(APP_DEFINITION_DIR, appDefRes);
-
-    for (Package pkg : getMetaInfo(roleGroup).getApplication().getPackages()) {
-      Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
-      if (!fileSystem.isFile(pkgPath)) {
-        pkgPath = fileSystem.buildResourcePath(getClusterName(),
-            pkg.getName());
-      }
-      if (!fileSystem.isFile(pkgPath)) {
-        throw new IOException("Package doesn't exist as a resource: " +
-            pkg.getName());
-      }
-      log.info("Adding resource {}", pkg.getName());
-      LocalResourceType type = LocalResourceType.FILE;
-      if ("archive".equals(pkg.getType())) {
-        type = LocalResourceType.ARCHIVE;
-      }
-      LocalResource packageResource = fileSystem.createAmResource(
-          pkgPath, type);
-      launcher.addLocalResource(APP_PACKAGES_DIR, packageResource);
-    }
-
-    String agentConf = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AGENT_CONF, "");
-    if (SliderUtils.isSet(agentConf)) {
-      LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
-                                                                   .getFileSystem().resolvePath(new Path(agentConf)),
-                                                               LocalResourceType.FILE);
-      launcher.addLocalResource(AGENT_CONFIG_FILE, agentConfRes);
-    }
-
-    String agentVer = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AGENT_VERSION, null);
-    if (agentVer != null) {
-      LocalResource agentVerRes = fileSystem.createAmResource(
-          fileSystem.getFileSystem().resolvePath(new Path(agentVer)),
-          LocalResourceType.FILE);
-      launcher.addLocalResource(AGENT_VERSION_FILE, agentVerRes);
-    }
-
-    if (SliderUtils.isHadoopClusterSecure(getConfig())) {
-      providerUtils.localizeServiceKeytabs(launcher, instanceDefinition,
-          fileSystem, getClusterName());
-    }
-
-    MapOperations amComponent = instanceDefinition.
-        getAppConfOperations().getComponent(COMPONENT_AM);
-    if (providerUtils.hasTwoWaySSLEnabled(amComponent)) {
-      providerUtils.localizeContainerSSLResources(launcher, container,
-          fileSystem, getClusterName());
-    }
-
-    if (providerUtils.areStoresRequested(appComponent)) {
-      providerUtils.localizeContainerSecurityStores(launcher, container,
-          roleName, fileSystem, instanceDefinition, appComponent,
-          getClusterName());
-    }
-
-    //add the configuration resources
-    launcher.addLocalResources(fileSystem.submitDirectory(
-        generatedConfPath,
-        PROPAGATED_CONF_DIR_NAME));
-
-    if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) {
-      // build and localize configuration files
-      Map<String, Map<String, String>> configurations =
-          buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
-              instanceDefinition.getInternalOperations(),
-              container.getId().toString(), roleName, roleGroup);
-      for (ConfigFile configFile : getMetaInfo(roleGroup)
-          .getComponentConfigFiles(roleGroup)) {
-        localizeConfigFile(launcher, roleName, roleGroup, configFile,
-            configurations, launcher.getEnv(), fileSystem);
-      }
-    }
-
-    String label = getContainerLabel(container, roleName, roleGroup);
-    CommandLineBuilder operation = new CommandLineBuilder();
-
-    String pythonExec = instanceDefinition.getAppConfOperations()
-        .getGlobalOptions().getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
-                                      PYTHON_EXE);
-
-    operation.add(pythonExec);
-
-    operation.add(scriptPath);
-    operation.add(ARG_LABEL, label);
-    operation.add(ARG_ZOOKEEPER_QUORUM);
-    operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM));
-    operation.add(ARG_ZOOKEEPER_REGISTRY_PATH);
-    operation.add(getZkRegistryPath());
-
-    String debugCmd = agentLaunchParameter.getNextLaunchParameter(roleGroup);
-    if (SliderUtils.isSet(debugCmd)) {
-      operation.add(ARG_DEBUG);
-      operation.add(debugCmd);
-    }
-
-    operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
-        + AGENT_OUT_FILE + " 2>&1");
-
-    launcher.addCommand(operation.build());
-
-    // localize addon package
-    String addonAppDefString = instanceDefinition.getAppConfOperations()
-        .getGlobalOptions().getOption(ADDONS, null);
-    log.debug("All addon appdefs: {}", addonAppDefString);
-    if (addonAppDefString != null) {
-      Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
-      while (scanner.hasNext()) {
-        String addonAppDef = scanner.next();
-        String addonAppDefPath = instanceDefinition
-            .getAppConfOperations().getGlobalOptions().get(addonAppDef);
-        log.debug("Addon package {} is stored at: {}", addonAppDef, addonAppDefPath);
-        LocalResource addonPkgRes = fileSystem.createAmResource(
-            fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)),
-            LocalResourceType.ARCHIVE);
-        launcher.addLocalResource(ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes);
-      }
-      log.debug("Metainfo map for master and addon: {}",
-          packageMetainfo.toString());
-    }    
-
-    // Additional files to localize in addition to the application def
-    String appResourcesString = instanceDefinition.getAppConfOperations()
-        .getGlobalOptions().getOption(APP_RESOURCES, null);
-    log.info("Configuration value for extra resources to localize: {}", appResourcesString);
-    if (null != appResourcesString) {
-      try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) {
-        while (scanner.hasNext()) {
-          String resource = scanner.next();
-          Path resourcePath = new Path(resource);
-          LocalResource extraResource = fileSystem.createAmResource(
-              fileSystem.getFileSystem().resolvePath(resourcePath),
-              LocalResourceType.FILE);
-          String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName();
-          log.info("Localizing {} to {}", resourcePath, destination);
-          // TODO Can we try harder to avoid collisions?
-          launcher.addLocalResource(destination, extraResource);
-        }
-      }
-    }
-
-    // initialize addon pkg states for all componentInstanceStatus
-    Map<String, State> pkgStatuses = new TreeMap<>();
-    for (Metainfo appPkg : packageMetainfo.values()) {
-      // check each component of that addon to see if they apply to this
-      // component 'role'
-      for (ComponentsInAddonPackage comp : appPkg.getApplicationPackage()
-          .getComponents()) {
-        log.debug("Current component: {} component in metainfo: {}", roleName,
-            comp.getName());
-        if (comp.getName().equals(roleGroup)
-            || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
-          pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT);
-        }
-      }
-    }
-    log.debug("For component: {} pkg status map: {}", roleName,
-        pkgStatuses.toString());
-    
-    // initialize the component instance state
-    getComponentStatuses().put(label,
-                               new ComponentInstanceState(
-                                   roleGroup,
-                                   container.getId(),
-                                   getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME),
-                                   pkgStatuses));
-  }
-
-  @VisibleForTesting
-  protected void localizeConfigFile(ContainerLauncher launcher,
-                                     String roleName, String roleGroup,
-                                     ConfigFile configFile,
-                                     Map<String, Map<String, String>> configs,
-                                     MapOperations env,
-                                     SliderFileSystem fileSystem)
-      throws IOException {
-    ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
-    providerUtils.localizeConfigFile(launcher, roleName, roleGroup,
-        configFile.getDictionaryName(), configFormat, configFile.getFileName(),
-        configs, env, fileSystem, getClusterName());
-  }
-
-  /**
-   * build the zookeeper registry path.
-   * 
-   * @return the path the service registered at
-   * @throws NullPointerException if the service has not yet registered
-   */
-  private String getZkRegistryPath() {
-    Preconditions.checkNotNull(yarnRegistry, "Yarn registry not bound");
-    String path = yarnRegistry.getAbsoluteSelfRegistrationPath();
-    Preconditions.checkNotNull(path, "Service record path not defined");
-    return path;
-  }
-
-  @Override
-  public void rebuildContainerDetails(List<Container> liveContainers,
-                                      String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
-    for (Container container : liveContainers) {
-      // get the role name and label
-      ProviderRole role = providerRoleMap.get(ContainerPriority
-                                                  .extractRole(container));
-      if (role != null) {
-        String roleName = role.name;
-        String roleGroup = role.group;
-        String label = getContainerLabel(container, roleName, roleGroup);
-        log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
-                 container.getId(), roleName, applicationId);
-        getComponentStatuses().put(label,
-            new ComponentInstanceState(roleGroup, container.getId(),
-                                       applicationId));
-      } else {
-        log.warn("Role not found for container {} in cluster {}",
-                 container.getId(), applicationId);
-      }
-    }
-  }
-
-  @Override
-  public boolean isSupportedRole(String role) {
-    return true;
-  }
-
-  /**
-   * Handle registration calls from the agents
-   *
-   * @param registration registration entry
-   *
-   * @return response
-   */
-  @Override
-  public RegistrationResponse handleRegistration(Register registration) {
-    log.info("Handling registration: {}", registration);
-    RegistrationResponse response = new RegistrationResponse();
-    String label = registration.getLabel();
-    String pkg = registration.getPkg();
-    State agentState = registration.getActualState();
-    String appVersion = registration.getAppVersion();
-
-    log.info("label: {} pkg: {}", label, pkg);
-
-    if (getComponentStatuses().containsKey(label)) {
-      response.setResponseStatus(RegistrationStatus.OK);
-      ComponentInstanceState componentStatus = getComponentStatuses().get(label);
-      componentStatus.heartbeat(System.currentTimeMillis());
-      updateComponentStatusWithAgentState(componentStatus, agentState);
-
-      String roleName = getRoleName(label);
-      String roleGroup = getRoleGroup(label);
-      String containerId = getContainerId(label);
-
-      if (SliderUtils.isSet(registration.getTags())) {
-        tags.recordAssignedTag(roleName, containerId, registration.getTags());
-      } else {
-        response.setTags(tags.getTag(roleName, containerId));
-      }
-
-      String hostFqdn = registration.getPublicHostname();
-      Map<String, String> ports = registration.getAllocatedPorts();
-      if (ports != null && !ports.isEmpty()) {
-        processAllocatedPorts(hostFqdn, roleName, roleGroup, containerId, ports);
-      }
-
-      Map<String, String> folders = registration.getLogFolders();
-      if (folders != null && !folders.isEmpty()) {
-        publishFolderPaths(folders, containerId, roleName, hostFqdn);
-      }
-
-      // Set app version if empty. It gets unset during upgrade - why?
-      checkAndSetContainerAppVersion(containerId, appVersion);
-    } else {
-      response.setResponseStatus(RegistrationStatus.FAILED);
-      response.setLog("Label not recognized.");
-      log.warn("Received registration request from unknown label {}", label);
-    }
-    log.info("Registration response: {}", response);
-    return response;
-  }
-
-  // Checks if app version is empty. Sets it to the version as reported by the
-  // container during registration phase.
-  private void checkAndSetContainerAppVersion(String containerId,
-      String appVersion) {
-    StateAccessForProviders amState = getAmState();
-    try {
-      RoleInstance role = amState.getOwnedContainer(containerId);
-      if (role != null) {
-        String currentAppVersion = role.appVersion;
-        log.debug("Container = {}, app version current = {} new = {}",
-            containerId, currentAppVersion, appVersion);
-        if (currentAppVersion == null
-            || currentAppVersion.equals(APP_VERSION_UNKNOWN)) {
-          amState.getOwnedContainer(containerId).appVersion = appVersion;
-        }
-      }
-    } catch (NoSuchNodeException e) {
-      // ignore - there is nothing to do if we don't find a container
-      log.warn("Owned container {} not found - {}", containerId, e);
-    }
-  }
-
-  /**
-   * Handle heartbeat response from agents
-   *
-   * @param heartBeat incoming heartbeat from Agent
-   *
-   * @return response to send back
-   */
-  @Override
-  public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
-    log.debug("Handling heartbeat: {}", heartBeat);
-    HeartBeatResponse response = new HeartBeatResponse();
-    long id = heartBeat.getResponseId();
-    response.setResponseId(id + 1L);
-
-    String label = heartBeat.getHostname();
-    String pkg = heartBeat.getPackage();
-
-    log.debug("package received: " + pkg);
-    
-    String roleName = getRoleName(label);
-    String roleGroup = getRoleGroup(label);
-    String containerId = getContainerId(label);
-    boolean doUpgrade = false;
-    if (isInUpgradeMode && upgradeContainers.contains(containerId)) {
-      doUpgrade = true;
-    }
-
-    CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup);
-    List<ComponentCommand> commands = getApplicationComponent(roleGroup).getCommands();
-
-    if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup)
-        && (cmdScript == null || cmdScript.getScript() == null)
-        && commands.size() == 0) {
-      log.error(
-          "role.script is unavailable for {}. Commands will not be sent.",
-          roleName);
-      return response;
-    }
-
-    String scriptPath = null;
-    long timeout = 600L;
-    if (cmdScript != null) {
-      scriptPath = cmdScript.getScript();
-      timeout = cmdScript.getTimeout();
-    }
-
-    if (timeout == 0L) {
-      timeout = 600L;
-    }
-
-    if (!getComponentStatuses().containsKey(label)) {
-      // container is completed but still heart-beating, send terminate signal
-      log.info(
-          "Sending terminate signal to completed container (still heartbeating): {}",
-          label);
-      response.setTerminateAgent(true);
-      return response;
-    }
-
-    List<ComponentStatus> statuses = heartBeat.getComponentStatus();
-    if (statuses != null && !statuses.isEmpty()) {
-      log.info("status from agent: " + statuses.toString());
-      for(ComponentStatus status : statuses){
-        providerUtils.updateServiceRecord(getAmState(), yarnRegistry,
-            containerId, roleName, Collections.singletonList(status.getIp()),
-            status.getHostname());
-      }
-    }
-
-    Boolean isMaster = isMaster(roleGroup);
-    ComponentInstanceState componentStatus = getComponentStatuses().get(label);
-    componentStatus.heartbeat(System.currentTimeMillis());
-    if (doUpgrade) {
-      switch (componentStatus.getState()) {
-      case STARTED:
-        componentStatus.setTargetState(State.UPGRADED);
-        break;
-      case UPGRADED:
-        componentStatus.setTargetState(State.STOPPED);
-        break;
-      case STOPPED:
-        componentStatus.setTargetState(State.TERMINATING);
-        break;
-      default:
-        break;
-      }
-      log.info("Current state = {} target state {}",
-          componentStatus.getState(), componentStatus.getTargetState());
-    }
-
-    if (appStopInitiated && !componentStatus.isStopInitiated()) {
-      log.info("Stop initiated for label {}", label);
-      componentStatus.setTargetState(State.STOPPED);
-      componentStatus.setStopInitiated(true);
-    }
-
-    publishConfigAndExportGroups(heartBeat, componentStatus, roleGroup);
-    CommandResult result = null;
-    List<CommandReport> reports = heartBeat.getReports();
-    if (SliderUtils.isNotEmpty(reports)) {
-      CommandReport report = reports.get(0);
-      Map<String, String> ports = report.getAllocatedPorts();
-      if (SliderUtils.isNotEmpty(ports)) {
-        processAllocatedPorts(heartBeat.getFqdn(), roleName, roleGroup, containerId, ports);
-      }
-      result = CommandResult.getCommandResult(report.getStatus());
-      Command command = Command.getCommand(report.getRoleCommand());
-      componentStatus.applyCommandResult(result, command, pkg);
-      log.info("Component operation. Status: {}; new container state: {};"
-          + " new component state: {}", result,
-          componentStatus.getContainerState(), componentStatus.getState());
-
-      if (command == Command.INSTALL && SliderUtils.isNotEmpty(report.getFolders())) {
-        publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
-      }
-    }
-
-    int waitForCount = getAmState().getInstanceDefinitionSnapshot().
-        getAppConfOperations().getComponentOptInt(roleGroup, WAIT_HEARTBEAT, 0);
-
-    if (id < waitForCount) {
-      log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
-      getComponentStatuses().put(label, componentStatus);
-      return response;
-    }
-
-    Command command = componentStatus.getNextCommand(doUpgrade);
-    try {
-      if (Command.NOP != command) {
-        log.debug("For comp {} pkg {} issuing {}", roleName,
-            componentStatus.getNextPkgToInstall(), command.toString());
-        if (command == Command.INSTALL) {
-          log.info("Installing {} on {}.", roleName, containerId);
-          if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
-            addInstallDockerCommand(roleName, roleGroup, containerId,
-                response, null, timeout);
-          } else if (scriptPath != null) {
-            addInstallCommand(roleName, roleGroup, containerId, response,
-                scriptPath, null, timeout, null);
-          } else {
-            // commands
-            ComponentCommand installCmd = null;
-            for (ComponentCommand compCmd : commands) {
-              if (compCmd.getName().equals("INSTALL")) {
-                installCmd = compCmd;
-              }
-            }
-            addInstallCommand(roleName, roleGroup, containerId, response, null,
-                installCmd, timeout, null);
-          }
-          componentStatus.commandIssued(command);
-        } else if (command == Command.INSTALL_ADDON) {
-          String nextPkgToInstall = componentStatus.getNextPkgToInstall();
-          // retrieve scriptPath or command of that package for the component
-          for (ComponentsInAddonPackage comp : packageMetainfo
-              .get(nextPkgToInstall).getApplicationPackage().getComponents()) {
-            // given nextPkgToInstall and roleName is determined, the if below
-            // should only execute once per heartbeat
-            log.debug("Addon component: {} pkg: {} script: {}", comp.getName(),
-                nextPkgToInstall, comp.getCommandScript().getScript());
-            if (comp.getName().equals(roleGroup)
-                || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
-              scriptPath = comp.getCommandScript().getScript();
-              if (scriptPath != null) {
-                addInstallCommand(roleName, roleGroup, containerId, response,
-                    scriptPath, null, timeout, nextPkgToInstall);
-              } else {
-                ComponentCommand installCmd = null;
-                for (ComponentCommand compCmd : comp.getCommands()) {
-                  if (compCmd.getName().equals("INSTALL")) {
-                    installCmd = compCmd;
-                  }
-                }
-                addInstallCommand(roleName, roleGroup, containerId, response,
-                    null, installCmd, timeout, nextPkgToInstall);
-              }
-            }
-          }
-          componentStatus.commandIssued(command);
-        } else if (command == Command.START) {
-          // check against dependencies
-          boolean canExecute = commandOrder.canExecute(roleGroup, command, getComponentStatuses().values());
-          if (canExecute) {
-            log.info("Starting {} on {}.", roleName, containerId);
-            if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
-              addStartDockerCommand(roleName, roleGroup, containerId,
-                  response, null, timeout, false);
-            } else if (scriptPath != null) {
-              addStartCommand(roleName,
-                              roleGroup,
-                              containerId,
-                              response,
-                              scriptPath,
-                              null,
-                              null,
-                              timeout,
-                              isMarkedAutoRestart(roleGroup));
-            } else {
-              ComponentCommand startCmd = null;
-              for (ComponentCommand compCmd : commands) {
-                if (compCmd.getName().equals("START")) {
-                  startCmd = compCmd;
-                }
-              }
-              ComponentCommand stopCmd = null;
-              for (ComponentCommand compCmd : commands) {
-                if (compCmd.getName().equals("STOP")) {
-                  stopCmd = compCmd;
-                }
-              }
-              addStartCommand(roleName, roleGroup, containerId, response, null,
-                  startCmd, stopCmd, timeout, false);
-            }
-            componentStatus.commandIssued(command);
-          } else {
-            log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
-          }
-        } else if (command == Command.UPGRADE) {
-          addUpgradeCommand(roleName, roleGroup, containerId, response,
-              scriptPath, timeout);
-          componentStatus.commandIssued(command, true);
-        } else if (command == Command.STOP) {
-          log.info("Stop command being sent to container with id {}",
-              containerId);
-          addStopCommand(roleName, roleGroup, containerId, response, scriptPath,
-              timeout, doUpgrade);
-          componentStatus.commandIssued(command);
-        } else if (command == Command.TERMINATE) {
-          log.info("A formal terminate command is being sent to container {}"
-              + " in state {}", label, componentStatus.getState());
-          response.setTerminateAgent(true);
-        }
-      }
-
-      // if there is no outstanding command then retrieve config
-      if (isMaster && componentStatus.getState() == State.STARTED
-          && command == Command.NOP) {
-        if (!componentStatus.getConfigReported()) {
-          log.info("Requesting applied config for {} on {}.", roleName, containerId);
-          if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
-            addGetConfigDockerCommand(roleName, roleGroup, containerId, response);
-          } else {
-            addGetConfigCommand(roleName, roleGroup, containerId, response);
-          }
-        }
-      }
-      
-      // if restart is required then signal
-      response.setRestartEnabled(false);
-      if (componentStatus.getState() == State.STARTED
-          && command == Command.NOP && isMarkedAutoRestart(roleGroup)) {
-        response.setRestartEnabled(true);
-      }
-
-      //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail
-      if (componentStatus.getState() == State.INSTALL_FAILED
-         && command == Command.NOP) {
-        log.warn("Sending terminate signal to container that failed installation: {}", label);
-        response.setTerminateAgent(true);
-      }
-
-    } catch (SliderException e) {
-      log.warn("Component instance failed operation.", e);
-      componentStatus.applyCommandResult(CommandResult.FAILED, command, null);
-    }
-
-    log.debug("Heartbeat response: " + response);
-    return response;
-  }
-
-  private boolean isDockerContainer(String roleGroup) {
-    String type = getApplicationComponent(roleGroup).getType();
-    if (SliderUtils.isSet(type)) {
-      return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
-    }
-    return false;
-  }
-
-  private boolean isYarnDockerContainer(String roleGroup) {
-    String type = getApplicationComponent(roleGroup).getType();
-    if (SliderUtils.isSet(type)) {
-      return type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
-    }
-    return false;
-  }
-
-  protected void processAllocatedPorts(String fqdn,
-                                       String roleName,
-                                       String roleGroup,
-                                       String containerId,
-                                       Map<String, String> ports) {
-    RoleInstance instance;
-    try {
-      instance = getAmState().getOwnedContainer(containerId);
-    } catch (NoSuchNodeException e) {
-      log.warn("Failed to locate instance of container {}", containerId, e);
-      instance = null;
-    }
-    for (Map.Entry<String, String> port : ports.entrySet()) {
-      String portname = port.getKey();
-      String portNo = port.getValue();
-      log.info("Recording allocated port for {} as {}", portname, portNo);
-
-      // add the allocated ports to the global list as well as per container list
-      // per container allocation will over-write each other in the global
-      this.getAllocatedPorts().put(portname, portNo);
-      this.getAllocatedPorts(containerId).put(portname, portNo);
-      if (instance != null) {
-        try {
-          // if the returned value is not a single port number then there are no
-          // meaningful way for Slider to use it during export
-          // No need to error out as it may not be the responsibility of the component
-          // to allocate port or the component may need an array of ports
-          instance.registerPortEndpoint(Integer.valueOf(portNo), portname);
-        } catch (NumberFormatException e) {
-          log.warn("Failed to parse {}", portNo, e);
-        }
-      }
-    }
-
-    processAndPublishComponentSpecificData(ports, containerId, fqdn, roleGroup);
-    processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName, roleGroup);
-
-    // and update registration entries
-    if (instance != null) {
-      queueAccess.put(new RegisterComponentInstance(instance.getId(),
-          roleName, roleGroup, 0, TimeUnit.MILLISECONDS));
-    }
-  }
-
-  private void updateComponentStatusWithAgentState(
-      ComponentInstanceState componentStatus, State agentState) {
-    if (agentState != null) {
-      componentStatus.setState(agentState);
-    }
-  }
-
-  @Override
-  public Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterDesc) {
-    Map<String, MonitorDetail> details = super.buildMonitorDetails(clusterDesc);
-    buildRoleHostDetails(details);
-    return details;
-  }
-
-  public void applyInitialRegistryDefinitions(URL amWebURI,
-      URL agentOpsURI,
-      URL agentStatusURI,
-      ServiceRecord serviceRecord)
-    throws IOException {
-    super.applyInitialRegistryDefinitions(amWebURI,
-                                          serviceRecord);
-
-    try {
-      URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS);
-      URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS);
-
-      serviceRecord.addInternalEndpoint(
-          new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API,
-                       ProtocolTypes.PROTOCOL_REST,
-                       restURL.toURI()));
-      serviceRecord.addInternalEndpoint(
-          new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API,
-                       ProtocolTypes.PROTOCOL_REST,
-                       agentStatusURL.toURI()));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-
-    // identify client component
-    Component client = null;
-    for (Component component : getMetaInfo().getApplication().getComponents()) {
-      if (component.getCategory().equals("CLIENT")) {
-        client = component;
-        break;
-      }
-    }
-    if (client == null) {
-      log.info("No client component specified, not publishing client configs");
-      return;
-    }
-
-    // register AM-generated client configs
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    MapOperations clientOperations = appConf.getOrAddComponent(client.getName());
-    appConf.resolve();
-    if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION,
-        false)) {
-      log.info("AM config generation is false, not publishing client configs");
-      return;
-    }
-
-    // build and localize configuration files
-    Map<String, Map<String, String>> configurations = new TreeMap<>();
-    Map<String, String> tokens = providerUtils.getStandardTokenMap(appConf,
-        getAmState().getInternalsSnapshot(), client.getName(),
-        client.getName(), getClusterName());
-
-    for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
-      addNamedConfiguration(configFile.getDictionaryName(),
-          appConf.getGlobalOptions().options, configurations, tokens, null,
-          client.getName(), client.getName());
-      if (appConf.getComponent(client.getName()) != null) {
-        addNamedConfiguration(configFile.getDictionaryName(),
-            appConf.getComponent(client.getName()).options, configurations,
-            tokens, null, client.getName(), client.getName());
-      }
-    }
-
-    //do a final replacement of re-used configs
-    dereferenceAllConfigs(configurations);
-
-    for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
-      ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
-
-      Map<String, String> config = configurations.get(configFile.getDictionaryName());
-      ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
-          fileSystem, getClusterName(),
-          new File(configFile.getFileName()).getName());
-      PublishedConfiguration publishedConfiguration =
-          new PublishedConfiguration(configFile.getDictionaryName(),
-              config.entrySet());
-      getAmState().getPublishedSliderConfigurations().put(
-          configFile.getDictionaryName(), publishedConfiguration);
-      log.info("Publishing AM configuration {}", configFile.getDictionaryName());
-    }
-  }
-
-  @Override
-  public void notifyContainerCompleted(ContainerId containerId) {
-    // containers get allocated and free'ed without being assigned to any
-    // component - so many of the data structures may not be initialized
-    if (containerId != null) {
-      String containerIdStr = containerId.toString();
-      if (getComponentInstanceData().containsKey(containerIdStr)) {
-        getComponentInstanceData().remove(containerIdStr);
-        log.info("Removing container specific data for {}", containerIdStr);
-        publishComponentInstanceData();
-      }
-
-      if (this.allocatedPorts.containsKey(containerIdStr)) {
-        Map<String, String> portsByContainerId = getAllocatedPorts(containerIdStr);
-        this.allocatedPorts.remove(containerIdStr);
-        // free up the allocations from global as well
-        // if multiple containers allocate global ports then last one
-        // wins and similarly first one removes it - its not supported anyway
-        for(String portName : portsByContainerId.keySet()) {
-          getAllocatedPorts().remove(portName);
-        }
-
-      }
-
-      String componentName = null;
-      synchronized (this.componentStatuses) {
-        for (String label : getComponentStatuses().keySet()) {
-          if (label.startsWith(containerIdStr)) {
-            componentName = getRoleName(label);
-            log.info("Removing component status for label {}", label);
-            getComponentStatuses().remove(label);
-          }
-        }
-      }
-
-      tags.releaseTag(componentName, containerIdStr);
-
-      synchronized (this.containerExportsMap) {
-        Set<String> containerExportSets = containerExportsMap.get(containerIdStr);
-        if (containerExportSets != null) {
-          for (String containerExportStr : containerExportSets) {
-            String[] parts = containerExportStr.split(":");
-            Map<String, List<ExportEntry>> exportGroup = getCurrentExports(parts[0]);
-            List<ExportEntry> exports = exportGroup.get(parts[1]);
-            List<ExportEntry> exportToRemove = new ArrayList<ExportEntry>();
-            for (ExportEntry export : exports) {
-              if (containerIdStr.equals(export.getContainerId())) {
-                exportToRemove.add(export);
-              }
-            }
-            exports.removeAll(exportToRemove);
-          }
-          log.info("Removing container exports for {}", containerIdStr);
-          containerExportsMap.remove(containerIdStr);
-        }
-      }
-    }
-  }
-
-  /**
-   * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
-   *
-   * @param instanceDefinition
-   */
-  private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
-    String hbMonitorInterval = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(HEARTBEAT_MONITOR_INTERVAL,
-                                     Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
-    try {
-      setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
-    } catch (NumberFormatException e) {
-      log.warn(
-          "Bad value {} for {}. Defaulting to ",
-          hbMonitorInterval,
-          HEARTBEAT_MONITOR_INTERVAL,
-          DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
-    }
-  }
-
-  /**
-   * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
-   *
-   * @param instanceDefinition
-   */
-  private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
-    String launchParameterStr = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AGENT_INSTANCE_DEBUG_DATA, "");
-    agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
-  }
-
-  @VisibleForTesting
-  protected Map<String, ExportEntry> getLogFolderExports() {
-    return logFolderExports;
-  }
-
-  @VisibleForTesting
-  protected Map<String, ExportEntry> getWorkFolderExports() {
-    return workFolderExports;
-  }
-
-  protected Metainfo getMetaInfo() {
-    return getMetaInfo(null);
-  }
-
-  @VisibleForTesting
-  protected Metainfo getMetaInfo(String roleGroup) {
-    String mapKey = DEFAULT_METAINFO_MAP_KEY;
-    if (roleGroup != null) {
-      ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-      mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
-          DEFAULT_METAINFO_MAP_KEY);
-    }
-    MetainfoHolder mh = this.metaInfoMap.get(mapKey);
-    if (mh == null) {
-      return null;
-    }
-    return mh.metaInfo;
-  }
-
-  @VisibleForTesting
-  protected Map<String, ComponentInstanceState> getComponentStatuses() {
-    return componentStatuses;
-  }
-
-  @VisibleForTesting
-  protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
-      String appDef, boolean addonPackage) throws IOException,
-      BadConfigException {
-    return AgentUtils.getApplicationMetainfo(fileSystem, appDef, addonPackage);
-  }
-
-  @VisibleForTesting
-  protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
-      String appDef) throws IOException, BadConfigException {
-    return getApplicationMetainfo(fileSystem, appDef, false);
-  }
-
-  @VisibleForTesting
-  protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
-    this.heartbeatMonitorInterval = heartbeatMonitorInterval;
-  }
-
-  public void setInUpgradeMode(boolean inUpgradeMode) {
-    this.isInUpgradeMode = inUpgradeMode;
-  }
-
-  public void addUpgradeContainers(Set<String> upgradeContainers) {
-    this.upgradeContainers.addAll(upgradeContainers);
-  }
-
-  public void setAppStopInitiated(boolean appStopInitiated) {
-    this.appStopInitiated = appStopInitiated;
-  }
-
-  /**
-   * Read all default configs
-   *
-   * @param fileSystem fs
-   * @param appDef app default path
-   * @param metainfo metadata
-   *
-   * @return configuration maps
-   * 
-   * @throws IOException
-   */
-  protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
-                                                                String appDef, Metainfo metainfo) throws IOException {
-    Map<String, DefaultConfig> defaultConfigMap = new HashMap<>();
-    if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) {
-      for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) {
-        DefaultConfig config = null;
-        try {
-          config = AgentUtils.getDefaultConfig(fileSystem, appDef, configFile.getDictionaryName() + ".xml");
-        } catch (IOException e) {
-          log.warn("Default config file not found. Only the config as input during create will be applied for {}",
-                   configFile.getDictionaryName());
-        }
-        if (config != null) {
-          defaultConfigMap.put(configFile.getDictionaryName(), config);
-        }
-      }
-    }
-
-    return defaultConfigMap;
-  }
-
-  protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) {
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
-        DEFAULT_METAINFO_MAP_KEY);
-    return metaInfoMap.get(mapKey).defaultConfigs;
-  }
-
-  private int getHeartbeatMonitorInterval() {
-    return this.heartbeatMonitorInterval;
-  }
-
-  private String getClusterName() {
-    if (SliderUtils.isUnset(clusterName)) {
-      clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME);
-    }
-    return clusterName;
-  }
-
-  @VisibleForTesting
-  protected void publishApplicationInstanceData(String name, String description,
-                                                Iterable<Map.Entry<String, String>> entries) {
-    providerUtils.publishApplicationInstanceData(name, description, entries,
-        getAmState());
-  }
-
-  /**
-   * Get a list of all hosts for all role/container per role
-   *
-   * @return the map of role->node
-   */
-  protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
-    return amState.getRoleClusterNodeMapping();
-  }
-
-  private String getContainerLabel(Container container, String role, String group) {
-    if (role.equals(group)) {
-      return container.getId().toString() + LABEL_MAKER + role;
-    } else {
-      return container.getId().toString() + LABEL_MAKER + role + LABEL_MAKER +
-          group;
-    }
-  }
-
-  protected String getClusterInfoPropertyValue(String name) {
-    StateAccessForProviders accessor = getAmState();
-    assert accessor.isApplicationLive();
-    ClusterDescription description = accessor.getClusterStatus();
-    return description.getInfo(name);
-  }
-
-  protected String getClusterOptionPropertyValue(String name)
-      throws BadConfigException {
-    StateAccessForProviders accessor = getAmState();
-    assert accessor.isApplicationLive();
-    ClusterDescription description = accessor.getClusterStatus();
-    return description.getMandatoryOption(name);
-  }
-
-  /**
-   * Lost heartbeat from the container - release it and ask for a replacement (async operation)
-   *
-   * @param label
-   * @param containerId
-   */
-  protected void lostContainer(
-      String label,
-      ContainerId containerId) {
-    getComponentStatuses().remove(label);
-    getQueueAccess().put(new ProviderReportedContainerLoss(containerId));
-  }
-
-  /**
-   * Build the provider status, can be empty
-   *
-   * @return the provider status - map of entries to add to the info section
-   */
-  public Map<String, String> buildProviderStatus() {
-    Map<String, String> stats = new HashMap<String, String>();
-    return stats;
-  }
-
-  @VisibleForTesting
-  protected void publishFolderPaths(
-      Map<String, String> folders, String containerId, String componentName, String hostFqdn) {
-    providerUtils.publishFolderPaths(folders, containerId, componentName, hostFqdn,
-        getAmState(), getLogFolderExports(), getWorkFolderExports());
-  }
-
-  /**
-   * Process return status for component instances
-   *
-   * @param heartBeat
-   * @param componentStatus
-   */
-  protected void publishConfigAndExportGroups(HeartBeat heartBeat,
-      ComponentInstanceState componentStatus, String componentGroup) {
-    List<ComponentStatus> statuses = heartBeat.getComponentStatus();
-    if (statuses != null && !statuses.isEmpty()) {
-      log.info("Processing {} status reports.", statuses.size());
-      for (ComponentStatus status : statuses) {
-        log.info("Status report: {}", status.toString());
-
-        if (status.getConfigs() != null) {
-          Application application = getMetaInfo(componentGroup).getApplication();
-
-          if ((!canAnyMasterPublishConfig(componentGroup) || canPublishConfig(componentGroup)) &&
-              !getAmState().getAppConfSnapshot().getComponentOptBool(
-                  componentGroup, AM_CONFIG_GENERATION, false)) {
-            // If no Master can explicitly publish then publish if its a master
-            // Otherwise, wait till the master that can publish is ready
-
-            Set<String> exportedConfigs = new HashSet();
-            String exportedConfigsStr = application.getExportedConfigs();
-            boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
-            if (!exportedAllConfigs) {
-              for (String exportedConfig : exportedConfigsStr.split(",")) {
-                if (exportedConfig.trim().length() > 0) {
-                  exportedConfigs.add(exportedConfig.trim());
-                }
-              }
-            }
-
-            for (String key : status.getConfigs().keySet()) {
-              if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
-                  exportedAllConfigs) {
-                Map<String, String> configs = status.getConfigs().get(key);
-                publishApplicationInstanceData(key, key, configs.entrySet());
-              }
-            }
-          }
-
-          List<ExportGroup> appExportGroups = application.getExportGroups();
-          boolean hasExportGroups = SliderUtils.isNotEmpty(appExportGroups);
-
-          Set<String> appExports = new HashSet();
-          String appExportsStr = getApplicationComponent(componentGroup).getAppExports();
-          if (SliderUtils.isSet(appExportsStr)) {
-            for (String appExport : appExportsStr.split(",")) {
-              if (!appExport.trim().isEmpty()) {
-                appExports.add(appExport.trim());
-              }
-            }
-          }
-
-          if (hasExportGroups && !appExports.isEmpty()) {
-            String configKeyFormat = "${site.%s.%s}";
-            String hostKeyFormat = "${%s_HOST}";
-
-            // publish export groups if any
-            Map<String, String> replaceTokens = new HashMap<String, String>();
-            for (Map.Entry<String, Map<String, ClusterNode>> entry : getRoleClusterNodeMapping().entrySet()) {
-              String hostName = providerUtils.getHostsList(
-                  entry.getValue().values(), true).iterator().next();
-              replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
-            }
-
-            for (String key : status.getConfigs().keySet()) {
-              Map<String, String> configs = status.getConfigs().get(key);
-              for (String configKey : configs.keySet()) {
-                String lookupKey = String.format(configKeyFormat, key, configKey);
-                replaceTokens.put(lookupKey, configs.get(configKey));
-              }
-            }
-
-            Set<String> modifiedGroups = new HashSet<String>();
-            for (ExportGroup exportGroup : appExportGroups) {
-              List<Export> exports = exportGroup.getExports();
-              if (SliderUtils.isNotEmpty(exports)) {
-                String exportGroupName = exportGroup.getName();
-                ConcurrentHashMap<String, List<ExportEntry>> map =
-                    (ConcurrentHashMap<String, List<ExportEntry>>)getCurrentExports(exportGroupName);
-                for (Export export : exports) {
-                  if (canBeExported(exportGroupName, export.getName(), appExports)) {
-                    String value = export.getValue();
-                    // replace host names
-                    for (String token : replaceTokens.keySet()) {
-                      if (value.contains(token)) {
-                        value = value.replace(token, replaceTokens.get(token));
-                      }
-                    }
-                    ExportEntry entry = new ExportEntry();
-                    entry.setLevel(APPLICATION_TAG);
-                    entry.setValue(value);
-                    entry.setUpdatedTime(new Date().toString());
-                    // over-write, app exports are singletons
-                    map.put(export.getName(), new ArrayList(Arrays.asList(entry)));
-                    log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
-                  }
-                }
-                modifiedGroups.add(exportGroupName);
-              }
-            }
-            publishModifiedExportGroups(modifiedGroups);
-          }
-
-          log.info("Received and processed config for {}", heartBeat.getHostname());
-          componentStatus.setConfigReported(true);
-
-        }
-      }
-    }
-  }
-
-  private boolean canBeExported(String exportGroupName, String name, Set<String> appExports) {
-    return appExports.contains(String.format("%s-%s", exportGroupName, name));
-  }
-
-  protected Map<String, List<ExportEntry>> getCurrentExports(String groupName) {
-    if (!this.exportGroups.containsKey(groupName)) {
-      synchronized (this.exportGroups) {
-        if (!this.exportGroups.containsKey(groupName)) {
-          this.exportGroups.put(groupName, new ConcurrentHashMap<String, List<ExportEntry>>());
-        }
-      }
-    }
-
-    return this.exportGroups.get(groupName);
-  }
-
-  private void publishModifiedExportGroups(Set<String> modifiedGroups) {
-    for (String roleGroup : modifiedGroups) {
-      Map<String, List<ExportEntry>> entries = this.exportGroups.get(roleGroup);
-      // Publish in old format for the time being
-      Map<String, String> simpleEntries = new HashMap<String, String>();
-      for (Entry<String, List<ExportEntry>> entry : entries.entrySet()) {
-        List<ExportEntry> exports = entry.getValue();
-        if (SliderUtils.isNotEmpty(exports)) {
-          // there is no support for multiple exports per name - so extract only the first one
-          simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
-        }
-      }
-      publishApplicationInstanceData(roleGroup, roleGroup,
-          simpleEntries.entrySet());
-
-      PublishedExports exports = new PublishedExports(roleGroup);
-      exports.setUpdated(new Date().getTime());
-      exports.putValues(entries.entrySet());
-      getAmState().getPublishedExportsSet().put(roleGroup, exports);
-    }
-  }
-
-  /** Publish component instance specific data if the component demands it */
-  protected void processAndPublishComponentSpecificData(Map<String, String> ports,
-                                                        String containerId,
-                                                        String hostFqdn,
-                                                        String componentGroup) {
-    String portVarFormat = "${site.%s}";
-    String hostNamePattern = "${THIS_HOST}";
-    Map<String, String> toPublish = new HashMap<String, String>();
-
-    Application application = getMetaInfo(componentGroup).getApplication();
-    for (Component component : application.getComponents()) {
-      if (component.getName().equals(componentGroup)) {
-        if (component.getComponentExports().size() > 0) {
-
-          for (ComponentExport export : component.getComponentExports()) {
-            String templateToExport = export.getValue();
-            for (String portName : ports.keySet()) {
-              boolean publishData = false;
-              String portValPattern = String.format(portVarFormat, portName);
-              if (templateToExport.contains(portValPattern)) {
-                templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
-                publishData = true;
-              }
-              if (templateToExport.contains(hostNamePattern)) {
-                templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
-                publishData = true;
-              }
-              if (publishData) {
-                toPublish.put(export.getName(), templateToExport);
-                log.info("Publishing {} for name {} and container {}",
-                         templateToExport, export.getName(), containerId);
-              }
-            }
-          }
-        }
-      }
-    }
-
-    if (toPublish.size() > 0) {
-      Map<String, String> perContainerData = null;
-      if (!getComponentInstanceData().containsKey(containerId)) {
-        perContainerData = new ConcurrentHashMap<String, String>();
-      } else {
-        perContainerData = getComponentInstanceData().get(containerId);
-      }
-      perContainerData.putAll(toPublish);
-      getComponentInstanceData().put(containerId, perContainerData);
-      publishComponentInstanceData();
-    }
-  }
-
-  /** Publish component instance specific data if the component demands it */
-  protected void processAndPublishComponentSpecificExports(Map<String, String> ports,
-                                                           String containerId,
-                                                           String hostFqdn,
-                                                           String compName,
-                                                           String compGroup) {
-    String portVarFormat = "${site.%s}";
-    String hostNamePattern = "${" + compGroup + "_HOST}";
-
-    List<ExportGroup> appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups();
-    Component component = getApplicationComponent(compGroup);
-    if (component != null && SliderUtils.isSet(component.getCompExports())
-        && SliderUtils.isNotEmpty(appExportGroups)) {
-
-      Set<String> compExports = new HashSet();
-      String compExportsStr = component.getCompExports();
-      for (String compExport : compExportsStr.split(",")) {
-        if (!compExport.trim().isEmpty()) {
-          compExports.add(compExport.trim());
-        }
-      }
-
-      Date now = new Date();
-      Set<String> modifiedGroups = new HashSet<String>();
-      for (ExportGroup exportGroup : appExportGroups) {
-        List<Export> exports = exportGroup.getExports();
-        if (SliderUtils.isNotEmpty(exports)) {
-          String exportGroupName = exportGroup.getName();
-          ConcurrentHashMap<String, List<ExportEntry>> map =
-              (ConcurrentHashMap<String, List<ExportEntry>>) getCurrentExports(exportGroupName);
-          for (Export export : exports) {
-            if (canBeExported(exportGroupName, export.getName(), compExports)) {
-              log.info("Attempting to publish {} of group {} for component type {}",
-                       export.getName(), exportGroupName, compName);
-              String templateToExport = export.getValue();
-              for (String portName : ports.keySet()) {
-                boolean publishData = false;
-                String portValPattern = String.format(portVarFormat, portName);
-                if (templateToExport.contains(portValPattern)) {
-                  templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
-                  publishData = true;
-                }
-                if (templateToExport.contains(hostNamePattern)) {
-                  templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
-                  publishData = true;
-                }
-                if (publishData) {
-                  ExportEntry entryToAdd = new ExportEntry();
-                  entryToAdd.setLevel(COMPONENT_TAG);
-                  entryToAdd.setValue(templateToExport);
-                  entryToAdd.setUpdatedTime(now.toString());
-                  entryToAdd.setContainerId(containerId);
-                  entryToAdd.setTag(tags.getTag(compName, containerId));
-
-                  List<ExportEntry> existingList =
-                      map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd)));
-
-                  // in-place edit, no lock needed
-                  if (existingList != null) {
-                    boolean updatedInPlace = false;
-                    for (ExportEntry entry : existingList) {
-                      if (containerId.toLowerCase(Locale.ENGLISH)
-                                     .equals(entry.getContainerId())) {
-                        entryToAdd.setValue(templateToExport);
-                        entryToAdd.setUpdatedTime(now.toString());
-                        updatedInPlace = true;
-                      }
-                    }
-                    if (!updatedInPlace) {
-                      existingList.add(entryToAdd);
-                    }
-                  }
-
-                  log.info("Publishing {} for name {} and container {}",
-                           templateToExport, export.getName(), containerId);
-                  modifiedGroups.add(exportGroupName);
-                  synchronized (containerExportsMap) {
-                    if (!containerExportsMap.containsKey(containerId)) {
-                      containerExportsMap.put(containerId, new HashSet<String>());
-                    }
-                    Set<String> containerExportMaps = containerExportsMap.get(containerId);
-                    containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName()));
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-      publishModifiedExportGroups(modifiedGroups);
-    }
-  }
-
-  private void publishComponentInstanceData() {
-    Map<String, String> dataToPublish = new HashMap<String, String>();
-    for (String container : getComponentInstanceData().keySet()) {
-      for (String prop : getComponentInstanceData().get(container).keySet()) {
-        dataToPublish.put(
-            container + "." + prop, getComponentInstanceData().get(container).get(prop));
-      }
-    }
-    publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
-  }
-
-  /**
-   * Return Component based on group
-   *
-   * @param roleGroup component group
-   *
-   * @return the component entry or null for no match
-   */
-  protected Component getApplicationComponent(String roleGroup) {
-    Metainfo metainfo = getMetaInfo(roleGroup);
-    if (metainfo == null) {
-      return null;
-    }
-    return metainfo.getApplicationComponent(roleGroup);
-  }
-
-  /**
-   * Extract script path from the application metainfo
-   *
-   * @param roleGroup component group
-   * @return the script path or null for no match
-   */
-  protected CommandScript getScriptPathForMasterPackage(String roleGroup) {
-    Component component = getApplicationComponent(roleGroup);
-    if (component != null) {
-      return component.getCommandScript();
-    }
-    return null;
-  }
-
-  /**
-   * Is the role of type MASTER
-   *
-   * @param roleGroup component group
-   *
-   * @return true if the role category is MASTER
-   */
-  protected boolean isMaster(String roleGroup) {
-    Component component = getApplicationComponent(roleGroup);
-    if (component != null) {
-      if (component.getCategory().equals("MASTER")) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Can the role publish configuration
-   *
-   * @param roleGroup component group
-   *
-   * @return true if it can be pubished
-   */
-  protected boolean canPublishConfig(String roleGroup) {
-    Component component = getApplicationComponent(roleGroup);
-    if (component != null) {
-      return Boolean.TRUE.toString().equals(component.getPublishConfig());
-    }
-    return false;
-  }
-
-  /**
-   * Checks if the role is marked auto-restart
-   *
-   * @param roleGroup component group
-   *
-   * @return true if it is auto-restart
-   */
-  protected boolean isMarkedAutoRestart(String roleGroup) {
-    Component component = getApplicationComponent(roleGroup);
-    if (component != null) {
-      return component.getAutoStartOnFailureBoolean();
-    }
-    return false;
-  }
-
-  /**
-   * Can any master publish config explicitly, if not a random master is used
-   *
-   * @return true if the condition holds
-   */
-  protected boolean canAnyMasterPublishConfig(String roleGroup) {
-    if (canAnyMasterPublish == null) {
-      Application application = getMetaInfo(roleGroup).getApplication();
-      if (application == null) {
-        log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
-      } else {
-        for (Component component : application.getComponents()) {
-          if (Boolean.TRUE.toString().equals(component.getPublishConfig()) &&
-              component.getCategory().equals("MASTER")) {
-            canAnyMasterPublish = true;
-          }
-        }
-      }
-    }
-
-    if (canAnyMasterPublish == null) {
-      canAnyMasterPublish = false;
-    }
-    return canAnyMasterPublish;
-  }
-
-  private String getRoleName(String label) {
-    int index1 = label.indexOf(LABEL_MAKER);
-    int index2 = label.lastIndexOf(LABEL_MAKER);
-    if (index1 == index2) {
-      return label.substring(index1 + LABEL_MAKER.length());
-    } else {
-      return label.substring(index1 + LABEL_MAKER.length(), index2);
-    }
-  }
-
-  private String getRoleGroup(String label) {
-    return label.substring(label.lastIndexOf(LABEL_MAKER) + LABEL_MAKER.length());
-  }
-
-  private String getContainerId(String label) {
-    return label.substring(0, label.indexOf(LABEL_MAKER));
-  }
-
-  /**
-   * Add install command to the heartbeat response
-   *
-   * @param roleName
-   * @param roleGroup
-   * @param containerId
-   * @param response
-   * @param scriptPath
-   * @param pkg
-   *          when this field is null, it indicates the command is for the
-   *          master package; while not null, for the package named by this
-   *          field
-   * @throws SliderException
-   */
-  @VisibleForTesting
-  protected void addInstallCommand(String roleName,
-                                   String roleGroup,
-                                   String containerId,
-                                   HeartBeatResponse response,
-                                   String scriptPath,
-                                   ComponentCommand compCmd,
-                                   long timeout,
-                                   String pkg)
-      throws SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
-    ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
-    prepareExecutionCommand(cmd);
-    String clusterName = getClusterName();
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(Command.INSTALL.toString());
-    cmd.setServiceName(clusterName);
-    cmd.setComponentName(roleName);
-    cmd.setRole(roleName);
-    cmd.setPkg(pkg);
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
-    hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    Map<String, Map<String, String>> configurations =
-        buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
-            containerId, roleName, roleGroup);
-    cmd.setConfigurations(configurations);
-    Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
-    cmd.setComponentConfigurations(componentConfigurations);
-
-    if (SliderUtils.isSet(scriptPath)) {
-      cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
-    } else {
-      // assume it to be default shell command
-      ComponentCommand effectiveCommand = compCmd;
-      if (effectiveCommand == null) {
-        effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL");
-      }
-      cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false));
-      configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
-    }
-
-    cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
-
-    response.addExecutionCommand(cmd);
-
-    log.debug("command looks like: {} ",  cmd);
-  }
-
-  @VisibleForTesting
-  protected void addInstallDockerCommand(String roleName,
-                                   String roleGroup,
-                                   String containerId,
-                                   HeartBeatResponse response,
-                                   ComponentCommand compCmd,
-                                   long timeout)
-      throws SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
-    ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
-    prepareExecutionCommand(cmd);
-    String clusterName = getClusterName();
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(Command.INSTALL.toString());
-    cmd.setServiceName(clusterName);
-    cmd.setComponentName(roleName);
-    cmd.setRole(roleName);
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
-        roleGroup);
-    cmd.setConfigurations(configurations);
-    Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
-    cmd.setComponentConfigurations(componentConfigurations);
-    
-    ComponentCommand effectiveCommand = compCmd;
-    if (compCmd == null) {
-      effectiveCommand = new ComponentCommand();
-      effectiveCommand.setName("INSTALL");
-      effectiveCommand.setExec("DEFAULT");
-    }
-    cmd.setCommandParams(setCommandParameters(effectiveCommand, timeout, false));
-    configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
-
-    cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
-    cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
-
-    Map<String, String> dockerConfig = new HashMap<String, String>();
-    if(isYarnDockerContainer(roleGroup)){
-      //put nothing
-      cmd.setYarnDockerMode(true);
-    } else {
-      dockerConfig.put(
-          "docker.command_path",
-          getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-              "commandPath"));
-      dockerConfig.put("docker.image_name",
-          getConfigFromMetaInfo(roleGroup, "image"));
-    }
-    configurations.put("docker", dockerConfig);
-
-    log.debug("Docker- command: {}", cmd.toString());
-
-    response.addExecutionCommand(cmd);
-  }
-
-  private Map<String, String> setCommandParameters(String scriptPath,
-      long timeout, boolean recordConfig) {
-    Map<String, String> cmdParams = new TreeMap<String, String>();
-    cmdParams.put("service_package_folder",
-        "${AGENT_WORK_ROOT}/work/app/definition/package");
-    cmdParams.put("script", scriptPath);
-    cmdParams.put("schema_version", "2.0");
-    cmdParams.put("command_timeout", Long.toString(timeout));
-    cmdParams.put("script_type", AbstractComponent.TYPE_PYTHON);
-    cmdParams.put("record_config", Boolean.toString(recordConfig));
-    return cmdParams;
-  }
-
-  private Map<String, String> setCommandParameters(ComponentCommand compCmd,
-      long timeout, boolean recordConfig) {
-    Map<String, String> cmdParams = new TreeMap<String, String>();
-    cmdParams.put("service_package_folder",
-        "${AGENT_WORK_ROOT}/work/app/definition/package");
-    cmdParams.put("command", compCmd.getExec());
-    cmdParams.put("schema_version", "2.0");
-    cmdParams.put("command_timeout", Long.toString(timeout));
-    cmdParams.put("script_type", compCmd.getType());
-    cmdParams.put("record_config", Boolean.toString(recordConfig));
-    return cmdParams;
-  }
-
-  private Map<String, Map<String, String>> buildComponentConfigurations(
-      ConfTreeOperations appConf) {
-    return appConf.getComponents();
-  }
-
-  protected static String getPackageListFromApplication(Application application) {
-    String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}";
-    String pkgListFormatString = "[%s]";
-    List<String> packages = new ArrayList<>();
-    if (application != null) {
-      if (application.getPackages().size() > 0) {
-        // no-op if there are packages that are not OS-specific, as these
-        // will be localized by AM rather than the Agent
-        // this should be backwards compatible, as there was previously an
-        // XML parsing bug that ensured non-OS-specific packages did not exist
-      } else {
-        List<OSSpecific> osSpecifics = application.getOSSpecifics();
-        if (osSpecifics != null && osSpecifics.size() > 0) {
-          for (OSSpecific osSpecific : osSpecifics) {
-            if (osSpecific.getOsType().equals("any")) {
-              for (OSPackage osPackage : osSpecific.getPackages()) {
-                packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
-              }
-            }
-          }
-        }
-      }
-    }
-
-    if (!packages.isEmpty()) {
-      return "[" + SliderUtils.join(packages, ",", false) + "]";
-    } else {
-      return "[]";
-    }
-  }
-
-  private String getPackageList(String roleGroup) {
-    return getPackageListFromApplication(getMetaInfo(roleGroup).getApplication());
-  }
-
-  private void prepareExecutionCommand(ExecutionCommand cmd) {
-    cmd.setTaskId(taskId.incrementAndGet());
-    cmd.setCommandId(cmd.getTaskId() + "-1");
-  }
-
-  private Map<String, String> commandParametersSet(String scriptPath, long timeout, boolean recordConfig) {
-    Map<String, String> cmdParams = new TreeMap<>();
-    cmdParams.put("service_package_folder",
-                  "${AGENT_WORK_ROOT}/work/app/definition/package");
-    cmdParams.put("script", scriptPath);
-    cmdParams.put("schema_version", "2.0");
-    cmdParams.put("command_timeout", Long.toString(timeout));
-    cmdParams.put("script_type", "PYTHON");
-    cmdParams.put("record_config", Boolean.toString(recordConfig));
-    return cmdParams;
-  }
-
-  private Map<String, String> commandParametersSet(ComponentCommand compCmd, long timeout, boolean recordConfig) {
-    Map<String, String> cmdParams = new TreeMap<>();
-    cmdParams.put("service_package_folder",
-                  "${AGENT_WORK_ROOT}/work/app/definition/package");
-    cmdParams.put("command", compCmd.getExec());
-    cmdParams.put("schema_version", "2.0");
-    cmdParams.put("command_timeout", Long.toString(timeout));
-    cmdParams.put("script_type", compCmd.getType());
-    cmdParams.put("record_config", Boolean.toString(recordConfig));
-    return cmdParams;
-  }
-
-  @VisibleForTesting
-  protected void addStatusCommand(String roleName,
-                                  String roleGroup,
-                                  String containerId,
-                                  HeartBeatResponse response,
-                                  String scriptPath,
-                                  long timeout)
-      throws SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)) {
-      addStatusDockerCommand(roleName, roleGroup, containerId, response,
-          scriptPath, timeout);
-      return;
-    }
-
-    StatusCommand cmd = new StatusCommand();
-    String clusterName = getClusterName();
-
-    cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
-    cmd.setComponentName(roleName);
-    cmd.setServiceName(clusterName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
-
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
-
-    Map<String, Map<String, String>> configurations =
-        buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
-            containerId, roleName, roleGroup);
-
-    cmd.setConfigurations(configurations);
-
-    response.addStatusCommand(cmd);
-  }
-
-  @VisibleForTesting
-  protected void addStatusDockerCommand(String roleName,
-                                  String roleGroup,
-                                  String containerId,
-                                  HeartBeatResponse response,
-                                  String scriptPath,
-                                  long timeout)
-      throws SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
-    StatusCommand cmd = new StatusCommand();
-    String clusterName = getClusterName();
-
-    cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
-    cmd.setComponentName(roleName);
-    cmd.setServiceName(clusterName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
-
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
-
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
-        roleGroup);
-    Map<String, String> dockerConfig = new HashMap<String, String>();
-    String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
-    if (statusCommand == null) {
-      if(isYarnDockerContainer(roleGroup)){
-        //should complain the required field is null
-        cmd.setYarnDockerMode(true);
-      } else {
-        statusCommand = "docker top "
-            + containerId
-            + " | grep \"\"";// default value
-      }
-    }
-    dockerConfig.put("docker.status_command",statusCommand);
-    configurations.put("docker", dockerConfig);
-    cmd.setConfigurations(configurations);
-    log.debug("Docker- status {}", cmd);
-    response.addStatusCommand(cmd);
-  }
-
-  @VisibleForTesting
-  protected void addGetConfigDockerCommand(String roleName, String roleGroup,
-      String containerId, HeartBeatResponse response) throws SliderException {
-    assert getAmState().isApplicationLive();
-
-    StatusCommand cmd = new StatusCommand();
-    String clusterName = getClusterName();
-
-    cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
-    cmd.setComponentName(roleName);
-    cmd.setServiceName(clusterName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    hostLevelParams.put(CONTAINER_ID, containerId);
-
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
-        roleGroup);
-    Map<String, String> dockerConfig = new HashMap<String, String>();
-    String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
-    if (statusCommand == null) {
-      if(isYarnDockerContainer(roleGroup)){
-        //should complain the required field is null
-        cmd.setYarnDockerMode(true);
-      } else {
-        statusCommand = "docker top "
-            + containerId
-            + " | grep \"\"";// default value
-      }
-    }
-    dockerConfig.put("docker.status_command",statusCommand);
-    configurations.put("docker", dockerConfig);
-
-    cmd.setConfigurations(configurations);
-    log.debug("Docker- getconfig command {}", cmd);
-    
-    response.addStatusCommand(cmd);
-  }
-  
-  private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup,
-      String configName){
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    String containerName = getApplicationComponent(roleGroup)
-        .getDockerContainers().get(0).getName();
-    String composedConfigName = null;
-    String appConfigValue = null;
-    //if the configName is about port , mount, inputfile, then check differently
-    if (configName.equals("containerPort") || configName.equals("hostPort")){
-      composedConfigName = containerName + ".ports." + configName;
-    } else 
-    if (configName.equals("containerMount")
-        || configName.equals("hostMount")){
-      composedConfigName = containerName + ".mounts." + configName;
-    } else
-    if (configName.equals("containerPath")
-        || configName.equals("fileLocalPath")) {
-      composedConfigName = containerName + ".inputFiles." + configName;
-    } else {
-      composedConfigName = containerName + "." + configName;
-    }
-    appConfigValue = appConf.getComponentOpt(roleGroup, composedConfigName,
-        null);
-    log.debug(
-        "Docker- value from appconfig component: {} configName: {} value: {}",
-        roleGroup, composedConfigName, appConfigValue);
-    if (appConfigValue == null) {
-      appConfigValue = getConfigFromMetaInfo(roleGroup, configName);
-      log.debug(
-          "Docker- value from metainfo component: {} configName: {} value: {}",
-          roleGroup, configName, appConfigValue);
-
-    }
-    return appConfigValue;
-  }
-
-  @VisibleForTesting
-  protected void addStartDockerCommand(String roleName, String roleGroup,
-      String containerId, HeartBeatResponse response,
-      ComponentCommand startCommand, long timeout, boolean isMarkedAutoRestart)
-      throws
-      SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
-    ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
-    prepareExecutionCommand(cmd);
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
-    String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
-    cmd.setHostname(hostName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(Command.START.toString());
-    cmd.setServiceName(clusterName);
-    cmd.setComponentName(roleName);
-    cmd.setRole(roleName);
-    Map<String, String> hostLevelParams = new TreeMap<>();
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    Map<String, String> roleParams = new TreeMap<>();
-    cmd.setRoleParams(roleParams);
-    cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
-    startCommand = new ComponentCommand();
-    startCommand.setName("START");
-    startCommand.setType("docker");
-    startCommand.setExec("exec");
-    cmd.setCommandParams(setCommandParameters(startCommand, timeout, true));
-    
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
-        roleGroup);
-    Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
-    cmd.setComponentConfigurations(componentConfigurations);
-    
-    Map<String, String> dockerConfig = new HashMap<String, String>();
-    if (isYarnDockerContainer(roleGroup)) {
-      dockerConfig.put(
-          "docker.startCommand",
-          getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-              "start_command"));
-      cmd.setYarnDockerMode(true);
-    } else {
-      dockerConfig.put(
-        "docker.command_path",
-        getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-            "commandPath"));
-
-      dockerConfig.put("docker.image_name",
-          getConfigFromMetaInfo(roleGroup, "image"));
-      // options should always have -d
-      String options = getConfigFromMetaInfoWithAppConfigOverriding(
-          roleGroup, "options");
-      if(options != null && !options.isEmpty()){
-        options = options + " -d";
-      } else {
-        options = "-d";
-      }
-      dockerConfig.put("docker.options", options);
-      // options should always have -d
-      dockerConfig.put(
-          "docker.containerPort",
-          getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-              "containerPort"));
-      dockerConfig
-          .put(
-              "docker.hostPort",
-              getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-                  "hostPort"));
-  
-      dockerConfig.put(
-          "docker.mounting_directory",
-          getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-              "containerMount"));
-      dockerConfig
-          .put(
-              "docker.host_mounting_directory",
-              getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
-                  "hostMount"));
-  
-      dockerConfig.put("docker.additional_param",
-          getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "additionalParam"));
-  
-      dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo(
-          roleGroup, "containerPath"));
-    }
-
-    String lifetime = getConfigFromMetaInfoWithAppConfigOverriding(
-        roleGroup, "lifetime");
-    dockerConfig.put("docker.lifetime", lifetime);
-    configurations.put("docker", dockerConfig);
-    String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(
-        roleGroup, "statusCommand");
-    if (statusCommand == null) {
-      if(isYarnDockerContainer(roleGroup)){
-        //should complain the required field is null
-      } else {
-        statusCommand = "docker top "
-          + containerId + " | grep \"\"";
-      }
-    }
-    dockerConfig.put("docker.status_command",statusCommand);
-    
-    cmd.setConfigurations(configurations);
-   // configurations.get("global").put("exec_cmd", startCommand.getExec());
-    cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
-
-    log.info("Docker- command: {}", cmd.toString());
-
-    response.addExecutionCommand(cmd);
-  }
-
-  private String getConfigFromMetaInfo(String roleGroup, String configName) {
-    String result = null;
-
-    List<DockerContainer> containers = getApplicationComponent(
-        roleGroup).getDockerContainers();// to support multi container per
-                                             // component later
-    log.debug("Docker- containers metainfo: {}", containers.toString());
-    if (containers.size() > 0) {
-      DockerContainer container = containers.get(0);
-      switch (configName) {
-      case "start_command":
-        result = container.getStartCommand();
-        break;
-      case "image":
-        result = container.getImage();
-        break;
-      case "network":
-        if (container.getNetwork() == null || container.getNetwork().isEmpty()) {
-          result = "none";
-        } else {
-          result = container.getNetwork();
-        }
-        break;
-      case "statusCommand":
-        result = container.getStatusCommand();
-        break;
-      case "commandPath":
-        result = container.getCommandPath();
-        break;
-      case "options":
-        result = container.getOptions();
-        break;
-      case "containerPort":
-        result = container.getPorts().size() > 0 ? container.getPorts().get(0)
-            .getContainerPort() : null;// to support
-        // multi port
-        // later
-        break;
-      case "hostPort":
-        result = container.getPorts().size() > 0 ? container.getPorts().get(0)
-            .getHostPort() : null;// to support multi
-        // port later
-        break;
-      case "containerMount":
-        result = container.getMounts().size() > 0 ? container.getMounts()
-            .get(0).getContainerMount() : null;// to support
-        // multi port
-        // later
-        break;
-      case "hostMount":
-        result = container.getMounts().size() > 0 ? container.getMounts()
-            .get(0).getHostMount() : null;// to support multi
-        // port later
-        break;
-      case "additionalParam":
-        result = container.getAdditionalParam();// to support multi port later
-        break;
-      case "runPriviledgedContainer":
-        if (container.getRunPrivilegedContainer() == null) {
-          result = "false";
-        } else {
-          result = container.getRunPrivilegedContainer();
-        }
-        break;
-      default:
-        break;
-      }
-    }
-    log.debug("Docker- component: {} configName: {} value: {}", roleGroup, configName, result);
-    return result;
-  }
-
-  @VisibleForTesting
-  protected void addGetConfigCommand(String roleName, String roleGroup,
-      String containerId, HeartBeatResponse response) throws SliderException {
-    assert getAmState().isApplicationLive();
-
-    StatusCommand cmd = new StatusCommand();
-    String clusterName = getClusterName();
-
-    cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
-    cmd.setComponentName(roleName);
-    cmd.setServiceName(clusterName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    hostLevelParams.put(CONTAINER_ID, containerId);
-
-    response.addStatusCommand(cmd);
-  }
-
-  @VisibleForTesting
-  protected void addStartCommand(String roleName, String roleGroup, String containerId,
-                                 HeartBeatResponse response,
-                                 String scriptPath, ComponentCommand startCommand,
-                                 ComponentCommand stopCommand,
-                                 long timeout, boolean isMarkedAutoRestart)
-      throws
-      SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
-    ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
-    prepareExecutionCommand(cmd);
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
-    String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
-    cmd.setHostname(hostName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(Command.START.toString());
-    cmd.setServiceName(clusterName);
-    cmd.setComponentName(roleName);
-    cmd.setRole(roleName);
-    Map<String, String> hostLevelParams = new TreeMap<>();
-    hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-
-    Map<String, String> roleParams = new TreeMap<>();
-    cmd.setRoleParams(roleParams);
-    cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
-
-    Map<String, Map<String, String>> configurations =
-        buildCommandConfigurations(appConf, internalsConf, containerId,
-            roleName, roleGroup);
-    cmd.setConfigurations(configurations);
-    Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
-    cmd.setComponentConfigurations(componentConfigurations);
-    
-    if (SliderUtils.isSet(scriptPath)) {
-      cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
-    } else {
-      if (startCommand == null) {
-        throw new SliderException("Expected START command not found for component " + roleName);
-      }
-      cmd.setCommandParams(commandParametersSet(startCommand, timeout, true));
-      configurations.get("global").put("exec_cmd", startCommand.getExec());
-    }
-
-    response.addExecutionCommand(cmd);
-
-    log.debug("command looks like: {}", cmd);
-    // With start command, the corresponding command for graceful stop needs to
-    // be sent. This will be used when a particular container is lost as per RM,
-    // but then the agent is still running and heart-beating to the Slider AM.
-    ExecutionCommand cmdStop = new ExecutionCommand(
-        AgentCommandType.EXECUTION_COMMAND);
-    cmdStop.setTaskId(taskId.get());
-    cmdStop.setCommandId(cmdStop.getTaskId() + "-1");
-    cmdStop.setHostname(hostName);
-    cmdStop.setClusterName(clusterName);
-    cmdStop.setRoleCommand(Command.STOP.toString());
-    cmdStop.setServiceName(clusterName);
-    cmdStop.setComponentName(roleName);
-    cmdStop.setRole(roleName);
-    Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
-    hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
-        .getOption(JAVA_HOME, ""));
-    hostLevelParamsStop.put(CONTAINER_ID, containerId);
-    cmdStop.setHostLevelParams(hostLevelParamsStop);
-
-    Map<String, String> roleParamsStop = new TreeMap<String, String>();
-    cmdStop.setRoleParams(roleParamsStop);
-    cmdStop.getRoleParams().put("auto_restart",
-                                Boolean.toString(isMarkedAutoRestart));
-
-    if (SliderUtils.isSet(scriptPath)) {
-      cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
-    } else {
-      if (stopCommand == null) {
-        stopCommand = ComponentCommand.getDefaultComponentCommand("STOP");
-      }
-      cmd.setCommandParams(commandParametersSet(stopCommand, timeout, true));
-      configurations.get("global").put("exec_cmd", startCommand.getExec());
-    }
-
-
-    Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
-        appConf, internalsConf, containerId, roleName, roleGroup);
-    cmdStop.setConfigurations(configurationsStop);
-    response.addExecutionCommand(cmdStop);
-  }
-
-  @VisibleForTesting
-  protected void addUpgradeCommand(String roleName, String roleGroup, String containerId,
-      HeartBeatResponse response, String scriptPath, long timeout)
-      throws SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
-    ExecutionCommand cmd = new ExecutionCommand(
-        AgentCommandType.EXECUTION_COMMAND);
-    prepareExecutionCommand(cmd);
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
-    String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
-    cmd.setHostname(hostName);
-    cmd.setClusterName(clusterName);
-    cmd.setRoleCommand(Command.UPGRADE.toString());
-    cmd.setServiceName(clusterName);
-    cmd.setComponentName(roleName);
-    cmd.setRole(roleName);
-    Map<String, String> hostLevelParams = new TreeMap<String, String>();
-    hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions()
-        .getMandatoryOption(JAVA_HOME));
-    hostLevelParams.put(CONTAINER_ID, containerId);
-    cmd.setHostLevelParams(hostLevelParams);
-    cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
-
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, internalsConf, containerId, roleName, roleGroup);
-    cmd.setConfigurations(configurations);
-    response.addExecutionCommand(cmd);
-  }
-    
-  protected void addStopCommand(String roleName, String roleGroup, String containerId,
-      HeartBeatResponse response, String scriptPath, long timeout,
-      boolean isInUpgradeMode) throws SliderException {
-    assert getAmState().isApplicationLive();
-    ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
-    ExecutionCommand cmdStop = new ExecutionCommand(
-        AgentCommandType.EXECUTION_COMMAND);
-    cmdStop.setTaskId(taskId.get());
-    cmdStop.setCommandId(cmdStop.getTaskId() + "-1");
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
-    String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
-    cmdStop.setHostname(hostName);
-    cmdStop.setClusterName(clusterName);
-    // Upgrade stop is differentiated by passing a transformed role command -
-    // UPGRADE_STOP
-    cmdStop.setRoleCommand(Command.transform(Command.STOP, isInUpgradeMode));
-    cmdStop.setServiceName(clusterName);
-    cmdStop.setComponentName(roleName);
-    cmdStop.setRole(roleName);
-    Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
-    hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
-        .getMandatoryOption(JAVA_HOME));
-    hostLevelParamsStop.put(CONTAINER_ID, containerId);
-    cmdStop.setHostLevelParams(hostLevelParamsStop);
-    cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
-
-    Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
-        appConf, internalsConf, containerId, roleName, roleGroup);
-    cmdStop.setConfigurations(configurationsStop);
-    response.addExecutionCommand(cmdStop);
-  }
-
-  protected static String getJDKDir() {
-    File javaHome = new File(System.getProperty("java.home")).getParentFile();
-    File jdkDirectory = null;
-    if (javaHome.getName().contains("jdk")) {
-      jdkDirectory = javaHome;
-    }
-    if (jdkDirectory != null) {
-      return jdkDirectory.getAbsolutePath();
-    } else {
-      return "";
-    }
-  }
-
-  protected Map<String, String> getAllocatedPorts() {
-    return getAllocatedPorts(SHARED_PORT_TAG);
-  }
-
-  protected Map<String, Map<String, String>> getComponentInstanceData() {
-    return this.componentInstanceData;
-  }
-
-  protected Map<String, String> getAllocatedPorts(String containerId) {
-    if (!this.allocatedPorts.containsKey(containerId)) {
-      synchronized (this.allocatedPorts) {
-        if (!this.allocatedPorts.containsKey(containerId)) {
-          this.allocatedPorts.put(containerId,
-                                  new ConcurrentHashMap<String, String>());
-        }
-      }
-    }
-    return this.allocatedPorts.get(containerId);
-  }
-
-  private Map<String, Map<String, String>> buildCommandConfigurations(
-      ConfTreeOperations appConf, ConfTreeOperations internalsConf,
-      String containerId, String roleName, String roleGroup)
-      throws SliderException {
-
-    Map<String, Map<String, String>> configurations = new TreeMap<>();
-    Map<String, String> tokens = providerUtils.getStandardTokenMap(appConf,
-        internalsConf, roleName, roleGroup, getClusterName());
-    tokens.put("${CONTAINER_ID}", containerId);
-
-    Set<String> configs = new HashSet<String>();
-    configs.addAll(getApplicationConfigurationTypes(roleGroup));
-    configs.addAll(getSystemConfigurationsRequested(appConf));
-
-    for (String configType : configs) {
-      addNamedConfiguration(configType, appConf.getGlobalOptions().options,
-                            configurations, tokens, containerId, roleName,
-                            roleGroup);
-      if (appConf.getComponent(roleGroup) != null) {
-        addNamedConfiguration(configType, appConf.getComponent(roleGroup).options,
-            configurations, tokens, containerId, roleName, roleGroup);
-      }
-    }
-
-    //do a final replacement of re-used configs
-    dereferenceAllConfigs(configurations);
-
-    return configurations;
-  }
-
-  @VisibleForTesting
-  protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
-    providerUtils.dereferenceAllConfigs(configurations);
-  }
-
-  @VisibleForTesting
-  protected Set<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) {
-    return providerUtils.getSystemConfigurationsRequested(appConf);
-  }
-
-  @VisibleForTesting
-  protected List<String> getApplicationConfigurationTypes(String roleGroup) {
-    List<String> configList = new ArrayList<String>();
-    configList.add(GLOBAL_CONFIG_TAG);
-
-    List<ConfigFile> configFiles = getMetaInfo(roleGroup).getApplication().getConfigFiles();
-    for (ConfigFile configFile : configFiles) {
-      log.info("Expecting config type {}.", configFile.getDictionaryName());
-      configList.add(configFile.getDictionaryName());
-    }
-    for (Component component : getMetaInfo(roleGroup).getApplication().getComponents()) {
-      if (!component.getName().equals(roleGroup)) {
-        continue;
-      }
-      if (component.getDockerContainers() == null) {
-        continue;
-      }
-      for (DockerContainer container : component.getDockerContainers()) {
-        if (container.getConfigFiles() == null) {
-          continue;
-        }
-        for (ConfigFile configFile : container.getConfigFiles()) {
-          log.info("Expecting config type {}.", configFile.getDictionaryName());
-          configList.add(configFile.getDictionaryName());
-        }
-      }
-    }
-
-    // remove duplicates.  mostly worried about 'global' being listed
-    return new ArrayList<String>(new HashSet<String>(configList));
-  }
-
-  private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
-                                     Map<String, Map<String, String>> configurations,
-                                     Map<String, String> tokens, String containerId,
-                                     String roleName, String roleGroup) {
-    Map<String, String> config = new HashMap<String, String>();
-    if (configName.equals(GLOBAL_CONFIG_TAG)) {
-      addDefaultGlobalConfig(config, containerId, roleName);
-    }
-    // add role hosts to tokens
-    addRoleRelatedTokens(tokens);
-    providerUtils.propagateSiteOptions(sourceConfig, config, configName, tokens);
-
-    //apply any port updates
-    if (!this.getAllocatedPorts().isEmpty()) {
-      for (String key : config.keySet()) {
-        String value = config.get(key);
-        String lookupKey = configName + "." + key;
-        if (!value.contains(PER_CONTAINER_TAG)) {
-          // If the config property is shared then pass on the already allocated value
-          // from any container
-          if (this.getAllocatedPorts().containsKey(lookupKey)) {
-            config.put(key, getAllocatedPorts().get(lookupKey));
-          }
-        } else {
-          if (this.getAllocatedPorts(containerId).containsKey(lookupKey)) {
-            config.put(key, getAllocatedPorts(containerId).get(lookupKey));
-          }
-        }
-      }
-    }
-
-    //apply defaults only if the key is not present and value is not empty
-    if (getDefaultConfigs(roleGroup).containsKey(configName)) {
-      log.info("Adding default configs for type {}.", configName);
-      for (PropertyInfo defaultConfigProp : getDefaultConfigs(roleGroup).get(configName).getPropertyInfos()) {
-        if (!config.containsKey(defaultConfigProp.getName())) {
-          if (!defaultConfigProp.getName().isEmpty() &&
-              defaultConfigProp.getValue() != null &&
-              !defaultConfigProp.getValue().isEmpty()) {
-            config.put(defaultConfigProp.getName(), defaultConfigProp.getValue());
-          }
-        }
-      }
-    }
-
-    configurations.put(configName, config);
-  }
-
-  @VisibleForTesting
-  protected void addRoleRelatedTokens(Map<String, String> tokens) {
-    providerUtils.addRoleRelatedTokens(tokens, getAmState());
-  }
-
-  private void addDefaultGlobalConfig(Map<String, String> config, String containerId, String roleName) {
-    config.put("app_log_dir", "${AGENT_LOG_ROOT}");
-    config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
-    config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
-    config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + APP_CONF_DIR);
-    config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + PROPAGATED_CONF_DIR_NAME);
-    config.put("app_container_id", containerId);
-    config.put("app_container_tag", tags.getTag(roleName, containerId));
-
-    // add optional parameters only if they are not already provided
-    if (!config.containsKey("pid_file")) {
-      config.put("pid_file", "${AGENT_WORK_ROOT}/app/run/component.pid");
-    }
-    if (!config.containsKey("app_root")) {
-      config.put("app_root", "${AGENT_WORK_ROOT}/app/install");
-    }
-  }
-
-  private void buildRoleHostDetails(Map<String, MonitorDetail> details) {
-    for (Map.Entry<String, Map<String, ClusterNode>> entry :
-        getRoleClusterNodeMapping().entrySet()) {
-      details.put(entry.getKey() + " Host(s)/Container(s)",
-                  new MonitorDetail(providerUtils.getHostsList(
-                      entry.getValue().values(), false).toString(), false));
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java

@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AgentRoles {
-
-  /**
-   * List of roles Agent provider does not have any roles by default. All roles are read from the application
-   * specification.
-   */
-  protected static final List<ProviderRole> ROLES =
-      new ArrayList<ProviderRole>();
-
-  public static List<ProviderRole> getRoles() {
-    return ROLES;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java

@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
-/**
- *
- */
-public class AgentUtils {
-  private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
-
-  public static Metainfo getApplicationMetainfoFromSummaryFile(
-      SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) {
-    FileSystem fs = fileSystem.getFileSystem();
-    Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
-    Path appPathJson = new Path(metainfoPath + ".metainfo.json");
-    Path appPathUsed = null;
-    try {
-      FSDataInputStream appStream = null;
-      if (fs.exists(appPathXML)) {
-        appPathUsed = appPathXML;
-        appStream = fs.open(appPathXML);
-        return parseMetainfo(appStream, metainfoForAddon, "xml");
-      } else if (fs.exists(appPathJson)) {
-        appPathUsed = appPathJson;
-        appStream = fs.open(appPathJson);
-        return parseMetainfo(appStream, metainfoForAddon, "json");
-      }
-    } catch (IOException e) {
-      log.info("Failed to get metainfo from summary file {} - {}", appPathUsed,
-          e.getMessage());
-      log.debug("Failed to get metainfo", e);
-    }
-    return null;
-  }
-
-  public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
-      String metainfoPath, boolean metainfoForAddon) throws IOException,
-      BadConfigException {
-    log.info("Reading metainfo at {}", metainfoPath);
-    Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
-        metainfoPath, metainfoForAddon);
-    if (metainfo != null) {
-      log.info("Got metainfo from summary file");
-      return metainfo;
-    }
-
-    FileSystem fs = fileSystem.getFileSystem();
-    Path appPath = new Path(metainfoPath);
-
-    InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
-        fs, appPath, "metainfo.json");
-    if (metainfoJsonStream == null) {
-      InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
-          fs, appPath, "metainfo.xml");
-      if (metainfoXMLStream != null) {
-        metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
-      }
-    } else {
-      metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
-    }
-
-    if (metainfo == null) {
-      log.error("metainfo is unavailable at {}.", metainfoPath);
-      throw new FileNotFoundException("metainfo.xml/json is required in app package. " +
-                                      appPath);
-    }
-    return metainfo;
-  }
-
-  private static Metainfo parseMetainfo(InputStream stream,
-      boolean metainfoForAddon, String type) throws IOException {
-    AbstractMetainfoParser metainfoParser = null;
-    if (metainfoForAddon) {
-      metainfoParser = new AddonPackageMetainfoParser();
-    } else {
-      metainfoParser = new MetainfoParser();
-    }
-    if (type.equals("xml")) {
-      return metainfoParser.fromXmlStream(stream);
-    } else if (type.equals("json")) {
-      return metainfoParser.fromJsonStream(stream);
-    }
-    return null;
-  }
-
-  static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
-                                        String appDef, String configFileName)
-      throws IOException {
-    // this is the path inside the zip file
-    String fileToRead = "configuration/" + configFileName;
-    log.info("Reading default config file {} at {}", fileToRead, appDef);
-    InputStream configStream = SliderUtils.getApplicationResourceInputStream(
-        fileSystem.getFileSystem(), new Path(appDef), fileToRead);
-    if (configStream == null) {
-      log.error("{} is unavailable at {}.", fileToRead, appDef);
-      throw new IOException("Expected config file " + fileToRead + " is not available.");
-    }
-
-    return new DefaultConfigParser().parse(configStream);
-  }
-
-  static String getMetainfoComponentName(String roleGroup,
-      ConfTreeOperations appConf) throws BadConfigException {
-    String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null);
-    if (prefix == null) {
-      return roleGroup;
-    }
-    if (!roleGroup.startsWith(prefix)) {
-      throw new BadConfigException("Component " + roleGroup + " doesn't start" +
-          " with prefix " + prefix);
-    }
-    return roleGroup.substring(prefix.length());
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java

@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum Command {
-  NOP,           // do nothing
-  INSTALL,       // Install the component
-  INSTALL_ADDON, // Install add on packages if any
-  START,         // Start the component
-  STOP,          // Stop the component
-  UPGRADE,       // The component will undergo upgrade
-  TERMINATE;     // Send terminate signal to agent
-
-  public static Command getCommand(String commandVal) {
-    if (commandVal.equals(Command.START.toString())) {
-      return Command.START;
-    }
-    if (commandVal.equals(Command.INSTALL.toString())) {
-      return Command.INSTALL;
-    }
-    if (commandVal.equals(Command.STOP.toString())) {
-      return Command.STOP;
-    }
-    if (commandVal.equals(Command.UPGRADE.toString())) {
-      return Command.UPGRADE;
-    }
-    if (commandVal.equals(Command.TERMINATE.toString())) {
-      return Command.TERMINATE;
-    }
-
-    return Command.NOP;
-  }
-
-  public static String transform(Command command, boolean isUpgrade) {
-    switch (command) {
-    case STOP:
-      return isUpgrade ? "UPGRADE_STOP" : command.name();
-    default:
-      return command.name();
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java

@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** Command results. */
-public enum CommandResult {
-  IN_PROGRESS,  // Command is in progress
-  COMPLETED,    // Command has successfully completed
-  FAILED;        // Command has failed
-
-  public static CommandResult getCommandResult(String commandResVal) {
-    if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
-      return CommandResult.COMPLETED;
-    }
-    if (commandResVal.equals(CommandResult.FAILED.toString())) {
-      return CommandResult.FAILED;
-    }
-    if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
-      return CommandResult.IN_PROGRESS;
-    }
-
-    throw new IllegalArgumentException("Unrecognized value " + commandResVal);
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java

@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
-/**
- * Stores the command dependency order for all components in a service. <commandOrder>
- * <command>SUPERVISOR-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> Means, SUPERVISOR START
- * requires NIMBUS to be STARTED
- */
-public class ComponentCommandOrder {
-  public static final Logger log =
-      LoggerFactory.getLogger(ComponentCommandOrder.class);
-  private static char SPLIT_CHAR = '-';
-  Map<Command, Map<String, List<ComponentState>>> dependencies =
-      new HashMap<Command, Map<String, List<ComponentState>>>();
-  Map<String, Collection<String>> prefixRoleMap = new HashMap<>();
-  Map<String, String> rolePrefixMap = new HashMap<>();
-
-  public ComponentCommandOrder() {}
-
-  public ComponentCommandOrder(List<CommandOrder> commandOrders,
-      ConfTreeOperations resources) {
-    mergeCommandOrders(commandOrders, resources);
-  }
-
-  void mergeCommandOrders(List<CommandOrder> commandOrders,
-      ConfTreeOperations resources) {
-    for (String component : resources.getComponentNames()) {
-      String prefix = SliderUtils.trimPrefix(
-          resources.getComponentOpt(component, ROLE_PREFIX, null));
-      if (prefix != null) {
-        rolePrefixMap.put(component, prefix);
-        if (!prefixRoleMap.containsKey(prefix)) {
-          prefixRoleMap.put(prefix, new HashSet<String>());
-        }
-        prefixRoleMap.get(prefix).add(component);
-      }
-    }
-    if (commandOrders != null && commandOrders.size() > 0) {
-      for (CommandOrder commandOrder : commandOrders) {
-        ComponentCommand componentCmd = getComponentCommand(
-            commandOrder.getCommand(), resources);
-        String requires = commandOrder.getRequires();
-        List<ComponentState> requiredStates = parseRequiredStates(requires,
-            resources);
-        if (requiredStates.size() > 0) {
-          Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
-          if (compDep == null) {
-            compDep = new HashMap<>();
-            dependencies.put(componentCmd.command, compDep);
-          }
-
-          List<ComponentState> requirements = compDep.get(componentCmd.componentName);
-          if (requirements == null) {
-            requirements = new ArrayList<>();
-            compDep.put(componentCmd.componentName, requirements);
-          }
-
-          requirements.addAll(requiredStates);
-        }
-      }
-    }
-  }
-
-  private List<ComponentState> parseRequiredStates(String requires,
-      ConfTreeOperations resources) {
-    if (requires == null || requires.length() < 2) {
-      throw new IllegalArgumentException("Input cannot be null and must contain component and state.");
-    }
-
-    String[] componentStates = requires.split(",");
-    List<ComponentState> retList = new ArrayList<ComponentState>();
-    for (String componentStateStr : componentStates) {
-      retList.add(getComponentState(componentStateStr, resources));
-    }
-
-    return retList;
-  }
-
-  private ComponentCommand getComponentCommand(String compCmdStr,
-      ConfTreeOperations resources) {
-    if (compCmdStr == null || compCmdStr.trim().length() < 2) {
-      throw new IllegalArgumentException("Input cannot be null and must contain component and command.");
-    }
-
-    compCmdStr = compCmdStr.trim();
-    int splitIndex = compCmdStr.lastIndexOf(SPLIT_CHAR);
-    if (splitIndex == -1 || splitIndex == 0 || splitIndex == compCmdStr.length() - 1) {
-      throw new IllegalArgumentException("Input does not appear to be well-formed.");
-    }
-    String compStr = compCmdStr.substring(0, splitIndex);
-    String cmdStr = compCmdStr.substring(splitIndex + 1);
-
-    if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
-      throw new IllegalArgumentException("Component " + compStr + " specified" +
-          " in command order does not exist");
-    }
-
-    Command cmd = Command.valueOf(cmdStr);
-
-    if (cmd != Command.START) {
-      throw new IllegalArgumentException("Dependency order can only be specified for START.");
-    }
-    return new ComponentCommand(compStr, cmd);
-  }
-
-  private ComponentState getComponentState(String compStStr,
-      ConfTreeOperations resources) {
-    if (compStStr == null || compStStr.trim().length() < 2) {
-      throw new IllegalArgumentException("Input cannot be null.");
-    }
-
-    compStStr = compStStr.trim();
-    int splitIndex = compStStr.lastIndexOf(SPLIT_CHAR);
-    if (splitIndex == -1 || splitIndex == 0 || splitIndex == compStStr.length() - 1) {
-      throw new IllegalArgumentException("Input does not appear to be well-formed.");
-    }
-    String compStr = compStStr.substring(0, splitIndex);
-    String stateStr = compStStr.substring(splitIndex + 1);
-
-    if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
-      throw new IllegalArgumentException("Component " + compStr + " specified" +
-          " in command order does not exist");
-    }
-
-    State state = State.valueOf(stateStr);
-    if (state != State.STARTED && state != State.INSTALLED) {
-      throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED.");
-    }
-    return new ComponentState(compStr, state);
-  }
-
-  // dependency is still on component level, but not package level
-  // so use component name to check dependency, not component-package
-  public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) {
-    if (!dependencies.containsKey(command)) {
-      return true;
-    }
-    List<ComponentState> required = new ArrayList<>();
-    if (dependencies.get(command).containsKey(component)) {
-      required.addAll(dependencies.get(command).get(component));
-    }
-    String prefix = rolePrefixMap.get(component);
-    if (prefix != null && dependencies.get(command).containsKey(prefix)) {
-      required.addAll(dependencies.get(command).get(prefix));
-    }
-
-    for (ComponentState stateToMatch : required) {
-      for (ComponentInstanceState currState : currentStates) {
-        log.debug("Checking schedule {} {} against dependency {} is {}",
-            component, command, currState.getComponentName(), currState.getState());
-        if (currState.getComponentName().equals(stateToMatch.componentName) ||
-            (prefixRoleMap.containsKey(stateToMatch.componentName) &&
-                prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) {
-          if (currState.getState() != stateToMatch.state) {
-            if (stateToMatch.state == State.STARTED) {
-              log.info("Cannot schedule {} {} as dependency {} is {}",
-                  component, command, currState.getComponentName(), currState.getState());
-              return false;
-            } else {
-              //state is INSTALLED
-              if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
-                log.info("Cannot schedule {} {} as dependency {} is {}",
-                    component, command, currState.getComponentName(), currState.getState());
-                return false;
-              }
-            }
-          }
-        }
-      }
-    }
-    return true;
-  }
-
-  static class ComponentState {
-    public String componentName;
-    public State state;
-
-    public ComponentState(String componentName, State state) {
-      this.componentName = componentName;
-      this.state = state;
-    }
-  }
-
-  static class ComponentCommand {
-    public String componentName;
-    public Command command;
-
-    public ComponentCommand(String componentName, Command command) {
-      this.componentName = componentName;
-      this.command = command;
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java

@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** This class implements a simple state machine for component instances. */
-public class ComponentInstanceState {
-  public static final Logger log =
-      LoggerFactory.getLogger(ComponentInstanceState.class);
-  private static int MAX_FAILURE_TOLERATED = 3;
-  private static String INVALID_TRANSITION_ERROR =
-      "Result %s for command %s is not expected for component %s in state %s.";
-
-  private final String componentName;
-  private final ContainerId containerId;
-  private final String containerIdAsString;
-  private final String applicationId;
-  private State state = State.INIT;
-  private State targetState = State.STARTED;
-  private int failuresSeen = 0;
-  private Boolean configReported = false;
-  private long lastHeartbeat = 0;
-  private String ip;
-  private String hostname;
-  private ContainerState containerState;
-
-  private Map<String, State> pkgStatuses;
-  private String nextPkgToInstall;
-
-  private boolean stopInitiated;
-
-  public ComponentInstanceState(String componentName,
-      ContainerId containerId,
-      String applicationId) {
-    this(componentName, containerId, applicationId,
-        new TreeMap<String, State>());
-  }
-
-  public ComponentInstanceState(String componentName,
-      ContainerId containerId,
-      String applicationId, Map<String, State> pkgStatuses) {
-    this.componentName = componentName;
-    this.containerId = containerId;
-    this.containerIdAsString = containerId.toString();
-    this.applicationId = applicationId;
-    this.containerState = ContainerState.INIT;
-    this.lastHeartbeat = System.currentTimeMillis();
-    this.pkgStatuses = pkgStatuses;
-  }
-  
-  public String getComponentName() {
-    return componentName;
-  }
-
-  public Boolean getConfigReported() {
-    return configReported;
-  }
-
-  public void setConfigReported(Boolean configReported) {
-    this.configReported = configReported;
-  }
-
-  public ContainerState getContainerState() {
-    return containerState;
-  }
-
-  public void setContainerState(ContainerState containerState) {
-    this.containerState = containerState;
-  }
-
-  public long getLastHeartbeat() {
-    return lastHeartbeat;
-  }
-
-  /**
-   * Update the heartbeat, and change container state
-   * to mark as healthy if appropriate
-   * @param heartbeatTime last time the heartbeat was seen
-   * @return the current container state
-   */
-  public ContainerState heartbeat(long heartbeatTime) {
-    this.lastHeartbeat = heartbeatTime;
-    if(containerState == ContainerState.UNHEALTHY ||
-       containerState == ContainerState.INIT) {
-      containerState = ContainerState.HEALTHY;
-    }
-    return containerState;
-  }
-  
-
-  public ContainerId getContainerId() {
-    return containerId;
-  }
-
-  public void commandIssued(Command command) {
-    commandIssued(command, false);
-  }
-
-  public void commandIssued(Command command, boolean isInUpgradeMode) {
-    Command expected = getNextCommand(isInUpgradeMode);
-    if (expected != command) {
-      throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state);
-    }
-    if (expected == Command.INSTALL_ADDON) {
-      // for add on packages, the pkg must be nextPkgToInstall
-      State currentState = pkgStatuses.get(nextPkgToInstall);
-      log.debug("Command issued: component: {} is in {}", componentName,
-          currentState);
-      State nextState = currentState.getNextState(command);
-      pkgStatuses.put(nextPkgToInstall, nextState);
-      log.debug("Command issued: component: {} is now in {}", componentName,
-          nextState);
-    } else {
-      // for master package
-      state = state.getNextState(command);
-    }
-  }
-
-  public void applyCommandResult(CommandResult result, Command command,
-      String pkg) {
-    // if the heartbeat is for a package
-    // update that package's state in the component status
-    // and don't bother with the master pkg
-    if (StringUtils.isNotEmpty(pkg)
-        && !Component.MASTER_PACKAGE_NAME.equals(pkg)) {
-      log.debug("This result is for component: {} pkg: {}", componentName, pkg);
-      State previousPkgState = pkgStatuses.get(pkg);
-      log.debug("Currently component: {} pkg: {} is in state: {}",
-          componentName, pkg, previousPkgState.toString());
-      State nextPkgState = previousPkgState.getNextState(result);
-      pkgStatuses.put(pkg, nextPkgState);
-      log.debug("Component: {} pkg: {} next state: {}", componentName, pkg,
-          nextPkgState);
-    } else {
-      log.debug("This result is for component: {} master package",
-          componentName);
-      applyCommandResult(result, command);
-    }
-  }
-
-  public void applyCommandResult(CommandResult result, Command command) {
-    if (!this.state.couldHaveIssued(command)) {
-      throw new IllegalStateException("Invalid command " + command + " for state " + this.state);
-    }
-
-    try {
-      if (result == CommandResult.FAILED) {
-        failuresSeen++;
-      } else if (result == CommandResult.COMPLETED) {
-        failuresSeen = 0;
-      }
-      state = state.getNextState(result);
-    } catch (IllegalArgumentException e) {
-      String message = String.format(INVALID_TRANSITION_ERROR,
-                                     result.toString(),
-                                     command.toString(),
-                                     componentName,
-                                     state.toString());
-      log.warn(message);
-      throw new IllegalStateException(message);
-    }
-  }
-
-  public boolean hasPendingCommand() {
-    if (state.canIssueCommands() &&
-        state != targetState &&
-        failuresSeen < MAX_FAILURE_TOLERATED) {
-      return true;
-    }
-
-    return false;
-  }
-
-  public Command getNextCommand() {
-    return getNextCommand(false);
-  }
-
-  public Command getNextCommand(boolean isInUpgradeMode) {
-    if (!hasPendingCommand()) {
-      nextPkgToInstall = null;
-      return Command.NOP;
-    }
-
-    log.debug("In getNextCommand, checking for component: {} ", componentName);
-    // if the master pkg is just installed, check if any add on pkg need to be
-    // installed
-    nextPkgToInstall = null;
-    if (state == State.INSTALLED) {
-      for (Map.Entry<String, State> pkgStatus : pkgStatuses.entrySet()) {
-        String pkg = pkgStatus.getKey();
-        State pkgState = pkgStatus.getValue();
-        log.debug("In getNextCommand, pkg: {} is in {}", pkg, pkgState);
-        if (pkgState == State.INSTALLING) {
-          // first check if any pkg is install in progress, if so, wait
-          // so we don't need to do anything, just return NOP
-          log.debug("In getNextCommand, pkg: {} we are issuing NOP", pkg);
-          nextPkgToInstall = pkg;
-          return Command.NOP;
-        } else if (pkgState == State.INIT) {
-          // temporarily storing pkg here
-          // in case no pkg in 'installing' state
-          // will return the package to install
-          nextPkgToInstall = pkg;
-        }
-      }
-      // when we reach here, no pkg is in 'installing' state
-      if (nextPkgToInstall != null) {
-        // nextPkgToInstall != null means some pkg is in INIT state 
-        // issue 'install' to the pkg we have stored in nextPkgToInstall
-        log.debug("In getNextCommand, pkg: {} we are issuing install addon",
-            nextPkgToInstall);
-        return Command.INSTALL_ADDON;
-      }
-    }
-    return this.state.getSupportedCommand(isInUpgradeMode, stopInitiated);
-  }
-
-  public State getState() {
-    return state;
-  }
-
-  @VisibleForTesting
-  protected void setState(State state) {
-    this.state = state;
-  }
-
-  public State getTargetState() {
-    return targetState;
-  }
-
-  public void setTargetState(State targetState) {
-    this.targetState = targetState;
-  }
-
-  public String getNextPkgToInstall() {
-    return nextPkgToInstall;
-  }
-
-  public boolean isStopInitiated() {
-    return stopInitiated;
-  }
-
-  public void setStopInitiated(boolean stopInitiated) {
-    this.stopInitiated = stopInitiated;
-  }
-
-  @Override
-  public int hashCode() {
-    int hashCode = 1;
-
-    hashCode = hashCode ^ (componentName != null ? componentName.hashCode() : 0);
-    hashCode = hashCode ^ (containerIdAsString != null ? containerIdAsString.hashCode() : 0);
-    hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0);
-    return hashCode;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-
-    if (o == null || getClass() != o.getClass()) return false;
-
-    ComponentInstanceState that = (ComponentInstanceState) o;
-
-    if (this.componentName != null ?
-        !this.componentName.equals(that.componentName) : this.componentName != null) {
-      return false;
-    }
-
-    if (this.containerIdAsString != null ?
-        !this.containerIdAsString.equals(that.containerIdAsString) : this.containerIdAsString != null) {
-      return false;
-    }
-
-    if (this.applicationId != null ?
-        !this.applicationId.equals(that.applicationId) : this.applicationId != null) {
-      return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("ComponentInstanceState{");
-    sb.append("containerIdAsString='").append(containerIdAsString).append('\'');
-    sb.append(", state=").append(state);
-    sb.append(", failuresSeen=").append(failuresSeen);
-    sb.append(", lastHeartbeat=").append(lastHeartbeat);
-    sb.append(", containerState=").append(containerState);
-    sb.append(", componentName='").append(componentName).append('\'');
-    sb.append(", ip=").append(ip);
-    sb.append(", hostname='").append(hostname).append('\'');
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public String getIp() {
-    return ip;
-  }
-
-  public void setIp(String ip) {
-    this.ip = ip;
-  }
-
-  public String getHostname() {
-    return hostname;
-  }
-
-  public void setHostname(String hostname) {
-    this.hostname = hostname;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java

@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentHashMap;
-
-/** A simple tag provider that attempts to associate tags from 1-N to all container of a component */
-public class ComponentTagProvider {
-  private static final Logger log = LoggerFactory.getLogger(ComponentTagProvider.class);
-  private static String FREE = "free";
-  private final ConcurrentHashMap<String, ConcurrentHashMap<String, String>> allTags;
-
-  public ComponentTagProvider() {
-    allTags = new ConcurrentHashMap<String, ConcurrentHashMap<String, String>>();
-  }
-
-  /**
-   * Record an assigned tag to a container
-   *
-   * @param component
-   * @param containerId
-   * @param tag
-   */
-  public void recordAssignedTag(String component, String containerId, String tag) {
-    if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
-      Integer key = null;
-      try {
-        key = Integer.valueOf(tag);
-      } catch (NumberFormatException nfe) {
-        //ignore
-      }
-      if (key != null && key > 0) {
-        ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
-        synchronized (compTags) {
-          for (int index = 1; index <= key.intValue(); index++) {
-            String tempKey = new Integer(index).toString();
-            if (!compTags.containsKey(tempKey)) {
-              compTags.put(tempKey, FREE);
-            }
-          }
-          compTags.put(key.toString(), containerId);
-        }
-      }
-    }
-  }
-
-  /**
-   * Get a tag for container
-   *
-   * @param component
-   * @param containerId
-   *
-   * @return
-   */
-  public String getTag(String component, String containerId) {
-    if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
-      ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
-      synchronized (compTags) {
-        for (String key : compTags.keySet()) {
-          if (compTags.get(key).equals(containerId)) {
-            return key;
-          }
-        }
-        for (String key : compTags.keySet()) {
-          if (compTags.get(key).equals(FREE)) {
-            compTags.put(key, containerId);
-            return key;
-          }
-        }
-        String newKey = new Integer(compTags.size() + 1).toString();
-        compTags.put(newKey, containerId);
-        return newKey;
-      }
-    }
-    return "";
-  }
-
-  /**
-   * Release a tag associated with a container
-   *
-   * @param component
-   * @param containerId
-   */
-  public void releaseTag(String component, String containerId) {
-    if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
-      ConcurrentHashMap<String, String> compTags = allTags.get(component);
-      if (compTags != null) {
-        synchronized (compTags) {
-          for (String key : compTags.keySet()) {
-            if (compTags.get(key).equals(containerId)) {
-              compTags.put(key, FREE);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  private ConcurrentHashMap<String, String> getComponentSpecificTags(String component) {
-    if (!allTags.containsKey(component)) {
-      synchronized (allTags) {
-        if (!allTags.containsKey(component)) {
-          allTags.put(component, new ConcurrentHashMap<String, String>());
-        }
-      }
-    }
-    return allTags.get(component);
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java

@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum ContainerState {
-  INIT,           // Container is not net activated
-  HEALTHY,     // Agent is heartbeating
-  UNHEALTHY,      // Container is unhealthy - no heartbeat for some interval
-  HEARTBEAT_LOST;  // Container is lost - request a new instance
-
-  /**
-   * Indicates whether or not it is a valid state to produce a command.
-   *
-   * @return true if command can be issued for this state.
-   */
-  public boolean canIssueCommands() {
-    switch (this) {
-      case HEALTHY:
-        return true;
-      default:
-        return false;
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java

@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-/** Monitors the container state and heartbeats. */
-public class HeartbeatMonitor implements Runnable {
-  protected static final Logger log =
-      LoggerFactory.getLogger(HeartbeatMonitor.class);
-  private final int threadWakeupInterval; //1 minute
-  private final AgentProviderService provider;
-  private volatile boolean shouldRun = true;
-  private Thread monitorThread = null;
-
-  public HeartbeatMonitor(AgentProviderService provider, int threadWakeupInterval) {
-    this.provider = provider;
-    this.threadWakeupInterval = threadWakeupInterval;
-  }
-
-  public void shutdown() {
-    shouldRun = false;
-  }
-
-  public void start() {
-    log.info("Starting heartbeat monitor with interval {}", threadWakeupInterval);
-    monitorThread = new Thread(this);
-    monitorThread.start();
-  }
-
-  void join(long millis) throws InterruptedException {
-    if (isAlive()) {
-      monitorThread.join(millis);
-    }
-  }
-
-  public boolean isAlive() {
-    return monitorThread != null && monitorThread.isAlive();
-  }
-
-  @Override
-  public void run() {
-    while (shouldRun) {
-      try {
-        log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " +
-                  "milliseconds");
-        Thread.sleep(threadWakeupInterval);
-        doWork(System.currentTimeMillis());
-      } catch (InterruptedException ex) {
-        log.warn("Scheduler thread is interrupted going to stop", ex);
-        shouldRun = false;
-      } catch (Exception ex) {
-        log.warn("Exception received", ex);
-      } catch (Throwable t) {
-        log.warn("ERROR", t);
-      }
-    }
-  }
-
-  /**
-   * Every interval the current state of the container are checked. If the state is INIT or HEALTHY and no HB are
-   * received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not
-   * communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is
-   * declared unavailable
-   * @param now current time in milliseconds ... tests can set this explicitly
-   */
-  @VisibleForTesting
-  public void doWork(long now) {
-    Map<String, ComponentInstanceState> componentStatuses = provider.getComponentStatuses();
-    if (componentStatuses != null) {
-      for (String containerLabel : componentStatuses.keySet()) {
-        ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel);
-        long timeSinceLastHeartbeat = now - componentInstanceState.getLastHeartbeat();
-
-        if (timeSinceLastHeartbeat > threadWakeupInterval) {
-          switch (componentInstanceState.getContainerState()) {
-            case INIT:
-            case HEALTHY:
-              componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
-              log.warn(
-                  "Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
-                  componentInstanceState,
-                  componentInstanceState.getLastHeartbeat(),
-                  timeSinceLastHeartbeat);
-              break;
-            case UNHEALTHY:
-              if (timeSinceLastHeartbeat > threadWakeupInterval * 2) {
-                componentInstanceState.setContainerState(
-                    ContainerState.HEARTBEAT_LOST);
-                log.warn(
-                    "Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
-                    componentInstanceState, componentInstanceState.getLastHeartbeat(),
-                    timeSinceLastHeartbeat);
-                ContainerId containerId =
-                    componentInstanceState.getContainerId();
-                provider.lostContainer(containerLabel, containerId);
-              }
-              break;
-            case HEARTBEAT_LOST:
-              // unexpected case
-              log.warn("Heartbeat from lost component: {}", componentInstanceState);
-              break;
-          }
-            
-        }
-      }
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java

@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum State {
-  INIT,           // Not installed
-  INSTALLING,     // Being installed
-  INSTALLED,      // Installed (or stopped)
-  STARTING,       // Starting
-  STARTED,        // Started
-  INSTALL_FAILED, // Install failed, start failure in INSTALLED
-  UPGRADING,      // Undergoing upgrade, perform necessary pre-upgrade steps
-  UPGRADED,       // Pre-upgrade steps completed
-  STOPPING,       // Stop has been issued
-  STOPPED,        // Agent has stopped
-  TERMINATING;    // Terminate signal to ask the agent to kill itself
-                  // No need for state TERMINATED (as the agent is dead by then)
-
-  /**
-   * Indicates whether or not it is a valid state to produce a command.
-   *
-   * @return true if command can be issued for this state.
-   */
-  public boolean canIssueCommands() {
-    switch (this) {
-      case INSTALLING:
-      case STARTING:
-      case UPGRADING:
-      case STOPPING:
-      case TERMINATING:
-        return false;
-      default:
-        return true;
-    }
-  }
-
-  /**
-   * Returns valid command in this state.
-   *
-   * @return command allowed in this state.
-   */
-  public Command getSupportedCommand() {
-    return getSupportedCommand(false);
-  }
-
-  public Command getSupportedCommand(boolean isInUpgradeMode) {
-    return getSupportedCommand(isInUpgradeMode, false);
-  }
-
-  public Command getSupportedCommand(boolean isInUpgradeMode,
-      boolean stopInitiated) {
-    switch (this) {
-      case INIT:
-      case INSTALL_FAILED:
-        return Command.INSTALL;
-      case INSTALLED:
-        return Command.START;
-      case STARTED:
-      return isInUpgradeMode ? Command.UPGRADE : (stopInitiated) ? Command.STOP
-          : Command.NOP;
-      case UPGRADED:
-        return Command.STOP;
-      case STOPPED:
-        return Command.TERMINATE;
-      default:
-        return Command.NOP;
-    }
-  }
-
-  /**
-   * Returns next state based on the command result.
-   *
-   * @return next state.
-   */
-  public State getNextState(CommandResult result) throws IllegalArgumentException {
-    switch (result) {
-      case IN_PROGRESS:
-        if (this == State.INSTALLING || this == State.STARTING
-            || this == State.UPGRADING || this == State.STOPPING
-            || this == State.TERMINATING) {
-          return this;
-        } else {
-          throw new IllegalArgumentException(result + " is not valid for " + this);
-        }
-      case COMPLETED:
-        if (this == State.INSTALLING) {
-          return State.INSTALLED;
-        } else if (this == State.STARTING) {
-          return State.STARTED;
-        } else if (this == State.UPGRADING) {
-          return State.UPGRADED;
-        } else if (this == State.STOPPING) {
-          return State.STOPPED;
-        } else {
-          throw new IllegalArgumentException(result + " is not valid for " + this);
-        }
-      case FAILED:
-        if (this == State.INSTALLING) {
-          return State.INSTALL_FAILED;
-        } else if (this == State.STARTING) {
-          return State.INSTALLED;
-        } else if (this == State.UPGRADING) {
-          // if pre-upgrade failed, force stop now, so mark it upgraded
-          // what other options can be exposed to app owner?
-          return State.UPGRADED;
-        } else if (this == State.STOPPING) {
-          // if stop fails, force mark it stopped (and let container terminate)
-          return State.STOPPED;
-        } else if (this == State.STOPPED) {
-          // if in stopped state, force mark it as terminating
-          return State.TERMINATING;
-        } else {
-          throw new IllegalArgumentException(result + " is not valid for " + this);
-        }
-      default:
-        throw new IllegalArgumentException("Bad command result " + result);
-    }
-  }
-
-  /**
-   * Returns next state based on the command.
-   *
-   * @return next state.
-   */
-  public State getNextState(Command command) throws IllegalArgumentException {
-    switch (command) {
-      case INSTALL:
-        if (this == State.INIT || this == State.INSTALL_FAILED) {
-          return State.INSTALLING;
-        } else {
-          throw new IllegalArgumentException(command + " is not valid for " + this);
-        }
-      case INSTALL_ADDON:
-          if (this == State.INIT || this == State.INSTALL_FAILED) {
-            return State.INSTALLING;
-          } else {
-            throw new IllegalArgumentException(command + " is not valid for " + this);
-          }
-      case START:
-        if (this == State.INSTALLED) {
-          return State.STARTING;
-        } else {
-          throw new IllegalArgumentException(command + " is not valid for " + this);
-        }
-      case UPGRADE:
-        if (this == State.STARTED) {
-          return State.UPGRADING;
-        } else {
-          throw new IllegalArgumentException(command + " is not valid for " + this);
-        }
-      case STOP:
-        if (this == State.STARTED || this == State.UPGRADED) {
-          return State.STOPPING;
-        } else {
-          throw new IllegalArgumentException(command + " is not valid for " + this);
-        }
-      case TERMINATE:
-        if (this == State.STOPPED) {
-          return State.TERMINATING;
-        } else {
-          throw new IllegalArgumentException(command + " is not valid for " + this);
-        }
-      case NOP:
-        return this;
-      default:
-        throw new IllegalArgumentException("Bad command " + command);
-    }
-  }
-
-  public boolean couldHaveIssued(Command command) {
-    if ((this == State.INSTALLING && command == Command.INSTALL)
-        || (this == State.STARTING && command == Command.START)
-        || (this == State.UPGRADING && command == Command.UPGRADE)
-        || (this == State.STOPPING 
-           && (command == Command.STOP || command == Command.NOP))
-        || (this == State.TERMINATING && command == Command.TERMINATE)
-       ) {
-      return true;
-    }
-    return false;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java

@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-/**
- *  Component defined in master package metainfo.json
- */
-public abstract class AbstractComponent implements Validate {
-  public static final String TYPE_STANDARD = "STANDARD";
-  public static final String TYPE_DOCKER = "DOCKER";
-  public static final String TYPE_PYTHON = "PYTHON";
-  public static final String CATEGORY_MASTER = "MASTER";
-  public static final String CATEGORY_SLAVE = "SLAVE";
-  public static final String CATEGORY_CLIENT = "CLIENT";
-  public static final String MASTER_PACKAGE_NAME = "MASTER";
-
-  protected String name;
-  protected CommandScript commandScript;
-  protected List<ComponentCommand> commands = new ArrayList<>();
-
-  public AbstractComponent() {
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public CommandScript getCommandScript() {
-    return commandScript;
-  }
-
-  public void addCommandScript(CommandScript commandScript) {
-    this.commandScript = commandScript;
-  }
-
-  @JsonProperty("commands")
-  public List<ComponentCommand> getCommands() {
-    return commands;
-  }
-
-  public void setCommands(List<ComponentCommand> commands) {
-    this.commands = commands;
-  }
-
-  public void addCommand(ComponentCommand command) {
-    commands.add(command);
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb = new StringBuilder("{");
-    sb.append("\n\"name\": ").append(name);
-    sb.append(",\n\"commandScript\" :").append(commandScript);
-    sb.append('}');
-    return sb.toString();
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java

@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-import org.apache.commons.digester.Digester;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-
-/**
- * This abstract class provide common functionality to parse metainfo.json for
- * either master package or add on packages.
- */
-public abstract class AbstractMetainfoParser {
-  protected final GsonBuilder gsonBuilder = new GsonBuilder();
-  protected final Gson gson;
-  private static final Logger log = LoggerFactory
-      .getLogger(AbstractMetainfoParser.class);
-
-  public AbstractMetainfoParser() {
-    gson = gsonBuilder.create();
-  }
-
-  /**
-   * Convert to a JSON string
-   *
-   * @return a JSON string description
-   *
-   * @throws IOException Problems mapping/writing the object
-   */
-  public String toJsonString(Metainfo metaInfo) throws IOException {
-    return gson.toJson(metaInfo);
-  }
-
-  /**
-   * Convert from JSON
-   *
-   * @param json input
-   *
-   * @return the parsed JSON
-   *
-   * @throws IOException IO
-   */
-  public Metainfo fromJsonString(String json)
-      throws IOException {
-    return gson.fromJson(json, Metainfo.class);
-  }
-
-  /**
-   * Parse metainfo from an IOStream
-   *
-   * @param is
-   *
-   * @return
-   *
-   * @throws IOException
-   */
-  public Metainfo fromJsonStream(InputStream is) throws IOException {
-    log.debug("loading from xml stream");
-    StringWriter writer = new StringWriter();
-    IOUtils.copy(is, writer);
-    return fromJsonString(writer.toString());
-  }
-
-  /**
-   * Parse metainfo from an XML formatted IOStream
-   *
-   * @param metainfoStream
-   *
-   * @return
-   *
-   * @throws IOException
-   */
-  public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException {
-    log.debug("loading from xml stream");
-    Digester digester = new Digester();
-    digester.setValidating(false);
-
-    composeSchema(digester);
-
-    try {
-      return (Metainfo) digester.parse(metainfoStream);
-    } catch (IOException e) {
-      log.debug("IOException in metainfoparser during fromXmlStream: "
-          + e.getMessage());
-    } catch (SAXException e) {
-      log.debug("SAXException in metainfoparser during fromXmlStream: "
-          + e.getMessage());
-    } finally {
-      if (metainfoStream != null) {
-        metainfoStream.close();
-      }
-    }
-
-    return null;
-  }
-
-  /**
-   * Compose the schema for the metainfo
-   *
-   * @param Digester - The Digester object we passed in to compose the schema
-   *
-   * @return
-   *
-   * @throws IOException
-   */
-  abstract protected void composeSchema(Digester digester);
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java

@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application type defined in the metainfo
- */
-public abstract class AbstractMetainfoSchema implements Validate {
-  protected String name;
-  protected String comment;
-  protected String version;
-  protected List<ConfigFile> configFiles = new ArrayList<>();
-
-  public AbstractMetainfoSchema() {
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getComment() {
-    return comment;
-  }
-
-  public void setComment(String comment) {
-    this.comment = comment;
-  }
-
-  public String getVersion() {
-    return version;
-  }
-
-  public void setVersion(String version) {
-    this.version = version;
-  }
-
-  public void addConfigFile(ConfigFile configFile) {
-    this.configFiles.add(configFile);
-  }
-
-  @JsonProperty("configFiles")
-  public List<ConfigFile> getConfigFiles() {
-    return configFiles;
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java

@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-
-/**
- *
- */
-public class AddonPackageMetainfoParser extends AbstractMetainfoParser {
-
-  protected void composeSchema(Digester digester) {
-    digester.addObjectCreate("metainfo", Metainfo.class);
-    digester.addBeanPropertySetter("metainfo/schemaVersion");
-
-    digester.addObjectCreate("*/applicationPackage", ApplicationPackage.class);
-    digester.addBeanPropertySetter("*/applicationPackage/name");
-    digester.addBeanPropertySetter("*/applicationPackage/comment");
-    digester.addBeanPropertySetter("*/applicationPackage/version");
-
-    digester.addObjectCreate("*/component", ComponentsInAddonPackage.class);
-    digester.addBeanPropertySetter("*/component/name");
-    digester.addSetNext("*/component", "addComponent");
-
-    digester.addObjectCreate("*/commandScript", CommandScript.class);
-    digester.addBeanPropertySetter("*/commandScript/script");
-    digester.addBeanPropertySetter("*/commandScript/scriptType");
-    digester.addBeanPropertySetter("*/commandScript/timeout");
-    digester.addSetNext("*/commandScript", "addCommandScript");
-
-    digester.addObjectCreate("*/configFile", ConfigFile.class);
-    digester.addBeanPropertySetter("*/configFile/type");
-    digester.addBeanPropertySetter("*/configFile/fileName");
-    digester.addBeanPropertySetter("*/configFile/dictionaryName");
-    digester.addSetNext("*/configFile", "addConfigFile");
-
-    digester.addSetRoot("*/applicationPackage", "setApplicationPackage");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java

@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application type defined in the metainfo
- */
-public class Application extends AbstractMetainfoSchema {
-  String exportedConfigs;
-  List<ExportGroup> exportGroups = new ArrayList<>();
-  List<OSSpecific> osSpecifics = new ArrayList<>();
-  List<CommandOrder> commandOrders = new ArrayList<>();
-  List<Package> packages = new ArrayList<>();
-  private List<Component> components = new ArrayList<>();
-
-  public Application() {
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getComment() {
-    return comment;
-  }
-
-  public void setComment(String comment) {
-    this.comment = comment;
-  }
-
-  public String getVersion() {
-    return version;
-  }
-
-  public void setVersion(String version) {
-    this.version = version;
-  }
-
-  public String getExportedConfigs() {
-    return exportedConfigs;
-  }
-
-  public void setExportedConfigs(String exportedConfigs) {
-    this.exportedConfigs = exportedConfigs;
-  }
-
-  public void addConfigFile(ConfigFile configFile) {
-    this.configFiles.add(configFile);
-  }
-
-  @JsonProperty("configFiles")
-  public List<ConfigFile> getConfigFiles() {
-    return configFiles;
-  }
-
-  public void addComponent(Component component) {
-    components.add(component);
-  }
-
-  @JsonProperty("components")
-  public List<Component> getComponents() {
-    return components;
-  }
-
-  public void addExportGroup(ExportGroup exportGroup) {
-    exportGroups.add(exportGroup);
-  }
-
-  @JsonProperty("exportGroups")
-  public List<ExportGroup> getExportGroups() {
-    return exportGroups;
-  }
-
-  public void addOSSpecific(OSSpecific osSpecific) {
-    osSpecifics.add(osSpecific);
-  }
-
-  @JsonIgnore
-  public List<OSSpecific> getOSSpecifics() {
-    return osSpecifics;
-  }
-
-  public void addCommandOrder(CommandOrder commandOrder) {
-    commandOrders.add(commandOrder);
-  }
-
-  @JsonProperty("commandOrders")
-  public List<CommandOrder> getCommandOrders() {
-    return commandOrders;
-  }
-
-  public void addPackage(Package pkg) {
-    packages.add(pkg);
-  }
-
-  @JsonProperty("packages")
-  public List<Package> getPackages() {
-    return packages;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append(",\n\"name\": ").append(name);
-    sb.append(",\n\"comment\": ").append(comment);
-    sb.append(",\n\"version\" :").append(version);
-    sb.append(",\n\"components\" : {");
-    for (Component component : components) {
-      sb.append("\n").append(component.toString());
-    }
-    sb.append("\n},");
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public void validate(String version) throws SliderException {
-    if(SliderUtils.isUnset(version)) {
-      throw new BadCommandArgumentsException("schema version cannot be null");
-    }
-
-    Metainfo.checkNonNull(getName(), "name", "application");
-
-    Metainfo.checkNonNull(getVersion(), "version", "application");
-
-    if(getComponents().size() == 0) {
-      throw new SliderException("application must contain at least one component");
-    }
-
-    if(version.equals(Metainfo.VERSION_TWO_ZERO)) {
-      if(getPackages().size() > 0) {
-        throw new SliderException("packages is not supported in version " + version);
-      }
-    }
-
-    if(version.equals(Metainfo.VERSION_TWO_ONE)) {
-      if(getOSSpecifics().size() > 0) {
-        throw new SliderException("osSpecifics is not supported in version " + version);
-      }
-    }
-
-    for(CommandOrder co : getCommandOrders()) {
-      co.validate(version);
-    }
-
-    for(Component comp : getComponents()) {
-      comp.validate(version);
-    }
-
-    for(ConfigFile cf : getConfigFiles()) {
-      cf.validate(version);
-    }
-
-    for(ExportGroup eg : getExportGroups()) {
-      eg.validate(version);
-    }
-
-    for(Package pkg : getPackages()) {
-      pkg.validate(version);
-    }
-
-    for(OSSpecific os : getOSSpecifics()) {
-      os.validate(version);
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java

@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-public class ApplicationPackage extends AbstractMetainfoSchema{
-  private List<ComponentsInAddonPackage> components = new ArrayList<ComponentsInAddonPackage>();
-
-  public void addComponent(ComponentsInAddonPackage component) {
-    components.add(component);
-  }
-
-  // we must override getcomponent() as well. otherwise it is pointing to the
-  // overriden components of type List<Component>
-  public List<ComponentsInAddonPackage> getComponents(){
-    return this.components;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb = new StringBuilder("{");
-    sb.append("\n\"name\": ").append(name);
-    sb.append(",\n\"comment\": ").append(comment);
-    sb.append(",\n\"version\" :").append(version);
-    sb.append(",\n\"components\" : {");
-    for (ComponentsInAddonPackage component : components) {
-      sb.append("\n").append(component);
-    }
-    sb.append("\n},");
-    sb.append('}');
-    return sb.toString();
-  }
-
-  @Override
-  public void validate(String version) throws SliderException {
-    if (name == null || name.isEmpty()) {
-      throw new SliderException(
-          "Missing name in metainfo.json for add on packages");
-    }
-    if (components.isEmpty()) {
-      throw new SliderException(
-          "Missing components in metainfo.json for add on packages");
-    }
-    for (ComponentsInAddonPackage component : components) {
-      if (component.name == null || component.name.isEmpty()) {
-        throw new SliderException(
-            "Missing name of components in metainfo.json for add on packages");
-      }
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java

@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class CommandOrder implements Validate {
-  String command;
-  String requires;
-
-  public CommandOrder() {
-  }
-
-  public String getCommand() {
-    return command;
-  }
-
-  public void setCommand(String command) {
-    this.command = command;
-  }
-
-  public String getRequires() {
-    return requires;
-  }
-
-  public void setRequires(String requires) {
-    this.requires = requires;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append(",\n\"command\": ").append(command);
-    sb.append(",\n\"requires\": ").append(requires);
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getCommand(), "command", "package");
-    Metainfo.checkNonNull(getRequires(), "requires", "package");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java

@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- * CommandScript that implements all component commands
- */
-public class CommandScript implements Validate {
-  String script;
-  String scriptType;
-  long timeout;
-
-  public CommandScript() {
-
-  }
-
-  public String getScript() {
-    return script;
-  }
-
-  public void setScript(String script) {
-    this.script = script;
-  }
-
-  public String getScriptType() {
-    return scriptType;
-  }
-
-  public void setScriptType(String scriptType) {
-    this.scriptType = scriptType;
-  }
-
-  public long getTimeout() {
-    return timeout;
-  }
-
-  public void setTimeout(long timeout) {
-    this.timeout = timeout;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append(",\n\"script\": ").append(script);
-    sb.append(",\n\"scriptType\": ").append(scriptType);
-    sb.append(",\n\"timeout\" :").append(timeout);
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getScript(), "script", "commandScript");
-    Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java

@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Component defined in master package metainfo.json
- */
-public class Component extends AbstractComponent {
-
-  String category = CATEGORY_MASTER;
-  String publishConfig = Boolean.FALSE.toString();
-  String minInstanceCount = "0";
-  String maxInstanceCount;
-  String autoStartOnFailure = Boolean.FALSE.toString();
-  String appExports;
-  String compExports;
-  String type = TYPE_STANDARD;
-  List<ComponentExport> componentExports = new ArrayList<>();
-  List<DockerContainer> dockerContainers = new ArrayList<>();
-  List<ConfigFile> configFiles = new ArrayList<>();
-
-  public Component() {
-  }
-
-  public String getType() {
-    return type;
-  }
-
-  public void setType(String type) {
-    this.type = type;
-  }
-
-  public String getCategory() {
-    return category;
-  }
-
-  public void setCategory(String category) {
-    this.category = category;
-  }
-
-  public String getPublishConfig() {
-    return publishConfig;
-  }
-
-  public void setPublishConfig(String publishConfig) {
-    this.publishConfig = publishConfig;
-  }
-
-  public String getAutoStartOnFailure() {
-    return autoStartOnFailure;
-  }
-
-  public void setAutoStartOnFailure(String autoStartOnFailure) {
-    this.autoStartOnFailure = autoStartOnFailure;
-  }
-
-  public String getAppExports() {
-    return appExports;
-  }
-
-  public void setAppExports(String appExports) {
-    this.appExports = appExports;
-  }
-
-  public String getCompExports() {
-    return compExports;
-  }
-
-  public void setCompExports(String compExports) {
-    this.compExports = compExports;
-  }
-
-  public String getMinInstanceCount() {
-    return minInstanceCount;
-  }
-  
-  @JsonProperty("dockerContainers")
-  public List<DockerContainer> getDockerContainers() {
-     return this.dockerContainers;
-  }
-  
-  public Boolean getAutoStartOnFailureBoolean() {
-    if (SliderUtils.isUnset(getAutoStartOnFailure())) {
-      return Boolean.FALSE;
-    }
-
-    return Boolean.parseBoolean(getAutoStartOnFailure());
-  }
-
-  public int getMinInstanceCountInt() throws BadConfigException {
-    if (SliderUtils.isUnset(minInstanceCount)) {
-      return 0;
-    }
-
-    try {
-      return Integer.parseInt(minInstanceCount);
-    } catch (NumberFormatException nfe) {
-      throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
-    }
-  }
-
-  public int getMaxInstanceCountInt() throws BadConfigException {
-    if (SliderUtils.isUnset(maxInstanceCount)) {
-      return Integer.MAX_VALUE;
-    }
-
-    try {
-      return Integer.parseInt(maxInstanceCount);
-    } catch (NumberFormatException nfe) {
-      throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
-    }
-  }
-
-  public void setMinInstanceCount(String minInstanceCount) {
-    this.minInstanceCount = minInstanceCount;
-  }
-
-  public String getMaxInstanceCount() {
-    return maxInstanceCount;
-  }
-
-  public void setMaxInstanceCount(String maxInstanceCount) {
-    this.maxInstanceCount = maxInstanceCount;
-  }
-
-  public void addComponentExport(ComponentExport export) {
-    componentExports.add(export);
-  }
-
-  public List<ComponentExport> getComponentExports() {
-    return componentExports;
-  }
-
-  public Boolean getRequiresAutoRestart() {
-    return Boolean.parseBoolean(this.autoStartOnFailure);
-  }
-
-  public void addConfigFile(ConfigFile configFile) {
-    this.configFiles.add(configFile);
-  }
-
-  @JsonProperty("configFiles")
-  public List<ConfigFile> getConfigFiles() {
-    return configFiles;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append("\n\"name\": ").append(name);
-    sb.append(",\n\"category\": ").append(category);
-    sb.append(",\n\"commandScript\" :").append(commandScript);
-    for(DockerContainer dc : dockerContainers){
-      sb.append(",\n\"container\" :").append(dc.toString());
-    }    
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "component");
-    Metainfo.checkNonNull(getCategory(), "category", "component");
-    if (!getCategory().equals(CATEGORY_MASTER)
-        && !getCategory().equals(CATEGORY_SLAVE)
-        && !getCategory().equals(CATEGORY_CLIENT)) {
-      throw new SliderException("Invalid category for the component " + getCategory());
-    }
-
-    Metainfo.checkNonNull(getType(), "type", "component");
-    if (!getType().equals(TYPE_DOCKER)
-        && !getType().equals(TYPE_STANDARD)) {
-      throw new SliderException("Invalid type for the component " + getType());
-    }
-
-    if (version.equals(Metainfo.VERSION_TWO_ZERO)) {
-      if (getType().equals(TYPE_DOCKER)) {
-        throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO);
-      }
-
-      if (getCommands().size() > 0) {
-        throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO);
-      }
-    }
-
-    if (commandScript != null) {
-      commandScript.validate(version);
-    }
-
-    if (version.equals(Metainfo.VERSION_TWO_ONE)) {
-      for (ComponentCommand cc : getCommands()) {
-        cc.validate(version);
-      }
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java

@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents the metadata associated with the application.
- */
-public class ComponentCommand implements Validate {
-  protected static final Logger
-      log = LoggerFactory.getLogger(ComponentCommand.class);
-
-
-  private String exec;
-  private String name = "START";
-  private String type = "SHELL";
-
-  /**
-   * Creator.
-   */
-  public ComponentCommand() {
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setExec(String exec) {
-    this.exec = exec;
-  }
-
-  public String getExec() {
-    return exec;
-  }
-
-  public void setType(String type) {
-    this.type = type;
-  }
-
-  public String getType() {
-    return type;
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "componentCommand");
-
-    Metainfo.checkNonNull(getType(), "version", "application");
-  }
-
-  public static ComponentCommand getDefaultComponentCommand() {
-    ComponentCommand cc = new ComponentCommand();
-    cc.setExec("DEFAULT");
-    return cc;
-  }
-
-  public static ComponentCommand getDefaultComponentCommand(String commandName) {
-    ComponentCommand cc = new ComponentCommand();
-    cc.setExec("DEFAULT");
-    cc.setName(commandName);
-    return cc;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java

@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-/**
- *
- */
-public class ComponentExport {
-  String name;
-  String value;
-
-  public ComponentExport() {
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getValue() {
-    return value;
-  }
-
-  public void setValue(String value) {
-    this.value = value;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append(",\n\"name\": ").append(name);
-    sb.append(",\n\"value\": ").append(value);
-    sb.append('}');
-    return sb.toString();
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java

@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-public class ComponentsInAddonPackage extends AbstractComponent {
-
-  @Override
-  public void validate(String version) throws SliderException {
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java

@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class ConfigFile implements Validate {
-  String type;
-  String fileName;
-  String dictionaryName;
-
-  public ConfigFile() {
-  }
-
-  public String getType() {
-    return type;
-  }
-
-  public void setType(String type) {
-    this.type = type;
-  }
-
-  public String getFileName() {
-    return fileName;
-  }
-
-  public void setFileName(String fileName) {
-    this.fileName = fileName;
-  }
-
-  public String getDictionaryName() {
-    return dictionaryName;
-  }
-
-  public void setDictionaryName(String dictionaryName) {
-    this.dictionaryName = dictionaryName;
-  }
-
-  public void validate(String version) throws SliderException {
-
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java

@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application default config
- */
-public class DefaultConfig {
-  List<PropertyInfo> propertyInfos;
-
-  public DefaultConfig() {
-    propertyInfos = new ArrayList<PropertyInfo>();
-  }
-
-  public void addPropertyInfo(PropertyInfo propertyInfo) {
-    propertyInfos.add(propertyInfo);
-  }
-
-  public List<PropertyInfo> getPropertyInfos() {
-    return propertyInfos;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java

@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class DefaultConfigParser {
-
-  public DefaultConfig parse(InputStream configFileStream) throws IOException {
-    Digester digester = new Digester();
-    digester.setValidating(false);
-
-    digester.addObjectCreate("configuration", DefaultConfig.class);
-
-    digester.addObjectCreate("*/property", PropertyInfo.class);
-    digester.addBeanPropertySetter("*/property/name");
-    digester.addBeanPropertySetter("*/property/value");
-    digester.addBeanPropertySetter("*/property/description");
-    digester.addSetNext("*/property", "addPropertyInfo");
-
-    try {
-      return (DefaultConfig) digester.parse(configFileStream);
-    } catch (IOException e) {
-
-    } catch (SAXException e) {
-
-    } finally {
-      configFileStream.close();
-    }
-
-    return null;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java

@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Represents a docker container
- */
-public class DockerContainer implements Validate {
-  protected static final Logger
-      log = LoggerFactory.getLogger(DockerContainer.class);
-
-  private String name;
-  private String image;
-  private String network;
-  private String useNetworkScript;
-  private String options;
-  private List<DockerContainerMount> mounts = new ArrayList<>();
-  private List<DockerContainerPort> ports = new ArrayList<>();
-  private String statusCommand;
-  private String startCommand;
-  private String commandPath;
-  private String additionalParam;
-  private String runPrivilegedContainer;
-  private List<DockerContainerInputFile> inputFiles = new ArrayList<>();
-  private List<ConfigFile> configFiles = new ArrayList<>();
-
-  public DockerContainer() {
-  }
-
-  @JsonProperty("mounts")
-  public List<DockerContainerMount> getMounts() { return this.mounts; }
-
-  @JsonProperty("ports")
-  public List<DockerContainerPort> getPorts() {
-    return this.ports;
-  }
-
-  @JsonProperty("inputFiles")
-  public List<DockerContainerInputFile> getInputFiles() {
-    return this.inputFiles;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getImage() {
-    return image;
-  }
-
-  public void setImage(String image) {
-    this.image = image;
-  }
-
-  public String getNetwork() {
-    return network;
-  }
-
-  public void setNetwork(String network) {
-    this.network = network;
-  }
-
-  public String getUseNetworkScript() {
-    return useNetworkScript;
-  }
-
-  public void setUseNetworkScript(String useNetworkScript) {
-    this.useNetworkScript = useNetworkScript;
-  }
-
-  public String getOptions() {
-    return options;
-  }
-
-  public void setOptions(String options) {
-    this.options = options;
-  }
-
-  @Override
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "dockerContainer");
-    Metainfo.checkNonNull(getImage(), "image", "dockerContainer");
-    for (DockerContainerMount dcm : getMounts()) {
-      dcm.validate(version);
-    }
-    for (DockerContainerPort dcp : getPorts()) {
-      dcp.validate(version);
-    }
-  }
-
-  @JsonProperty("statusCommand")
-  public String getStatusCommand() {
-    return statusCommand;
-  }
-
-  @JsonProperty("statusCommand")
-  public void setStatusCommand(String statusCommand) {
-    this.statusCommand = statusCommand;
-  }
-
-  public String getCommandPath() {
-    return commandPath;
-  }
-
-  public void setCommandPath(String commandPath) {
-    this.commandPath = commandPath;
-  }
-
-  public String getAdditionalParam() {
-    return additionalParam;
-  }
-
-  public void setAdditionalParam(String additionalParam) {
-    this.additionalParam = additionalParam;
-  }
-
-  @JsonProperty("startCommand")
-  public String getStartCommand() {
-    return startCommand;
-  }
-
-  @JsonProperty("startCommand")
-  public void setStartCommand(String startCommand) {
-    this.startCommand = startCommand;
-  }
-
-  @JsonProperty("runPrivilegedContainer")
-  public String getRunPrivilegedContainer() {
-    return runPrivilegedContainer;
-  }
-
-  @JsonProperty("runPrivilegedContainer")
-  public void setRunPrivilegedContainer(String runPrivilegedContainer) {
-    this.runPrivilegedContainer = runPrivilegedContainer;
-  }
-
-  public List<ConfigFile> getConfigFiles() {
-    return configFiles;
-  }
-
-  public void setConfigFiles(List<ConfigFile> configFiles) {
-    this.configFiles = configFiles;
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder result = new StringBuilder("DockerContainer [name=")
-        .append(name).append(", image=").append(image).append(", options=")
-        .append(options).append(", mounts=").append(mounts).append(", ports=")
-        .append(ports).append(", statusCommand=").append(statusCommand)
-        .append(", commandPath=").append(commandPath)
-        .append(", additionalParam=").append(additionalParam)
-        .append(", inputFiles=").append(inputFiles).append(", startCommand=")
-        .append(startCommand).append(", runPriviledgedContainer=")
-        .append(runPrivilegedContainer).append(", configFiles=")
-        .append(configFiles).append("]");
-    return result.toString();
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java

@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DockerContainerInputFile {
-  protected static final Logger log = LoggerFactory
-      .getLogger(DockerContainerInputFile.class);
-
-  private String containerPath;
-  private String fileLocalPath;
-
-  public DockerContainerInputFile() {
-  }
-
-  public String getContainerMount() {
-    return containerPath;
-  }
-
-  public void setContainerMount(String containerMount) {
-    this.containerPath = containerMount;
-  }
-
-  public String getFileLocalPath() {
-    return fileLocalPath;
-  }
-
-  public void setFileLocalPath(String fileLocalPath) {
-    this.fileLocalPath = fileLocalPath;
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java

@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents a docker container mount
- */
-public class DockerContainerMount implements Validate {
-  protected static final Logger
-      log = LoggerFactory.getLogger(DockerContainerMount.class);
-
-
-  private String containerMount;
-  private String hostMount;
-
-  public DockerContainerMount() {
-  }
-
-  public String getContainerMount() {
-    return containerMount;
-  }
-
-  public void setContainerMount(String containerMount) {
-    this.containerMount = containerMount;
-  }
-
-  public String getHostMount() {
-    return hostMount;
-  }
-
-  public void setHostMount(String hostMount) {
-    this.hostMount = hostMount;
-  }
-
-  @Override
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount");
-    Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java

@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents a docker container port
- */
-public class DockerContainerPort implements Validate {
-  protected static final Logger
-      log = LoggerFactory.getLogger(DockerContainerPort.class);
-
-
-  private String containerPort;
-  private String hostPort;
-
-  public DockerContainerPort() {
-  }
-
-  public String getContainerPort() {
-    return containerPort;
-  }
-
-  public void setContainerPort(String containerPort) {
-    this.containerPort = containerPort;
-  }
-
-  public String getHostPort() {
-    return hostPort;
-  }
-
-  public void setHostPort(String hostPort) {
-    this.hostPort = hostPort;
-  }
-
-  @Override
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort");
-    Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort");
-  }
-
-  @Override
-  public String toString() {
-    return "DockerContainerPort [containerPort=" + containerPort
-         + ", hostPort=" + hostPort + "]";
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java

@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class Export implements Validate {
-  String name;
-  String value;
-
-  public Export() {
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getValue() {
-    return value;
-  }
-
-  public void setValue(String value) {
-    this.value = value;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append(",\n\"name\": ").append(name);
-    sb.append(",\n\"value\": ").append(value);
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "export");
-    Metainfo.checkNonNull(getValue(), "value", "export");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java

@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class ExportGroup implements Validate {
-  String name;
-  List<Export> exports;
-
-  public ExportGroup() {
-    exports = new ArrayList<Export>();
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public void addExport(Export export) {
-    exports.add(export);
-  }
-
-  public List<Export> getExports() {
-    return exports;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb =
-        new StringBuilder("{");
-    sb.append(",\n\"name\": ").append(name);
-    sb.append(",\n\"exports\" : {");
-    for (Export export : exports) {
-      sb.append("\n").append(export);
-    }
-    sb.append("\n},");
-    sb.append('}');
-    return sb.toString();
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "exportGroup");
-    for(Export exp : getExports()) {
-      exp.validate(version);
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java

@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application metainfo uber class
- */
-public class Metainfo {
-  protected static final Logger log =
-      LoggerFactory.getLogger(Metainfo.class);
-  public static String VERSION_TWO_ZERO = "2.0";
-  public static String VERSION_TWO_ONE = "2.1";
-
-  String schemaVersion;
-  ApplicationPackage applicationPackage;
-  Application application;
-
-  public String getSchemaVersion() {
-    return schemaVersion;
-  }
-
-  public void setSchemaVersion(String schemaVersion) {
-    this.schemaVersion = schemaVersion;
-  }
-
-  public ApplicationPackage getApplicationPackage() {
-    return applicationPackage;
-  }
-
-  public void setApplicationPackage(ApplicationPackage pkg) {
-    this.applicationPackage = pkg;
-  }
-
-  public Application getApplication() {
-    return application;
-  }
-
-  public void setApplication(Application application) {
-    this.application = application;
-  }
-
-  public Component getApplicationComponent(String roleGroup) {
-    if (application == null) {
-      log.error("Malformed app definition: Expect application as the top level element for metainfo");
-    } else {
-      for (Component component : application.getComponents()) {
-        if (component.getName().equals(roleGroup)) {
-          return component;
-        }
-      }
-    }
-    return null;
-  }
-
-  public List<ConfigFile> getComponentConfigFiles(String roleGroup) {
-    List<ConfigFile> componentConfigFiles = new ArrayList<>();
-    componentConfigFiles.addAll(application.getConfigFiles());
-    Component component = getApplicationComponent(roleGroup);
-    if (component != null) {
-      componentConfigFiles.addAll(component.getConfigFiles());
-    }
-    return componentConfigFiles;
-  }
-
-  public void validate() throws SliderException {
-    if (!VERSION_TWO_ONE.equals(schemaVersion) &&
-        !VERSION_TWO_ZERO.equals(schemaVersion)) {
-      throw new SliderException("Unsupported version " + getSchemaVersion());
-    }
-    if (application != null) {
-      application.validate(schemaVersion);
-    }
-    if (applicationPackage != null) {
-      applicationPackage.validate(schemaVersion);
-    }
-  }
-
-  public static void checkNonNull(String value, String field, String type) throws SliderException {
-    if (SliderUtils.isUnset(value)) {
-      throw new SliderException(type + "." + field + " cannot be null");
-    }
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder builder = new StringBuilder();
-    builder.append("Metainfo [schemaVersion=");
-    builder.append(schemaVersion);
-    builder.append(", applicationPackage=");
-    builder.append(applicationPackage);
-    builder.append(", application=");
-    builder.append(application);
-    builder.append("]");
-    return builder.toString();
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java

@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-
-/**
- *
- */
-public class MetainfoParser extends AbstractMetainfoParser{
-  
-  protected void composeSchema(Digester digester){
-    digester.addObjectCreate("metainfo", Metainfo.class);
-    digester.addBeanPropertySetter("metainfo/schemaVersion");
-
-    digester.addObjectCreate("*/application", Application.class);
-    digester.addBeanPropertySetter("*/application/name");
-    digester.addBeanPropertySetter("*/application/comment");
-    digester.addBeanPropertySetter("*/application/version");
-    digester.addBeanPropertySetter("*/application/exportedConfigs");
-
-    digester.addObjectCreate("*/commandOrder", CommandOrder.class);
-    digester.addBeanPropertySetter("*/commandOrder/command");
-    digester.addBeanPropertySetter("*/commandOrder/requires");
-    digester.addSetNext("*/commandOrder", "addCommandOrder");
-
-    digester.addObjectCreate("*/exportGroup", ExportGroup.class);
-    digester.addBeanPropertySetter("*/exportGroup/name");
-    digester.addObjectCreate("*/export", Export.class);
-    digester.addBeanPropertySetter("*/export/name");
-    digester.addBeanPropertySetter("*/export/value");
-    digester.addSetNext("*/export", "addExport");
-    digester.addSetNext("*/exportGroup", "addExportGroup");
-
-    digester.addObjectCreate("*/component", Component.class);
-    digester.addBeanPropertySetter("*/component/name");
-    digester.addBeanPropertySetter("*/component/category");
-    digester.addBeanPropertySetter("*/component/publishConfig");
-    digester.addBeanPropertySetter("*/component/minInstanceCount");
-    digester.addBeanPropertySetter("*/component/maxInstanceCount");
-    digester.addBeanPropertySetter("*/component/autoStartOnFailure");
-    digester.addBeanPropertySetter("*/component/appExports");
-    digester.addBeanPropertySetter("*/component/compExports");
-    digester.addObjectCreate("*/componentExport", ComponentExport.class);
-    digester.addBeanPropertySetter("*/componentExport/name");
-    digester.addBeanPropertySetter("*/componentExport/value");
-    digester.addSetNext("*/componentExport", "addComponentExport");
-    digester.addSetNext("*/component", "addComponent");
-
-    digester.addObjectCreate("*/commandScript", CommandScript.class);
-    digester.addBeanPropertySetter("*/commandScript/script");
-    digester.addBeanPropertySetter("*/commandScript/scriptType");
-    digester.addBeanPropertySetter("*/commandScript/timeout");
-    digester.addSetNext("*/commandScript", "addCommandScript");
-
-    digester.addObjectCreate("*/command", ComponentCommand.class);
-    digester.addBeanPropertySetter("*/command/exec");
-    digester.addBeanPropertySetter("*/command/name");
-    digester.addBeanPropertySetter("*/command/type");
-    digester.addSetNext("*/command", "addCommand");
-
-    digester.addObjectCreate("*/osSpecific", OSSpecific.class);
-    digester.addBeanPropertySetter("*/osSpecific/osType");
-    digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class);
-    digester.addBeanPropertySetter("*/osSpecific/packages/package/type");
-    digester.addBeanPropertySetter("*/osSpecific/packages/package/name");
-    digester.addSetNext("*/osSpecific/packages/package", "addOSPackage");
-    digester.addSetNext("*/osSpecific", "addOSSpecific");
-
-    digester.addObjectCreate("*/application/packages/package", Package.class);
-    digester.addBeanPropertySetter("*/application/packages/package/type");
-    digester.addBeanPropertySetter("*/application/packages/package/name");
-    digester.addSetNext("*/application/packages/package", "addPackage");
-
-    digester.addObjectCreate("*/configFile", ConfigFile.class);
-    digester.addBeanPropertySetter("*/configFile/type");
-    digester.addBeanPropertySetter("*/configFile/fileName");
-    digester.addBeanPropertySetter("*/configFile/dictionaryName");
-    digester.addSetNext("*/configFile", "addConfigFile");
-
-    digester.addSetRoot("*/application", "setApplication");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java

@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class OSPackage implements Validate {
-  String type;
-  String name;
-
-  public OSPackage() {
-  }
-
-  public String getType() {
-    return type;
-  }
-
-  public void setType(String type) {
-    this.type = type;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "osPackage");
-    Metainfo.checkNonNull(getType(), "type", "osPackage");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java

@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class OSSpecific implements Validate {
-  String osType;
-  List<OSPackage> packages;
-
-  public OSSpecific() {
-    packages = new ArrayList<OSPackage>();
-  }
-
-  public String getOsType() {
-    return osType;
-  }
-
-  public void setOsType(String osType) {
-    this.osType = osType;
-  }
-
-  public void addOSPackage(OSPackage osPackage) {
-    packages.add(osPackage);
-  }
-
-  public List<OSPackage> getPackages() {
-    return packages;
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getOsType(), "osType", "osSpecific");
-    for (OSPackage opkg : getPackages()) {
-      opkg.validate(version);
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java

@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents package description.
- */
-public class Package implements Validate {
-  protected static final Logger
-      log = LoggerFactory.getLogger(Package.class);
-
-
-  private String name;
-  private String type;
-
-  public Package() {
-  }
-
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public String getType() {
-    return type;
-  }
-
-  public void setType(String type) {
-    this.type = type;
-  }
-
-  public void validate(String version) throws SliderException {
-    Metainfo.checkNonNull(getName(), "name", "package");
-    Metainfo.checkNonNull(getType(), "type", "package");
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java

@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-/**
- * Application config property info
- */
-public class PropertyInfo {
-  String name;
-  String value;
-  String description;
-
-  public PropertyInfo() {
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getValue() {
-    return value;
-  }
-
-  public void setValue(String value) {
-    this.value = value;
-  }
-
-  public String getDescription() {
-    return description;
-  }
-
-  public void setDescription(String description) {
-    this.description = description;
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java

@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- * Implementer provides a validate method
- */
-public interface Validate {
-
-  public void validate(String version) throws SliderException;
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md

@@ -1,22 +0,0 @@
-<!---
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-  
-# todo list
-
-* Retry on failure
-  * Agent can toleate a configurable number of failures (e.g. 3) before giving up
-* Agent should separate out hostname and label that is received for registration

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java

@@ -20,12 +20,8 @@ package org.apache.slider.server.appmaster;
 
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.health.HealthCheckRegistry;
-import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
-import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
-import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.google.common.base.Preconditions;
 import com.google.protobuf.BlockingService;
-
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -36,8 +32,14 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
 import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
@@ -66,18 +68,9 @@ import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
-
 import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
@@ -122,18 +115,16 @@ import org.apache.slider.providers.ProviderCompleted;
 import org.apache.slider.providers.ProviderRole;
 import org.apache.slider.providers.ProviderService;
 import org.apache.slider.providers.SliderProviderFactory;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.providers.agent.AgentProviderService;
 import org.apache.slider.providers.slideram.SliderAMClientProvider;
 import org.apache.slider.providers.slideram.SliderAMProviderService;
 import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
-import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
-import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
-import org.apache.slider.server.appmaster.actions.QueueExecutor;
-import org.apache.slider.server.appmaster.actions.QueueService;
 import org.apache.slider.server.appmaster.actions.ActionStopSlider;
 import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers;
 import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
+import org.apache.slider.server.appmaster.actions.QueueExecutor;
+import org.apache.slider.server.appmaster.actions.QueueService;
+import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
 import org.apache.slider.server.appmaster.actions.RenewingAction;
 import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
 import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
@@ -143,26 +134,24 @@ import org.apache.slider.server.appmaster.management.YarnServiceHealthCheck;
 import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
 import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
 import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
 import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
 import org.apache.slider.server.appmaster.rpc.RpcBinder;
 import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
 import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.rpc.SliderIPCService;
 import org.apache.slider.server.appmaster.security.SecurityConfiguration;
 import org.apache.slider.server.appmaster.state.AppState;
 import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
 import org.apache.slider.server.appmaster.state.ContainerAssignment;
 import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
 import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.web.AgentService;
-import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
 import org.apache.slider.server.appmaster.web.SliderAMWebApp;
 import org.apache.slider.server.appmaster.web.WebAppApi;
 import org.apache.slider.server.appmaster.web.WebAppApiImpl;
+import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
 import org.apache.slider.server.appmaster.web.rest.RestPaths;
 import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory;
 import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
@@ -181,7 +170,6 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
@@ -199,6 +187,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
 
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
+
 /**
  * This is the AM, which directly implements the callbacks from the AM and NM
  */
@@ -765,11 +756,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
                                     appMasterContainerID.toString(),
                                     clustername);
       certificateManager.setPassphrase(instanceDefinition.getPassphrase());
- 
-      if (component.getOptionBool(
-          AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
-        uploadServerCertForLocalization(clustername, fs);
-      }
 
       // Web service endpoints: initialize
       WebAppApiImpl webAppApi =
@@ -784,10 +770,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
               contentCache);
       initAMFilterOptions(serviceConf);
 
-      if (providerService instanceof AgentProviderService) {
-        // start the agent web app
-        startAgentWebApp(appInformation, serviceConf, webAppApi);
-      }
       int webAppPort = deployWebApplication(webAppApi);
 
       String scheme = WebAppUtils.HTTP_PREFIX;
@@ -1165,26 +1147,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     return portScanner.getAvailablePort();
   }
 
-  private void uploadServerCertForLocalization(String clustername,
-                                               SliderFileSystem fs)
-      throws IOException {
-    Path certsDir = fs.buildClusterSecurityDirPath(clustername);
-    if (!fs.getFileSystem().exists(certsDir)) {
-      fs.getFileSystem().mkdirs(certsDir,
-        new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
-    }
-    Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME);
-    if (!fs.getFileSystem().exists(destPath)) {
-      fs.getFileSystem().copyFromLocalFile(
-          new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()),
-          destPath);
-      log.info("Uploaded server cert to localization path {}", destPath);
-    }
-
-    fs.getFileSystem().setPermission(destPath,
-        new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
-  }
-
   protected void login(String principal, File localKeytabFile)
       throws IOException, SliderException {
     log.info("Logging in as {} with keytab {}", principal, localKeytabFile);
@@ -1220,53 +1182,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     }
   }
 
-  /**
-   * Set up and start the agent web application 
-   * @param appInformation application information
-   * @param serviceConf service configuration
-   * @param webAppApi web app API instance to bind to
-   * @throws IOException
-   */
-  private void startAgentWebApp(MapOperations appInformation,
-      Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException, SliderException {
-    URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
-    StringBuilder sb = new StringBuilder("AM classpath:");
-    for (URL url : urls) {
-      sb.append("\n").append(url.toString());
-    }
-    LOG_YARN.debug(sb.append("\n").toString());
-    initAMFilterOptions(serviceConf);
-
-
-    // Start up the agent web app and track the URL for it
-    MapOperations appMasterConfig = getInstanceDefinition()
-        .getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
-    AgentWebApp agentWebApp = AgentWebApp.$for(AgentWebApp.BASE_PATH,
-        webAppApi,
-        RestPaths.AGENT_WS_CONTEXT)
-        .withComponentConfig(appMasterConfig)
-        .withPort(getPortToRequest())
-        .withSecuredPort(getPortToRequest())
-            .start();
-    agentOpsUrl =
-        "https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
-    agentStatusUrl =
-        "https://" + appMasterHostname + ":" + agentWebApp.getPort();
-    AgentService agentService =
-      new AgentService("slider-agent", agentWebApp);
-
-    agentService.init(serviceConf);
-    agentService.start();
-    addService(agentService);
-
-    appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/");
-    appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/");
-    appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT,
-                       agentWebApp.getPort());
-    appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT,
-                       agentWebApp.getSecuredPort());
-  }
-
   /**
    * Set up the AM filter 
    * @param serviceConf configuration to patch
@@ -1327,17 +1242,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
         serviceRecord);
 
     // provider service dynamic definitions.
-    if (providerService instanceof AgentProviderService) {
-      URL agentOpsURI = new URL(agentOpsUrl);
-      URL agentStatusURI = new URL(agentStatusUrl);
-      ((AgentProviderService)providerService).applyInitialRegistryDefinitions(
-          amWebURI,
-          agentOpsURI,
-          agentStatusURI,
-          serviceRecord);
-    } else {
-      providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord);
-    }
+    providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord);
+
 
     // set any provided attributes
     setProvidedServiceRecordAttributes(
@@ -1800,11 +1706,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     }
     LOG_YARN.info("Final list of containers to be upgraded (total {}) : {}",
         containers.size(), containers);
-    if (providerService instanceof AgentProviderService) {
-      AgentProviderService agentProviderService = (AgentProviderService) providerService;
-      agentProviderService.setInUpgradeMode(true);
-      agentProviderService.addUpgradeContainers(containers);
-    }
   }
 
   // create a reverse map of roles -> set of all live containers
@@ -1968,11 +1869,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    * Shutdown operation: release all containers
    */
   private void releaseAllContainers() {
-    if (providerService instanceof AgentProviderService) {
-      log.info("Setting stopInitiated flag to true");
-      AgentProviderService agentProviderService = (AgentProviderService) providerService;
-      agentProviderService.setAppStopInitiated(true);
-    }
     // Add the sleep here (before releasing containers) so that applications get
     // time to perform graceful shutdown
     try {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java

@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
-import org.apache.slider.server.services.workflow.ClosingService;
-import org.apache.slider.server.services.workflow.WorkflowCompositeService;
-
-/**
- * agent service gives the agent webapp lifecycle integration
- */
-public class AgentService extends ClosingService<AgentWebApp> {
-
-
-  public AgentService(String name) {
-    super(name);
-  }
-
-  public AgentService(String name, AgentWebApp app) {
-    super(name, app);
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java

@@ -17,7 +17,6 @@
 package org.apache.slider.server.appmaster.web;
 
 import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.api.SliderClusterProtocol;
 import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.AppMasterActionOperations;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
@@ -25,7 +24,6 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.state.AppState;
 import org.apache.slider.server.appmaster.state.RoleStatus;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
 import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
 import org.apache.slider.server.services.security.CertificateManager;
 
@@ -57,11 +55,6 @@ public interface WebAppApi {
    * is a computed value and not just a getter
    */
   Map<String, RoleStatus> getRoleStatusByName();
-
-  /**
-   * Returns an interface that can support the agent-based REST operations.
-   */
-  AgentRestOperations getAgentRestOperations();
   
   /**
    * Registry operations accessor

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java

@@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.state.RoleStatus;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
 import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
 import org.apache.slider.server.services.security.CertificateManager;
 import org.slf4j.Logger;
@@ -96,11 +95,6 @@ public class WebAppApiImpl implements WebAppApi {
     return map;
   }
 
-  @Override
-  public AgentRestOperations getAgentRestOperations() {
-    return provider.getAgentRestOperations();
-  }
-
   @Override
   public RegistryOperations getRegistryOperations() {
     return registryOperations;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java

@@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.rest;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
 import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource;
 import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
 import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java

@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-public enum AgentCommandType {
-  EXECUTION_COMMAND,
-  STATUS_COMMAND,
-  REGISTRATION_COMMAND
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java

@@ -1,376 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import com.google.gson.annotations.SerializedName;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class AgentEnv {
-
-  /**
-   * Various directories, configurable in <code>ambari-agent.ini</code>
-   */
-  private Directory[] stackFoldersAndFiles = new Directory[0];
-
-  /**
-   * Directories that match name <code>/etc/alternatives/*conf</code>
-   */
-  private Alternative[] alternatives = new Alternative[0];
-
-  /**
-   * List of existing users
-   */
-  private ExistingUser[] existingUsers = new ExistingUser[0];
-
-  /**
-   * List of repos
-   */
-  private String[] existingRepos = new String[0];
-
-  /**
-   * List of packages
-   */
-  private PackageDetail[] installedPackages = new PackageDetail[0];
-
-  /**
-   * The host health report
-   */
-  private HostHealth hostHealth = new HostHealth();
-
-  private Integer umask;
-
-  private Boolean iptablesIsRunning;
-
-  public Integer getUmask() {
-    return umask;
-  }
-
-  public void setUmask(Integer umask) {
-    this.umask = umask;
-  }
-
-  public Directory[] getStackFoldersAndFiles() {
-    return stackFoldersAndFiles;
-  }
-
-  public void setStackFoldersAndFiles(Directory[] dirs) {
-    stackFoldersAndFiles = dirs;
-  }
-
-  public void setExistingUsers(ExistingUser[] users) {
-    existingUsers = users;
-  }
-
-  public ExistingUser[] getExistingUsers() {
-    return existingUsers;
-  }
-
-  public void setAlternatives(Alternative[] dirs) {
-    alternatives = dirs;
-  }
-
-  public Alternative[] getAlternatives() {
-    return alternatives;
-  }
-
-  public void setExistingRepos(String[] repos) {
-    existingRepos = repos;
-  }
-
-  public String[] getExistingRepos() {
-    return existingRepos;
-  }
-
-  public void setInstalledPackages(PackageDetail[] packages) {
-    installedPackages = packages;
-  }
-
-  public PackageDetail[] getInstalledPackages() {
-    return installedPackages;
-  }
-
-  public void setHostHealth(HostHealth healthReport) {
-    hostHealth = healthReport;
-  }
-
-  public HostHealth getHostHealth() {
-    return hostHealth;
-  }
-
-  public Boolean getIptablesIsRunning() {
-    return iptablesIsRunning;
-  }
-
-  public void setIptablesIsRunning(Boolean iptablesIsRunning) {
-    this.iptablesIsRunning = iptablesIsRunning;
-  }
-
-  public static class HostHealth {
-    /**
-     * Java processes running on the system.  Default empty array.
-     */
-    @SerializedName("activeJavaProcs")
-    private JavaProc[] activeJavaProcs = new JavaProc[0];
-
-    /**
-     * The current time when agent send the host check report
-     */
-    @SerializedName("agentTimeStampAtReporting")
-    private long agentTimeStampAtReporting = 0;
-
-    /**
-     * The current time when host check report was received
-     */
-    @SerializedName("serverTimeStampAtReporting")
-    private long serverTimeStampAtReporting = 0;
-
-    /**
-     * Live services running on the agent
-     */
-    @SerializedName("liveServices")
-    private LiveService[] liveServices = new LiveService[0];
-
-    public void setAgentTimeStampAtReporting(long currentTime) {
-      agentTimeStampAtReporting = currentTime;
-    }
-
-    public long getAgentTimeStampAtReporting() {
-      return agentTimeStampAtReporting;
-    }
-
-    public void setServerTimeStampAtReporting(long currentTime) {
-      serverTimeStampAtReporting = currentTime;
-    }
-
-    public long getServerTimeStampAtReporting() {
-      return serverTimeStampAtReporting;
-    }
-
-    public void setActiveJavaProcs(JavaProc[] procs) {
-      activeJavaProcs = procs;
-    }
-
-    public JavaProc[] getActiveJavaProcs() {
-      return activeJavaProcs;
-    }
-
-    public void setLiveServices(LiveService[] services) {
-      liveServices = services;
-    }
-
-    public LiveService[] getLiveServices() {
-      return liveServices;
-    }
-  }
-
-  public static class PackageDetail {
-    @SerializedName("name")
-    private String pkgName;
-    @SerializedName("version")
-    private String pkgVersion;
-    @SerializedName("repoName")
-    private String pkgRepoName;
-
-    public void setName(String name) {
-      pkgName = name;
-    }
-
-    public String getName() {
-      return pkgName;
-    }
-
-    public void setVersion(String version) {
-      pkgVersion = version;
-    }
-
-    public String getVersion() {
-      return pkgVersion;
-    }
-
-    public void setRepoName(String repoName) {
-      pkgRepoName = repoName;
-    }
-
-    public String getRepoName() {
-      return pkgRepoName;
-    }
-  }
-
-  /**
-   * Represents information about a directory of interest.
-   */
-  public static class Directory {
-    @SerializedName("name")
-    private String dirName;
-    @SerializedName("type")
-    private String dirType;
-
-    public void setName(String name) {
-      dirName = name;
-    }
-
-    public String getName() {
-      return dirName;
-    }
-
-    public void setType(String type) {
-      dirType = type;
-    }
-
-    public String getType() {
-      return dirType;
-    }
-  }
-
-  /**
-   * Represents information about running java processes.
-   */
-  public static class JavaProc {
-    @SerializedName("user")
-    private String user;
-    @SerializedName("pid")
-    private int pid = 0;
-    @SerializedName("hadoop")
-    private boolean is_hadoop = false;
-    @SerializedName("command")
-    private String command;
-
-    public void setUser(String user) {
-      this.user = user;
-    }
-
-    public String getUser() {
-      return user;
-    }
-
-    public void setPid(int pid) {
-      this.pid = pid;
-    }
-
-    public int getPid() {
-      return pid;
-    }
-
-    public void setHadoop(boolean hadoop) {
-      is_hadoop = hadoop;
-    }
-
-    public boolean isHadoop() {
-      return is_hadoop;
-    }
-
-    public void setCommand(String cmd) {
-      command = cmd;
-    }
-
-    public String getCommand() {
-      return command;
-    }
-  }
-
-  public static class Alternative {
-    @SerializedName("name")
-    private String altName;
-    @SerializedName("target")
-    private String altTarget;
-
-    public void setName(String name) {
-      altName = name;
-    }
-
-    public String getName() {
-      return altName;
-    }
-
-    public void setTarget(String target) {
-      altTarget = target;
-    }
-
-    public String getTarget() {
-      return altTarget;
-    }
-  }
-
-  public static class LiveService {
-    @SerializedName("name")
-    private String svcName;
-    @SerializedName("status")
-    private String svcStatus;
-    @SerializedName("desc")
-    private String svcDesc;
-
-    public void setName(String name) {
-      svcName = name;
-    }
-
-    public String getName() {
-      return svcName;
-    }
-
-    public void setStatus(String status) {
-      svcStatus = status;
-    }
-
-    public String getStatus() {
-      return svcStatus;
-    }
-
-    public void setDesc(String desc) {
-      svcDesc = desc;
-    }
-
-    public String getDesc() {
-      return svcDesc;
-    }
-  }
-
-  public static class ExistingUser {
-    @SerializedName("name")
-    private String name;
-    @SerializedName("homeDir")
-    private String homeDir;
-    @SerializedName("status")
-    private String status;
-
-    public void setUserName(String userName) {
-      name = userName;
-    }
-
-    public String getUserName() {
-      return name;
-    }
-
-    public void setUserHomeDir(String userHomeDir) {
-      homeDir = userHomeDir;
-    }
-
-    public String getUserHomeDir() {
-      return homeDir;
-    }
-
-    public void setUserStatus(String userStatus) {
-      status = userStatus;
-    }
-
-    public String getUserStatus() {
-      return status;
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java

@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.services.security.SignCertResponse;
-import org.apache.slider.server.services.security.SignMessage;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class AgentResource extends AbstractSliderResource {
-
-  private String agent_name;
-
-  public AgentResource(WebAppApi slider) {
-    super(slider);
-  }
-
-  private void init(HttpServletResponse res) {
-    res.setContentType(null);
-  }
-
-  @GET
-  @Path("/agent/register")
-  public Response endpointAgentRegister() {
-    Response response = Response.status(200).entity("/agent/register").build();
-    return response;
-  }
-
-  @GET
-  @Path("/agent")
-  public Response endpointAgent() {
-    Response response = Response.status(200).entity("/agent").build();
-    return response;
-  }
-  @GET
-  @Path("/")
-  public Response endpointRoot() {
-    Response response = Response.status(200).entity("/").build();
-    return response;
-  }
-
-  @POST
-  @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register")
-  @Consumes({MediaType.APPLICATION_JSON})
-  @Produces({MediaType.APPLICATION_JSON})
-  public RegistrationResponse register(Register registration,
-                                       @Context HttpServletResponse res,
-                                       @PathParam("agent_name") String agent_name) {
-    init(res);
-    this.agent_name = agent_name;
-    AgentRestOperations ops = slider.getAgentRestOperations();
-    return ops.handleRegistration(registration);
-
-  }
-
-  @POST
-  @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat")
-  @Consumes(MediaType.APPLICATION_JSON)
-  @Produces({MediaType.APPLICATION_JSON})
-  public HeartBeatResponse heartbeat(HeartBeat message,
-                                     @Context HttpServletResponse res,
-                                     @PathParam("agent_name") String agent_name) {
-    init(res);
-    AgentRestOperations ops = slider.getAgentRestOperations();
-    return ops.handleHeartBeat(message);
-  }
-
-  @GET
-  @Path("/cert/ca")
-  @Produces({MediaType.TEXT_PLAIN})
-  public String downloadSrvrCrt() {
-    return slider.getCertificateManager().getServerCert();
-  }
-
-  @Path("/certs/{hostName}")
-  @POST
-  @Consumes(MediaType.APPLICATION_JSON)
-  @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
-  public SignCertResponse signAgentCrt(@PathParam("hostName") String hostname,
-                                       SignMessage message, @Context HttpServletRequest req) {
-    return slider.getCertificateManager().signAgentCrt(hostname,
-                                                       message.getCsr(),
-                                                       message.getPassphrase());
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java

@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-/**
- *
- */
-public interface AgentRestOperations {
-
-  RegistrationResponse handleRegistration(Register registration);
-
-  HeartBeatResponse handleHeartBeat(HeartBeat heartBeat);
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java

@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.spi.container.WebApplication;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
-import com.sun.jersey.spi.container.servlet.WebConfig;
-import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.services.security.SecurityUtils;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSelectChannelConnector;
-import org.mortbay.jetty.servlet.Context;
-import org.mortbay.jetty.servlet.ServletHolder;
-import org.mortbay.thread.QueuedThreadPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.ext.Provider;
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-import java.net.BindException;
-import java.util.Set;
-
-/**
- *
- */
-public class AgentWebApp implements Closeable {
-  protected static final Logger LOG = LoggerFactory.getLogger(AgentWebApp.class);
-  private int port;
-  private int securedPort;
-  private static Server agentServer;
-  public static final String BASE_PATH = "slideragent";
-
-  public static class Builder {
-    final String name;
-    final String wsName;
-    final WebAppApi application;
-    int port;
-    int securedPort;
-    MapOperations configsMap;
-
-    public Builder(String name, String wsName, WebAppApi application) {
-      this.name = name;
-      this.wsName = wsName;
-      this.application = application;
-    }
-
-    public Builder withComponentConfig(MapOperations appMasterConfig) {
-      this.configsMap = appMasterConfig;
-      return this;
-    }
-
-    public Builder withPort (int port) {
-      this.port = port;
-      return this;
-    }
-
-    public Builder withSecuredPort (int securedPort) {
-      this.securedPort = securedPort;
-      return this;
-    }
-
-    public AgentWebApp start() throws IOException {
-      if (configsMap == null) {
-        throw new IllegalStateException("No SSL Configuration Available");
-      }
-
-      agentServer = new Server();
-      agentServer.setThreadPool(
-          new QueuedThreadPool(
-              configsMap.getOptionInt("agent.threadpool.size.max", 25)));
-      agentServer.setStopAtShutdown(true);
-      agentServer.setGracefulShutdown(1000);
-
-      SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port);
-      SslSelectChannelConnector ssl2WayConnector =
-          createSSLConnector(Boolean.valueOf(
-              configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED,
-                                   "false")), securedPort);
-      agentServer.setConnectors(new Connector[]{ssl1WayConnector,
-          ssl2WayConnector});
-
-      ServletHolder agent = new ServletHolder(new AgentServletContainer());
-      Context agentRoot = new Context(agentServer, "/", Context.SESSIONS);
-
-      agent.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
-                             "com.sun.jersey.api.core.PackagesResourceConfig");
-      agent.setInitParameter("com.sun.jersey.config.property.packages",
-                             "org.apache.slider.server.appmaster.web.rest.agent");
-      agent.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature",
-                             "true");
-//      agent.setInitParameter("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
-//      agent.setInitParameter("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
-//      agent.setInitParameter("com.sun.jersey.config.feature.Trace", "true");
-      agentRoot.addServlet(agent, "/*");
-
-      try {
-        openListeners();
-        agentServer.start();
-      } catch (IOException e) {
-        LOG.error("Unable to start agent server", e);
-        throw e;
-      } catch (Exception e) {
-        LOG.error("Unable to start agent server", e);
-        throw new IOException("Unable to start agent server: " + e, e);
-      }
-
-      AgentWebApp webApp = new AgentWebApp();
-      webApp.setPort(getConnectorPort(agentServer, 0));
-      webApp.setSecuredPort(getConnectorPort(agentServer, 1));
-      return webApp;
-
-    }
-
-    private void openListeners() throws Exception {
-      // from HttpServer2.openListeners()
-      for (Connector listener : agentServer.getConnectors()) {
-        if (listener.getLocalPort() != -1) {
-          // This listener is either started externally or has been bound
-          continue;
-        }
-        int port = listener.getPort();
-        while (true) {
-          // jetty has a bug where you can't reopen a listener that previously
-          // failed to open w/o issuing a close first, even if the port is changed
-          try {
-            listener.close();
-            listener.open();
-            LOG.info("Jetty bound to port " + listener.getLocalPort());
-            break;
-          } catch (BindException ex) {
-            if (port == 0) {
-              BindException be = new BindException("Port in use: "
-                  + listener.getHost() + ":" + listener.getPort());
-              be.initCause(ex);
-              throw be;
-            }
-          }
-          // try the next port number
-          listener.setPort(++port);
-          Thread.sleep(100);
-        }
-      }
-    }
-
-    private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) {
-      SslSelectChannelConnector sslConnector = new
-          SslSelectChannelConnector();
-
-      String keystore = SecurityUtils.getSecurityDir() +
-                        File.separator + "keystore.p12";
-      String srvrCrtPass = SecurityUtils.getKeystorePass();
-      sslConnector.setKeystore(keystore);
-      sslConnector.setTruststore(keystore);
-      sslConnector.setPassword(srvrCrtPass);
-      sslConnector.setKeyPassword(srvrCrtPass);
-      sslConnector.setTrustPassword(srvrCrtPass);
-      sslConnector.setKeystoreType("PKCS12");
-      sslConnector.setTruststoreType("PKCS12");
-      sslConnector.setNeedClientAuth(needClientAuth);
-
-      sslConnector.setPort(port);
-      sslConnector.setAcceptors(2);
-      return sslConnector;
-    }
-
-    @Provider
-    public class WebAppApiProvider extends
-        SingletonTypeInjectableProvider<javax.ws.rs.core.Context, WebAppApi> {
-
-      public WebAppApiProvider () {
-        super(WebAppApi.class, application);
-      }
-    }
-
-    public class AgentServletContainer extends ServletContainer {
-      public AgentServletContainer() {
-        super();
-      }
-
-      @Override
-      protected void configure(WebConfig wc,
-                               ResourceConfig rc,
-                               WebApplication wa) {
-        super.configure(wc, rc, wa);
-        Set<Object> singletons = rc.getSingletons();
-        singletons.add(new WebAppApiProvider());
-      }
-    }
-
-    private int getConnectorPort(Server webServer, int index) {
-      Preconditions.checkArgument(index >= 0);
-      if (index > webServer.getConnectors().length)
-        throw new IllegalStateException("Illegal connect index requested");
-
-      Connector c = webServer.getConnectors()[index];
-      if (c.getLocalPort() == -1) {
-        // The connector is not bounded
-        throw new IllegalStateException("The connector is not bound to a port");
-      }
-
-      return c.getLocalPort();
-    }
-  }
-
-  public static Builder $for(String name, WebAppApi app, String wsPrefix) {
-    return new Builder(name, wsPrefix, app);
-  }
-
-  public int getPort() {
-    return port;
-  }
-
-  public void setPort(int port) {
-    this.port = port;
-  }
-
-  public void setSecuredPort(int securedPort) {
-    this.securedPort = securedPort;
-  }
-
-  public int getSecuredPort() {
-    return securedPort;
-  }
-
-  public void close() throws IOException{
-    //need to stop server and reset injector
-    try {
-      agentServer.stop();
-    } catch (IOException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new IOException(e.toString(), e);
-    }
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java

@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-
-import javax.ws.rs.Path;
-import javax.ws.rs.core.Context;
-
-/** The available agent REST services exposed by a slider AM. */
-@Path(RestPaths.SLIDER_AGENT_CONTEXT_ROOT)
-public class AgentWebServices {
-  /** AM/WebApp info object */
-  @Context
-  private WebAppApi slider;
-
-  public AgentWebServices() {
-  }
-
-  @Path(RestPaths.SLIDER_SUBPATH_AGENTS)
-  public AgentResource getAgentResource () {
-    return new AgentResource(slider);
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java

@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class CommandReport {
-
-  int exitCode;
-  private String role;
-  private String actionId;
-  private String stdout;
-  private String stderr;
-  private String structuredOut;
-  private String status;
-  private String clusterName;
-  private String serviceName;
-  private long taskId;
-  private String roleCommand;
-  private Map<String, String> folders;
-  private Map<String, String> allocatedPorts;
-  private Map<String, Map<String, String>> configurationTags;
-
-  @JsonProperty("taskId")
-  public long getTaskId() {
-    return taskId;
-  }
-
-  @JsonProperty("taskId")
-  public void setTaskId(long taskId) {
-    this.taskId = taskId;
-  }
-
-  @JsonProperty("clusterName")
-  public String getClusterName() {
-    return this.clusterName;
-  }
-
-  @JsonProperty("clusterName")
-  public void setClusterName(String clusterName) {
-    this.clusterName = clusterName;
-  }
-
-  @JsonProperty("actionId")
-  public String getActionId() {
-    return this.actionId;
-  }
-
-  @JsonProperty("actionId")
-  public void setActionId(String actionId) {
-    this.actionId = actionId;
-  }
-
-  @JsonProperty("stderr")
-  public String getStdErr() {
-    return this.stderr;
-  }
-
-  @JsonProperty("stderr")
-  public void setStdErr(String stderr) {
-    this.stderr = stderr;
-  }
-
-  @JsonProperty("exitcode")
-  public int getExitCode() {
-    return this.exitCode;
-  }
-
-  @JsonProperty("exitcode")
-  public void setExitCode(int exitCode) {
-    this.exitCode = exitCode;
-  }
-
-  @JsonProperty("stdout")
-  public String getStdOut() {
-    return this.stdout;
-  }
-
-  @JsonProperty("stdout")
-  public void setStdOut(String stdout) {
-    this.stdout = stdout;
-  }
-
-  @JsonProperty("structuredOut")
-  public String getStructuredOut() {
-    return this.structuredOut;
-  }
-
-  @JsonProperty("structuredOut")
-  public void setStructuredOut(String structuredOut) {
-    this.structuredOut = structuredOut;
-  }
-
-  @JsonProperty("roleCommand")
-  public String getRoleCommand() {
-    return this.roleCommand;
-  }
-
-  @JsonProperty("roleCommand")
-  public void setRoleCommand(String roleCommand) {
-    this.roleCommand = roleCommand;
-  }
-
-  @JsonProperty("role")
-  public String getRole() {
-    return role;
-  }
-
-  @JsonProperty("role")
-  public void setRole(String role) {
-    this.role = role;
-  }
-
-  @JsonProperty("status")
-  public String getStatus() {
-    return status;
-  }
-
-  @JsonProperty("status")
-  public void setStatus(String status) {
-    this.status = status;
-  }
-
-  @JsonProperty("serviceName")
-  public String getServiceName() {
-    return serviceName;
-  }
-
-  @JsonProperty("serviceName")
-  public void setServiceName(String serviceName) {
-    this.serviceName = serviceName;
-  }
-
-  /** @return the config tags that match this command, or <code>null</code> if none are present */
-  @JsonProperty("configurationTags")
-  public Map<String, Map<String, String>> getConfigurationTags() {
-    return configurationTags;
-  }
-
-  /** @param tags the config tags that match this command */
-  @JsonProperty("configurationTags")
-  public void setConfigurationTags(Map<String, Map<String, String>> tags) {
-    configurationTags = tags;
-  }
-
-  /** @return the allocated ports, or <code>null</code> if none are present */
-  @JsonProperty("allocatedPorts")
-  public Map<String, String> getAllocatedPorts() {
-    return allocatedPorts;
-  }
-
-  /** @param ports allocated ports */
-  @JsonProperty("allocatedPorts")
-  public void setAllocatedPorts(Map<String, String> ports) {
-    this.allocatedPorts = ports;
-  }
-
-  /** @return the folders, or <code>null</code> if none are present */
-  @JsonProperty("folders")
-  public Map<String, String> getFolders() {
-    return folders;
-  }
-
-  /** @param folders allocated ports */
-  @JsonProperty("folders")
-  public void setFolders(Map<String, String> folders) {
-    this.folders = folders;
-  }
-
-  @Override
-  public String toString() {
-    return "CommandReport{" +
-           "role='" + role + '\'' +
-           ", actionId='" + actionId + '\'' +
-           ", status='" + status + '\'' +
-           ", exitCode=" + exitCode +
-           ", clusterName='" + clusterName + '\'' +
-           ", serviceName='" + serviceName + '\'' +
-           ", taskId=" + taskId +
-           ", roleCommand=" + roleCommand +
-           ", configurationTags=" + configurationTags +
-           '}';
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java

@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ComponentStatus {
-  String componentName;
-  String msg;
-  String status;
-  String serviceName;
-  String clusterName;
-  String roleCommand;
-  String ip;
-  String hostname;
-  @JsonProperty("configurations")
-  private Map<String, Map<String, String>> configurations;
-
-  public String getRoleCommand() {
-    return roleCommand;
-  }
-
-  public void setRoleCommand(String roleCommand) {
-    this.roleCommand = roleCommand;
-  }
-
-  public String getComponentName() {
-    return this.componentName;
-  }
-
-  public void setComponentName(String componentName) {
-    this.componentName = componentName;
-  }
-
-  public String getMessage() {
-    return this.msg;
-  }
-
-  public void setMessage(String msg) {
-    this.msg = msg;
-  }
-
-  public String getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(String status) {
-    this.status = status;
-  }
-
-  public String getServiceName() {
-    return serviceName;
-  }
-
-  public void setServiceName(String serviceName) {
-    this.serviceName = serviceName;
-  }
-
-  public String getClusterName() {
-    return clusterName;
-  }
-
-  public void setClusterName(String clusterName) {
-    this.clusterName = clusterName;
-  }
-
-  /** @return the config tags that match this command, or <code>null</code> if none are present */
-  public Map<String, Map<String, String>> getConfigs() {
-    return configurations;
-  }
-
-  /** @param configs the config tags that match this status */
-  public void setConfigs(Map<String, Map<String, String>> configs) {
-    this.configurations = configs;
-  }
-
-  @Override
-  public String toString() {
-    return "ComponentStatus{" +
-           "componentName='" + componentName + '\'' +
-           ", msg='" + msg + '\'' +
-           ", status='" + status + '\'' +
-           ", serviceName='" + serviceName + '\'' +
-           ", clusterName='" + clusterName + '\'' +
-           ", roleCommand='" + roleCommand + '\'' +
-           ", ip='" + ip + '\'' +
-           ", hostname='" + hostname + '\'' +
-           '}';
-  }
-
-  public String getIp() {
-    return ip;
-  }
-
-  public void setIp(String ip) {
-    this.ip = ip;
-  }
-
-  public String getHostname() {
-    return hostname;
-  }
-
-  public void setHostname(String hostname) {
-    this.hostname = hostname;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java

@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class DiskInfo {
-  String available;
-  String mountpoint;
-  String device;
-  String used;
-  String percent;
-  String size;
-  String type;
-
-  /**
-   * DiskInfo object that tracks information about a disk.
-   * @param mountpoint
-   * @param available
-   * @param used
-   * @param percent
-   * @param size
-   */
-  public DiskInfo(String device, String mountpoint, String available,
-                  String used, String percent, String size, String type) {
-    this.device = device;
-    this.mountpoint = mountpoint;
-    this.available = available;
-    this.used = used;
-    this.percent = percent;
-    this.size = size;
-    this.type = type;
-  }
-
-  /**
-   * Needed for Serialization
-   */
-  public DiskInfo() {}
-
-  @JsonProperty("available")
-  public void setAvailable(String available) {
-    this.available = available;
-  }
-
-  @JsonProperty("available")
-  public String getAvailable() {
-    return this.available;
-  }
-
-  @JsonProperty("mountpoint")
-  public String getMountPoint() {
-    return this.mountpoint;
-  }
-
-  @JsonProperty("mountpoint")
-  public void setMountPoint(String mountpoint) {
-    this.mountpoint = mountpoint;
-  }
-
-  @JsonProperty("type")
-  public String getType() {
-    return this.type;
-  }
-
-  @JsonProperty("type")
-  public void setType(String type) {
-    this.type = type;
-  }
-
-  @JsonProperty("used")
-  public String getUsed() {
-    return this.used;
-  }
-
-  @JsonProperty("used")
-  public void setUsed(String used) {
-    this.used = used;
-  }
-
-  @JsonProperty("percent")
-  public String getPercent() {
-    return this.percent;
-  }
-
-  @JsonProperty("percent")
-  public void setPercent(String percent) {
-    this.percent = percent;
-  }
-
-  @JsonProperty("size")
-  public String getSize() {
-    return this.size;
-  }
-
-  @JsonProperty("size")
-  public void setSize(String size) {
-    this.size = size;
-  }
-
-  @Override
-  public String toString() {
-    return "available=" + this.available + ",mountpoint=" + this.mountpoint
-           + ",used=" + this.used + ",percent=" + this.percent + ",size=" +
-           this.size + ",device=" + this.device +
-           ",type=" + this.type;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java

@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.DockerContainer;
-import org.apache.slider.providers.agent.application.metadata.DockerContainerInputFile;
-import org.apache.slider.providers.agent.application.metadata.DockerContainerMount;
-import org.apache.slider.providers.agent.application.metadata.DockerContainerPort;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ExecutionCommand {
-  protected static final Logger log =
-      LoggerFactory.getLogger(ExecutionCommand.class);
-  private AgentCommandType commandType = AgentCommandType.EXECUTION_COMMAND;
-  private String clusterName;
-  private long taskId;
-  private String commandId;
-  //TODO Remove hostname from being set in the command
-  private String hostname;
-  private String role;
-  private Map<String, String> hostLevelParams = new HashMap<String, String>();
-  private Map<String, String> roleParams = null;
-  private String roleCommand;
-  private Map<String, Map<String, String>> configurations;
-  private Map<String, Map<String, String>> componentConfigurations;
-  private Map<String, String> commandParams;
-  private String serviceName;
-  private String componentName;
-  private String componentType;
-  private List<DockerContainer> containers = new ArrayList<>();
-  private String pkg;
-  private boolean yarnDockerMode = false;
-
-  public ExecutionCommand(AgentCommandType commandType) {
-    this.commandType = commandType;
-  }
-
-  @JsonProperty("commandType")
-  public AgentCommandType getCommandType() {
-    return commandType;
-  }
-
-  @JsonProperty("commandType")
-  public void setCommandType(AgentCommandType commandType) {
-    this.commandType = commandType;
-  }
-
-  @JsonProperty("commandId")
-  public String getCommandId() {
-    return this.commandId;
-  }
-
-  @JsonProperty("commandId")
-  public void setCommandId(String commandId) {
-    this.commandId = commandId;
-  }
-
-  @JsonProperty("taskId")
-  public long getTaskId() {
-    return taskId;
-  }
-
-  @JsonProperty("taskId")
-  public void setTaskId(long taskId) {
-    this.taskId = taskId;
-  }
-
-  @JsonProperty("role")
-  public String getRole() {
-    return role;
-  }
-
-  @JsonProperty("role")
-  public void setRole(String role) {
-    this.role = role;
-  }
-
-  @JsonProperty("roleParams")
-  public Map<String, String> getRoleParams() {
-    return roleParams;
-  }
-
-  @JsonProperty("roleParams")
-  public void setRoleParams(Map<String, String> roleParams) {
-    this.roleParams = roleParams;
-  }
-
-  @JsonProperty("roleCommand")
-  public String getRoleCommand() {
-    return roleCommand;
-  }
-
-  @JsonProperty("roleCommand")
-  public void setRoleCommand(String cmd) {
-    this.roleCommand = cmd;
-  }
-
-  @JsonProperty("clusterName")
-  public String getClusterName() {
-    return clusterName;
-  }
-
-  @JsonProperty("clusterName")
-  public void setClusterName(String clusterName) {
-    this.clusterName = clusterName;
-  }
-
-  @JsonProperty("componentType")
-  public String getComponentType() {
-    return componentType;
-  }
-
-  @JsonProperty("componentType")
-  public void setComponentType(String componentType) {
-    this.componentType = componentType;
-  }
-
-  @JsonProperty("hostname")
-  public String getHostname() {
-    return hostname;
-  }
-
-  @JsonProperty("hostname")
-  public void setHostname(String hostname) {
-    this.hostname = hostname;
-  }
-
-  @JsonProperty("hostLevelParams")
-  public Map<String, String> getHostLevelParams() {
-    return hostLevelParams;
-  }
-
-  @JsonProperty("hostLevelParams")
-  public void setHostLevelParams(Map<String, String> params) {
-    this.hostLevelParams = params;
-  }
-
-  @JsonProperty("configurations")
-  public Map<String, Map<String, String>> getConfigurations() {
-    return configurations;
-  }
-
-  @JsonProperty("configurations")
-  public void setConfigurations(Map<String, Map<String, String>> configurations) {
-    this.configurations = configurations;
-  }
-
-  @JsonProperty("commandParams")
-  public Map<String, String> getCommandParams() {
-    return commandParams;
-  }
-
-  @JsonProperty("commandParams")
-  public void setCommandParams(Map<String, String> commandParams) {
-    this.commandParams = commandParams;
-  }
-
-  @JsonProperty("serviceName")
-  public String getServiceName() {
-    return serviceName;
-  }
-
-  @JsonProperty("serviceName")
-  public void setServiceName(String serviceName) {
-    this.serviceName = serviceName;
-  }
-
-  @JsonProperty("componentName")
-  public String getComponentName() {
-    return componentName;
-  }
-
-  @JsonProperty("componentName")
-  public void setComponentName(String componentName) {
-    this.componentName = componentName;
-  }
-
-  @JsonProperty("package")
-  public String getPkg() {
-    return pkg;
-  }
-
-  @JsonProperty("package")
-  public void setPkg(String pkg) {
-    this.pkg = pkg;
-  }
-
-  @JsonProperty("componentConfig")
-  public Map<String, Map<String, String>> getComponentConfigurations() {
-    return componentConfigurations;
-  }
-
-  @JsonProperty("componentConfig")
-  public void setComponentConfigurations(
-      Map<String, Map<String, String>> componentConfigurations) {
-    this.componentConfigurations = componentConfigurations;
-  }
-
-  @JsonProperty("containers")
-  public List<DockerContainer> getContainers() {
-    return containers;
-  }
-
-  @JsonProperty("yarnDockerMode")
-  public boolean isYarnDockerMode() {
-    return yarnDockerMode ;
-  }
-
-  @JsonProperty("yarnDockerMode")
-  public void setYarnDockerMode(boolean yarnDockerMode) {
-    this.yarnDockerMode = yarnDockerMode;
-  }
-  @Override
-  public String toString() {
-    StringBuilder builder = new StringBuilder();
-    builder.append("ExecutionCommand [commandType=").append(commandType)
-        .append(", clusterName=").append(clusterName).append(", taskId=")
-        .append(taskId).append(", commandId=").append(commandId)
-        .append(", hostname=").append(hostname).append(", role=").append(role)
-        .append(", hostLevelParams=").append(hostLevelParams)
-        .append(", roleParams=").append(roleParams).append(", roleCommand=")
-        .append(roleCommand).append(", configurations=").append(configurations)
-        .append(", commandParams=").append(commandParams)
-        .append(", serviceName=").append(serviceName)
-        .append(", componentName=").append(componentName)
-        .append(", componentType=").append(componentType)
-        .append(", yarnDockerMode=").append(yarnDockerMode).append(", pkg=")
-        .append(pkg).append("]");
-    return builder.toString();
-  }
-  
-  public void addContainerDetails(String componentGroup, Metainfo metaInfo) {
-    Component component = metaInfo.getApplicationComponent(componentGroup);
-    this.setComponentType(component.getType());
-    log.info("Adding container details for {}", componentGroup, " from ",
-        metaInfo.toString());
-    for (DockerContainer metaContainer : component.getDockerContainers()) {
-      DockerContainer container = new DockerContainer();
-      container.setImage(metaContainer.getImage());
-      container.setNetwork(metaContainer.getNetwork());
-      container.setUseNetworkScript(metaContainer.getUseNetworkScript());
-      container.setName(metaContainer.getName());
-      container.setOptions(metaContainer.getOptions());
-      container.setAdditionalParam(metaContainer.getAdditionalParam());
-      container.setCommandPath(metaContainer.getAdditionalParam());
-      container.setStatusCommand(metaContainer.getStatusCommand());
-      container.setStartCommand(metaContainer.getStartCommand());
-      if (metaContainer.getMounts().size() > 0) {
-        for (DockerContainerMount metaContMount : metaContainer.getMounts()) {
-          DockerContainerMount contMnt = new DockerContainerMount();
-          contMnt.setContainerMount(metaContMount.getContainerMount());
-          contMnt.setHostMount(metaContMount.getHostMount());
-          container.getMounts().add(contMnt);
-        }
-      }
-      if (metaContainer.getPorts().size() > 0) {
-        for (DockerContainerPort metaCntPort : metaContainer.getPorts()) {
-          DockerContainerPort cntPort = new DockerContainerPort();
-          cntPort.setContainerPort(metaCntPort.getContainerPort());
-          cntPort.setHostPort(metaCntPort.getHostPort());
-          container.getPorts().add(cntPort);
-        }
-      }
-      if (metaContainer.getInputFiles().size() > 0) {
-        for (DockerContainerInputFile metaInpFile : metaContainer
-            .getInputFiles()) {
-          DockerContainerInputFile inpFile = new DockerContainerInputFile();
-          inpFile.setContainerMount(metaInpFile.getContainerMount());
-          inpFile.setFileLocalPath(metaInpFile.getFileLocalPath());
-          container.getInputFiles().add(inpFile);
-        }
-      }
-      if (metaContainer.getConfigFiles() != null) {
-        container.setConfigFiles(metaContainer.getConfigFiles());
-      }
-      log.info("Docker container meta info ready: " + container.toString());
-      this.getContainers().add(container);
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java

@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- *
- * Data model for agent heartbeat for server (ambari or app master).
- *
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HeartBeat {
-  private long responseId = -1;
-  private long timestamp;
-  private String hostname;
-  List<CommandReport> reports = new ArrayList<CommandReport>();
-  List<ComponentStatus> componentStatus = new ArrayList<ComponentStatus>();
-  private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
-  HostStatus nodeStatus;
-  private AgentEnv agentEnv = null;
-  private String fqdn;
-  private String pkg;
-
-  public long getResponseId() {
-    return responseId;
-  }
-
-  public void setResponseId(long responseId) {
-    this.responseId=responseId;
-  }
-
-  public long getTimestamp() {
-    return timestamp;
-  }
-
-  public void setTimestamp(long timestamp) {
-    this.timestamp = timestamp;
-  }
-
-  public String getHostname() {
-    return hostname;
-  }
-
-  public void setHostname(String hostname) {
-    this.hostname = hostname;
-  }
-
-  public String getFqdn() {
-    return fqdn;
-  }
-
-  public void setFqdn(String fqdn) {
-    this.fqdn = fqdn;
-  }
-
-  @JsonProperty("reports")
-  public List<CommandReport> getReports() {
-    return this.reports;
-  }
-
-  @JsonProperty("reports")
-  public void setReports(List<CommandReport> reports) {
-    this.reports = reports;
-  }
-
-  public HostStatus getNodeStatus() {
-    return nodeStatus;
-  }
-
-  public void setNodeStatus(HostStatus nodeStatus) {
-    this.nodeStatus = nodeStatus;
-  }
-
-  public AgentEnv getAgentEnv() {
-    return agentEnv;
-  }
-
-  public void setAgentEnv(AgentEnv env) {
-    agentEnv = env;
-  }
-
-  @JsonProperty("componentStatus")
-  public List<ComponentStatus> getComponentStatus() {
-    return componentStatus;
-  }
-
-  @JsonProperty("componentStatus")
-  public void setComponentStatus(List<ComponentStatus> componentStatus) {
-    this.componentStatus = componentStatus;
-  }
-
-  @JsonProperty("mounts")
-  public List<DiskInfo> getMounts() {
-    return this.mounts;
-  }
-
-  @JsonProperty("mounts")
-  public void setMounts(List<DiskInfo> mounts) {
-    this.mounts = mounts;
-  }
-
-  @JsonProperty("package")
-  public String getPackage() {
-    return pkg;
-  }
-
-  @JsonProperty("package")
-  public void setPackage(String pkg) {
-    this.pkg = pkg;
-  }
-
-  @Override
-  public String toString() {
-    return "HeartBeat{" +
-           "responseId=" + responseId +
-           ", timestamp=" + timestamp +
-           ", hostname='" + hostname + '\'' +
-           ", reports=" + reports +
-           ", componentStatus=" + componentStatus +
-           ", package=" + pkg +
-           ", nodeStatus=" + nodeStatus +
-           '}';
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java

@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- * Controller to Agent response data model.
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HeartBeatResponse {
-
-  private long responseId;
-
-  List<ExecutionCommand> executionCommands = new ArrayList<ExecutionCommand>();
-  List<StatusCommand> statusCommands = new ArrayList<StatusCommand>();
-
-  RegistrationCommand registrationCommand;
-
-  boolean yarnDockerMode = false;
-  boolean restartAgent = false;
-  boolean restartEnabled = true;
-  boolean hasMappedComponents = false;
-  boolean terminateAgent = false;
-
-  @JsonProperty("responseId")
-  public long getResponseId() {
-    return responseId;
-  }
-
-  @JsonProperty("responseId")
-  public void setResponseId(long responseId) {
-    this.responseId=responseId;
-  }
-
-  @JsonProperty("executionCommands")
-  public List<ExecutionCommand> getExecutionCommands() {
-    return executionCommands;
-  }
-
-  @JsonProperty("executionCommands")
-  public void setExecutionCommands(List<ExecutionCommand> executionCommands) {
-    this.executionCommands = executionCommands;
-  }
-
-  @JsonProperty("statusCommands")
-  public List<StatusCommand> getStatusCommands() {
-    return statusCommands;
-  }
-
-  @JsonProperty("statusCommands")
-  public void setStatusCommands(List<StatusCommand> statusCommands) {
-    this.statusCommands = statusCommands;
-  }
-
-  @JsonProperty("registrationCommand")
-  public RegistrationCommand getRegistrationCommand() {
-    return registrationCommand;
-  }
-
-  @JsonProperty("registrationCommand")
-  public void setRegistrationCommand(RegistrationCommand registrationCommand) {
-    this.registrationCommand = registrationCommand;
-  }
-
-  @JsonProperty("restartAgent")
-  public boolean isRestartAgent() {
-    return restartAgent;
-  }
-
-  @JsonProperty("restartAgent")
-  public void setRestartAgent(boolean restartAgent) {
-    this.restartAgent = restartAgent;
-  }
-
-  @JsonProperty("restartEnabled")
-  public boolean getRstartEnabled() {
-    return restartEnabled;
-  }
-
-  @JsonProperty("restartEnabled")
-  public void setRestartEnabled(boolean restartEnabled) {
-    this.restartEnabled = restartEnabled;
-  }
-
-  @JsonProperty("hasMappedComponents")
-  public boolean hasMappedComponents() {
-    return hasMappedComponents;
-  }
-
-  @JsonProperty("hasMappedComponents")
-  public void setHasMappedComponents(boolean hasMappedComponents) {
-    this.hasMappedComponents = hasMappedComponents;
-  }
-
-  @JsonProperty("terminateAgent")
-  public boolean isTerminateAgent() {
-    return terminateAgent;
-  }
-
-  @JsonProperty("terminateAgent")
-  public void setTerminateAgent(boolean terminateAgent) {
-    this.terminateAgent = terminateAgent;
-  }
-
-  public void addExecutionCommand(ExecutionCommand execCmd) {
-    executionCommands.add(execCmd);
-  }
-
-  public void addStatusCommand(StatusCommand statCmd) {
-    statusCommands.add(statCmd);
-  }
-
-  @Override
-  public String toString() {
-    return "HeartBeatResponse{" +
-           "responseId=" + responseId +
-           ", executionCommands=" + executionCommands +
-           ", statusCommands=" + statusCommands +
-           ", registrationCommand=" + registrationCommand +
-           ", restartAgent=" + restartAgent +
-           ", terminateAgent=" + terminateAgent +
-           '}';
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java

@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HostInfo {
-  private String architecture;
-  private String domain;
-  private String fqdn;
-  private String hardwareisa;
-  private String hardwaremodel;
-  private String hostname;
-  private String id;
-  private String interfaces;
-  private String ipaddress;
-  private String kernel;
-  private String kernelmajversion;
-  private String kernelrelease;
-  private String kernelversion;
-  private String macaddress;
-  private long memoryfree;
-  private long memorysize;
-  private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
-  private long memorytotal;
-  private String netmask;
-  private String operatingsystem;
-  private String operatingsystemrelease;
-  private String osfamily;
-  private int physicalprocessorcount;
-  private int processorcount;
-  private boolean selinux;
-  private String swapfree;
-  private String swapsize;
-  private String timezone;
-  private String uptime;
-  private long uptime_days;
-  private long uptime_hours;
-
-
-  @JsonProperty("architecture")
-  public String getArchitecture() {
-    return this.architecture;
-  }
-
-  @JsonProperty("architecture")
-  public void setArchitecture(String architecture) {
-    this.architecture = architecture;
-  }
-
-  @JsonProperty("domain")
-  public String getDomain() {
-    return this.domain;
-  }
-
-  @JsonProperty("domain")
-  public void setDomain(String domain) {
-    this.domain = domain;
-  }
-
-  @JsonProperty("fqdn")
-  public String getFQDN() {
-    return this.fqdn;
-  }
-
-  @JsonProperty("fqdn")
-  public void setFQDN(String fqdn) {
-    this.fqdn = fqdn;
-  }
-
-  @JsonProperty("hardwareisa")
-  public String getHardwareIsa() {
-    return hardwareisa;
-  }
-
-  @JsonProperty("hardwareisa")
-  public void setHardwareIsa(String hardwareisa) {
-    this.hardwareisa = hardwareisa;
-  }
-
-  @JsonProperty("hardwaremodel")
-  public String getHardwareModel() {
-    return this.hardwaremodel;
-  }
-
-  @JsonProperty("hardwaremodel")
-  public void setHardwareModel(String hardwaremodel) {
-    this.hardwaremodel = hardwaremodel;
-  }
-
-  @JsonProperty("hostname")
-  public String getHostName() {
-    return this.hostname;
-  }
-
-  @JsonProperty("hostname")
-  public void setHostName(String hostname) {
-    this.hostname = hostname;
-  }
-
-  @JsonProperty("id")
-  public String getAgentUserId() {
-    return id;
-  }
-
-  @JsonProperty("id")
-  public void setAgentUserId(String id) {
-    this.id = id;
-  }
-
-  @JsonProperty("interfaces")
-  public String getInterfaces() {
-    return this.interfaces;
-  }
-
-  @JsonProperty("interfaces")
-  public void setInterfaces(String interfaces) {
-    this.interfaces = interfaces;
-  }
-
-  @JsonProperty("ipaddress")
-  public String getIPAddress() {
-    return this.ipaddress;
-  }
-
-  @JsonProperty("ipaddress")
-  public void setIPAddress(String ipaddress) {
-    this.ipaddress = ipaddress;
-  }
-
-  @JsonProperty("kernel")
-  public String getKernel() {
-    return this.kernel;
-  }
-
-  @JsonProperty("kernel")
-  public void setKernel(String kernel) {
-    this.kernel = kernel;
-  }
-
-  @JsonProperty("kernelmajversion")
-  public String getKernelMajVersion() {
-    return this.kernelmajversion;
-  }
-
-  @JsonProperty("kernelmajversion")
-  public void setKernelMajVersion(String kernelmajversion) {
-    this.kernelmajversion = kernelmajversion;
-  }
-
-  @JsonProperty("kernelrelease")
-  public String getKernelRelease() {
-    return this.kernelrelease;
-  }
-
-  @JsonProperty("kernelrelease")
-  public void setKernelRelease(String kernelrelease) {
-    this.kernelrelease = kernelrelease;
-  }
-
-  @JsonProperty("kernelversion")
-  public String getKernelVersion() {
-    return this.kernelversion;
-  }
-
-  @JsonProperty("kernelversion")
-  public void setKernelVersion(String kernelversion) {
-    this.kernelversion = kernelversion;
-  }
-
-  @JsonProperty("macaddress")
-  public String getMacAddress() {
-    return this.macaddress;
-  }
-
-  @JsonProperty("macaddress")
-  public void setMacAddress(String macaddress) {
-    this.macaddress = macaddress;
-  }
-
-  @JsonProperty("memoryfree")
-  public long getFreeMemory() {
-    return this.memoryfree;
-  }
-
-  @JsonProperty("memoryfree")
-  public void setFreeMemory(long memoryfree) {
-    this.memoryfree = memoryfree;
-  }
-
-  @JsonProperty("memorysize")
-  public long getMemorySize() {
-    return this.memorysize;
-  }
-
-  @JsonProperty("memorysize")
-  public void setMemorySize(long memorysize) {
-    this.memorysize = memorysize;
-  }
-
-  @JsonProperty("mounts")
-  public List<DiskInfo> getMounts() {
-    return this.mounts;
-  }
-
-  @JsonProperty("mounts")
-  public void setMounts(List<DiskInfo> mounts) {
-    this.mounts = mounts;
-  }
-
-  @JsonProperty("memorytotal")
-  public long getMemoryTotal() {
-    return this.memorytotal;
-  }
-
-  @JsonProperty("memorytotal")
-  public void setMemoryTotal(long memorytotal) {
-    this.memorytotal = memorytotal;
-  }
-
-  @JsonProperty("netmask")
-  public String getNetMask() {
-    return this.netmask;
-  }
-
-  @JsonProperty("netmask")
-  public void setNetMask(String netmask) {
-    this.netmask = netmask;
-  }
-
-  @JsonProperty("operatingsystem")
-  public String getOS() {
-    return this.operatingsystem;
-  }
-
-  @JsonProperty("operatingsystem")
-  public void setOS(String operatingsystem) {
-    this.operatingsystem = operatingsystem;
-  }
-
-  @JsonProperty("operatingsystemrelease")
-  public String getOSRelease() {
-    return this.operatingsystemrelease;
-  }
-
-  @JsonProperty("operatingsystemrelease")
-  public void setOSRelease(String operatingsystemrelease) {
-    this.operatingsystemrelease = operatingsystemrelease;
-  }
-
-  @JsonProperty("osfamily")
-  public String getOSFamily() {
-    return this.osfamily;
-  }
-
-  @JsonProperty("osfamily")
-  public void setOSFamily(String osfamily) {
-    this.osfamily = osfamily;
-  }
-
-  @JsonProperty("physicalprocessorcount")
-  public int getPhysicalProcessorCount() {
-    return this.physicalprocessorcount;
-  }
-
-  @JsonProperty("physicalprocessorcount")
-  public void setPhysicalProcessorCount(int physicalprocessorcount) {
-    this.physicalprocessorcount = physicalprocessorcount;
-  }
-
-  @JsonProperty("processorcount")
-  public int getProcessorCount() {
-    return this.processorcount;
-  }
-
-  @JsonProperty("processorcount")
-  public void setProcessorCount(int processorcount) {
-    this.processorcount = processorcount;
-  }
-
-  @JsonProperty("selinux")
-  public boolean getSeLinux() {
-    return selinux;
-  }
-
-  @JsonProperty("selinux")
-  public void setSeLinux(boolean selinux) {
-    this.selinux = selinux;
-  }
-
-  @JsonProperty("swapfree")
-  public String getSwapFree() {
-    return this.swapfree;
-  }
-
-  @JsonProperty("swapfree")
-  public void setSwapFree(String swapfree) {
-    this.swapfree = swapfree;
-  }
-
-  @JsonProperty("swapsize")
-  public String getSwapSize() {
-    return swapsize;
-  }
-
-  @JsonProperty("swapsize")
-  public void setSwapSize(String swapsize) {
-    this.swapsize = swapsize;
-  }
-
-  @JsonProperty("timezone")
-  public String getTimeZone() {
-    return this.timezone;
-  }
-
-  @JsonProperty("timezone")
-  public void setTimeZone(String timezone) {
-    this.timezone = timezone;
-  }
-
-  @JsonProperty("uptime")
-  public String getUptime() {
-    return this.uptime;
-  }
-
-  @JsonProperty("uptime")
-  public void setUpTime(String uptime) {
-    this.uptime = uptime;
-  }
-
-  @JsonProperty("uptime_hours")
-  public long getUptimeHours() {
-    return this.uptime_hours;
-  }
-
-  @JsonProperty("uptime_hours")
-  public void setUpTimeHours(long uptime_hours) {
-    this.uptime_hours = uptime_hours;
-  }
-
-  @JsonProperty("uptime_days")
-  public long getUpTimeDays() {
-    return this.uptime_days;
-  }
-
-  @JsonProperty("uptime_days")
-  public void setUpTimeDays(long uptime_days) {
-    this.uptime_days = uptime_days;
-  }
-
-  private String getDiskString() {
-    if (mounts == null) {
-      return null;
-    }
-    StringBuilder ret = new StringBuilder();
-    for (DiskInfo diskInfo : mounts) {
-      ret.append("(").append(diskInfo.toString()).append(")");
-    }
-    return ret.toString();
-  }
-
-  public String toString() {
-    return "[" +
-           "hostname=" + this.hostname + "," +
-           "fqdn=" + this.fqdn + "," +
-           "domain=" + this.domain + "," +
-           "architecture=" + this.architecture + "," +
-           "processorcount=" + this.processorcount + "," +
-           "physicalprocessorcount=" + this.physicalprocessorcount + "," +
-           "osname=" + this.operatingsystem + "," +
-           "osversion=" + this.operatingsystemrelease + "," +
-           "osfamily=" + this.osfamily + "," +
-           "memory=" + this.memorytotal + "," +
-           "uptime_hours=" + this.uptime_hours + "," +
-           "mounts=" + getDiskString() + "]\n";
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java

@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HostStatus {
-  public HostStatus(Status status, String cause) {
-    super();
-    this.status = status;
-    this.cause = cause;
-  }
-  public HostStatus() {
-    super();
-  }
-
-  public enum Status {
-    HEALTHY,
-    UNHEALTHY
-  }
-  Status status;
-  String cause;
-  public Status getStatus() {
-    return status;
-  }
-  public void setStatus(Status status) {
-    this.status = status;
-  }
-  public String getCause() {
-    return cause;
-  }
-  public void setCause(String cause) {
-    this.cause = cause;
-  }
-
-  @Override
-  public String toString() {
-    return "HostStatus{" +
-           "status=" + status +
-           ", cause='" + cause + '\'' +
-           '}';
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java

@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.providers.agent.State;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/** Data model for agent to send heartbeat to ambari and/or app master. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class Register {
-  private int responseId = -1;
-  private long timestamp;
-  private String label;
-  private int currentPingPort;
-  private HostInfo hardwareProfile;
-  private String publicHostname;
-  private String tags;
-  private AgentEnv agentEnv;
-  private String agentVersion;
-  private State actualState;
-  private State expectedState;
-  private Map<String, String> allocatedPorts;
-  private Map<String, String> logFolders;
-  private String pkg;
-  private String appVersion;
-
-  @JsonProperty("responseId")
-  public int getResponseId() {
-    return responseId;
-  }
-
-  @JsonProperty("responseId")
-  public void setResponseId(int responseId) {
-    this.responseId = responseId;
-  }
-
-  public long getTimestamp() {
-    return timestamp;
-  }
-
-  public void setTimestamp(long timestamp) {
-    this.timestamp = timestamp;
-  }
-
-  public String getLabel() {
-    return label;
-  }
-
-  public void setLabel(String label) {
-    this.label = label;
-  }
-
-  public String getTags() {
-    return tags;
-  }
-
-  public void setTags(String tags) {
-    this.tags = tags;
-  }
-
-  public HostInfo getHardwareProfile() {
-    return hardwareProfile;
-  }
-
-  public void setHardwareProfile(HostInfo hardwareProfile) {
-    this.hardwareProfile = hardwareProfile;
-  }
-
-  public String getPublicHostname() {
-    return publicHostname;
-  }
-
-  public void setPublicHostname(String name) {
-    this.publicHostname = name;
-  }
-
-  public AgentEnv getAgentEnv() {
-    return agentEnv;
-  }
-
-  public void setAgentEnv(AgentEnv env) {
-    this.agentEnv = env;
-  }
-
-  public String getAgentVersion() {
-    return agentVersion;
-  }
-
-  public void setAgentVersion(String agentVersion) {
-    this.agentVersion = agentVersion;
-  }
-
-  public int getCurrentPingPort() {
-    return currentPingPort;
-  }
-
-  public void setCurrentPingPort(int currentPingPort) {
-    this.currentPingPort = currentPingPort;
-  }
-
-  public State getActualState() {
-    return actualState;
-  }
-
-  public void setActualState(State actualState) {
-    this.actualState = actualState;
-  }
-
-  public State getExpectedState() {
-    return expectedState;
-  }
-
-  public void setExpectedState(State expectedState) {
-    this.expectedState = expectedState;
-  }
-
-  /** @return the allocated ports, or <code>null</code> if none are present */
-  @JsonProperty("allocatedPorts")
-  public Map<String, String> getAllocatedPorts() {
-    return allocatedPorts;
-  }
-
-  /** @param ports allocated ports */
-  @JsonProperty("allocatedPorts")
-  public void setAllocatedPorts(Map<String, String> ports) {
-    this.allocatedPorts = ports;
-  }
-
-  /** @return the log folders, or <code>null</code> if none are present */
-  @JsonProperty("logFolders")
-  public Map<String, String> getLogFolders() {
-    return logFolders;
-  }
-
-  /** @param logFolders assigned log folders */
-  @JsonProperty("logFolders")
-  public void setLogFolders(Map<String, String> logFolders) {
-    this.logFolders = logFolders;
-  }
-
-  public String getPkg() {
-    return pkg;
-  }
-
-  public void setPkg(String pkg) {
-    this.pkg = pkg;
-  }
-
-  @JsonProperty("appVersion")
-  public String getAppVersion() {
-    return appVersion;
-  }
-
-  @JsonProperty("appVersion")
-  public void setAppVersion(String appVersion) {
-    this.appVersion = appVersion;
-  }
-
-  @Override
-  public String toString() {
-    String ret = "responseId=" + responseId + "\n" +
-                 "timestamp=" + timestamp + "\n" +
-                 "label=" + label + "\n" +
-                 "hostname=" + publicHostname + "\n" +
-                 "expectedState=" + expectedState + "\n" +
-                 "actualState=" + actualState + "\n" +
-                 "appVersion=" + appVersion + "\n";
-
-    if (hardwareProfile != null) {
-      ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
-    }
-    return ret;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java

@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegistrationCommand {
-
-  private String command;
-
-  public String getCommand() {
-    return command;
-  }
-
-  public void setCommand(String command) {
-    this.command = command;
-  }
-
-  public RegistrationCommand(String command) {
-
-    this.command = command;
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java

@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.List;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegistrationResponse {
-
-  @JsonProperty("response")
-  private RegistrationStatus response;
-
-  /**
-   * exitstatus is a code of error which was rised on server side. exitstatus
-   * = 0 (OK - Default) exitstatus = 1 (Registration failed because different
-   * version of agent and server)
-   */
-  @JsonProperty("exitstatus")
-  private int exitstatus;
-
-  /** log - message, which will be printed to agents log */
-  @JsonProperty("log")
-  private String log;
-
-  /** tags - tags associated with the container */
-  @JsonProperty("tags")
-  private String tags;
-  
-  @JsonProperty("package")
-  private String pkg;
-
-  //Response id to start with, usually zero.
-  @JsonProperty("responseId")
-  private long responseId;
-
-  @JsonProperty("statusCommands")
-  private List<StatusCommand> statusCommands = null;
-
-  public RegistrationResponse() {
-  }
-
-  public RegistrationStatus getResponse() {
-    return response;
-  }
-
-  public void setResponse(RegistrationStatus response) {
-    this.response = response;
-  }
-
-  public int getExitstatus() {
-    return exitstatus;
-  }
-
-  public void setExitstatus(int exitstatus) {
-    this.exitstatus = exitstatus;
-  }
-
-  public RegistrationStatus getResponseStatus() {
-    return response;
-  }
-
-  public void setResponseStatus(RegistrationStatus response) {
-    this.response = response;
-  }
-
-  public List<StatusCommand> getStatusCommands() {
-    return statusCommands;
-  }
-
-  public void setStatusCommands(List<StatusCommand> statusCommands) {
-    this.statusCommands = statusCommands;
-  }
-
-  public long getResponseId() {
-    return responseId;
-  }
-
-  public void setResponseId(long responseId) {
-    this.responseId = responseId;
-  }
-
-  public String getTags() {
-    return tags;
-  }
-
-  public void setTags(String tags) {
-    this.tags = tags;
-  }
-
-  public String getLog() {
-    return log;
-  }
-
-  public void setLog(String log) {
-    this.log = log;
-  }
-
-  public String getPkg() {
-    return pkg;
-  }
-
-  public void setPkg(String pkg) {
-    this.pkg = pkg;
-  }
-
-  @Override
-  public String toString() {
-    return "RegistrationResponse{" +
-           "response=" + response +
-           ", responseId=" + responseId +
-           ", statusCommands=" + statusCommands +
-           '}';
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java

@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-public enum RegistrationStatus {
-    OK,
-    FAILED
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java

@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Command to report the status of a list of services in roles.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class StatusCommand {
-  public static String STATUS_COMMAND = "STATUS";
-  public static String GET_CONFIG_COMMAND = "GET_CONFIG";
-
-  AgentCommandType agentCommandType;
-
-  private String clusterName;
-  private String serviceName;
-  private String componentName;
-  private Map<String, Map<String, String>> configurations;
-  private Map<String, String> commandParams = new HashMap<String, String>();
-  private Map<String, String> hostLevelParams = new HashMap<String, String>();
-  private String roleCommand;
-  private boolean yarnDockerMode;
-
-  public StatusCommand() {
-    this.agentCommandType = AgentCommandType.STATUS_COMMAND;
-  }
-
-  @JsonProperty("clusterName")
-  public String getClusterName() {
-    return clusterName;
-  }
-
-  @JsonProperty("clusterName")
-  public void setClusterName(String clusterName) {
-    this.clusterName = clusterName;
-  }
-
-  @JsonProperty("serviceName")
-  public String getServiceName() {
-    return serviceName;
-  }
-
-  @JsonProperty("serviceName")
-  public void setServiceName(String serviceName) {
-    this.serviceName = serviceName;
-  }
-
-  @JsonProperty("componentName")
-  public String getComponentName() {
-    return componentName;
-  }
-
-  @JsonProperty("componentName")
-  public void setComponentName(String componentName) {
-    this.componentName = componentName;
-  }
-
-  @JsonProperty("configurations")
-  public Map<String, Map<String, String>> getConfigurations() {
-    return configurations;
-  }
-
-  @JsonProperty("configurations")
-  public void setConfigurations(Map<String, Map<String, String>> configurations) {
-    this.configurations = configurations;
-  }
-
-  @JsonProperty("hostLevelParams")
-  public Map<String, String> getHostLevelParams() {
-    return hostLevelParams;
-  }
-
-  @JsonProperty("hostLevelParams")
-  public void setHostLevelParams(Map<String, String> params) {
-    this.hostLevelParams = params;
-  }
-
-  @JsonProperty("commandParams")
-  public Map<String, String> getCommandParams() {
-    return commandParams;
-  }
-
-  @JsonProperty("commandParams")
-  public void setCommandParams(Map<String, String> commandParams) {
-    this.commandParams = commandParams;
-  }
-
-  @JsonProperty("commandType")
-  public AgentCommandType getCommandType() {
-    return agentCommandType;
-  }
-
-  @JsonProperty("commandType")
-  public void setCommandType(AgentCommandType commandType) {
-    this.agentCommandType = commandType;
-  }
-
-  @JsonProperty("roleCommand")
-  public String getRoleCommand() {
-    return roleCommand;
-  }
-
-  @JsonProperty("roleCommand")
-  public void setRoleCommand(String roleCommand) {
-    this.roleCommand = roleCommand;
-  }
-  
-  @JsonProperty("yarnDockerMode")
-  public boolean isYarnDockerMode() {
-    return yarnDockerMode;
-  }
-
-  @JsonProperty("yarnDockerMode")
-  public void setYarnDockerMode(boolean yarnDockerMode) {
-    this.yarnDockerMode = yarnDockerMode;
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder builder = new StringBuilder();
-    builder.append("StatusCommand [agentCommandType=").append(agentCommandType)
-        .append(", clusterName=").append(clusterName).append(", serviceName=")
-        .append(serviceName).append(", componentName=").append(componentName)
-        .append(", configurations=").append(configurations)
-        .append(", commandParams=").append(commandParams)
-        .append(", hostLevelParams=").append(hostLevelParams)
-        .append(", roleCommand=").append(roleCommand).append("]");
-    return builder.toString();
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java

@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.tools.TestUtility;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Set;
-
-/**
- *
- */
-public class TestAgentClientProvider {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestAgentClientProvider.class);
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-
-  @Test
-  public void testGetApplicationTags() throws Exception {
-    Configuration configuration = new Configuration();
-    FileSystem fs = FileSystem.getLocal(configuration);
-    SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
-    AgentClientProvider provider = new AgentClientProvider(null);
-    String zipFileName = TestUtility.createAppPackage(
-        folder,
-        "testpkg",
-        "test.zip",
-        "target/test-classes/org/apache/slider/common/tools/test");
-    Set<String> tags = provider.getApplicationTags(sliderFileSystem, zipFileName);
-    assert tags != null;
-    assert !tags.isEmpty();
-    assert tags.contains("Name: STORM");
-    assert tags.contains("Description: Apache Hadoop Stream processing framework");
-    assert tags.contains("Version: 0.9.1.2.1");
-
-  }
-
-  @Test
-  public void testValidateInstanceDefinition() throws Exception {
-    AgentClientProvider provider = new AgentClientProvider(null);
-    AggregateConf instanceDefinition = new AggregateConf();
-
-    try {
-      provider.validateInstanceDefinition(instanceDefinition, null);
-      Assert.assertFalse("Should fail with BadConfigException", true);
-    } catch (BadConfigException e) {
-      log.info(e.toString());
-      Assert.assertTrue(e.getMessage().contains("Application definition must be provided"));
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java

@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- *
- */
-public class TestAgentLaunchParameter {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestAgentLaunchParameter.class);
-
-  @Test
-  public void testTestAgentLaunchParameter() throws Exception {
-    AgentLaunchParameter alp = new AgentLaunchParameter("");
-    Assert.assertEquals("", alp.getNextLaunchParameter("abc"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
-
-    alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE");
-    Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
-    Assert.assertEquals("2", alp.getNextLaunchParameter("a"));
-    Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
-    Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
-
-    Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("6", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("c"));
-
-    alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:");
-    Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
-    Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
-    Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
-
-    Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-
-    alp = new AgentLaunchParameter("|:");
-    Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("a"));
-
-    alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:");
-    Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
-    Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
-    Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
-
-    alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::");
-    Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
-    Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
-    Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
-    Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
-    Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java

@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.tools.TestUtility;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestAgentUtils {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestAgentUtils.class);
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-  private static final String metainfo_str = "<metainfo>\n"
-      + "  <schemaVersion>2.0</schemaVersion>\n"
-      + "  <application>\n"
-      + "      <name>MYTESTAPPLICATION</name>\n"
-      + "      <comment>\n"
-      + "        My Test Application\n"
-      + "      </comment>\n"
-      + "      <version>1.0</version>\n"
-      + "      <type>YARN-APP</type>\n"
-      + "      <components>\n"
-      + "        <component>\n"
-      + "          <name>REST</name>\n"
-      + "          <category>MASTER</category>\n"
-      + "          <commandScript>\n"
-      + "            <script>scripts/rest.py</script>\n"
-      + "            <scriptType>PYTHON</scriptType>\n"
-      + "            <timeout>600</timeout>\n"
-      + "          </commandScript>\n"
-      + "        </component>\n"
-      + "      </components>\n"
-      + "  </application>\n"
-      + "</metainfo>";
-
-  @Test
-  public void testGetApplicationMetainfo() throws Exception {
-    String zipFileName = TestUtility.createAppPackage(
-        folder,
-        "testpkg",
-        "test.zip",
-        "target/test-classes/org/apache/slider/common/tools/test");
-    Configuration configuration = new Configuration();
-    FileSystem fs = FileSystem.getLocal(configuration);
-    log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
-    SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
-    // Without accompany metainfo file, read metainfo from the zip file
-    Metainfo metainfo = AgentUtils.getApplicationMetainfo(
-        sliderFileSystem, zipFileName, false);
-    Assert.assertNotNull(metainfo.getApplication());
-    Assert.assertEquals("STORM", metainfo.getApplication().getName());
-
-    // With accompany metainfo file, read metainfo from the accompany file
-    String acompanyFileName = zipFileName + ".metainfo.xml";
-    File f = new File(acompanyFileName);
-    try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
-      writer.write(metainfo_str);
-    }
-    metainfo = AgentUtils.getApplicationMetainfo(
-        sliderFileSystem, zipFileName, false);
-    Assert.assertNotNull(metainfo.getApplication());
-    Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java

@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import com.google.common.io.Files;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.common.params.ActionCreateArgs;
-import org.apache.slider.common.params.AddonArgsDelegate;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.persist.AppDefinitionPersister;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class TestAppDefinitionPersister {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestAppDefinitionPersister.class);
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-
-  /**
-   * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration();
-   * BasicConfigurator.configure(); }*
-   */
-
-
-  @Test
-  public void testAppDefinitionPersister() throws Exception {
-    Configuration configuration = new Configuration();
-    FileSystem fs = FileSystem.getLocal(configuration);
-    log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
-    SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
-    AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem);
-    String clustername = "c1";
-    ActionCreateArgs buildInfo = new ActionCreateArgs();
-    buildInfo.appMetaInfo = null;
-    buildInfo.appDef = null;
-    buildInfo.addonDelegate = new AddonArgsDelegate();
-
-    // nothing to do
-    adp.processSuppliedDefinitions(clustername, buildInfo, null);
-    adp.persistPackages();
-    List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions();
-    Assert.assertTrue(appDefinitions.size() == 0);
-
-    ConfTree ct = new ConfTree();
-    ConfTreeOperations appConf = new ConfTreeOperations(ct);
-    final File tempDir = Files.createTempDir();
-    final File metainfo = new File(tempDir, "metainfo.json");
-
-    // unreadable metainfo
-    buildInfo.appMetaInfo = metainfo;
-
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains(
-          "Path specified with "
-              + "--metainfo either cannot be read or is not a file"));
-    }
-
-    try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
-      writer.println("{");
-      writer.println("}");
-    }
-    buildInfo.appDef = metainfo;
-
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains(
-          "Both --metainfo and --appdef cannot be specified"));
-    }
-
-    // both --metainfojson and --appdef cannot be specified
-    buildInfo.appMetaInfo = null;
-    buildInfo.appMetaInfoJson = "{}";
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains(
-          "Both --metainfojson and --appdef cannot be specified"));
-    }
-
-    buildInfo.appDef = null;
-
-    buildInfo.appMetaInfoJson = "";
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains(
-          "Empty string specified with --metainfojson"));
-    }
-    buildInfo.appMetaInfo = metainfo;
-
-    // both --metainfo and --metainfojson cannot be specified
-    buildInfo.appMetaInfoJson = "{}";
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains(
-          "Both --metainfo and --metainfojson cannot be specified"));
-    }
-    buildInfo.appMetaInfoJson = null;
-
-    appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
-
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains(
-          "application.def cannot "
-              + "not be set if --metainfo is specified in the cmd line"));
-    }
-
-    appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-
-    adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    appDefinitions = adp.getAppDefinitions();
-    Assert.assertTrue(appDefinitions.size() == 1);
-    Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
-    log.info(appDefinitions.get(0).toString());
-    Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default"));
-    Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
-    Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
-
-    buildInfo.appDef = tempDir;
-    buildInfo.appMetaInfo = null;
-
-    appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
-
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided"));
-    }
-
-    adp.getAppDefinitions().clear();
-    appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-    adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    appDefinitions = adp.getAppDefinitions();
-    Assert.assertTrue(appDefinitions.size() == 1);
-    Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
-    log.info(appDefinitions.get(0).toString());
-    Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
-    Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
-    Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
-
-    adp.getAppDefinitions().clear();
-    buildInfo.appDef = null;
-    buildInfo.appMetaInfo = null;
-    appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-
-    ArrayList<String> list = new ArrayList<String>() {{
-      add("addon1");
-      add("");
-      add("addon2");
-      add(metainfo.getAbsolutePath());
-    }};
-
-    buildInfo.addonDelegate.addonTuples = list;
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified"));
-    }
-
-    buildInfo.appMetaInfo = metainfo;
-
-    try {
-      adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    } catch (BadConfigException bce) {
-      log.info(bce.getMessage());
-      Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1"));
-    }
-
-    appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-
-    list = new ArrayList<String>() {{
-      add("addon1");
-      add(tempDir.getAbsolutePath());
-      add("addon2");
-      add(metainfo.getAbsolutePath());
-    }};
-
-    buildInfo.addonDelegate.addonTuples = list;
-    adp.getAppDefinitions().clear();
-
-    adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
-    appDefinitions = adp.getAppDefinitions();
-
-    Assert.assertTrue(appDefinitions.size() == 3);
-    Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
-    Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains(
-        "addons/addon1/addon_addon1.zip"));
-    Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains(
-        "addons/addon2/addon_addon2.zip"));
-    log.info(appConf.getGlobalOptions().get("application.addons"));
-    Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains(
-        "application.addon.addon2,application.addon.addon1")
-                      || appConf.getGlobalOptions().get("application.addons").contains(
-        "application.addon.addon1,application.addon.addon2"));
-    int seen = 0;
-    for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) {
-      if (adp_ad.pkgName.equals("appPkg.zip")) {
-        log.info(adp_ad.toString());
-        Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default"));
-        Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef"));
-        seen++;
-      }
-      if (adp_ad.pkgName.equals("addon_addon1.zip")) {
-        log.info(adp_ad.toString());
-        Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
-        Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1"));
-        seen++;
-      }
-      if (adp_ad.pkgName.equals("addon_addon2.zip")) {
-        log.info(adp_ad.toString());
-        Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json"));
-        Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2"));
-        seen++;
-      }
-    }
-    Assert.assertEquals(3, seen);
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java

@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class TestComponentTagProvider {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestComponentTagProvider.class);
-
-  @Test
-  public void testTagProvider() throws Exception {
-    ComponentTagProvider ctp = new ComponentTagProvider();
-    Assert.assertEquals("", ctp.getTag(null, null));
-    Assert.assertEquals("", ctp.getTag(null, "cid"));
-    Assert.assertEquals("", ctp.getTag("comp1", null));
-
-    Assert.assertEquals("1", ctp.getTag("comp1", "cid1"));
-    Assert.assertEquals("2", ctp.getTag("comp1", "cid2"));
-    Assert.assertEquals("3", ctp.getTag("comp1", "cid3"));
-    ctp.releaseTag("comp1", "cid2");
-    Assert.assertEquals("2", ctp.getTag("comp1", "cid22"));
-
-    ctp.releaseTag("comp1", "cid4");
-    ctp.recordAssignedTag("comp1", "cid5", "5");
-    Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
-    Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
-    Assert.assertEquals("6", ctp.getTag("comp1", "cid6"));
-
-    ctp.recordAssignedTag("comp1", "cid55", "5");
-    Assert.assertEquals("5", ctp.getTag("comp1", "cid55"));
-
-    ctp.recordAssignedTag("comp2", "cidb3", "3");
-    Assert.assertEquals("1", ctp.getTag("comp2", "cidb1"));
-    Assert.assertEquals("2", ctp.getTag("comp2", "cidb2"));
-    Assert.assertEquals("4", ctp.getTag("comp2", "cidb4"));
-
-    ctp.recordAssignedTag("comp2", "cidb5", "six");
-    ctp.recordAssignedTag("comp2", "cidb5", "-55");
-    ctp.recordAssignedTag("comp2", "cidb5", "tags");
-    ctp.recordAssignedTag("comp2", "cidb5", null);
-    ctp.recordAssignedTag("comp2", "cidb5", "");
-    ctp.recordAssignedTag("comp2", "cidb5", "5");
-    Assert.assertEquals("6", ctp.getTag("comp2", "cidb6"));
-
-    ctp.recordAssignedTag("comp2", null, "5");
-    ctp.recordAssignedTag(null, null, "5");
-    ctp.releaseTag("comp1", null);
-    ctp.releaseTag(null, "cid4");
-    ctp.releaseTag(null, null);
-  }
-
-  @Test
-  public void testTagProviderWithThread() throws Exception {
-    ComponentTagProvider ctp = new ComponentTagProvider();
-    Thread thread = new Thread(new Taggged(ctp));
-    Thread thread2 = new Thread(new Taggged(ctp));
-    Thread thread3 = new Thread(new Taggged(ctp));
-    thread.start();
-    thread2.start();
-    thread3.start();
-    ctp.getTag("comp1", "cid50");
-    thread.join();
-    thread2.join();
-    thread3.join();
-    Assert.assertEquals("101", ctp.getTag("comp1", "cid101"));
-  }
-
-  public class Taggged implements Runnable {
-    private final ComponentTagProvider ctp;
-
-    public Taggged(ComponentTagProvider ctp) {
-      this.ctp = ctp;
-    }
-
-    public void run() {
-      for (int i = 0; i < 100; i++) {
-        String containerId = "cid" + (i + 1);
-        this.ctp.getTag("comp1", containerId);
-      }
-      for (int i = 0; i < 100; i++) {
-        String containerId = "cid" + (i + 1);
-        this.ctp.getTag("comp1", containerId);
-      }
-      for (int i = 0; i < 100; i += 2) {
-        String containerId = "cid" + (i + 1);
-        this.ctp.releaseTag("comp1", containerId);
-      }
-      for (int i = 0; i < 100; i += 2) {
-        String containerId = "cid" + (i + 1);
-        this.ctp.getTag("comp1", containerId);
-      }
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java

@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestState {
-  protected static final Logger log = LoggerFactory.getLogger(TestState.class);
-
-  @Test
-  public void testState() throws Exception {
-    State state = State.STARTED;
-    Assert.assertEquals(Command.STOP, state.getSupportedCommand(false, true));
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java

@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class TestConfigParser {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestConfigParser.class);
-  private static final String config_1_str = "<configuration>\n"
-                                             + "  <property>\n"
-                                             + "    <name>security.client.protocol.acl</name>\n"
-                                             + "    <value>*</value>\n"
-                                             + "    <description>ACL for HRegionInterface protocol implementations (ie. \n"
-                                             + "    clients talking to HRegionServers)\n"
-                                             + "    The ACL is a comma-separated list of user and group names. The user and \n"
-                                             + "    group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
-                                             + "    A special value of \"*\" means all users are allowed.</description>\n"
-                                             + "  </property>\n"
-                                             + "\n"
-                                             + "  <property>\n"
-                                             + "    <name>security.admin.protocol.acl</name>\n"
-                                             + "    <value>*</value>\n"
-                                             + "    <description>ACL for HMasterInterface protocol implementation (ie. \n"
-                                             + "    clients talking to HMaster for admin operations).\n"
-                                             + "    The ACL is a comma-separated list of user and group names. The user and \n"
-                                             + "    group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
-                                             + "    A special value of \"*\" means all users are allowed.</description>\n"
-                                             + "  </property>\n"
-                                             + "\n"
-                                             + "  <property>\n"
-                                             + "    <name>security.masterregion.protocol.acl</name>\n"
-                                             + "    <value>*</value>\n"
-                                             + "    <description>ACL for HMasterRegionInterface protocol implementations\n"
-                                             + "    (for HRegionServers communicating with HMaster)\n"
-                                             + "    The ACL is a comma-separated list of user and group names. The user and \n"
-                                             + "    group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
-                                             + "    A special value of \"*\" means all users are allowed.</description>\n"
-                                             + "  </property>\n"
-                                             + "  <property>\n"
-                                             + "    <name>emptyVal</name>\n"
-                                             + "    <value></value>\n"
-                                             + "    <description>non-empty-desc</description>\n"
-                                             + "  </property>\n"
-                                             + "  <property>\n"
-                                             + "    <name>emptyDesc</name>\n"
-                                             + "    <value></value>\n"
-                                             + "    <description></description>\n"
-                                             + "  </property>\n"
-                                             + "  <property>\n"
-                                             + "    <name>noDesc</name>\n"
-                                             + "    <value></value>\n"
-                                             + "  </property>\n"
-                                             + "</configuration>";
-
-  @Test
-  public void testParse() throws IOException {
-
-    InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes());
-    DefaultConfig config = new DefaultConfigParser().parse(config_1);
-    Assert.assertNotNull(config);
-    Assert.assertNotNull(config.getPropertyInfos());
-    Assert.assertEquals(6, config.getPropertyInfos().size());
-    for (PropertyInfo pInfo : config.getPropertyInfos()) {
-      if (pInfo.getName().equals("security.client.protocol.acl")) {
-        Assert.assertEquals("*", pInfo.getValue());
-        Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface "));
-      }
-      if (pInfo.getName().equals("emptyVal")) {
-        Assert.assertEquals("", pInfo.getValue());
-        Assert.assertEquals("non-empty-desc", pInfo.getDescription());
-      }
-      if (pInfo.getName().equals("emptyDesc")) {
-        Assert.assertEquals("", pInfo.getValue());
-        Assert.assertEquals("", pInfo.getDescription());
-      }
-      if (pInfo.getName().equals("noDesc")) {
-        Assert.assertEquals("", pInfo.getValue());
-        Assert.assertNull(pInfo.getDescription());
-      }
-    }
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java

@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.providers.agent.AgentProviderService;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-
-import static org.mockito.Mockito.doReturn;
-
-/**
- *
- */
-public class TestMetainfoParser {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestMetainfoParser.class);
-  public static final String METAINFO_XML =
-      "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
-
-  @Test
-  public void testParse() throws IOException {
-
-    InputStream resStream = this.getClass().getResourceAsStream(
-        METAINFO_XML);
-    MetainfoParser parser = new MetainfoParser();
-    Metainfo metainfo = parser.fromXmlStream(resStream);
-    Assert.assertNotNull(metainfo);
-    Assert.assertNotNull(metainfo.getApplication());
-    Application application = metainfo.getApplication();
-    assert "STORM".equals(application.getName());
-    assert 6 == application.getComponents().size();
-    OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
-    assert "tarball".equals(pkg.getType());
-    assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
-    boolean found = false;
-    for (Component comp : application.getComponents()) {
-      if (comp != null && comp.getName().equals("NIMBUS")) {
-        found = true;
-        Assert.assertEquals(0, comp.getComponentExports().size());
-      }
-      if (comp != null && comp.getName().equals("SUPERVISOR")) {
-        Assert.assertEquals(1, comp.getComponentExports().size());
-      }
-      if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) {
-        assert 2 == comp.getCommands().size();
-        assert "start command".equals(comp.getCommands().get(0).getExec());
-        assert "START".equals(comp.getCommands().get(0).getName());
-        assert "stop command".equals(comp.getCommands().get(1).getExec());
-        assert "STOP".equals(comp.getCommands().get(1).getName());
-      }
-    }
-    assert found;
-    Assert.assertEquals(0, application.getConfigFiles().size());
-    assert 1 == application.getPackages().size();
-    Package p = application.getPackages().get(0);
-    assert "tarball".equals(p.getType());
-    assert "test-tarball-name.tgz".equals(p.getName());
-  }
-
-  @Test
-  public void testJsonParse() throws IOException {
-    String metaInfo1_json = "{\n"
-                            + "\"schemaVersion\":\"2.2\",\n"
-                            + "\"application\":{\n"
-                            +     "\"name\": \"MEMCACHED\","
-                            +     "\"exportGroups\": ["
-                            +        "{"
-                            +          "\"name\": \"Servers\","
-                            +          "\"exports\": ["
-                            +            "{"
-                            +               "\"name\": \"host_port\","
-                            +               "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\""
-                            +            "}"
-                            +          "]"
-                            +        "}"
-                            +      "],"
-                            +     "\"components\": ["
-                            +        "{"
-                            +          "\"name\": \"MEMCACHED\","
-                            +          "\"compExports\": \"Servers-host_port\","
-                            +          "\"commands\": ["
-                            +            "{"
-                            +               "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\""
-                            +            "}"
-                            +          "]"
-                            +        "},"
-                            +        "{"
-                            +          "\"name\": \"MEMCACHED2\","
-                            +          "\"commands\": ["
-                            +            "{"
-                            +               "\"exec\": \"scripts/config.py\","
-                            +               "\"type\": \"PYTHON\","
-                            +               "\"name\": \"CONFIGURE\""
-                            +            "}"
-                            +          "],"
-                            +          "\"dockerContainers\": ["
-                            +            "{"
-                            +               "\"name\": \"redis\","
-                            +               "\"image\": \"dockerhub/redis\","
-                            +               "\"options\": \"-net=bridge\","
-                            +               "\"mounts\": ["
-                            +                 "{"
-                            +                   "\"containerMount\": \"/tmp/conf\","
-                            +                   "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\""
-                            +                 "}"
-                            +               "]"
-                            +            "}"
-                            +          "]"
-                            +        "}"
-                            +      "]"
-                            +   "}"
-                            + "}";
-
-    MetainfoParser parser = new MetainfoParser();
-    Metainfo mInfo = parser.fromJsonString(metaInfo1_json);
-    Assert.assertEquals("2.2", mInfo.getSchemaVersion());
-
-    Application app = mInfo.getApplication();
-    Assert.assertNotNull(app);
-
-    Assert.assertEquals("MEMCACHED", app.getName());
-    List<ExportGroup> egs = app.getExportGroups();
-    Assert.assertEquals(1, egs.size());
-    ExportGroup eg = egs.get(0);
-    Assert.assertEquals("Servers", eg.getName());
-    List<Export> exports = eg.getExports();
-    Assert.assertEquals(1, exports.size());
-    Export export = exports.get(0);
-    Assert.assertEquals("host_port", export.getName());
-    Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue());
-
-    List<Component> components = app.getComponents();
-    Assert.assertEquals(2, components.size());
-
-    Component c1 = mInfo.getApplicationComponent("MEMCACHED");
-    Assert.assertNotNull(c1);
-    Assert.assertEquals("MEMCACHED", c1.getName());
-    Assert.assertEquals("Servers-host_port", c1.getCompExports());
-    Assert.assertEquals(1, c1.getCommands().size());
-    ComponentCommand cmd = c1.getCommands().get(0);
-    Assert.assertEquals("START", cmd.getName());
-    Assert.assertEquals("SHELL", cmd.getType());
-    Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main",
-                        cmd.getExec());
-
-    Component c2 = mInfo.getApplicationComponent("MEMCACHED2");
-    Assert.assertNotNull(c2);
-    Assert.assertEquals("MEMCACHED2", c2.getName());
-    Assert.assertEquals(1, c2.getCommands().size());
-    cmd = c2.getCommands().get(0);
-    Assert.assertEquals("CONFIGURE", cmd.getName());
-    Assert.assertEquals("PYTHON", cmd.getType());
-    Assert.assertEquals("scripts/config.py", cmd.getExec());
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java

@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.providers.agent.AgentProviderService;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class TestAgentProviderService extends AgentProviderService {
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestAgentProviderService.class);
-
-  public TestAgentProviderService() {
-    super();
-    log.info("TestAgentProviderService created");
-  }
-
-  @Override
-  public void bind(StateAccessForProviders stateAccessor,
-      QueueAccess queueAccess,
-      List<Container> liveContainers) {
-    super.bind(stateAccessor, queueAccess, liveContainers);
-    Map<String,String> dummyProps = new HashMap<String, String>();
-    dummyProps.put("prop1", "val1");
-    dummyProps.put("prop2", "val2");
-    log.info("publishing dummy-site.xml with values {}", dummyProps);
-    publishApplicationInstanceData("dummy-site", "dummy configuration",
-                                   dummyProps.entrySet());
-    // publishing global config for testing purposes
-    publishApplicationInstanceData("global", "global configuration",
-                                   stateAccessor.getAppConfSnapshot()
-                                       .getGlobalOptions().entrySet());
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java

@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.agent.AgentProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestSliderProviderFactory extends AgentProviderFactory{
-  protected static final Logger log =
-      LoggerFactory.getLogger(TestSliderProviderFactory.class);
-
-  public TestSliderProviderFactory() {
-    log.info("Created TestSliderProviderFactory");
-  }
-
-  @Override
-  public ProviderService createServerProvider() {
-    log.info("Creating TestAgentProviderService");
-    return new TestAgentProviderService();
-  }
-}