YARN-6160. Create an agent-less docker-less provider in the native services framework. Contributed by Billie Rinaldi

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java

@@ -133,7 +133,6 @@ import org.apache.slider.core.zk.BlockingZKWatcher;
 import org.apache.slider.core.zk.ZKIntegration;
 import org.apache.slider.providers.AbstractClientProvider;
 import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.SliderProviderFactory;
 import org.apache.slider.server.appmaster.SliderAppMaster;
 import org.apache.slider.server.appmaster.rpc.RpcBinder;
 import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
@@ -257,8 +256,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     config = super.bindArgs(config, args);
     serviceArgs = new ClientArgs(args);
     serviceArgs.parse();
-    // add the slider XML config
-    ConfigHelper.injectSliderXMLResource();
     // yarn-ify
     YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
     return patchConfiguration(yarnConfiguration);
@@ -622,14 +619,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     return EXIT_SUCCESS;
   }
 
-  @Override
-  public AbstractClientProvider createClientProvider(String provider)
-    throws SliderException {
-    SliderProviderFactory factory =
-      SliderProviderFactory.createSliderProviderFactory(provider);
-    return factory.createClientProvider();
-  }
-
   private Application getApplicationFromArgs(String clusterName,
       AbstractClusterBuildingActionArgs args) throws IOException {
     File file = args.getAppDef();
@@ -893,7 +882,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 
   private void persistApp(Path appDir, Application application)
       throws IOException, SliderException {
-    FsPermission appDirPermission = new FsPermission("777");
+    FsPermission appDirPermission = new FsPermission("750");
     sliderFileSystem.createWithPermissions(appDir, appDirPermission);
     Path appJson = new Path(appDir, application.getName() + ".json");
     jsonSerDeser
@@ -1190,17 +1179,18 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
       }
     }
 
+    // TODO handle client install
     // Only INSTALL is supported
-    AbstractClientProvider
-        provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
-    provider.processClientOperation(sliderFileSystem,
-        getRegistryOperations(),
-        getConfig(),
-        "INSTALL",
-        clientInfo.installLocation,
-        pkgFile,
-        config,
-        clientInfo.name);
+    //    ClientProvider
+    //        provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
+    //    provider.processClientOperation(sliderFileSystem,
+    //        getRegistryOperations(),
+    //        getConfig(),
+    //        "INSTALL",
+    //        clientInfo.installLocation,
+    //        pkgFile,
+    //        config,
+    //        clientInfo.name);
     return EXIT_SUCCESS;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java

@@ -42,7 +42,6 @@ import org.apache.slider.common.params.ActionThawArgs;
 import org.apache.slider.common.params.ActionUpgradeArgs;
 import org.apache.slider.core.exceptions.BadCommandArgumentsException;
 import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.providers.AbstractClientProvider;
 
 import java.io.IOException;
 
@@ -62,15 +61,6 @@ public interface SliderClientAPI extends Service {
   int actionAmSuicide(String clustername,
       ActionAMSuicideArgs args) throws YarnException, IOException;
 
-  /**
-   * Get the provider for this cluster
-   * @param provider the name of the provider
-   * @return the provider instance
-   * @throws SliderException problems building the provider
-   */
-  AbstractClientProvider createClientProvider(String provider)
-    throws SliderException;
-
   /**
    * Manage keytabs leveraged by slider
    *

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java

@@ -18,8 +18,6 @@
 
 package org.apache.slider.common;
 
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -158,12 +156,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
    */
   String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
                                     HISTORY_FILENAME_SUFFIX;
-  /**
-   * XML resource listing the standard Slider providers
-   * {@value}
-   */
-  String SLIDER_XML = "org/apache/slider/slider.xml";
-  
+
   String CLUSTER_DIRECTORY = "cluster";
 
   String PACKAGE_DIRECTORY = "package";
@@ -311,5 +304,12 @@ public interface SliderKeys extends SliderXmlConfKeys {
   String APP_RESOURCES = "application.resources";
   String APP_RESOURCES_DIR = "app/resources";
 
-  String APP_PACKAGES_DIR = "app/packages";
+  String APP_INSTALL_DIR = "app/install";
+
+  String OUT_FILE = "stdout.txt";
+  String ERR_FILE = "stderr.txt";
+
+  String QUICK_LINKS = "quicklinks";
+
+  String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java

@@ -50,7 +50,6 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * Methods to aid in config, both in the Configuration class and
@@ -62,11 +61,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class ConfigHelper {
   private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
 
-  private static AtomicBoolean sliderResourceInjected =
-      new AtomicBoolean(false);
-  private static AtomicBoolean sliderResourceInjectionAttempted =
-      new AtomicBoolean(false);
-  
   /**
    * Dump the (sorted) configuration
    * @param conf config
@@ -614,37 +608,4 @@ public class ConfigHelper {
   public static void registerDeprecatedConfigItems() {
   }
 
-  /**
-   * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
-   * included
-   * @return a configuration instance
-   */
-  public static Configuration loadSliderConfiguration() {
-    Configuration conf = new Configuration();
-    conf.addResource(SliderKeys.SLIDER_XML);
-    return conf;
-  }
-
-  /**
-   * Inject the {@link SliderKeys#SLIDER_XML} resource
-   * into the configuration resources <i>of all configurations</i>.
-   * <p>
-   *   This operation is idempotent.
-   * <p>
-   * If the resource is not on the classpath, downgrades, rather than
-   * fails.
-   * @return true if the resource was found and loaded.
-   */
-  public static synchronized boolean injectSliderXMLResource() {
-    if (sliderResourceInjectionAttempted.getAndSet(true)) {
-      return sliderResourceInjected.get();
-    }
-    URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
-    if (resourceUrl != null) {
-      Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
-      sliderResourceInjected.set(true);
-    }
-    return sliderResourceInjected.get();
-  }
-  
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java

@@ -19,32 +19,26 @@
 package org.apache.slider.providers;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.exceptions.SliderException;
 import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-public abstract class AbstractClientProvider extends Configured {
-  private static final Logger log =
-    LoggerFactory.getLogger(AbstractClientProvider.class);
+public abstract class AbstractClientProvider {
 
-  public AbstractClientProvider(Configuration conf) {
-    super(conf);
+  public AbstractClientProvider() {
   }
 
-  public abstract String getName();
-
-  public abstract List<ProviderRole> getRoles();
-
   /**
    * Generates a fixed format of application tags given one or more of
    * application name, version and description. This allows subsequent query for
@@ -70,7 +64,29 @@ public abstract class AbstractClientProvider extends Configured {
   }
 
   /**
-   * Process client operations for applications such as install, configure
+   * Validate the artifact.
+   * @param artifact
+   */
+  public abstract void validateArtifact(Artifact artifact, FileSystem
+      fileSystem) throws IOException;
+
+  protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException;
+
+  /**
+   * Validate the config files.
+   * @param configFiles config file list
+   * @param fileSystem file system
+   */
+  public void validateConfigFiles(List<ConfigFile> configFiles, FileSystem
+      fileSystem) throws IOException {
+    for (ConfigFile configFile : configFiles) {
+      validateConfigFile(configFile, fileSystem);
+    }
+  }
+
+  /**
+   * Process client operations for applications such as install, configure.
    * @param fileSystem
    * @param registryOperations
    * @param configuration

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java

@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ContainerState;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.slider.util.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService extends AbstractService
+    implements ProviderService, SliderKeys {
+
+  protected static final Logger log =
+      LoggerFactory.getLogger(AbstractProviderService.class);
+  private static final ProviderUtils providerUtils = new ProviderUtils(log);
+  protected StateAccessForProviders amState;
+  protected YarnRegistryViewForProviders yarnRegistry;
+  private ServiceTimelinePublisher serviceTimelinePublisher;
+
+  protected AbstractProviderService(String name) {
+    super(name);
+  }
+
+  public abstract void processArtifact(ContainerLauncher launcher, Component
+      component, SliderFileSystem fileSystem) throws IOException;
+
+  @Override
+  public void setAMState(StateAccessForProviders stateAccessor) {
+    this.amState = stateAccessor;
+  }
+
+  @Override
+  public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
+    this.yarnRegistry = yarnRegistry;
+  }
+
+  public void buildContainerLaunchContext(ContainerLauncher launcher,
+      Application application, Container container, ProviderRole providerRole,
+      SliderFileSystem fileSystem, RoleInstance roleInstance)
+      throws IOException, SliderException {
+    Component component = providerRole.component;
+    processArtifact(launcher, component, fileSystem);
+
+    // Generate tokens (key-value pair) for config substitution.
+    // Get pre-defined tokens
+    Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
+    Map<String, String> tokensForSubstitution = providerUtils
+        .initCompTokensForSubstitute(roleInstance);
+    tokensForSubstitution.putAll(globalTokens);
+    // Set the environment variables in launcher
+    launcher.putEnv(SliderUtils
+        .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+    launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+    launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    if (System.getenv(HADOOP_USER_NAME) != null) {
+      launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+    }
+    launcher.setEnv("LANG", "en_US.UTF-8");
+    launcher.setEnv("LC_ALL", "en_US.UTF-8");
+    launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+    for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+      tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+    }
+    providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+    // create config file on hdfs and add local resource
+    providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+        component, tokensForSubstitution, roleInstance, amState);
+
+    // substitute launch command
+    String launchCommand = ProviderUtils
+        .substituteStrWithTokens(component.getLaunchCommand(),
+            tokensForSubstitution);
+    CommandLineBuilder operation = new CommandLineBuilder();
+    operation.add(launchCommand);
+    operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+    launcher.addCommand(operation.build());
+
+    // publish exports
+    providerUtils
+        .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
+    PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
+        application.getQuicklinks().entrySet());
+    amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
+    if (serviceTimelinePublisher != null) {
+      serviceTimelinePublisher.serviceAttemptUpdated(application);
+    }
+  }
+
+  public boolean processContainerStatus(ContainerId containerId,
+      ContainerStatus status) {
+    log.debug("Handling container status: {}", status);
+    if (SliderUtils.isEmpty(status.getIPs()) ||
+        SliderUtils.isUnset(status.getHost())) {
+      return true;
+    }
+    RoleInstance instance = amState.getOwnedContainer(containerId);
+    if (instance == null) {
+      // container is completed?
+      return false;
+    }
+
+    try {
+      providerUtils.updateServiceRecord(amState, yarnRegistry,
+          containerId.toString(), instance.role, status.getIPs(), status.getHost());
+    } catch (IOException e) {
+      // could not write service record to ZK, log and retry
+      log.warn("Error updating container {} service record in registry, " +
+          "retrying", containerId, e);
+      return true;
+    }
+    // TODO publish ip and host
+    org.apache.slider.api.resource.Container container =
+        instance.providerRole.component.getContainer(containerId.toString());
+    if (container != null) {
+      container.setIp(StringUtils.join(",", status.getIPs()));
+      container.setHostname(status.getHost());
+      container.setState(ContainerState.READY);
+    } else {
+      log.warn(containerId + " not found in Application!");
+    }
+    return false;
+  }
+
+  @Override
+  public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
+    this.serviceTimelinePublisher = publisher;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class DefaultClientProvider extends AbstractClientProvider {
+
+  public DefaultClientProvider() {
+  }
+
+  @Override
+  public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+  }
+
+  @Override
+  protected void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException {
+    // validate dest_file is not absolute
+    if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+      throw new IllegalArgumentException(
+          "Dest_file must not be absolute path: " + configFile.getDestFile());
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+public final class DefaultProviderFactory extends SliderProviderFactory {
+  private static final SliderProviderFactory FACTORY = new
+      DefaultProviderFactory();
+
+  private DefaultProviderFactory() {}
+
+  private static class Client {
+    static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
+  }
+
+  private static class Server {
+    static final ProviderService PROVIDER = new DefaultProviderService();
+  }
+
+  @Override
+  public AbstractClientProvider createClientProvider() {
+    return Client.PROVIDER;
+  }
+
+  @Override
+  public ProviderService createServerProvider() {
+    return Server.PROVIDER;
+  }
+
+  public static SliderProviderFactory getInstance() {
+    return FACTORY;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.AbstractProviderService;
+
+import java.io.IOException;
+
+public class DefaultProviderService extends AbstractProviderService {
+
+  protected DefaultProviderService() {
+    super(DefaultProviderService.class.getSimpleName());
+  }
+
+  @Override
+  public void processArtifact(ContainerLauncher launcher, Component
+      component, SliderFileSystem fileSystem) throws IOException {
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java

@@ -18,93 +18,60 @@
 
 package org.apache.slider.providers;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.api.resource.Artifact;
 import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.providers.docker.DockerProviderFactory;
+import org.apache.slider.providers.tarball.TarballProviderFactory;
+import org.apache.slider.util.RestApiErrorMessages;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Base class for factories
+ * Base class for factories.
  */
-public abstract class SliderProviderFactory extends Configured {
-
-  public static final String DEFAULT_CLUSTER_TYPE = AgentKeys.PROVIDER_AGENT;
-  
+public abstract class SliderProviderFactory {
   protected static final Logger log =
-    LoggerFactory.getLogger(SliderProviderFactory.class);
-  public static final String PROVIDER_NOT_FOUND =
-    "Unable to find provider of application type %s";
-
-  public SliderProviderFactory(Configuration conf) {
-    super(conf);
-  }
+      LoggerFactory.getLogger(SliderProviderFactory.class);
 
-  protected SliderProviderFactory() {
-  }
+  protected SliderProviderFactory() {}
 
   public abstract AbstractClientProvider createClientProvider();
 
   public abstract ProviderService createServerProvider();
 
+  public static synchronized ProviderService getProviderService(Artifact
+      artifact) {
+    return createSliderProviderFactory(artifact).createServerProvider();
+  }
+
+  public static synchronized AbstractClientProvider getClientProvider(Artifact
+      artifact) {
+    return createSliderProviderFactory(artifact).createClientProvider();
+  }
+
   /**
    * Create a provider for a specific application
-   * @param application app
-   * @return app instance
+   * @param artifact artifact
+   * @return provider factory
    * @throws SliderException on any instantiation problem
    */
-  public static SliderProviderFactory createSliderProviderFactory(String application) throws
-      SliderException {
-    Configuration conf = loadSliderConfiguration();
-    if (application == null) {
-      application = DEFAULT_CLUSTER_TYPE;
+  public static synchronized SliderProviderFactory createSliderProviderFactory(
+      Artifact artifact) {
+    if (artifact == null || artifact.getType() == null) {
+      log.info("Loading service provider type default");
+      return DefaultProviderFactory.getInstance();
     }
-    String providerKey =
-      String.format(SliderXmlConfKeys.KEY_PROVIDER, application);
-    if (application.contains(".")) {
-      log.debug("Treating {} as a classname", application);
-      String name = "classname.key";
-      conf.set(name, application);
-      providerKey = name;
+    log.info("Loading service provider type {}", artifact.getType());
+    switch (artifact.getType()) {
+      // TODO add handling for custom types?
+      // TODO handle application
+      case DOCKER:
+        return DockerProviderFactory.getInstance();
+      case TARBALL:
+        return TarballProviderFactory.getInstance();
+      default:
+        throw new IllegalArgumentException(
+            RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
     }
-    
-    Class<? extends SliderProviderFactory> providerClass;
-    try {
-      providerClass = conf.getClass(providerKey, null, SliderProviderFactory.class);
-    } catch (RuntimeException e) {
-      throw new BadClusterStateException(e, "Failed to load provider %s: %s", application, e);
-    }
-    if (providerClass == null) {
-      throw new BadClusterStateException(PROVIDER_NOT_FOUND, application);
-    }
-
-    Exception ex;
-    try {
-      SliderProviderFactory providerFactory = providerClass.newInstance();
-      providerFactory.setConf(conf);
-      return providerFactory;
-    } catch (Exception e) {
-      ex = e;
-    }
-    //by here the operation failed and ex is set to the value 
-    throw new BadClusterStateException(ex,
-                              "Failed to create an instance of %s : %s",
-                              providerClass,
-                              ex);
-  }
-
-  /**
-   * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
-   * included
-   * @return a configuration instance
-   */
-  public static Configuration loadSliderConfiguration() {
-    Configuration conf = new Configuration();
-    conf.addResource(SliderKeys.SLIDER_XML);
-    return conf;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java

@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/*
-
- */
-public interface AgentKeys {
-
-  String AGENT_TAR = "slider-agent.tar.gz";
-  String PROVIDER_AGENT = "agent";
-  String ROLE_NODE = "echo";
-
-  /**
-   * Template stored in the slider classpath -to use if there is
-   * no site-specific template
-   * {@value}
-   */
-  String CONF_RESOURCE = "org/apache/slider/providers/agent/conf/";
-  /*  URL to talk back to Agent Controller*/
-  String CONTROLLER_URL = "agent.controller.url";
-  /**
-   * The location of pre-installed agent path.
-   * This can be also be dynamically computed based on Yarn installation of agent.
-   */
-  String PACKAGE_PATH = "agent.package.root";
-  /**
-   * The location of the script implementing the command.
-   */
-  String SCRIPT_PATH = "agent.script";
-  /**
-   * Execution home for the agent.
-   */
-  String APP_HOME = "app.home";
-  String APP_ROOT = "site.global.app_root";
-  String APP_CLIENT_ROOT = "client_root";
-  /**
-   * Runas user of the application
-   */
-  String RUNAS_USER = "site.global.app_user";
-  /**
-   * Name of the service.
-   */
-  String SERVICE_NAME = "app.name";
-  String ARG_LABEL = "--label";
-  String ARG_HOST = "--host";
-  String ARG_PORT = "--port";
-  String ARG_SECURED_PORT = "--secured_port";
-  String ARG_ZOOKEEPER_QUORUM = "--zk-quorum";
-  String ARG_ZOOKEEPER_REGISTRY_PATH = "--zk-reg-path";
-  String ARG_DEBUG = "--debug";
-  String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/";
-  String AGENT_JINJA2_ROOT = "./infra/agent/slider-agent/jinja2";
-  String AGENT_MAIN_SCRIPT = "agent/main.py";
-
-  String APP_DEF = "application.def";
-  String APP_DEF_ORIGINAL = "application.def.original";
-  String ADDON_PREFIX = "application.addon.";
-  String ADDONS = "application.addons";
-  String AGENT_VERSION = "agent.version";
-  String AGENT_CONF = "agent.conf";
-  String ADDON_FOR_ALL_COMPONENTS = "ALL";
-
-  String AGENT_INSTALL_DIR = "infra/agent";
-  String APP_DEFINITION_DIR = "app/definition";
-  String ADDON_DEFINITION_DIR = "addon/definition";
-  String AGENT_CONFIG_FILE = "infra/conf/agent.ini";
-  String AGENT_VERSION_FILE = "infra/version";
-
-  String PACKAGE_LIST = "package_list";
-  String WAIT_HEARTBEAT = "wait.heartbeat";
-  String PYTHON_EXE = "python";
-  String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
-  String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
-  String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
-  String AGENT_OUT_FILE = "slider-agent.out";
-  String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
-  String INFRA_RUN_SECURITY_DIR = "infra/run/security/";
-  String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
-  String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
-  String TEST_RELAX_VERIFICATION = "test.relax.validation";
-
-  String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
-}
-
-

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java

@@ -17,37 +17,37 @@
  */
 package org.apache.slider.providers.docker;
 
-import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.slider.util.RestApiErrorMessages;
 
-import java.util.Collections;
-import java.util.List;
+import java.io.IOException;
 
 public class DockerClientProvider extends AbstractClientProvider
     implements SliderKeys {
 
-  protected static final Logger log =
-      LoggerFactory.getLogger(DockerClientProvider.class);
-  private static final ProviderUtils providerUtils = new ProviderUtils(log);
-  protected static final String NAME = "docker";
-
-  public DockerClientProvider(Configuration conf) {
-    super(conf);
+  public DockerClientProvider() {
+    super();
   }
 
   @Override
-  public String getName() {
-    return NAME;
+  public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+    if (artifact == null) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+    }
+    if (StringUtils.isEmpty(artifact.getId())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+    }
   }
 
   @Override
-  public List<ProviderRole> getRoles() {
-    return Collections.emptyList();
+  protected void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException {
   }
-
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java

@@ -27,7 +27,4 @@ public interface DockerKeys {
 
   String DEFAULT_DOCKER_NETWORK = "bridge";
   Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
-
-  String OUT_FILE = "stdout.txt";
-  String ERR_FILE = "stderr.txt";
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java

@@ -17,27 +17,36 @@
  */
 package org.apache.slider.providers.docker;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.slider.providers.AbstractClientProvider;
 import org.apache.slider.providers.ProviderService;
 import org.apache.slider.providers.SliderProviderFactory;
 
 public class DockerProviderFactory extends SliderProviderFactory {
+  private static final SliderProviderFactory FACTORY = new
+      DockerProviderFactory();
 
-  public DockerProviderFactory() {
+  private DockerProviderFactory() {
   }
 
-  public DockerProviderFactory(Configuration conf) {
-    super(conf);
+  private static class Client {
+    static final AbstractClientProvider PROVIDER = new DockerClientProvider();
+  }
+
+  private static class Server {
+    static final ProviderService PROVIDER = new DockerProviderService();
   }
 
   @Override
   public AbstractClientProvider createClientProvider() {
-    return new DockerClientProvider(getConf());
+    return Client.PROVIDER;
   }
 
   @Override
   public ProviderService createServerProvider() {
-    return new DockerProviderService();
+    return Server.PROVIDER;
+  }
+
+  public static SliderProviderFactory getInstance() {
+    return FACTORY;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java

@@ -17,159 +17,31 @@
  */
 package org.apache.slider.providers.docker;
 
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
 import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.apache.slider.providers.AbstractProviderService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
 
-import static org.apache.slider.util.ServiceApiUtil.$;
-
-public class DockerProviderService extends AbstractService
-    implements ProviderService, DockerKeys, SliderKeys {
+public class DockerProviderService extends AbstractProviderService
+    implements DockerKeys {
 
   protected static final Logger log =
       LoggerFactory.getLogger(DockerProviderService.class);
-  private static final ProviderUtils providerUtils = new ProviderUtils(log);
-  private static final String QUICK_LINKS = "quicklinks";
-  protected StateAccessForProviders amState;
-  protected YarnRegistryViewForProviders yarnRegistry;
-  private ServiceTimelinePublisher serviceTimelinePublisher;
 
   protected DockerProviderService() {
-    super("DockerProviderService");
-  }
-
-  @Override
-  public void setAMState(StateAccessForProviders stateAccessor) {
-    this.amState = stateAccessor;
-  }
-
-  @Override
-  public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
-    this.yarnRegistry = yarnRegistry;
+    super(DockerProviderService.class.getSimpleName());
   }
 
-
-  public void buildContainerLaunchContext(ContainerLauncher launcher,
-      Application application, Container container, ProviderRole providerRole,
-      SliderFileSystem fileSystem, RoleInstance roleInstance)
-      throws IOException, SliderException {
-    Component component = providerRole.component;
+  public void processArtifact(ContainerLauncher launcher, Component
+      component, SliderFileSystem fileSystem) throws IOException {
     launcher.setYarnDockerMode(true);
     launcher.setDockerImage(component.getArtifact().getId());
     launcher.setDockerNetwork(component.getConfiguration()
         .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
     launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer());
-
-    // Generate tokens (key-value pair) for config substitution.
-    // Get pre-defined tokens
-    Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
-    Map<String, String> tokensForSubstitution = providerUtils
-        .initCompTokensForSubstitute(roleInstance);
-    tokensForSubstitution.putAll(globalTokens);
-    // Set the environment variables in launcher
-    launcher.putEnv(SliderUtils
-        .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
-    launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
-    launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
-    if (System.getenv(HADOOP_USER_NAME) != null) {
-      launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
-    }
-    launcher.setEnv("LANG", "en_US.UTF-8");
-    launcher.setEnv("LC_ALL", "en_US.UTF-8");
-    launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
-    for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
-      tokensForSubstitution.put($(entry.getKey()), entry.getValue());
-    }
-    providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
-
-    // create config file on hdfs and add local resource
-    providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
-        component, tokensForSubstitution, roleInstance, amState);
-
-    // substitute launch command
-    String launchCommand = ProviderUtils
-        .substituteStrWithTokens(component.getLaunchCommand(),
-            tokensForSubstitution);
-    CommandLineBuilder operation = new CommandLineBuilder();
-    operation.add(launchCommand);
-    operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
-    launcher.addCommand(operation.build());
-
-    // publish exports
-    providerUtils
-        .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
-    PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
-        application.getQuicklinks().entrySet());
-    amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
-    if (serviceTimelinePublisher != null) {
-      serviceTimelinePublisher.serviceAttemptUpdated(application);
-    }
-  }
-
-  public boolean processContainerStatus(ContainerId containerId,
-      ContainerStatus status) {
-    log.debug("Handling container status: {}", status);
-    if (SliderUtils.isEmpty(status.getIPs()) ||
-        SliderUtils.isUnset(status.getHost())) {
-      return true;
-    }
-    RoleInstance instance = amState.getOwnedContainer(containerId);
-    if (instance == null) {
-      // container is completed?
-      return false;
-    }
-
-    try {
-      providerUtils.updateServiceRecord(amState, yarnRegistry,
-          containerId.toString(), instance.role, status.getIPs(), status.getHost());
-    } catch (IOException e) {
-      // could not write service record to ZK, log and retry
-      log.warn("Error updating container {} service record in registry, " +
-          "retrying", containerId, e);
-      return true;
-    }
-    // TODO publish ip and host
-    org.apache.slider.api.resource.Container container =
-        instance.providerRole.component.getContainer(containerId.toString());
-    if (container != null) {
-      container.setIp(StringUtils.join(",", status.getIPs()));
-      container.setHostname(status.getHost());
-      container.setState(ContainerState.READY);
-    } else {
-      log.warn(containerId + " not found in Application!");
-    }
-    return false;
-  }
-
-  @Override
-  public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
-    this.serviceTimelinePublisher = publisher;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.tarball;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.util.RestApiErrorMessages;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class TarballClientProvider extends AbstractClientProvider
+    implements SliderKeys {
+
+  public TarballClientProvider() {
+  }
+
+  @Override
+  public void validateArtifact(Artifact artifact, FileSystem fs)
+      throws IOException {
+    if (artifact == null) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+    }
+    if (StringUtils.isEmpty(artifact.getId())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+    }
+    Path p = new Path(artifact.getId());
+    if (!fs.exists(p)) {
+      throw new IllegalArgumentException( "Artifact tarball does not exist "
+          + artifact.getId());
+    }
+  }
+
+  @Override
+  protected void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException {
+    // validate dest_file is not absolute
+    if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+      throw new IllegalArgumentException(
+          "Dest_file must not be absolute path: " + configFile.getDestFile());
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.tarball;
+
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.SliderProviderFactory;
+
+public class TarballProviderFactory extends SliderProviderFactory {
+  private static final SliderProviderFactory FACTORY = new
+      TarballProviderFactory();
+
+  private TarballProviderFactory() {
+  }
+
+  private static class Client {
+    static final AbstractClientProvider PROVIDER = new TarballClientProvider();
+  }
+
+  private static class Server {
+    static final ProviderService PROVIDER = new TarballProviderService();
+  }
+
+  @Override
+  public AbstractClientProvider createClientProvider() {
+    return Client.PROVIDER;
+  }
+
+  @Override
+  public ProviderService createServerProvider() {
+    return Server.PROVIDER;
+  }
+
+  public static SliderProviderFactory getInstance() {
+    return FACTORY;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.tarball;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.AbstractProviderService;
+
+import java.io.IOException;
+
+public class TarballProviderService extends AbstractProviderService {
+
+  protected TarballProviderService() {
+    super(TarballProviderService.class.getSimpleName());
+  }
+
+  @Override
+  public void processArtifact(ContainerLauncher launcher, Component
+      component, SliderFileSystem fileSystem) throws IOException {
+    Path artifact =  new Path(component.getArtifact().getId());
+    if (!fileSystem.isFile(artifact)) {
+      throw new IOException("Package doesn't exist as a resource: " +
+          artifact.toString());
+    }
+    log.info("Adding resource {}", artifact.toString());
+    LocalResourceType type = LocalResourceType.ARCHIVE;
+    LocalResource packageResource = fileSystem.createAmResource(
+        artifact, type);
+    launcher.addLocalResource(APP_INSTALL_DIR, packageResource);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java

@@ -27,7 +27,7 @@ import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.providers.ProviderRole;
 import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.providers.SliderProviderFactory;
 import org.apache.slider.server.appmaster.actions.ActionStartContainer;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.state.ContainerAssignment;
@@ -43,6 +43,8 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
+
 /**
  * A service for launching containers
  */
@@ -59,11 +61,6 @@ public class RoleLaunchService
    */
   private final QueueAccess actionQueue;
 
-  /**
-   * Provider building up the command
-   */
-  private final ProviderService provider;
-  
   /**
    * Filesystem to use for the launch
    */
@@ -75,16 +72,14 @@ public class RoleLaunchService
   /**
    * Construct an instance of the launcher
    * @param queueAccess
-   * @param provider the provider
    * @param fs filesystem
    * @param envVars environment variables
    */
-  public RoleLaunchService(QueueAccess queueAccess, ProviderService provider,
-      SliderFileSystem fs, Map<String, String> envVars) {
+  public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs,
+      Map<String, String> envVars) {
     super(ROLE_LAUNCH_SERVICE);
     this.actionQueue = queueAccess;
     this.fs = fs;
-    this.provider = provider;
     this.envVars = envVars;
   }
 
@@ -167,11 +162,13 @@ public class RoleLaunchService
         instance.roleId = role.id;
         instance.environment = envDescription;
 
+        ProviderService provider = SliderProviderFactory.getProviderService(
+            role.component.getArtifact());
         provider.buildContainerLaunchContext(containerLauncher, application,
             container, role, fs, instance);
 
         long delay = role.component.getConfiguration()
-            .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0);
+            .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0);
         long maxDelay = getConfig()
             .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
                 YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java

@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.TimelineV2Client;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -342,7 +341,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    * ProviderService of this cluster
    */
   @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-  private ProviderService providerService;
+  private List<ProviderService> providers = new ArrayList<>();
 
   /**
    * The YARN registry service
@@ -523,8 +522,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   public Configuration bindArgs(Configuration config, String... args) throws Exception {
     // let the superclass process it
     Configuration superConf = super.bindArgs(config, args);
-    // add the slider XML config
-    ConfigHelper.injectSliderXMLResource();
 
     //yarn-ify
     YarnConfiguration yarnConfiguration = new YarnConfiguration(
@@ -603,12 +600,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     // obtain security state
     // set the global security flag for the instance definition
 
-    //get our provider
-    SliderProviderFactory factory =
-      SliderProviderFactory.createSliderProviderFactory("docker");
-    providerService = factory.createServerProvider();
-    // init the provider BUT DO NOT START IT YET
-    initAndAddService(providerService);
+    // initialize our providers
+    for (Component component : application.getComponents()) {
+      SliderProviderFactory factory = SliderProviderFactory
+          .createSliderProviderFactory(component.getArtifact());
+      ProviderService providerService = factory.createServerProvider();
+      // init the provider BUT DO NOT START IT YET
+      initAndAddService(providerService);
+      providers.add(providerService);
+    }
 
     InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
     log.info("RM is at {}", rmSchedulerAddress);
@@ -667,7 +667,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
         serviceTimelinePublisher.init(getConfig());
         serviceTimelinePublisher.start();
 
-        providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
+        for (ProviderService providerService : providers) {
+          providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
+        }
         appState.setServiceTimelinePublisher(serviceTimelinePublisher);
         log.info("ServiceTimelinePublisher started.");
       }
@@ -707,7 +709,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       WebAppApiImpl webAppApi =
           new WebAppApiImpl(
               stateForProviders,
-              providerService, registryOperations,
+              registryOperations,
               metricsAndMonitoring,
               actionQueues);
       initAMFilterOptions(serviceConf);
@@ -843,13 +845,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
 
     //launcher service
     launchService = new RoleLaunchService(actionQueues,
-                                          providerService,
                                           fs, envVars);
 
     deployChildService(launchService);
 
     //Give the provider access to the state, and AM
-    providerService.setAMState(stateForProviders);
+    for (ProviderService providerService : providers) {
+      providerService.setAMState(stateForProviders);
+    }
 
     // chaos monkey
     maybeStartMonkey();
@@ -1119,7 +1122,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
         SliderKeys.APP_TYPE,
         instanceName,
         appAttemptID);
-    providerService.bindToYarnRegistry(yarnRegistryOperations);
+    for (ProviderService providerService : providers) {
+      providerService.bindToYarnRegistry(yarnRegistryOperations);
+    }
 
     // Yarn registry
     ServiceRecord serviceRecord = new ServiceRecord();
@@ -1859,7 +1864,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   protected synchronized void launchProviderService()
       throws IOException, SliderException {
     // didn't start, so don't register
-    providerService.start();
+    for (ProviderService providerService : providers) {
+      providerService.start();
+    }
     // and send the started event ourselves
     eventCallbackEvent(null);
   }
@@ -1959,19 +1966,23 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       ContainerStatus containerStatus) {
     LOG_YARN.debug("Container Status: id={}, status={}", containerId,
         containerStatus);
+    RoleInstance cinfo = appState.getOwnedContainer(containerId);
+    if (cinfo == null) {
+      LOG_YARN.error("Owned container not found for {}", containerId);
+      return;
+    }
+    ProviderService providerService = SliderProviderFactory
+        .getProviderService(cinfo.providerRole.component.getArtifact());
     if (providerService.processContainerStatus(containerId, containerStatus)) {
       try {
         Thread.sleep(1000);
       } catch (InterruptedException e) {
       }
-      RoleInstance cinfo = appState.getOwnedContainer(containerId);
-      if (cinfo != null) {
-        LOG_YARN.info("Re-requesting status for role {}, {}",
-            cinfo.role, containerId);
-        //trigger another async container status
-        nmClientAsync.getContainerStatusAsync(containerId,
-            cinfo.container.getNodeId());
-      }
+      LOG_YARN.info("Re-requesting status for role {}, {}",
+          cinfo.role, containerId);
+      //trigger another async container status
+      nmClientAsync.getContainerStatusAsync(containerId,
+          cinfo.container.getNodeId());
     } else if (timelineServiceEnabled) {
       RoleInstance instance = appState.getOwnedContainer(containerId);
       if (instance != null) {
@@ -1997,11 +2008,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     LOG_YARN.warn("Failed to stop Container {}", containerId);
   }
 
-
-  public ProviderService getProviderService() {
-    return providerService;
-  }
-
   /**
    * Queue an action for immediate execution in the executor thread
    * @param action action to execute

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java

@@ -17,7 +17,6 @@
 package org.apache.slider.server.appmaster.web;
 
 import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.state.AppState;
@@ -33,11 +32,6 @@ public interface WebAppApi {
    */
   StateAccessForProviders getAppState();
   
-  /**
-   * The {@link ProviderService} for the current cluster
-   */
-  ProviderService getProviderService();
-  
   /**
    * Registry operations accessor
    * @return registry access

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java

@@ -17,7 +17,6 @@
 package org.apache.slider.server.appmaster.web;
 
 import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -33,21 +32,18 @@ public class WebAppApiImpl implements WebAppApi {
   private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class);
 
   protected final StateAccessForProviders appState;
-  protected final ProviderService provider;
   private final RegistryOperations registryOperations;
   private final MetricsAndMonitoring metricsAndMonitoring;
   private final QueueAccess queues;
 
   public WebAppApiImpl(StateAccessForProviders appState,
-      ProviderService provider, RegistryOperations registryOperations,
+      RegistryOperations registryOperations,
       MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) {
     checkNotNull(appState);
-    checkNotNull(provider);
     this.queues = queues;
 
     this.registryOperations = registryOperations;
     this.appState = appState;
-    this.provider = provider;
     this.metricsAndMonitoring = metricsAndMonitoring;
   }
 
@@ -56,11 +52,6 @@ public class WebAppApiImpl implements WebAppApi {
     return appState;
   }
 
-  @Override
-  public ProviderService getProviderService() {
-    return provider;
-  }
-
   @Override
   public RegistryOperations getRegistryOperations() {
     return registryOperations;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java

@@ -19,7 +19,6 @@
 package org.apache.slider.server.appmaster.web.view;
 
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
 import org.apache.slider.server.appmaster.web.WebAppApi;
 import org.apache.slider.server.appmaster.web.rest.RestPaths;
@@ -33,12 +32,10 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_
 public abstract class SliderHamletBlock extends HtmlBlock  {
 
   protected final StateAccessForProviders appState;
-  protected final ProviderService providerService;
   protected final RestPaths restPaths = new RestPaths();
   
   public SliderHamletBlock(WebAppApi slider) {
     this.appState = slider.getAppState();
-    this.providerService = slider.getProviderService();
   }
 
   protected String rootPath(String absolutePath) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml

@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~  or more contributor license agreements.  See the NOTICE file
-  ~  distributed with this work for additional information
-  ~  regarding copyright ownership.  The ASF licenses this file
-  ~  to you under the Apache License, Version 2.0 (the
-  ~  "License"); you may not use this file except in compliance
-  ~  with the License.  You may obtain a copy of the License at
-  ~
-  ~       http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~  Unless required by applicable law or agreed to in writing, software
-  ~  distributed under the License is distributed on an "AS IS" BASIS,
-  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~  See the License for the specific language governing permissions and
-  ~  limitations under the License.
-  -->
-
-<configuration>
-  <property>
-    <name>slider.config.loaded</name>
-    <value>true</value>
-  </property>
-  <property>
-    <name>slider.provider.docker</name>
-    <value>org.apache.slider.providers.docker.DockerProviderFactory</value>
-  </property>
-</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java

@@ -18,11 +18,16 @@
 
 package org.apache.slider.providers;
 
-import org.apache.slider.providers.docker.DockerKeys;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.Artifact.TypeEnum;
+import org.apache.slider.providers.docker.DockerClientProvider;
 import org.apache.slider.providers.docker.DockerProviderFactory;
+import org.apache.slider.providers.docker.DockerProviderService;
+import org.apache.slider.providers.tarball.TarballClientProvider;
+import org.apache.slider.providers.tarball.TarballProviderFactory;
+import org.apache.slider.providers.tarball.TarballProviderService;
 import org.junit.Test;
 
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -30,25 +35,38 @@ import static org.junit.Assert.assertTrue;
  */
 public class TestProviderFactory {
   @Test
-  public void testLoadAgentProvider() throws Throwable {
+  public void testDockerFactory() throws Throwable {
     SliderProviderFactory factory = SliderProviderFactory
-        .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
+        .createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER));
     assertTrue(factory instanceof DockerProviderFactory);
+    assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
+    assertTrue(factory.createServerProvider() instanceof DockerProviderService);
+    assertTrue(SliderProviderFactory.getProviderService(new Artifact()
+        .type(TypeEnum.DOCKER)) instanceof DockerProviderService);
   }
 
   @Test
-  public void testCreateClientProvider() throws Throwable {
+  public void testTarballFactory() throws Throwable {
     SliderProviderFactory factory = SliderProviderFactory
-        .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
-    assertNotNull(factory.createClientProvider());
+        .createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL));
+    assertTrue(factory instanceof TarballProviderFactory);
+    assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
+    assertTrue(factory.createServerProvider() instanceof
+        TarballProviderService);
+    assertTrue(SliderProviderFactory.getProviderService(new Artifact()
+        .type(TypeEnum.TARBALL)) instanceof TarballProviderService);
   }
 
   @Test
-  public void testCreateProviderByClassname() throws Throwable {
+  public void testDefaultFactory() throws Throwable {
     SliderProviderFactory factory = SliderProviderFactory
-        .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
-    assertNotNull(factory.createServerProvider());
-    assertTrue(factory instanceof DockerProviderFactory);
+        .createSliderProviderFactory(null);
+    assertTrue(factory instanceof DefaultProviderFactory);
+    assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
+    assertTrue(factory.createServerProvider() instanceof
+        DefaultProviderService);
+    assertTrue(SliderProviderFactory.getProviderService(null) instanceof
+        DefaultProviderService);
   }
 
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java

@@ -20,9 +20,7 @@ import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockProviderService;
 import org.apache.slider.server.appmaster.state.ProviderAppState;
 import org.apache.slider.server.appmaster.web.WebAppApi;
 import org.apache.slider.server.appmaster.web.WebAppApiImpl;
@@ -45,11 +43,9 @@ public class TestClusterSpecificationBlock extends BaseMockAppStateTest {
     ProviderAppState providerAppState = new ProviderAppState(
         "undefined",
         appState);
-    ProviderService providerService = new MockProviderService();
 
     WebAppApiImpl inst = new WebAppApiImpl(
         providerAppState,
-        providerService,
         null,
         null, null);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java

@@ -27,12 +27,10 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
 import org.apache.hadoop.yarn.webapp.hamlet.HamletImpl.EImp;
 import org.apache.slider.api.ClusterNode;
-import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
 import org.apache.slider.server.appmaster.model.mock.MockContainer;
 import org.apache.slider.server.appmaster.model.mock.MockContainerId;
 import org.apache.slider.server.appmaster.model.mock.MockNodeId;
-import org.apache.slider.server.appmaster.model.mock.MockProviderService;
 import org.apache.slider.server.appmaster.model.mock.MockResource;
 import org.apache.slider.server.appmaster.state.ProviderAppState;
 import org.apache.slider.server.appmaster.state.RoleInstance;
@@ -65,14 +63,12 @@ public class TestContainerStatsBlock extends BaseMockAppStateTest {
   @Before
   public void setup() throws Exception {
     super.setup();
-    ProviderService providerService = new MockProviderService();
     ProviderAppState providerAppState = new ProviderAppState(
         "undefined",
         appState);
 
     WebAppApiImpl inst = new WebAppApiImpl(
         providerAppState,
-        providerService,
         null,
         METRICS, null);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java

@@ -22,12 +22,10 @@ import com.google.inject.Injector;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.slider.providers.ProviderService;
 import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest;
 import org.apache.slider.server.appmaster.model.mock.MockContainer;
 import org.apache.slider.server.appmaster.model.mock.MockContainerId;
 import org.apache.slider.server.appmaster.model.mock.MockNodeId;
-import org.apache.slider.server.appmaster.model.mock.MockProviderService;
 import org.apache.slider.server.appmaster.model.mock.MockResource;
 import org.apache.slider.server.appmaster.state.ContainerOutcome;
 import org.apache.slider.server.appmaster.state.OutstandingRequest;
@@ -58,14 +56,12 @@ public class TestIndexBlock extends BaseMockAppStateAATest {
   public void setup() throws Exception {
     super.setup();
     assertNotNull(appState);
-    ProviderService providerService = new MockProviderService();
     ProviderAppState providerAppState = new ProviderAppState(
         "undefined",
         appState);
 
     WebAppApiImpl inst = new WebAppApiImpl(
         providerAppState,
-        providerService,
         null,
         METRICS, null);