AMBARI-4643. Restart All fails for Client only components. (swagle)

ambari-agent/src/main/python/resource_management/libraries/script/script.py

@@ -180,9 +180,20 @@ class Script(object):
     """
     Default implementation of restart command is to call stop and start methods
     Feel free to override restart() method with your implementation.
+    For client components we call install
     """
-    self.stop(env)
-    self.start(env)
+    config = self.get_config()
+    componentCategory = None
+    try :
+      componentCategory = config['roleParams']['component_category']
+    except KeyError:
+      pass
+
+    if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower():
+      self.install(env)
+    else:
+      self.stop(env)
+      self.start(env)
 
   def configure(self, env):
     """

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -40,7 +40,6 @@ public class ExecutionCommand extends AgentCommand {
     super(AgentCommandType.EXECUTION_COMMAND);
   }
 
-
   private String clusterName;
   private long taskId;
   private String commandId;
@@ -259,6 +258,7 @@ public class ExecutionCommand extends AgentCommand {
     String AMBARI_DB_RCA_DRIVER = "ambari_db_rca_driver";
     String AMBARI_DB_RCA_USERNAME = "ambari_db_rca_username";
     String AMBARI_DB_RCA_PASSWORD = "ambari_db_rca_password";
+    String COMPONENT_CATEGORY = "component_category";
 
     String SERVICE_CHECK = "SERVICE_CHECK"; // TODO: is it standart command? maybe add it to RoleCommand enum?
     String CUSTOM_COMMAND = "custom_command";

ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java

@@ -33,6 +33,7 @@ public class ActionExecutionContext {
   private final String actionName;
   private final String serviceName;
   private final String componentName;
+  private final String componentCategory;
   private final List<String> hosts;
   private final Map<String, String> parameters;
   private final TargetHostType targetType;
@@ -42,12 +43,14 @@ public class ActionExecutionContext {
    * Create an ActionExecutionContext to execute an action from a request
    */
   public ActionExecutionContext(String clusterName, String actionName, String serviceName,
-                                String componentName, List<String> hosts, Map<String, String> parameters,
+                                String componentName, String componentCategory,
+                                List<String> hosts, Map<String, String> parameters,
                                 TargetHostType targetType, Short timeout) {
     this.clusterName = clusterName;
     this.actionName = actionName;
     this.serviceName = serviceName;
     this.componentName = componentName;
+    this.componentCategory = componentCategory;
     this.parameters = parameters;
     this.hosts = new ArrayList<String>();
     if (hosts != null) {
@@ -88,4 +91,8 @@ public class ActionExecutionContext {
   public Short getTimeout() {
     return timeout;
   }
+
+  public String getComponentCategory() {
+    return componentCategory;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java

@@ -41,6 +41,7 @@ import org.apache.ambari.server.utils.StageUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -50,6 +51,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_T
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCHEMA_VERSION;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
 
 /**
  * Helper class containing logic to process custom action execution requests
@@ -73,6 +75,7 @@ public class AmbariActionExecutionHelper {
     this.ambariMetaInfo = amcImpl.getAmbariMetaInfo();
   }
 
+  // TODO: validate should not return context, should make it consistent with Command Execution helper
   /**
    * Validates the request to execute an action
    *
@@ -128,6 +131,7 @@ public class AmbariActionExecutionHelper {
     }
 
     String targetComponent = expectedComponent;
+    String componentCategory = "";
     if (targetComponent == null || targetComponent.isEmpty()) {
       targetComponent = actualComponent;
     }
@@ -150,6 +154,7 @@ public class AmbariActionExecutionHelper {
         throw new AmbariException("Action " + actionRequest.getActionName() + " targets component " + targetComponent +
             " that does not exist.");
       }
+      componentCategory = compInfo.getCategory();
     }
 
     if (actionDef.getInputs() != null) {
@@ -182,9 +187,9 @@ public class AmbariActionExecutionHelper {
         + ", request=" + actionRequest.toString());
 
     ActionExecutionContext actionExecutionContext = new ActionExecutionContext(
-        actionRequest.getClusterName(), actionRequest.getActionName(), targetService, targetComponent,
-        actionRequest.getHosts(), actionRequest.getParameters(), actionDef.getTargetType(),
-        actionDef.getDefaultTimeout());
+      actionRequest.getClusterName(), actionRequest.getActionName(),
+      targetService, targetComponent, componentCategory, actionRequest.getHosts(),
+      actionRequest.getParameters(), actionDef.getTargetType(), actionDef.getDefaultTimeout());
 
     return actionExecutionContext;
   }
@@ -274,9 +279,6 @@ public class AmbariActionExecutionHelper {
           new ServiceComponentHostOpInProgressEvent(actionContext.getActionName(), hostName,
               System.currentTimeMillis()), clusterName, actionContext.getServiceName());
 
-      stage.getExecutionCommandWrapper(hostName, actionContext.getActionName()).getExecutionCommand()
-          .setRoleParams(actionContext.getParameters());
-
       Cluster cluster = clusters.getCluster(clusterName);
 
       Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
@@ -305,6 +307,14 @@ public class AmbariActionExecutionHelper {
       execCmd.setServiceName(serviceName);
       execCmd.setComponentName(componentName);
 
+      Map<String, String> roleParams = execCmd.getRoleParams();
+      if (roleParams == null) {
+        roleParams = new TreeMap<String, String>();
+      }
+      roleParams.putAll(actionContext.getParameters());
+      roleParams.put(COMPONENT_CATEGORY, actionContext.getComponentCategory());
+      execCmd.setRoleParams(roleParams);
+
       // Generate cluster host info
       execCmd.setClusterHostInfo(
           StageUtils.getClusterHostInfo(clusters.getHostsForCluster(clusterName), cluster));

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java

@@ -83,6 +83,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_P
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
 
 
 /**
@@ -123,9 +124,9 @@ public class AmbariCustomCommandExecutionHelper {
   private AmbariMetaInfo ambariMetaInfo;
   @Inject
   private ConfigHelper configHelper;
-  ;
 
-  private Boolean isServiceCheckCommand(String command, String service) {
+  private Boolean isServiceCheckCommand(String
+                                          command, String service) {
     List<String> actions = actionMetadata.getActions(service);
     if (actions == null || actions.size() == 0) {
       return false;
@@ -247,6 +248,8 @@ public class AmbariCustomCommandExecutionHelper {
 
     for (String hostName : actionRequest.getHosts()) {
 
+      Host host = clusters.getHost(hostName);
+
       stage.addHostRoleExecutionCommand(hostName, Role.valueOf(componentName),
           RoleCommand.CUSTOM_COMMAND,
           new ServiceComponentHostOpInProgressEvent(componentName,
@@ -273,6 +276,8 @@ public class AmbariCustomCommandExecutionHelper {
           StageUtils.getClusterHostInfo(clusters.getHostsForCluster(clusterName), cluster));
 
       hostLevelParams.put(CUSTOM_COMMAND, commandName);
+      // Set parameters required for re-installing clients on restart
+      hostLevelParams.put(REPO_INFO, getRepoInfo(cluster, host));
       execCmd.setHostLevelParams(hostLevelParams);
 
       Map<String, String> commandParams = new TreeMap<String, String>();
@@ -285,11 +290,12 @@ public class AmbariCustomCommandExecutionHelper {
 
       String commandTimeout = configs.getDefaultAgentTaskTimeout();
 
+      ComponentInfo componentInfo = ambariMetaInfo.getComponent(
+        stackId.getStackName(), stackId.getStackVersion(),
+        serviceName, componentName);
+
       if (serviceInfo.getSchemaVersion().equals(AmbariMetaInfo.SCHEMA_VERSION_2)) {
         // Service check command is not custom command
-        ComponentInfo componentInfo = ambariMetaInfo.getComponent(
-            stackId.getStackName(), stackId.getStackVersion(),
-            serviceName, componentName);
         CommandScriptDefinition script = componentInfo.getCommandScript();
 
         if (script != null) {
@@ -313,6 +319,13 @@ public class AmbariCustomCommandExecutionHelper {
       commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
 
       execCmd.setCommandParams(commandParams);
+
+      Map<String, String> roleParams = execCmd.getRoleParams();
+      if (roleParams == null) {
+        roleParams = new TreeMap<String, String>();
+      }
+      roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory());
+      execCmd.setRoleParams(roleParams);
     }
   }
 
@@ -698,19 +711,7 @@ public class AmbariCustomCommandExecutionHelper {
 
     execCmd.setCommandParams(commandParams);
 
-    Map<String, List<RepositoryInfo>> repos = ambariMetaInfo.getRepository(
-        stackId.getStackName(), stackId.getStackVersion());
-    String repoInfo = "";
-    if (!repos.containsKey(host.getOsType())) {
-      // FIXME should this be an error?
-      LOG.warn("Could not retrieve repo information for host"
-          + ", hostname=" + scHost.getHostName()
-          + ", clusterName=" + cluster.getClusterName()
-          + ", stackInfo=" + stackId.getStackId());
-    } else {
-      repoInfo = gson.toJson(repos.get(host.getOsType()));
-    }
-
+    String repoInfo = getRepoInfo(cluster, host);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Sending repo information to agent"
           + ", hostname=" + scHost.getHostName()
@@ -720,7 +721,6 @@ public class AmbariCustomCommandExecutionHelper {
     }
 
     Map<String, String> hostParams = new TreeMap<String, String>();
-    // TODO: Move parameter population to org.apache.ambari.server.controller.AmbariManagementControllerImpl.createAction()
     hostParams.put(REPO_INFO, repoInfo);
     hostParams.put(JDK_LOCATION, amc.getJdkResourceUrl());
     hostParams.put(JAVA_HOME, amc.getJavaHome());
@@ -769,8 +769,24 @@ public class AmbariCustomCommandExecutionHelper {
       hostParams.put(DB_DRIVER_FILENAME, configs.getMySQLJarName());
     }
     execCmd.setHostLevelParams(hostParams);
+  }
+
+  private String getRepoInfo(Cluster cluster, Host host) throws AmbariException {
+    StackId stackId = cluster.getDesiredStackVersion();
+
+    Map<String, List<RepositoryInfo>> repos = ambariMetaInfo.getRepository(
+      stackId.getStackName(), stackId.getStackVersion());
+    String repoInfo = "";
+    if (!repos.containsKey(host.getOsType())) {
+      // FIXME should this be an error?
+      LOG.warn("Could not retrieve repo information for host"
+        + ", hostname=" + host.getHostName()
+        + ", clusterName=" + cluster.getClusterName()
+        + ", stackInfo=" + stackId.getStackId());
+    } else {
+      repoInfo = gson.toJson(repos.get(host.getOsType()));
+    }
 
-    Map<String, String> roleParams = new TreeMap<String, String>();
-    execCmd.setRoleParams(roleParams);
+    return repoInfo;
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -2088,7 +2088,6 @@ public class AmbariManagementControllerImpl implements
     String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
     Stage stage = createNewStage(cluster, actionManager.getNextRequestId(), requestContext, clusterHostInfoJson);
 
-
     Map<String, String> params = createDefaultHostParams(cluster);
 
     if (actionRequest.isCommand()) {

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -3886,6 +3886,95 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals("h3", taskStatus.getHostName());
   }
 
+  @Test
+  public void testComponentCategorySentWithRestart() throws AmbariException {
+    setupClusterWithHosts("c1", "HDP-2.0.7",
+      new ArrayList<String>() {{
+        add("h1");
+      }},
+      "centos5");
+
+    Cluster cluster = clusters.getCluster("c1");
+    cluster.setDesiredStackVersion(new StackId("HDP-2.0.7"));
+    cluster.setCurrentStackVersion(new StackId("HDP-2.0.7"));
+
+    ConfigFactory cf = injector.getInstance(ConfigFactory.class);
+    Config config1 = cf.createNew(cluster, "global",
+      new HashMap<String, String>() {{
+        put("key1", "value1");
+      }});
+    config1.setVersionTag("version1");
+
+    Config config2 = cf.createNew(cluster, "core-site",
+      new HashMap<String, String>() {{
+        put("key1", "value1");
+      }});
+    config2.setVersionTag("version1");
+
+    cluster.addConfig(config1);
+    cluster.addConfig(config2);
+
+    Service hdfs = cluster.addService("HDFS");
+    hdfs.persist();
+
+    hdfs.addServiceComponent(Role.HDFS_CLIENT.name()).persist();
+    hdfs.addServiceComponent(Role.NAMENODE.name()).persist();
+    hdfs.addServiceComponent(Role.DATANODE.name()).persist();
+
+    hdfs.getServiceComponent(Role.HDFS_CLIENT.name()).addServiceComponentHost("h1").persist();
+    hdfs.getServiceComponent(Role.NAMENODE.name()).addServiceComponentHost("h1").persist();
+    hdfs.getServiceComponent(Role.DATANODE.name()).addServiceComponentHost("h1").persist();
+
+    installService("c1", "HDFS", false, false);
+
+    startService("c1", "HDFS", false, false);
+
+    Cluster c = clusters.getCluster("c1");
+    Service s = c.getService("HDFS");
+
+    Assert.assertEquals(State.STARTED, s.getDesiredState());
+    for (ServiceComponent sc : s.getServiceComponents().values()) {
+      for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) {
+        if (sc.isClientComponent()) {
+          Assert.assertEquals(State.INSTALLED, sch.getDesiredState());
+        } else {
+          Assert.assertEquals(State.STARTED, sch.getDesiredState());
+        }
+      }
+    }
+
+    Map<String, String> params = new HashMap<String, String>() {{
+      put("test", "test");
+    }};
+    ExecuteActionRequest actionRequest = new ExecuteActionRequest("c1",
+      "RESTART", null, "HDFS", "HDFS_CLIENT",
+      new ArrayList<String>() {{ add("h1"); }},
+      params);
+
+    Map<String, String> requestProperties = new HashMap<String, String>();
+    requestProperties.put(REQUEST_CONTEXT_PROPERTY, "Called from a test");
+
+    RequestStatusResponse response = controller.createAction(actionRequest, requestProperties);
+
+    List<Stage> stages = actionDB.getAllStages(response.getRequestId());
+    Assert.assertNotNull(stages);
+
+    HostRoleCommand hrc = null;
+    for (Stage stage : stages) {
+      for (HostRoleCommand cmd : stage.getOrderedHostRoleCommands()) {
+        if (cmd.getRole().equals(Role.HDFS_CLIENT)) {
+          hrc = cmd;
+        }
+      }
+    }
+    Assert.assertNotNull(hrc);
+    Map<String, String> roleParams = hrc.getExecutionCommandWrapper()
+      .getExecutionCommand().getRoleParams();
+
+    Assert.assertNotNull(roleParams);
+    Assert.assertEquals("CLIENT", roleParams.get(ExecutionCommand.KeyNames.COMPONENT_CATEGORY));
+  }
+
   @SuppressWarnings("serial")
   @Test
   public void testCreateActionsFailures() throws Exception {

ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py

@@ -19,6 +19,7 @@ limitations under the License.
 '''
 from mock.mock import MagicMock, call, patch
 from stacks.utils.RMFTestCase import *
+from resource_management.libraries.script.script import Script
 
 class TestYarnClient(RMFTestCase):
 
@@ -221,3 +222,104 @@ class TestYarnClient(RMFTestCase):
     )
     self.assertNoMoreResources()
 
+  def test_restart_client(self):
+    self.executeScript("2.0.6/services/YARN/package/scripts/yarn_client.py",
+                       classname = "YarnClient",
+                       command = "restart",
+                       config_file="default.json",
+                       config_overrides = { 'roleParams' : { "component_category": "CLIENT" } }
+    )
+
+    self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn',
+      owner = 'yarn',
+      group = 'hadoop',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn',
+      owner = 'yarn',
+      group = 'hadoop',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred',
+      owner = 'mapred',
+      group = 'hadoop',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred',
+      owner = 'mapred',
+      group = 'hadoop',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/local',
+      owner = 'yarn',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
+      owner = 'yarn',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/log',
+      owner = 'yarn',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
+      owner = 'yarn',
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
+      owner = 'yarn',
+      recursive = True,
+    )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+      owner = 'hdfs',
+      group = 'hadoop',
+      mode = 420,
+      conf_dir = '/etc/hadoop/conf',
+      configurations = self.getConfig()['configurations']['core-site'],
+    )
+    self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
+      owner = 'yarn',
+      group = 'hadoop',
+      mode = 420,
+      conf_dir = '/etc/hadoop/conf',
+      configurations = self.getConfig()['configurations']['mapred-site'],
+    )
+    self.assertResourceCalled('XmlConfig', 'yarn-site.xml',
+      owner = 'yarn',
+      group = 'hadoop',
+      mode = 420,
+      conf_dir = '/etc/hadoop/conf',
+      configurations = self.getConfig()['configurations']['yarn-site'],
+    )
+    self.assertResourceCalled('XmlConfig', 'capacity-scheduler.xml',
+      owner = 'yarn',
+      group = 'hadoop',
+      mode = 420,
+      conf_dir = '/etc/hadoop/conf',
+      configurations = self.getConfig()['configurations']['capacity-scheduler'],
+    )
+    self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
+      owner = 'yarn',
+      group = 'hadoop',
+    )
+    self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
+      owner = 'mapred',
+      group = 'hadoop',
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
+      content = Template('yarn.conf.j2'),
+      mode = 420,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/mapreduce.conf',
+      content = Template('mapreduce.conf.j2'),
+      mode = 420,
+    )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh',
+      content = Template('yarn-env.sh.j2'),
+      owner = 'yarn',
+      group = 'hadoop',
+      mode = 493,
+    )
+    self.assertNoMoreResources()
+
+

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -37,7 +37,8 @@ PATH_TO_STACK_TESTS = os.path.normpath("test/python/stacks/")
 
 class RMFTestCase(TestCase):
   def executeScript(self, path, classname=None, command=None, config_file=None,
-                    # common mocks for all the scripts 
+                    # common mocks for all the scripts
+                    config_overrides = None,
                     shell_mock_value = (0, "OK."), 
                     os_type=('Suse','11','Final'),
                     kinit_path_local="/usr/bin/kinit"
@@ -49,13 +50,19 @@ class RMFTestCase(TestCase):
     configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
     script_path = os.path.join(stacks_path, norm_path)
     config_file_path = os.path.join(configs_path, config_file)
-    
+
     try:
       with open(config_file_path, "r") as f:
-        self.config_dict = ConfigDictionary(json.load(f))
+        self.config_dict = json.load(f)
     except IOError:
       raise RuntimeError("Can not read config file: "+ config_file_path)
-    
+
+    if config_overrides:
+      for key, value in config_overrides.iteritems():
+        self.config_dict[key] = value
+
+    self.config_dict = ConfigDictionary(self.config_dict)
+
     # append basedir to PYTHONPATH
     scriptsdir = os.path.dirname(script_path)
     basedir = os.path.dirname(scriptsdir)