AMBARI-4302. Add support for decommissioning NodeManager

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java

@@ -44,6 +44,7 @@ import org.apache.ambari.server.state.ServiceComponentHostEvent;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.ServiceOsSpecific;
 import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.State;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
 import org.apache.ambari.server.utils.StageUtils;
 import org.slf4j.Logger;
@@ -241,15 +242,6 @@ public class AmbariCustomCommandExecutionHelper {
       execCmd.setClusterHostInfo(
           StageUtils.getClusterHostInfo(clusters.getHostsForCluster(clusterName), cluster));
 
-      if (hostLevelParams == null) {
-        hostLevelParams = new TreeMap<String, String>();
-      }
-      hostLevelParams.put(JDK_LOCATION, amc.getJdkResourceUrl());
-      hostLevelParams.put(JAVA_HOME, amc.getJavaHome());
-      hostLevelParams.put(JDK_NAME, amc.getJDKName());
-      hostLevelParams.put(JCE_NAME, amc.getJCEName());
-      hostLevelParams.put(STACK_NAME, stackId.getStackName());
-      hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
       hostLevelParams.put(CUSTOM_COMMAND, commandName);
       execCmd.setHostLevelParams(hostLevelParams);
 
@@ -283,7 +275,6 @@ public class AmbariCustomCommandExecutionHelper {
           serviceInfo.getServiceMetadataFolder());
 
       execCmd.setCommandParams(commandParams);
-
     }
   }
 
@@ -333,7 +324,6 @@ public class AmbariCustomCommandExecutionHelper {
           .iterator().next();
     }
 
-
     addServiceCheckAction(stage, hostName, smokeTestRole, nowTimestamp,
         serviceName, componentName, actionParameters,
         hostLevelParams);
@@ -387,13 +377,6 @@ public class AmbariCustomCommandExecutionHelper {
     if (hostLevelParams == null) {
       hostLevelParams = new TreeMap<String, String>();
     }
-    hostLevelParams.put(JDK_LOCATION, amc.getJdkResourceUrl());
-    hostLevelParams.put(JAVA_HOME, amc.getJavaHome());
-    hostLevelParams.put(JDK_NAME, amc.getJDKName());
-    hostLevelParams.put(JCE_NAME, amc.getJCEName());
-    hostLevelParams.put(STACK_NAME, stackId.getStackName());
-    hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
-    hostLevelParams.putAll(amc.getRcaParameters());
     execCmd.setHostLevelParams(hostLevelParams);
 
     Map<String, String> commandParams = new TreeMap<String, String>();
@@ -500,6 +483,14 @@ public class AmbariCustomCommandExecutionHelper {
       throw new AmbariException("Component " + slaveCompType + " is not supported for decommissioning.");
     }
 
+    // Decommission only if the sch is in state STARTED or INSTALLED
+    for (ServiceComponentHost sch : svcComponents.get(slaveCompType).getServiceComponentHosts().values()) {
+      if (excludedHosts.contains(sch.getHostName()) && sch.getState() != State.STARTED) {
+        throw new AmbariException("Component " + slaveCompType + " on host " + sch.getHostName() + " cannot be " +
+            "decommissioned as its not in STARTED state. Aborting the whole request.");
+      }
+    }
+
     // Set/reset decommissioned flag on all components
     for (ServiceComponentHost sch : svcComponents.get(slaveCompType).getServiceComponentHosts().values()) {
       if (excludedHosts.contains(sch.getHostName())) {

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -744,7 +744,6 @@ public class AmbariManagementControllerImpl implements
     return response;
   }
 
-
   private Set<ConfigurationResponse> getConfigurations(
       ConfigurationRequest request) throws AmbariException {
     if (request.getClusterName() == null) {
@@ -1277,11 +1276,10 @@ public class AmbariManagementControllerImpl implements
               + ", serviceCheckRole=" + smokeTestRole);
           continue;
         }
-        Configuration configuration = injector.getInstance(Configuration.class);
+
         customCommandExecutionHelper.addServiceCheckAction(stage, clientHost,
             smokeTestRole, nowTimestamp, serviceName,
-            null, null, null);
-
+            null, null, createDefaultHostParams(cluster));
       }
 
       RoleCommandOrder rco = getRoleCommandOrder(cluster);
@@ -1293,6 +1291,19 @@ public class AmbariManagementControllerImpl implements
     return null;
   }
 
+  private TreeMap<String, String> createDefaultHostParams(Cluster cluster) {
+    StackId stackId = cluster.getDesiredStackVersion();
+    TreeMap<String, String> hostLevelParams = new TreeMap<String, String>();
+    hostLevelParams.put(JDK_LOCATION, getJdkResourceUrl());
+    hostLevelParams.put(JAVA_HOME, getJavaHome());
+    hostLevelParams.put(JDK_NAME, getJDKName());
+    hostLevelParams.put(JCE_NAME, getJCEName());
+    hostLevelParams.put(STACK_NAME, stackId.getStackName());
+    hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
+    hostLevelParams.putAll(getRcaParameters());
+    return hostLevelParams;
+  }
+
   private void persistStages(List<Stage> stages) throws AmbariException {
     if (stages != null && !stages.isEmpty()) {
       persistRequest(requestFactory.createNewFromStages(stages));
@@ -2044,11 +2055,8 @@ public class AmbariManagementControllerImpl implements
     String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
     Stage stage = createNewStage(cluster, actionManager.getNextRequestId(), requestContext, clusterHostInfoJson);
 
-    Map<String, String> params = new TreeMap<String, String>();
-    // TODO : Update parameter population to be done only here
-    params.put(JDK_LOCATION, this.jdkResourceUrl);
-    params.put(STACK_VERSION, cluster.getDesiredStackVersion().getStackVersion());
-    params.putAll(getRcaParameters());
+
+    Map<String, String> params = createDefaultHostParams(cluster);
 
     if (actionRequest.isCommand()) {
       customCommandExecutionHelper.addAction(actionRequest, stage, params);

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java

@@ -73,8 +73,8 @@ public class HostComponentResourceProvider extends AbstractControllerResourcePro
       = PropertyHelper.getPropertyId("HostRoles", "actual_configs");
   protected static final String HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID
       = PropertyHelper.getPropertyId("HostRoles", "stale_configs");
-  protected static final String HOST_COMPONENT_ADMIN_STATE_PROPERTY_ID
-      = PropertyHelper.getPropertyId("HostRoles", "admin_state");
+  protected static final String HOST_COMPONENT_DESIRED_ADMIN_STATE_PROPERTY_ID
+      = PropertyHelper.getPropertyId("HostRoles", "desired_admin_state");
   //Component name mappings
   private static final Map<String, PropertyProvider> HOST_COMPONENT_PROPERTIES_PROVIDER = new HashMap<String, PropertyProvider>();
   private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT = 1500;   //milliseconds
@@ -193,7 +193,7 @@ public class HostComponentResourceProvider extends AbstractControllerResourcePro
       setResourceProperty(resource, HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID,
           Boolean.valueOf(response.isStaleConfig()), requestedIds);
       if (response.getAdminState() != null) {
-        setResourceProperty(resource, HOST_COMPONENT_ADMIN_STATE_PROPERTY_ID,
+        setResourceProperty(resource, HOST_COMPONENT_DESIRED_ADMIN_STATE_PROPERTY_ID,
             response.getAdminState(), requestedIds);
       }
 
@@ -302,9 +302,9 @@ public class HostComponentResourceProvider extends AbstractControllerResourcePro
       serviceComponentHostRequest.setStaleConfig(
           properties.get(HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID).toString().toLowerCase());
     }
-    if (properties.get(HOST_COMPONENT_ADMIN_STATE_PROPERTY_ID) != null) {
+    if (properties.get(HOST_COMPONENT_DESIRED_ADMIN_STATE_PROPERTY_ID) != null) {
       serviceComponentHostRequest.setAdminState(
-          properties.get(HOST_COMPONENT_ADMIN_STATE_PROPERTY_ID).toString());
+          properties.get(HOST_COMPONENT_DESIRED_ADMIN_STATE_PROPERTY_ID).toString());
     }
 
     return serviceComponentHostRequest;

ambari-server/src/main/python/ambari-server.py

@@ -2618,7 +2618,7 @@ def upgrade_local_repo_db(args, dbkey, dbvalue):
     client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]
     client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INSERT_METAINFO_SCRIPTS[DATABASE_INDEX], args.database_username,
                                                                        BLIND_PASSWORD, args.database_name)
-    #TODO temporarty code
+    #TODO temporary code
     if not args.database == "oracle":
       raise FatalException(-20, "Upgrade for remote database only supports Oracle.")
 

ambari-server/src/main/resources/properties.json

@@ -64,7 +64,7 @@
         "params/run_smoke_test",
         "HostRoles/nagios_alerts",
         "HostRoles/stale_configs",
-        "HostRoles/admin_state",
+        "HostRoles/desired_admin_state",
         "_"
     ],
     "Configuration":[

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml

@@ -21,7 +21,7 @@
   <services>
     <service>
       <name>MAPREDUCE</name>
-      <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
+      <comment>Apache Hadoop Distributed Processing Framework</comment>
       <version>1.2.0.1.3.3.0</version>
       <components>
 

ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py

@@ -194,11 +194,12 @@ def decommission():
 
   hdfs_user = params.hdfs_user
   conf_dir = params.hadoop_conf_dir
+  user_group = params.user_group
 
   File(params.exclude_file_path,
        content=Template("exclude_hosts_list.j2"),
-       owner=params.hdfs_user,
-       group=params.user_group
+       owner=hdfs_user,
+       group=user_group
   )
 
   ExecuteHadoop('dfsadmin -refreshNodes',

ambari-server/src/main/resources/stacks/HDP/2.0.8/services/YARN/configuration/yarn-site.xml

@@ -323,4 +323,15 @@
     </description>
   </property>
 
+  <property>
+    <name>yarn.resourcemanager.nodes.exclude-path</name>
+    <value>/etc/hadoop/conf/yarn.exclude</value>
+    <description>
+      Names a file that contains a list of hosts that are
+      not permitted to connect to the resource manager.  The full pathname of the
+      file must be specified.  If the value is empty, no hosts are
+      excluded.
+    </description>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.8/services/YARN/package/scripts/params.py

@@ -82,4 +82,8 @@ yarn_bin = "/usr/lib/hadoop-yarn/sbin"
 user_group = config['configurations']['global']['user_group']
 limits_conf_dir = "/etc/security/limits.d"
 hadoop_conf_dir = "/etc/hadoop/conf"
-yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
+yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
+
+#exclude file
+exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
+exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path']

ambari-server/src/main/resources/stacks/HDP/2.0.8/services/YARN/package/scripts/resourcemanager.py

@@ -48,6 +48,7 @@ from resource_management import *
 from yarn import yarn
 from service import service
 
+
 class Resourcemanager(Script):
   def install(self, env):
     self.install_packages(env)
@@ -55,11 +56,13 @@ class Resourcemanager(Script):
 
   def configure(self, env):
     import params
+
     env.set_params(params)
     yarn()
 
   def start(self, env):
     import params
+
     env.set_params(params)
     self.configure(env) # FOR SECURITY
     service('resourcemanager',
@@ -68,6 +71,7 @@ class Resourcemanager(Script):
 
   def stop(self, env):
     import params
+
     env.set_params(params)
 
     service('resourcemanager',
@@ -76,8 +80,33 @@ class Resourcemanager(Script):
 
   def status(self, env):
     import status_params
+
     env.set_params(status_params)
     check_process_status(status_params.resourcemanager_pid_file)
+    pass
+
+  def decommission(self, env):
+    import params
+
+    env.set_params(params)
+
+    yarn_user = params.yarn_user
+    conf_dir = params.config_dir
+    user_group = params.user_group
+
+    yarn_refresh_cmd = format("/usr/bin/yarn --config {conf_dir} rmadmin -refreshNodes")
+
+    File(params.exclude_file_path,
+         content=Template("exclude_hosts_list.j2"),
+         owner=yarn_user,
+         group=user_group
+    )
+
+    Execute(yarn_refresh_cmd,
+            user=yarn_user
+    )
+    pass
+
 
 if __name__ == "__main__":
-  Resourcemanager().execute()
+  Resourcemanager().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.8/services/YARN/package/templates/exclude_hosts_list.j2

@@ -0,0 +1,3 @@
+{% for host in exclude_hosts %}
+{{host}}
+{% endfor %}

ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Oracle-UPGRADE.sql

@@ -26,7 +26,7 @@ ALTER TABLE hostconfigmapping ADD (user_name VARCHAR2 (255) DEFAULT '_db');
 ALTER TABLE stage ADD (cluster_host_info BLOB DEFAULT NULL);
 
 -- add decommission state
-ALTER TABLE hostcomponentdesiredstate ADD (component_attribute VARCHAR2 (255) DEFAULT NULL);
+ALTER TABLE hostcomponentdesiredstate ADD (admin_state VARCHAR2 (32) DEFAULT NULL);
 
 -- DML
 --Upgrade version to current

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -3829,6 +3829,14 @@ public class AmbariManagementControllerTest {
     expectActionCreationErrorWithMessage(actionRequest, requestProperties,
         "Decommission command cannot be issued with target host(s) specified.");
 
+    hdfs.getServiceComponent(Role.DATANODE.name()).getServiceComponentHost("h1").setState(State.INSTALLED);
+    params2 = new HashMap<String, String>() {{
+      put("excluded_hosts", "h1 ");
+    }};
+    actionRequest = new ExecuteActionRequest("c1", "DECOMMISSION", null, "HDFS", "NAMENODE", null, params2);
+    expectActionCreationErrorWithMessage(actionRequest, requestProperties,
+        "Component DATANODE on host h1 cannot be decommissioned as its not in STARTED state");
+
     controller.getActionManager().createActionDefinition(
         "a1", ActionType.SYSTEM, "test,dirName", "Does file exist", "", "",
         TargetHostType.SPECIFIC, Short.valueOf("100"));