AMBARI-5544. Deleted slave components stick around from the service's perspective (dlysnichenko)

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -23,6 +23,21 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ClusterNotFoundException;
 import org.apache.ambari.server.DuplicateResourceException;
@@ -41,8 +56,32 @@ import org.apache.ambari.server.actionmanager.RequestFactory;
 import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.actionmanager.StageFactory;
 import org.apache.ambari.server.agent.ExecutionCommand;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_DRIVER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCHEMA_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.internal.RequestResourceFilter;
 import org.apache.ambari.server.controller.internal.RequestStageContainer;
 import org.apache.ambari.server.controller.internal.URLStreamProvider;
 import org.apache.ambari.server.customactions.ActionDefinition;
@@ -94,46 +133,6 @@ import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_DRIVER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCHEMA_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
-
 @Singleton
 public class AmbariManagementControllerImpl implements AmbariManagementController {
 
@@ -2187,11 +2186,20 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       }
       safeToRemoveSCHs.get(component).add(componentHost);
     }
-    
-    for (Entry<ServiceComponent, Set<ServiceComponentHost>> entry 
-           : safeToRemoveSCHs.entrySet()) {
+
+    for (Entry<ServiceComponent, Set<ServiceComponentHost>> entry
+            : safeToRemoveSCHs.entrySet()) {
+      ServiceComponent serviceComponent = entry.getKey();
       for (ServiceComponentHost componentHost : entry.getValue()) {
-        entry.getKey().deleteServiceComponentHosts(componentHost.getHostName());
+        removeFromExcludeHosts(serviceComponent, componentHost.getHostName());
+        serviceComponent.deleteServiceComponentHosts(
+                componentHost.getHostName());
+      }
+      if (serviceComponent.getServiceComponentHosts().isEmpty()) {
+        LOG.info("Change service component " + serviceComponent.getName()
+                + " state from " + serviceComponent.getDesiredState().toString()
+                + " to state " + State.INIT.toString());
+        serviceComponent.setDesiredState(State.INIT);
       }
     }
 
@@ -2202,6 +2210,31 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     return null;
   }
 
+  public RequestStatusResponse removeFromExcludeHosts(
+          ServiceComponent serviceComponent, String hostname)
+          throws AmbariException {
+    String clusterName = serviceComponent.getClusterName();
+    String serviceName = serviceComponent.getServiceName();
+    String serviceComponentName = serviceComponent.getName();
+    RequestStatusResponse response = null;
+    HashMap<String, String> requestProperties = new HashMap<String, String>();
+    HashMap<String, String> params = new HashMap<String, String>();
+    if ("YARN".equals(serviceName)) {
+      if ("NODEMANAGER".equals(serviceComponentName)) {
+        params.put("included_hosts", hostname);
+        params.put("slave_type", serviceComponentName);
+
+        RequestResourceFilter resourceFilter
+              = new RequestResourceFilter(serviceName, "RESOURCEMANAGER", null);
+        ExecuteActionRequest actionRequest = new ExecuteActionRequest(
+              clusterName, "DECOMMISSION", null,
+              Collections.singletonList(resourceFilter), params);
+        response = createAction(actionRequest, requestProperties);
+      }
+    }
+    return response;
+  }
+  
   @Override
   public void deleteUsers(Set<UserRequest> requests)
     throws AmbariException {

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -8316,6 +8316,95 @@ public class AmbariManagementControllerTest {
 
   }
 
+  @Test
+  public void testDeleteNodeManagers() throws Exception {
+    String clusterName = "foo1";
+    createCluster(clusterName);
+    Cluster cluster = clusters.getCluster(clusterName);
+    cluster.setDesiredStackVersion(new StackId("HDP-2.0.7"));
+    String serviceName = Service.Type.HDFS.toString();
+    createService(clusterName, serviceName, null);
+    String componentName1 = Role.NAMENODE.toString();
+    String componentName2 = Role.DATANODE.toString();
+    createServiceComponent(clusterName, serviceName, componentName1, State.INIT);
+    createServiceComponent(clusterName, serviceName, componentName2, State.INIT);
+    String host1 = "h1";
+    String host2 = "h2";
+    addHost(host2, clusterName);
+    addHost(host1, clusterName);
+    createServiceComponentHost(clusterName, null, componentName1, host1, null);
+    createServiceComponentHost(clusterName, serviceName, componentName2, host1, null);
+    String yarnService = Service.Type.YARN.toString();
+    createService(clusterName, yarnService, null);
+    String resourceManagerComponent = Role.RESOURCEMANAGER.toString();
+    String nodeManagerComponent = Role.NODEMANAGER.toString();
+    createServiceComponent(clusterName, yarnService, nodeManagerComponent, State.INIT);
+    createServiceComponentHost(clusterName, yarnService, nodeManagerComponent, host1, null);
+    createServiceComponentHost(clusterName, yarnService, nodeManagerComponent, host2, null);
+
+    // Install
+    installService(clusterName, serviceName, false, false);
+    installService(clusterName, yarnService, false, false);
+
+    // make them believe they are up
+    Map<String, ServiceComponentHost> hostComponents = cluster.getService(serviceName).getServiceComponent(componentName1).getServiceComponentHosts();
+    for (Map.Entry<String, ServiceComponentHost> entry : hostComponents.entrySet()) {
+      ServiceComponentHost cHost = entry.getValue();
+      cHost.handleEvent(new ServiceComponentHostInstallEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis(), cluster.getDesiredStackVersion().getStackId()));
+      cHost.handleEvent(new ServiceComponentHostOpSucceededEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis()));
+    }
+    hostComponents = cluster.getService(serviceName).getServiceComponent(componentName2).getServiceComponentHosts();
+    for (Map.Entry<String, ServiceComponentHost> entry : hostComponents.entrySet()) {
+      ServiceComponentHost cHost = entry.getValue();
+      cHost.handleEvent(new ServiceComponentHostInstallEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis(), cluster.getDesiredStackVersion().getStackId()));
+      cHost.handleEvent(new ServiceComponentHostOpSucceededEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis()));
+    }
+    hostComponents = cluster.getService(yarnService).getServiceComponent(nodeManagerComponent).getServiceComponentHosts();
+    for (Map.Entry<String, ServiceComponentHost> entry : hostComponents.entrySet()) {
+      ServiceComponentHost cHost = entry.getValue();
+      cHost.handleEvent(new ServiceComponentHostInstallEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis(), cluster.getDesiredStackVersion().getStackId()));
+      cHost.handleEvent(new ServiceComponentHostOpSucceededEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis()));
+    }
+
+    ServiceComponent serviceComponent = cluster.getService(yarnService).getServiceComponent(nodeManagerComponent);
+    Assert.assertEquals(State.INSTALLED, serviceComponent.getDesiredState());
+
+    Set<ServiceComponentHostRequest> schRequests = new HashSet<ServiceComponentHostRequest>();
+
+    // delete NodeManagers from all hosts if RESOURCEMANAGER not installed
+    schRequests.clear();
+    schRequests.add(new ServiceComponentHostRequest(clusterName, yarnService, nodeManagerComponent, host1, null));
+    schRequests.add(new ServiceComponentHostRequest(clusterName, yarnService, nodeManagerComponent, host2, null));
+
+    // verify if removeFromExcludeHosts was called while deletion
+    // it need RESOURCEMANAGER component
+    try {
+      controller.deleteHostComponents(schRequests);
+      Assert.fail("Expected a AmbariException.");
+    } catch (AmbariException e) {
+      // expected
+    }
+
+    createServiceComponent(clusterName, yarnService, resourceManagerComponent, State.INIT);
+    createServiceComponentHost(clusterName, yarnService, resourceManagerComponent, host1, null);
+    hostComponents = cluster.getService(yarnService).getServiceComponent(resourceManagerComponent).getServiceComponentHosts();
+    for (Map.Entry<String, ServiceComponentHost> entry : hostComponents.entrySet()) {
+      ServiceComponentHost cHost = entry.getValue();
+      cHost.handleEvent(new ServiceComponentHostInstallEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis(), cluster.getDesiredStackVersion().getStackId()));
+      cHost.handleEvent(new ServiceComponentHostOpSucceededEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis()));
+    }
+
+    // delete NodeManagers from all hosts
+    schRequests.clear();
+    schRequests.add(new ServiceComponentHostRequest(clusterName, yarnService, nodeManagerComponent, host1, null));
+    schRequests.add(new ServiceComponentHostRequest(clusterName, yarnService, nodeManagerComponent, host2, null));
+    controller.deleteHostComponents(schRequests);
+
+    Assert.assertEquals(3, cluster.getServiceComponentHosts(host1).size());
+    Assert.assertEquals(0, cluster.getServiceComponentHosts(host2).size());
+    Assert.assertEquals(State.INIT, serviceComponent.getDesiredState());
+  }
+  
   @Test
   public void testGetRootServices() throws Exception {