AMBARI-1259. Fix the host roles live status not go back to INSTALLED if it was in START_FAILED state. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1438313 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -194,6 +194,9 @@ Trunk (unreleased changes):
  AMBARI-1250. Upgrade the posgres connector to 9.1.
  (mahadev)
 
+ AMBARI-1259. Fix the host roles live status not go back to INSTALLED if it
+ was in START_FAILED state. (mahadev)
+
 AMBARI-1.2.0 branch:
 
  INCOMPATIBLE CHANGES

ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/server.pp

@@ -162,12 +162,14 @@ class hdp-nagios::server(
       notify => Class['hdp-nagios::server::services']
     }
 
+    class { 'hdp-nagios::server::enable_snmp': }
+
     class { 'hdp-nagios::server::web_permisssions': }
 
     class { 'hdp-nagios::server::services': ensure => $service_state}
 	
 	
-	Class['hdp-nagios::server::packages'] -> Hdp::Directory[$nagios_config_dir] -> Hdp::Directory[$plugins_dir] -> Hdp::Directory_recursive_create[$nagios_pid_dir] ->
+	Class['hdp-nagios::server::packages'] -> Class['hdp-nagios::server::enable_snmp']-> Hdp::Directory[$nagios_config_dir] -> Hdp::Directory[$plugins_dir] -> Hdp::Directory_recursive_create[$nagios_pid_dir] ->
 	Hdp::Directory[$nagios_obj_dir] -> Hdp::Directory_Recursive_Create[$nagios_var_dir] ->
 	Hdp::Directory_Recursive_Create[$check_result_path] -> Hdp::Directory_Recursive_Create[$nagios_rw_dir] ->
 	Class['hdp-nagios::server::config'] -> Class['hdp-nagios::server::web_permisssions'] -> Class['hdp-nagios::server::services'] -> Class['hdp-monitor-webserver']
@@ -217,3 +219,12 @@ class hdp-nagios::server::services($ensure)
     anchor{'hdp-nagios::server::services::begin':} ->  Service['nagios'] ->  anchor{'hdp-nagios::server::services::end':}
   }
 }
+
+class hdp-nagios::server::enable_snmp() {
+
+  exec { "enable_snmp":
+    command => "service snmpd start; chkconfig snmpd on",
+    path    => "/usr/local/bin/:/bin/:/sbin/",
+  }
+
+}

ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/server/packages.pp

@@ -28,9 +28,17 @@ class hdp-nagios::server::packages(
       ensure => 'uninstalled'
     }
   } elsif ($service_state in ['running','stopped','installed_and_configured']) {
-  
 
-  
+  hdp::package { 'perl':
+    ensure      => present,
+    java_needed => false
+  }
+
+  hdp::package { 'perl-Net-SNMP':
+    ensure      => present,
+    java_needed => false
+  }
+
   hdp::package { 'nagios-server':
     ensure      => present,
     java_needed => false

ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp

@@ -501,6 +501,16 @@ class hdp::params()
       64 => {'ALL' => 'hdp_mon_dashboard'}
     },
 
+    perl =>
+    {
+      64 => {'ALL' => 'perl'}
+    },
+
+    perl-Net-SNMP =>
+    {
+      64 => {'ALL' => 'perl-Net-SNMP'}
+    },
+
     nagios-server => {
       64 => {'ALL' => 'nagios-3.2.3'}
     },

ambari-agent/src/test/python/TestManifestGenerator.py

@@ -20,7 +20,6 @@ limitations under the License.
 import os
 
 from unittest import TestCase
-from numpy.testing import assert_
 from ambari_agent import manifestGenerator
 import ambari_agent.AmbariConfig
 import tempfile

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java

@@ -224,6 +224,12 @@ public class HeartBeatHandler {
                   || prevState.equals(State.START_FAILED)
                   || prevState.equals(State.STARTED)
                   || prevState.equals(State.STOP_FAILED)) {
+                if (prevState == State.START_FAILED
+                        && liveState == State.INSTALLED) {
+                  LOG.info("Ignoring INSTALLED state update for " +
+                          "START_FAILED component");
+                  continue;
+                }
                 scHost.setState(liveState);
                 if (!prevState.equals(liveState)) {
                   LOG.info("State of service component " + componentName

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -47,7 +47,7 @@ public class Configuration {
   public static final String BOOTSTRAP_DIR_DEFAULT = "/var/run/ambari-server/bootstrap";
   public static final String WEBAPP_DIR = "webapp.dir";
   public static final String BOOTSTRAP_SCRIPT = "bootstrap.script";
-    public static final String BOOTSTRAP_SCRIPT_DEFAULT =  "/usr/bin/ambari_bootstrap";
+  public static final String BOOTSTRAP_SCRIPT_DEFAULT =  "/usr/bin/ambari_bootstrap";
   public static final String BOOTSTRAP_SETUP_AGENT_SCRIPT = "bootstrap.setup_agent.script";
   public static final String BOOTSTRAP_SETUP_AGENT_PASSWORD = "bootstrap.setup_agent.password";
   public static final String BOOTSTRAP_MASTER_HOSTNAME = "bootstrap.master_host_name";
@@ -70,6 +70,7 @@ public class Configuration {
 
 
   public static final String CLIENT_SECURITY_KEY = "client.security";
+  public static final String CLIENT_API_PORT_KEY = "client.api.port";
   public static final String LDAP_USE_SSL_KEY = "authorization.ldap.useSSL";
   public static final String LDAP_PRIMARY_URL_KEY =
       "authorization.ldap.primaryUrl";
@@ -107,7 +108,7 @@ public class Configuration {
 
   public static final String SRVR_HOSTS_MAPPING = 
       "server.hosts.mapping";
-  
+
   private static final String SRVR_KSTR_DIR_DEFAULT = ".";
   public static final String SRVR_CRT_NAME_DEFAULT = "ca.crt";
   public static final String SRVR_KEY_NAME_DEFAULT = "ca.key";
@@ -119,6 +120,7 @@ public class Configuration {
       "/var/share/ambari/resources/";
 
   private static final String CLIENT_SECURITY_DEFAULT = "local";
+  private static final int CLIENT_API_PORT_DEFAULT = 8080;
 
   private static final String USER_ROLE_NAME_DEFAULT = "user";
   private static final String ADMIN_ROLE_NAME_DEFAULT = "admin";
@@ -133,6 +135,8 @@ public class Configuration {
   private static final String PERSISTENCE_IN_MEMORY_DEFAULT = "true";
 
 
+
+
   private static final Logger LOG = LoggerFactory.getLogger(
       Configuration.class);
 
@@ -166,7 +170,7 @@ public class Configuration {
     configsMap.put(KSTR_NAME_KEY, properties.getProperty(
         KSTR_NAME_KEY, KSTR_NAME_DEFAULT));
     configsMap.put(SRVR_CRT_PASS_FILE_KEY, properties.getProperty(
-     SRVR_CRT_PASS_FILE_KEY, SRVR_CRT_PASS_FILE_DEFAULT));
+        SRVR_CRT_PASS_FILE_KEY, SRVR_CRT_PASS_FILE_DEFAULT));
     configsMap.put(PASSPHRASE_ENV_KEY, properties.getProperty(
         PASSPHRASE_ENV_KEY, PASSPHRASE_ENV_DEFAULT));
     configsMap.put(PASSPHRASE_KEY, System.getenv(configsMap.get(
@@ -178,7 +182,7 @@ public class Configuration {
     configsMap.put(RESOURCES_DIR_KEY, properties.getProperty(
         RESOURCES_DIR_KEY, RESOURCES_DIR_DEFAULT));
     configsMap.put(SRVR_CRT_PASS_LEN_KEY, properties.getProperty(
-     SRVR_CRT_PASS_LEN_KEY, SRVR_CRT_PASS_LEN_DEFAULT));
+        SRVR_CRT_PASS_LEN_KEY, SRVR_CRT_PASS_LEN_DEFAULT));
 
     File passFile = new File(configsMap.get(SRVR_KSTR_DIR_KEY) + File.separator
         + configsMap.get(SRVR_CRT_PASS_FILE_KEY));
@@ -192,12 +196,12 @@ public class Configuration {
         FileUtils.writeStringToFile(passFile, randStr);
 
       } catch (IOException e) {
-          e.printStackTrace();
-          throw new RuntimeException(
+        e.printStackTrace();
+        throw new RuntimeException(
             "Error reading certificate password from file");
       }
     } else {
-        LOG.info("Reading password from existing file");
+      LOG.info("Reading password from existing file");
       try {
         randStr = FileUtils.readFileToString(passFile);
       } catch (IOException e) {
@@ -229,7 +233,7 @@ public class Configuration {
       LOG.info("No configuration file " + CONFIG_FILE + " found in classpath.", fnf);
     } catch (IOException ie) {
       throw new IllegalArgumentException("Can't read configuration file " +
-       CONFIG_FILE, ie);
+          CONFIG_FILE, ie);
     }
 
     return properties;
@@ -238,7 +242,7 @@ public class Configuration {
   public File getBootStrapDir() {
     String fileName = properties.getProperty(BOOTSTRAP_DIR);
     if (fileName == null) {
-        fileName = BOOTSTRAP_DIR_DEFAULT;
+      fileName = BOOTSTRAP_DIR_DEFAULT;
     }
     return new File(fileName);
   }
@@ -300,7 +304,7 @@ public class Configuration {
     LOG.info("Hosts Mapping File " +  properties.getProperty(SRVR_HOSTS_MAPPING));
     return properties.getProperty(SRVR_HOSTS_MAPPING);
   }
-  
+
   /**
    * Gets ambari stack-path
    * @return String
@@ -392,4 +396,9 @@ public class Configuration {
     return properties.getProperty(BOOTSTRAP_MASTER_HOSTNAME, defaultValue);
   }
 
+
+  public int getClientApiPort() {
+    return Integer.parseInt(properties.getProperty(CLIENT_API_PORT_KEY, String.valueOf(CLIENT_API_PORT_DEFAULT)));
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -30,6 +30,7 @@ import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.actionmanager.StageFactory;
 import org.apache.ambari.server.agent.ExecutionCommand;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.metadata.ActionMetadata;
 import org.apache.ambari.server.metadata.RoleCommandOrder;
 import org.apache.ambari.server.security.authorization.User;
@@ -108,6 +109,10 @@ public class AmbariManagementControllerImpl implements
   private Users users;
   @Inject
   private HostsMap hostsMap;
+  @Inject
+  private Configuration configs;
+
+
   
   final private String masterHostname;
 
@@ -126,9 +131,14 @@ public class AmbariManagementControllerImpl implements
     this.gson = injector.getInstance(Gson.class);
     LOG.info("Initializing the AmbariManagementControllerImpl");
     this.masterHostname =  InetAddress.getLocalHost().getCanonicalHostName();
-    this.jdkResourceUrl = "http://" + masterHostname + ":"
-        + AmbariServer.getResourcesPort()
-        + JDK_RESOURCE_LOCATION;
+    
+    if (configs != null) {
+      this.jdkResourceUrl = "http://" + masterHostname + ":"
+          + configs.getClientApiPort()
+          + JDK_RESOURCE_LOCATION; 
+    } else {
+    		this.jdkResourceUrl = null;
+    }
   }
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java

@@ -74,7 +74,6 @@ public class AmbariServer {
   public static final int AGENT_ONE_WAY_AUTH = 8440;
   public static final int AGENT_TWO_WAY_AUTH = 8441;
   public static final int CLIENT_SSL_API_PORT = 8443;
-  public static final int CLIENT_API_PORT = 8080;
 
   private Server server = null;
   private Server serverForAgent = null;
@@ -98,9 +97,6 @@ public class AmbariServer {
     return configs.getServerOsType();
   }
 
-  public static int getResourcesPort() {
-    return CLIENT_API_PORT;
-  }
 
   private static AmbariManagementController clusterController = null;
 
@@ -278,7 +274,7 @@ public class AmbariServer {
       } 
       else  {
         apiConnector = new SelectChannelConnector();
-        apiConnector.setPort(CLIENT_API_PORT);
+        apiConnector.setPort(configs.getClientApiPort());
       }
 
       server.addConnector(apiConnector);

ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java

@@ -17,67 +17,41 @@
  */
 package org.apache.ambari.server.agent;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.xml.bind.JAXBException;
-
-import junit.framework.Assert;
-
 import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
+import junit.framework.Assert;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
-import org.apache.ambari.server.actionmanager.ActionDBAccessor;
-import org.apache.ambari.server.actionmanager.ActionDBAccessorImpl;
-import org.apache.ambari.server.actionmanager.ActionDBInMemoryImpl;
-import org.apache.ambari.server.actionmanager.ActionManager;
-import org.apache.ambari.server.actionmanager.HostRoleStatus;
-import org.apache.ambari.server.actionmanager.Stage;
+import org.apache.ambari.server.actionmanager.*;
 import org.apache.ambari.server.agent.HostStatus.Status;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.HostsMap;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
-import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.HostState;
-import org.apache.ambari.server.state.Service;
-import org.apache.ambari.server.state.ServiceComponentHost;
-import org.apache.ambari.server.state.StackId;
-import org.apache.ambari.server.state.State;
+import org.apache.ambari.server.state.*;
 import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent;
 import org.apache.ambari.server.utils.StageUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.codehaus.jackson.JsonGenerationException;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.xml.bind.JAXBException;
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class TestHeartbeatHandler {
 
   private static final Logger log = LoggerFactory.getLogger(TestHeartbeatHandler.class);
@@ -228,6 +202,110 @@ public class TestHeartbeatHandler {
     assertEquals(State.INSTALLED, componentState2);
   }
 
+  @Test
+  public void testStartFailedStatusHeartbeat() throws Exception {
+    ActionManager am = new ActionManager(0, 0, null, null,
+            new ActionDBInMemoryImpl(), new HostsMap((String) null));
+    final String hostname = "host1";
+    String clusterName = "cluster1";
+    String serviceName = "HDFS";
+    String componentName1 = "DATANODE";
+    String componentName2 = "NAMENODE";
+    String componentName3 = "SECONDARY_NAMENODE";
+    //injector.injectMembers(this);
+
+    clusters.addHost(hostname);
+    clusters.getHost(hostname).setOsType("centos5");
+    clusters.getHost(hostname).persist();
+//    Host hostObject = clusters.getHost(hostname);
+//    hostObject.setIPv4("ipv4");
+//    hostObject.setIPv6("ipv6");
+//    hostObject.setOsType("centos5");
+//    hostObject.persist();
+    clusters.addCluster(clusterName);
+
+    Cluster cluster = clusters.getCluster(clusterName);
+    cluster.setDesiredStackVersion(new StackId("HDP-0.1"));
+
+    @SuppressWarnings("serial")
+    Set<String> hostNames = new HashSet<String>(){{
+      add(hostname);
+    }};
+    clusters.mapHostsToCluster(hostNames, clusterName);
+    Service hdfs = cluster.addService(serviceName);
+    hdfs.persist();
+    hdfs.addServiceComponent(Role.DATANODE.name()).persist();
+    hdfs.getServiceComponent(Role.DATANODE.name()).addServiceComponentHost(hostname).persist();
+    hdfs.addServiceComponent(Role.NAMENODE.name()).persist();
+    hdfs.getServiceComponent(Role.NAMENODE.name()).addServiceComponentHost(hostname).persist();
+    hdfs.addServiceComponent(Role.SECONDARY_NAMENODE.name()).persist();
+    hdfs.getServiceComponent(Role.SECONDARY_NAMENODE.name()).addServiceComponentHost(hostname).persist();
+
+    ActionQueue aq = new ActionQueue();
+    HeartBeatHandler handler = new HeartBeatHandler(clusters, aq, am, injector);
+
+    Register reg = new Register();
+    HostInfo hi = new HostInfo();
+    hi.setHostName("host1");
+    hi.setOS("CentOS");
+    hi.setOSRelease("5.8");
+    reg.setHostname(hostname);
+    reg.setResponseId(0);
+    reg.setHardwareProfile(hi);
+    handler.handleRegistration(reg);
+
+    ServiceComponentHost serviceComponentHost1 = clusters.getCluster(clusterName).getService(serviceName).
+            getServiceComponent(componentName1).getServiceComponentHost(hostname);
+    ServiceComponentHost serviceComponentHost2 = clusters.getCluster(clusterName).getService(serviceName).
+            getServiceComponent(componentName2).getServiceComponentHost(hostname);
+    ServiceComponentHost serviceComponentHost3 = clusters.getCluster(clusterName).getService(serviceName).
+            getServiceComponent(componentName3).getServiceComponentHost(hostname);
+    serviceComponentHost1.setState(State.INSTALLED);
+    serviceComponentHost2.setState(State.START_FAILED);
+    serviceComponentHost3.setState(State.STARTED);
+
+    HeartBeat hb = new HeartBeat();
+    hb.setTimestamp(System.currentTimeMillis());
+    hb.setResponseId(0);
+    hb.setHostname(hostname);
+    hb.setNodeStatus(new HostStatus(Status.HEALTHY, "I am ok"));
+    hb.setReports(new ArrayList<CommandReport>());
+    ArrayList<ComponentStatus> componentStatuses = new ArrayList<ComponentStatus>();
+    ComponentStatus componentStatus1 = new ComponentStatus();
+    componentStatus1.setClusterName(clusterName);
+    componentStatus1.setServiceName(serviceName);
+    componentStatus1.setMessage("I am ok");
+    componentStatus1.setStatus(State.START_FAILED.name());
+    componentStatus1.setComponentName(componentName1);
+    componentStatuses.add(componentStatus1);
+
+    ComponentStatus componentStatus2 = new ComponentStatus();
+    componentStatus2.setClusterName(clusterName);
+    componentStatus2.setServiceName(serviceName);
+    componentStatus2.setMessage("I am ok");
+    componentStatus2.setStatus(State.INSTALLED.name());
+    componentStatus2.setComponentName(componentName2);
+    componentStatuses.add(componentStatus2);
+
+    ComponentStatus componentStatus3 = new ComponentStatus();
+    componentStatus3.setClusterName(clusterName);
+    componentStatus3.setServiceName(serviceName);
+    componentStatus3.setMessage("I am ok");
+    componentStatus3.setStatus(State.INSTALLED.name());
+    componentStatus3.setComponentName(componentName3);
+    componentStatuses.add(componentStatus3);
+
+    hb.setComponentStatus(componentStatuses);
+
+    handler.handleHeartBeat(hb);
+    State componentState1 = serviceComponentHost1.getState();
+    State componentState2 = serviceComponentHost2.getState();
+    State componentState3 = serviceComponentHost3.getState();
+    assertEquals(State.START_FAILED, componentState1);
+    assertEquals(State.START_FAILED, componentState2);
+    assertEquals(State.INSTALLED, componentState3);
+  }
+
   @Test
   public void testCommandReport() throws AmbariException {
     String hostname = "host1";