AMBARI-9206 - Heartbeat Exceptions During Cluster Provisioning When Processing Alerts (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeat.java

@@ -65,17 +65,17 @@ public class HeartBeat {
   public void setHostname(String hostname) {
     this.hostname = hostname;
   }
-  
+
   @JsonProperty("reports")
   public List<CommandReport> getReports() {
-    return this.reports;
+    return reports;
   }
-  
+
   @JsonProperty("reports")
   public void setReports(List<CommandReport> reports) {
     this.reports = reports;
   }
-  
+
   public HostStatus getNodeStatus() {
     return nodeStatus;
   }
@@ -83,11 +83,11 @@ public class HeartBeat {
   public void setNodeStatus(HostStatus nodeStatus) {
     this.nodeStatus = nodeStatus;
   }
-  
+
   public AgentEnv getAgentEnv() {
     return agentEnv;
   }
-  
+
   public void setAgentEnv(AgentEnv env) {
     agentEnv = env;
   }
@@ -104,18 +104,22 @@ public class HeartBeat {
 
   @JsonProperty("mounts")
   public List<DiskInfo> getMounts() {
-    return this.mounts;
+    return mounts;
   }
 
   @JsonProperty("mounts")
   public void setMounts(List<DiskInfo> mounts) {
     this.mounts = mounts;
   }
-  
+
   public List<Alert> getAlerts() {
     return alerts;
   }
 
+  public void setAlerts(List<Alert> alerts) {
+    this.alerts = alerts;
+  }
+
   @Override
   public String toString() {
     return "HeartBeat{" +

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java

@@ -42,7 +42,6 @@ import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.bootstrap.DistributeRepositoriesStructuredOutput;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.MaintenanceStateHelper;
 import org.apache.ambari.server.events.ActionFinalReportReceivedEvent;
@@ -51,14 +50,6 @@ import org.apache.ambari.server.events.AlertReceivedEvent;
 import org.apache.ambari.server.events.publishers.AlertEventPublisher;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.metadata.ActionMetadata;
-import org.apache.ambari.server.orm.dao.HostDAO;
-import org.apache.ambari.server.orm.dao.HostVersionDAO;
-import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
-import org.apache.ambari.server.orm.entities.ClusterEntity;
-import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
-import org.apache.ambari.server.orm.entities.HostEntity;
-import org.apache.ambari.server.orm.entities.HostVersionEntity;
-import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
 import org.apache.ambari.server.serveraction.kerberos.KerberosActionDataFile;
 import org.apache.ambari.server.serveraction.kerberos.KerberosActionDataFileReader;
 import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction;
@@ -82,7 +73,6 @@ import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.StackInfo;
 import org.apache.ambari.server.state.State;
-import org.apache.ambari.server.state.UpgradeHelper;
 import org.apache.ambari.server.state.UpgradeState;
 import org.apache.ambari.server.state.alert.AlertDefinition;
 import org.apache.ambari.server.state.alert.AlertDefinitionHash;
@@ -103,8 +93,8 @@ import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
@@ -120,8 +110,12 @@ import com.google.inject.Singleton;
  */
 @Singleton
 public class HeartBeatHandler {
+  /**
+   * Logger.
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(HeartBeatHandler.class);
+
   private static final Pattern DOT_PATTERN = Pattern.compile("\\.");
-  private static Log LOG = LogFactory.getLog(HeartBeatHandler.class);
   private final Clusters clusterFsm;
   private final ActionQueue actionQueue;
   private final ActionManager actionManager;
@@ -272,16 +266,15 @@ public class HeartBeatHandler {
 
   /**
    * Extracts all of the {@link Alert}s from the heartbeat and fires
-   * {@link AlertEvent}s for each one.
+   * {@link AlertEvent}s for each one. If there is a problem looking up the
+   * cluster, then alerts will not be processed.
    *
    * @param heartbeat
    *          the heartbeat to process.
    * @param hostname
    *          the host that the heartbeat is for.
-   * @throws AmbariException
    */
-  protected void processAlerts(HeartBeat heartbeat, String hostname)
-      throws AmbariException {
+  protected void processAlerts(HeartBeat heartbeat, String hostname) {
 
     if (null == hostname || null == heartbeat) {
       return;
@@ -293,9 +286,16 @@ public class HeartBeatHandler {
           alert.setHost(hostname);
         }
 
-        Cluster cluster = clusterFsm.getCluster(alert.getCluster());
-        AlertEvent event = new AlertReceivedEvent(cluster.getClusterId(), alert);
-        alertEventPublisher.publish(event);
+        try {
+          Cluster cluster = clusterFsm.getCluster(alert.getCluster());
+          AlertEvent event = new AlertReceivedEvent(cluster.getClusterId(),
+              alert);
+          alertEventPublisher.publish(event);
+        } catch (AmbariException ambariException) {
+          LOG.warn(
+              "Unable to process alerts because the cluster {} does not exist",
+              alert.getCluster());
+        }
       }
     }
   }

ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java

@@ -32,8 +32,12 @@ import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HDFS;
 import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HDFS_CLIENT;
 import static org.apache.ambari.server.agent.DummyHeartbeatConstants.NAMENODE;
 import static org.apache.ambari.server.agent.DummyHeartbeatConstants.SECONDARY_NAMENODE;
-import static org.easymock.EasyMock.*;
+import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -76,13 +80,14 @@ import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.orm.OrmTestHelper;
 import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.RepositoryVersionState;
 import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.HostHealthStatus;
 import org.apache.ambari.server.state.HostState;
 import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.RepositoryVersionState;
 import org.apache.ambari.server.state.SecurityState;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponentHost;
@@ -2329,4 +2334,75 @@ public class TestHeartbeatHandler {
 
     Assert.assertEquals(Integer.valueOf(0), Integer.valueOf(sch.getProcesses().size()));
   }
+
+  /**
+   * Tests that if there is an invalid cluster in heartbeat data, the heartbeat
+   * doesn't fail.
+   *
+   * @throws Exception
+   */
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testHeartBeatWithAlertAndInvalidCluster() throws Exception {
+    ActionManager am = getMockActionManager();
+
+    expect(am.getTasks(anyObject(List.class))).andReturn(
+        new ArrayList<HostRoleCommand>());
+
+    replay(am);
+
+    Cluster cluster = getDummyCluster();
+
+    @SuppressWarnings("serial")
+    Set<String> hostNames = new HashSet<String>() {
+      {
+        add(DummyHostname1);
+      }
+    };
+    clusters.mapHostsToCluster(hostNames, DummyCluster);
+
+    Clusters fsm = clusters;
+    Host hostObject = clusters.getHost(DummyHostname1);
+    hostObject.setIPv4("ipv4");
+    hostObject.setIPv6("ipv6");
+    hostObject.setOsType(DummyOsType);
+
+    ActionQueue aq = new ActionQueue();
+
+    HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector);
+    Register reg = new Register();
+    HostInfo hi = new HostInfo();
+    hi.setHostName(DummyHostname1);
+    hi.setOS(DummyOs);
+    hi.setOSRelease(DummyOSRelease);
+    reg.setHostname(DummyHostname1);
+    reg.setHardwareProfile(hi);
+    reg.setAgentVersion(metaInfo.getServerVersion());
+    handler.handleRegistration(reg);
+
+    hostObject.setState(HostState.UNHEALTHY);
+
+    ExecutionCommand execCmd = new ExecutionCommand();
+    execCmd.setCommandId("2-34");
+    execCmd.setHostname(DummyHostname1);
+    aq.enqueue(DummyHostname1, new ExecutionCommand());
+
+    HeartBeat hb = new HeartBeat();
+    HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus);
+
+    hb.setResponseId(0);
+    hb.setNodeStatus(hs);
+    hb.setHostname(DummyHostname1);
+
+    Alert alert = new Alert("foo", "bar", "baz", "foobar", "foobarbaz",
+        AlertState.OK);
+
+    alert.setCluster("BADCLUSTER");
+
+    List<Alert> alerts = Collections.singletonList(alert);
+    hb.setAlerts(alerts);
+
+    // should NOT throw AmbariException from alerts.
+    handler.handleHeartBeat(hb);
+  }
 }