AMBARI-16134. Force running service checks for services before upgrade (dlysnichenko)

ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java

@@ -54,6 +54,13 @@ public enum CheckDescription {
             "The following issues have been detected on this cluster and should be addressed before upgrading: %s");
       }}),
 
+  SERVICE_CHECK(PrereqCheckType.SERVICE,
+      "Last Service Check should be more recent than the last configuration change for the given service",
+      new HashMap<String, String>() {{
+        put(AbstractCheckDescriptor.DEFAULT,
+            "The following service configurations have been updated and their Service Checks should be run again: %s");
+      }}),
+
   HOSTS_MAINTENANCE_MODE(PrereqCheckType.HOST,
       "Hosts in Maintenance Mode will be excluded from the upgrade.",
       new HashMap<String, String>() {{

ambari-server/src/main/java/org/apache/ambari/server/checks/HealthCheck.java

@@ -83,7 +83,11 @@ public class HealthCheck extends AbstractCheckDescriptor {
         String label = alertHistory.getAlertDefinition().getLabel();
         String hostName = alertHistory.getHostName();
 
-        errorMessages.add(state + ": " + label + ": " + hostName);
+        if (hostName == null) {
+          errorMessages.add(state + ": " + label);
+        } else {
+          errorMessages.add(state + ": " + label + ": " + hostName);
+        }
         prerequisiteCheck.getFailedDetail().add(new AlertDetail(state, label, hostName));
       }
     }

ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.checks;
+
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.RoleCommand;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.controller.internal.PageRequestImpl;
+import org.apache.ambari.server.controller.internal.RequestImpl;
+import org.apache.ambari.server.controller.spi.PageRequest;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.ServiceConfigDAO;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+import org.apache.ambari.server.orm.entities.ServiceConfigEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.stack.PrereqCheckStatus;
+import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+/**
+ * Checks that all Service Checks are less recent than last
+ * configuration update for given services.
+ * That is a potential problem when doing stack update.
+ */
+@Singleton
+@UpgradeCheck(group = UpgradeCheckGroup.DEFAULT, required = true)
+public class ServiceCheckValidityCheck extends AbstractCheckDescriptor {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ServiceCheckValidityCheck.class);
+
+  private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("dd-MM-yyyy hh:mm:ss");
+  private static final PageRequestImpl PAGE_REQUEST = new PageRequestImpl(PageRequest.StartingPoint.End, 1000, 0, null, null);
+  private static final RequestImpl REQUEST = new RequestImpl(null, null, null, null, null, PAGE_REQUEST);
+
+
+  @Inject
+  Provider<ServiceConfigDAO> serviceConfigDAOProvider;
+
+  @Inject
+  Provider<HostRoleCommandDAO> hostRoleCommandDAOProvider;
+
+  /**
+   * Constructor.
+   */
+  public ServiceCheckValidityCheck() {
+    super(CheckDescription.SERVICE_CHECK);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request)
+      throws AmbariException {
+
+    ServiceConfigDAO serviceConfigDAO = serviceConfigDAOProvider.get();
+    HostRoleCommandDAO hostRoleCommandDAO = hostRoleCommandDAOProvider.get();
+
+    final String clusterName = request.getClusterName();
+    final Cluster cluster = clustersProvider.get().getCluster(clusterName);
+    long clusterId = cluster.getClusterId();
+
+    Map<String, Long> lastServiceConfigUpdates = new HashMap<>();
+
+    for (Service service : cluster.getServices().values()) {
+      if (service.getMaintenanceState() != MaintenanceState.OFF || !hasAtLeastOneComponentVersionAdvertised(service)) {
+        continue;
+      }
+      ServiceConfigEntity lastServiceConfig = serviceConfigDAO.getLastServiceConfig(clusterId, service.getName());
+      lastServiceConfigUpdates.put(service.getName(), lastServiceConfig.getCreateTimestamp());
+    }
+
+    List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findAll(REQUEST, null);
+    Collections.reverse(commands);
+
+    LinkedHashSet<String> failedServiceNames = new LinkedHashSet<>();
+    for (Map.Entry<String, Long> serviceEntry : lastServiceConfigUpdates.entrySet()) {
+      String serviceName = serviceEntry.getKey();
+      Long configTimestamp = serviceEntry.getValue();
+      for (HostRoleCommandEntity command : commands) {
+        if (RoleCommand.SERVICE_CHECK.equals(command.getRoleCommand()) && command.getCommandDetail().contains(serviceName)) {
+          Long serviceCheckTimestamp = command.getStartTime();
+          if (serviceCheckTimestamp < configTimestamp) {
+            failedServiceNames.add(serviceName);
+            LOG.info("Service {} latest config change is {}, latest service check executed at {}",
+                serviceName,
+                DATE_FORMAT.format(new Date(configTimestamp)),
+                DATE_FORMAT.format(new Date(serviceCheckTimestamp))
+            );
+          }
+        }
+      }
+    }
+
+    if (!failedServiceNames.isEmpty()) {
+      prerequisiteCheck.setFailedOn(failedServiceNames);
+      prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL);
+      String failReason = getFailReason(prerequisiteCheck, request);
+      prerequisiteCheck.setFailReason(String.format(failReason, StringUtils.join(failedServiceNames, ", ")));
+    }
+  }
+
+  private boolean hasAtLeastOneComponentVersionAdvertised(Service service) {
+    Collection<ServiceComponent> components = service.getServiceComponents().values();
+    for (ServiceComponent component : components) {
+      if (component.isVersionAdvertised()) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/checks/ServiceCheckValidityCheckTest.java

@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.checks;
+
+import static java.util.Collections.singletonList;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.RoleCommand;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.ServiceConfigDAO;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+import org.apache.ambari.server.orm.entities.ServiceConfigEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.stack.PrereqCheckStatus;
+import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Provider;
+
+public class ServiceCheckValidityCheckTest {
+  private static final String CLUSTER_NAME = "cluster1";
+  private static final long CLUSTER_ID = 1L;
+  private static final String SERVICE_NAME = "HDFS";
+  private static final long CONFIG_CREATE_TIMESTAMP = 1461518722202L;
+  private static final String COMMAND_DETAIL = "HDFS service check";
+  private static final long SERVICE_CHECK_START_TIME = CONFIG_CREATE_TIMESTAMP - 2000L;
+  private static final String SERVICE_COMPONENT_NAME = "service component";
+  private ServiceCheckValidityCheck serviceCheckValidityCheck;
+
+  private ServiceConfigDAO serviceConfigDAO;
+  private HostRoleCommandDAO hostRoleCommandDAO;
+  private Service service;
+
+
+  @Before
+  public void setUp() throws Exception {
+    final Clusters clusters = mock(Clusters.class);
+    service = mock(Service.class);
+    serviceConfigDAO = mock(ServiceConfigDAO.class);
+    hostRoleCommandDAO = mock(HostRoleCommandDAO.class);
+
+    serviceCheckValidityCheck = new ServiceCheckValidityCheck();
+    serviceCheckValidityCheck.hostRoleCommandDAOProvider = new Provider<HostRoleCommandDAO>() {
+      @Override
+      public HostRoleCommandDAO get() {
+        return hostRoleCommandDAO;
+      }
+    };
+    serviceCheckValidityCheck.serviceConfigDAOProvider = new Provider<ServiceConfigDAO>() {
+      @Override
+      public ServiceConfigDAO get() {
+        return serviceConfigDAO;
+      }
+    };
+    serviceCheckValidityCheck.clustersProvider = new Provider<Clusters>() {
+      @Override
+      public Clusters get() {
+        return clusters;
+      }
+    };
+
+    Cluster cluster = mock(Cluster.class);
+    when(clusters.getCluster(CLUSTER_NAME)).thenReturn(cluster);
+    when(cluster.getClusterId()).thenReturn(CLUSTER_ID);
+    when(cluster.getServices()).thenReturn(ImmutableMap.of(SERVICE_NAME, service));
+
+    when(service.getName()).thenReturn(SERVICE_NAME);
+
+  }
+
+  @Test
+  public void testFailWhenServiceWithOutdatedServiceCheckExists() throws AmbariException {
+    ServiceComponent serviceComponent = mock(ServiceComponent.class);
+    when(serviceComponent.isVersionAdvertised()).thenReturn(true);
+
+    when(service.getMaintenanceState()).thenReturn(MaintenanceState.OFF);
+    when(service.getServiceComponents()).thenReturn(ImmutableMap.of(SERVICE_COMPONENT_NAME, serviceComponent));
+
+    ServiceConfigEntity serviceConfigEntity = new ServiceConfigEntity();
+    serviceConfigEntity.setServiceName(SERVICE_NAME);
+    serviceConfigEntity.setCreateTimestamp(CONFIG_CREATE_TIMESTAMP);
+
+    HostRoleCommandEntity hostRoleCommandEntity = new HostRoleCommandEntity();
+    hostRoleCommandEntity.setRoleCommand(RoleCommand.SERVICE_CHECK);
+    hostRoleCommandEntity.setCommandDetail(COMMAND_DETAIL);
+    hostRoleCommandEntity.setStartTime(SERVICE_CHECK_START_TIME);
+
+    when(serviceConfigDAO.getLastServiceConfig(eq(CLUSTER_ID), eq(SERVICE_NAME))).thenReturn(serviceConfigEntity);
+    when(hostRoleCommandDAO.findAll(any(Request.class), isNull(Predicate.class))).thenReturn(singletonList(hostRoleCommandEntity));
+
+    PrerequisiteCheck check = new PrerequisiteCheck(null, CLUSTER_NAME);
+    serviceCheckValidityCheck.perform(check, new PrereqCheckRequest(CLUSTER_NAME));
+    Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus());
+  }
+}