AMBARI-4291. To handle misfired trigger, the scheduler should start after a fixed delay. (swagle)

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -271,9 +271,13 @@ public class Configuration {
     "server.execution.scheduler.maxDbConnections";
   public static final String EXECUTION_SCHEDULER_MISFIRE_TOLERATION =
     "server.execution.scheduler.misfire.toleration.minutes";
+  public static final String EXECUTION_SCHEDULER_START_DELAY =
+    "server.execution.scheduler.start.delay.seconds";
+
   public static final String DEFAULT_SCHEDULER_THREAD_COUNT = "5";
   public static final String DEFAULT_SCHEDULER_MAX_CONNECTIONS = "5";
   public static final String DEFAULT_EXECUTION_SCHEDULER_MISFIRE_TOLERATION = "480";
+  public static final String DEFAULT_SCHEDULER_START_DELAY_SECONDS = "120";
 
   private static final Logger LOG = LoggerFactory.getLogger(
       Configuration.class);
@@ -899,4 +903,10 @@ public class Configuration {
         DEFAULT_EXECUTION_SCHEDULER_MISFIRE_TOLERATION);
     return Long.parseLong(limit);
   }
+
+  public Integer getExecutionSchedulerStartDelay() {
+    String delay = properties.getProperty(EXECUTION_SCHEDULER_START_DELAY,
+      DEFAULT_SCHEDULER_START_DELAY_SECONDS);
+    return Integer.parseInt(delay);
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/scheduler/AbstractLinearExecutionJob.java

@@ -97,7 +97,7 @@ public abstract class AbstractLinearExecutionJob implements ExecutionJob {
       return;
     }
 
-    int separationSeconds = jobDataMap.getIntValue((NEXT_EXECUTION_SEPARATION_SECONDS));
+    int separationSeconds = jobDataMap.getIntValue(NEXT_EXECUTION_SEPARATION_SECONDS);
 
     // Create trigger for next job execution
     Trigger trigger = newTrigger()

ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java

@@ -125,8 +125,10 @@ public class ExecutionScheduleManager {
   public void start() {
     LOG.info("Starting scheduler");
     try {
-      executionScheduler.startScheduler();
+      executionScheduler.startScheduler(configuration
+        .getExecutionSchedulerStartDelay());
       schedulerAvailable = true;
+
     } catch (AmbariException e) {
       LOG.warn("Unable to start scheduler. No recurring tasks will be " +
         "scheduled.");

ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduler.java

@@ -32,7 +32,7 @@ public interface ExecutionScheduler {
    * Initialize and start the scheduler to accept jobs.
    * @throws AmbariException
    */
-  public void startScheduler() throws AmbariException;
+  public void startScheduler(Integer delay) throws AmbariException;
 
   /**
    * Shutdown the scheduler threads and do not accept any more jobs.

ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionSchedulerImpl.java

@@ -148,7 +148,7 @@ public class ExecutionSchedulerImpl implements ExecutionScheduler {
   }
 
   @Override
-  public synchronized void startScheduler() throws AmbariException {
+  public synchronized void startScheduler(Integer delay) throws AmbariException {
     try {
       if (!isInitialized) {
         initializeScheduler();
@@ -161,7 +161,7 @@ public class ExecutionSchedulerImpl implements ExecutionScheduler {
       throw new AmbariException(msg);
     }
     try {
-      scheduler.start();
+      scheduler.startDelayed(delay != null ? delay : 0);
     } catch (SchedulerException e) {
       LOG.error("Failed to start scheduler", e);
       throw new AmbariException(e.getMessage());

ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java

@@ -92,7 +92,7 @@ public class ExecutionSchedulerTest {
     Scheduler scheduler = createNiceMock(Scheduler.class);
 
     expect(factory.getScheduler()).andReturn(scheduler);
-    expectPrivate(scheduler, "start").once();
+    expectPrivate(scheduler, "startDelayed", new Integer(180)).once();
     expectNew(StdSchedulerFactory.class).andReturn(factory);
     expectPrivate(scheduler, "shutdown").once();
 
@@ -100,7 +100,7 @@ public class ExecutionSchedulerTest {
 
     ExecutionSchedulerImpl executionScheduler = new ExecutionSchedulerImpl(configuration);
 
-    executionScheduler.startScheduler();
+    executionScheduler.startScheduler(180);
     executionScheduler.stopScheduler();
 
     PowerMock.verify(factory, StdSchedulerFactory.class, scheduler);