AMBARI-4527. Rolling Restart not working due to ntp changing server clock. (swagle)

ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java

@@ -137,8 +137,7 @@ public class ExecutionScheduleManager {
   public void start() {
     LOG.info("Starting scheduler");
     try {
-      executionScheduler.startScheduler(configuration
-        .getExecutionSchedulerStartDelay());
+      executionScheduler.startScheduler(configuration.getExecutionSchedulerStartDelay());
       schedulerAvailable = true;
 
     } catch (AmbariException e) {
@@ -215,6 +214,16 @@ public class ExecutionScheduleManager {
       throw new AmbariException("Scheduler unavailable.");
     }
 
+    // Check if scheduler is running, if not start immediately before scheduling jobs
+    try {
+      if (!executionScheduler.isSchedulerStarted()) {
+        executionScheduler.startScheduler(null);
+      }
+    } catch (SchedulerException e) {
+      LOG.error("Unable to determine scheduler state.", e);
+      throw new AmbariException("Scheduler unavailable.");
+    }
+
     // Create and persist jobs based on batches
     JobDetail firstJobDetail = persistBatch(requestExecution);
 
@@ -253,8 +262,6 @@ public class ExecutionScheduleManager {
           .endAt(endDate)
           .build();
 
-
-
       try {
         executionScheduler.scheduleJob(trigger);
         LOG.debug("Scheduled trigger next fire time: " + trigger.getNextFireTime());

ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduler.java

@@ -79,4 +79,10 @@ public interface ExecutionScheduler {
    */
   public List<? extends Trigger> getTriggersForJob(JobKey jobKey)
     throws SchedulerException;
+
+  /**
+   * Check whether the scheduler is already running.
+   * @return
+   */
+  public boolean isSchedulerStarted() throws SchedulerException;
 }

ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionSchedulerImpl.java

@@ -162,7 +162,20 @@ public class ExecutionSchedulerImpl implements ExecutionScheduler {
       throw new AmbariException(msg);
     }
     try {
-      scheduler.startDelayed(delay != null ? delay : 0);
+      if (!scheduler.isStarted()) {
+        // To avoid issue created due to change in server clock in between the
+        // scheduler initialization and scheduler start,
+        // start immediately if no delay provided.
+        if (delay != null) {
+          scheduler.startDelayed(delay);
+        } else {
+          scheduler.start();
+        }
+
+      } else {
+        LOG.info("Scheduler " + scheduler.getSchedulerInstanceId() +
+          " already started. Skipping start.");
+      }
     } catch (SchedulerException e) {
       LOG.error("Failed to start scheduler", e);
       throw new AmbariException(e.getMessage());
@@ -207,4 +220,9 @@ public class ExecutionSchedulerImpl implements ExecutionScheduler {
     return scheduler.getTriggersOfJob(jobKey);
   }
 
+  @Override
+  public boolean isSchedulerStarted() throws SchedulerException {
+    return scheduler.isStarted();
+  }
+
 }

ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java

@@ -39,7 +39,6 @@ import static org.powermock.api.easymock.PowerMock.expectNew;
 import static org.powermock.api.easymock.PowerMock.expectPrivate;
 
 @RunWith(PowerMockRunner.class)
-@PrepareForTest({ ExecutionSchedulerImpl.class })
 @PowerMockIgnore("javax.management.*")
 public class ExecutionSchedulerTest {
 
@@ -65,6 +64,7 @@ public class ExecutionSchedulerTest {
   }
 
   @Test
+  @PrepareForTest({ ExecutionSchedulerImpl.class })
   public void testSchedulerInitialize() throws Exception {
 
     ExecutionSchedulerImpl executionScheduler =
@@ -87,6 +87,7 @@ public class ExecutionSchedulerTest {
   }
 
   @Test
+  @PrepareForTest({ ExecutionSchedulerImpl.class })
   public void testSchedulerStartStop() throws Exception {
     StdSchedulerFactory factory = createNiceMock(StdSchedulerFactory.class);
     Scheduler scheduler = createNiceMock(Scheduler.class);
@@ -144,4 +145,28 @@ public class ExecutionSchedulerTest {
     Assert.assertEquals("org.quartz.impl.jdbcjobstore.oracle.OracleDelegate", subProps[0]);
     Assert.assertEquals("select 0 from dual", subProps[1]);
   }
+
+  @Test
+  @PrepareForTest({ ExecutionSchedulerImpl.class })
+  public void testSchedulerStartDelay() throws Exception {
+    StdSchedulerFactory factory = createNiceMock(StdSchedulerFactory.class);
+    Scheduler scheduler = createNiceMock(Scheduler.class);
+
+    expect(factory.getScheduler()).andReturn(scheduler).anyTimes();
+    expectNew(StdSchedulerFactory.class).andReturn(factory);
+    expect(scheduler.isStarted()).andReturn(false).anyTimes();
+    expectPrivate(scheduler, "startDelayed", new Integer(180)).once();
+    expectPrivate(scheduler, "start").once();
+
+    PowerMock.replay(factory, StdSchedulerFactory.class, scheduler);
+
+    ExecutionSchedulerImpl executionScheduler = new ExecutionSchedulerImpl(configuration);
+
+    executionScheduler.startScheduler(180);
+    executionScheduler.startScheduler(null);
+
+    PowerMock.verify(factory, StdSchedulerFactory.class, scheduler);
+
+    Assert.assertTrue(executionScheduler.isInitialized());
+  }
 }