ZOOKEEPER-1366 Zookeeper should be tolerant of clock adjustments (Hongchao Deng via michim)

git-svn-id: https://svn.apache.org/repos/asf/zookeeper/trunk@1657745 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -28,6 +28,9 @@ BUGFIXES:
   ZOOKEEPER-2111 Not isAlive states should be synchronized in ClientCnxn 
   (Hongchao via rakeshr)
 
+  ZOOKEEPER-1366 Zookeeper should be tolerant of clock adjustments (Hongchao
+  Deng via michim)
+
 IMPROVEMENTS:
   ZOOKEEPER-1660 Documentation for Dynamic Reconfiguration (Reed Wanderman-Milne via shralex)  
 

src/java/main/org/apache/zookeeper/ClientCnxn.java

@@ -64,6 +64,7 @@ import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.ZooKeeper.WatchRegistration;
 import org.apache.zookeeper.client.HostProvider;
 import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.proto.AuthPacket;
 import org.apache.zookeeper.proto.ConnectRequest;
 import org.apache.zookeeper.proto.Create2Response;
@@ -1103,7 +1104,7 @@ public class ClientCnxn {
             clientCnxnSocket.updateNow();
             clientCnxnSocket.updateLastSendAndHeard();
             int to;
-            long lastPingRwServer = System.currentTimeMillis();
+            long lastPingRwServer = Time.currentElapsedTime();
             final int MAX_SEND_PING_INTERVAL = 10000; //10 seconds
             while (state.isAlive()) {
                 try {
@@ -1178,7 +1179,7 @@ public class ClientCnxn {
 
                     // If we are in read-only mode, seek for read/write server
                     if (state == States.CONNECTEDREADONLY) {
-                        long now = System.currentTimeMillis();
+                        long now = Time.currentElapsedTime();
                         int idlePingRwServer = (int) (now - lastPingRwServer);
                         if (idlePingRwServer >= pingRwTimeout) {
                             lastPingRwServer = now;

src/java/main/org/apache/zookeeper/ClientCnxnSocket.java

@@ -27,6 +27,7 @@ import java.util.concurrent.LinkedBlockingDeque;
 
 import org.apache.jute.BinaryInputArchive;
 import org.apache.zookeeper.ClientCnxn.Packet;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.proto.ConnectResponse;
 import org.apache.zookeeper.server.ByteBufferInputStream;
 import org.slf4j.Logger;
@@ -77,7 +78,7 @@ abstract class ClientCnxnSocket {
     }
 
     void updateNow() {
-        now = System.currentTimeMillis();
+        now = Time.currentElapsedTime();
     }
 
     int getIdleRecv() {

src/java/main/org/apache/zookeeper/Login.java

@@ -34,6 +34,8 @@ import javax.security.auth.callback.CallbackHandler;
 
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.common.Time;
+
 import javax.security.auth.kerberos.KerberosTicket;
 import javax.security.auth.Subject;
 import java.util.Date;
@@ -71,7 +73,8 @@ public class Login {
     private String loginContextName = null;
     private String principal = null;
 
-    private long lastLogin = 0;
+    // Initialize 'lastLogin' to do a login at first time
+    private long lastLogin = Time.currentElapsedTime() - MIN_TIME_BEFORE_RELOGIN;
 
     /**
      * LoginThread constructor. The constructor starts the thread used
@@ -121,7 +124,7 @@ public class Login {
                 LOG.info("TGT refresh thread started.");
                 while (true) {  // renewal thread's main loop. if it exits from here, thread will exit.
                     KerberosTicket tgt = getTGT();
-                    long now = System.currentTimeMillis();
+                    long now = Time.currentWallTime();
                     long nextRefresh;
                     Date nextRefreshDate;
                     if (tgt == null) {
@@ -298,7 +301,7 @@ public class Login {
                 (TICKET_RENEW_WINDOW + (TICKET_RENEW_JITTER * rng.nextDouble())));
         if (proposedRefresh > expires) {
             // proposedRefresh is too far in the future: it's after ticket expires: simply return now.
-            return System.currentTimeMillis();
+            return Time.currentWallTime();
         }
         else {
             return proposedRefresh;
@@ -318,7 +321,7 @@ public class Login {
     }
 
     private boolean hasSufficientTimeElapsed() {
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         if (now - getLastLogin() < MIN_TIME_BEFORE_RELOGIN ) {
             LOG.warn("Not attempting to re-login since the last re-login was " +
                     "attempted less than " + (MIN_TIME_BEFORE_RELOGIN/1000) + " seconds"+

src/java/main/org/apache/zookeeper/Shell.java

@@ -39,10 +39,11 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.log4j.Logger;
+import org.apache.zookeeper.common.Time;
 
-/** 
+/**
  * A base class for running a Unix command.
- * 
+ *
  * <code>Shell</code> can be used to run unix commands like <code>du</code> or
  * <code>df</code>. It also offers facilities to gate commands by 
  * time-intervals.
@@ -146,7 +147,7 @@ abstract public class Shell {
 
   /** check to see if a command needs to be executed and execute if needed */
   protected void run() throws IOException {
-    if (lastTime + interval > System.currentTimeMillis())
+    if (lastTime + interval > Time.currentElapsedTime())
       return;
     exitCode = 0; // reset for next run
     runCommand();
@@ -245,7 +246,7 @@ abstract public class Shell {
         LOG.warn("Error while closing the error stream", ioe);
       }
       process.destroy();
-      lastTime = System.currentTimeMillis();
+      lastTime = Time.currentElapsedTime();
     }
   }
 

src/java/main/org/apache/zookeeper/ZKUtil.java

@@ -120,5 +120,4 @@ public class ZKUtil {
         }
         return tree;
     }
-    
 }

src/java/main/org/apache/zookeeper/common/Time.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.common;
+
+import java.util.Date;
+
+public class Time {
+    /**
+     * Returns time in milliseconds as does System.currentTimeMillis(),
+     * but uses elapsed time from an arbitrary epoch more like System.nanoTime().
+     * The difference is that if somebody changes the system clock,
+     * Time.currentElapsedTime will change but nanoTime won't. On the other hand,
+     * all of ZK assumes that time is measured in milliseconds.
+     * @return  The time in milliseconds from some arbitrary point in time.
+     */
+    public static long currentElapsedTime() {
+        return System.nanoTime() / 1000000;
+    }
+
+    /**
+     * Explicitly returns system dependent current wall time.
+     * @return Current time in msec.
+     */
+    public static long currentWallTime() {
+        return System.currentTimeMillis();
+    }
+
+    /**
+     * This is to convert the elapsedTime to a Date.
+     * @return A date object indicated by the elapsedTime.
+     */
+    public static Date elapsedTimeToDate(long elapsedTime) {
+        long wallTime = currentWallTime() + elapsedTime - currentElapsedTime();
+        return new Date(wallTime);
+    }
+}

src/java/main/org/apache/zookeeper/server/ConnectionBean.java

@@ -22,10 +22,10 @@ import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
-import java.util.Date;
 
 import javax.management.ObjectName;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.jmx.MBeanRegistry;
@@ -164,7 +164,7 @@ public class ConnectionBean implements ConnectionMXBean, ZKMBeanInfo {
     }
 
     public String getLastResponseTime() {
-        return new Date(stats.getLastResponseTime()).toString();
+        return Time.elapsedTimeToDate(stats.getLastResponseTime()).toString();
     }
 
     public long getLastLatency() {

src/java/main/org/apache/zookeeper/server/ExpiryQueue.java

@@ -27,6 +27,8 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.zookeeper.common.Time;
+
 /**
  * ExpiryQueue tracks elements in time sorted fixed duration buckets.
  * It's used by SessionTrackerImpl to expire sessions and NIOServerCnxnFactory
@@ -48,7 +50,7 @@ public class ExpiryQueue<E> {
 
     public ExpiryQueue(int expirationInterval) {
         this.expirationInterval = expirationInterval;
-        nextExpirationTime.set(roundToNextInterval(System.currentTimeMillis()));
+        nextExpirationTime.set(roundToNextInterval(Time.currentElapsedTime()));
     }
 
     private long roundToNextInterval(long time) {
@@ -84,7 +86,7 @@ public class ExpiryQueue<E> {
      */
     public Long update(E elem, int timeout) {
         Long prevExpiryTime = elemMap.get(elem);
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         Long newExpiryTime = roundToNextInterval(now + timeout);
 
         if (newExpiryTime.equals(prevExpiryTime)) {
@@ -123,7 +125,7 @@ public class ExpiryQueue<E> {
      * @return milliseconds until next expiration time, or 0 if has already past
      */
     public long getWaitTime() {
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         long expirationTime = nextExpirationTime.get();
         return now < expirationTime ? (expirationTime - now) : 0L;
     }
@@ -137,7 +139,7 @@ public class ExpiryQueue<E> {
      *         ready
      */
     public Set<E> poll() {
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         long expirationTime = nextExpirationTime.get();
         if (now < expirationTime) {
             return Collections.emptySet();
@@ -168,7 +170,7 @@ public class ExpiryQueue<E> {
             if (set != null) {
                 pwriter.print(set.size());
                 pwriter.print(" expire at ");
-                pwriter.print(new Date(time));
+                pwriter.print(Time.elapsedTimeToDate(time));
                 pwriter.println(":");
                 for (E elem : set) {
                     pwriter.print("\t");

src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java

@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Locale;
 
 import org.apache.jute.Record;
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.KeeperException;
@@ -182,7 +183,7 @@ public class FinalRequestProcessor implements RequestProcessor {
 
                 lastOp = "PING";
                 cnxn.updateStatsForResponse(request.cxid, request.zxid, lastOp,
-                        request.createTime, System.currentTimeMillis());
+                        request.createTime, Time.currentElapsedTime());
 
                 cnxn.sendResponse(new ReplyHeader(-2,
                         zks.getZKDatabase().getDataTreeLastProcessedZxid(), 0), null, "response");
@@ -193,7 +194,7 @@ public class FinalRequestProcessor implements RequestProcessor {
 
                 lastOp = "SESS";
                 cnxn.updateStatsForResponse(request.cxid, request.zxid, lastOp,
-                        request.createTime, System.currentTimeMillis());
+                        request.createTime, Time.currentElapsedTime());
 
                 zks.finishSessionInit(request.cnxn, true);
                 return;
@@ -459,7 +460,7 @@ public class FinalRequestProcessor implements RequestProcessor {
 
         zks.serverStats().updateLatency(request.createTime);
         cnxn.updateStatsForResponse(request.cxid, lastZxid, lastOp,
-                    request.createTime, System.currentTimeMillis());
+                    request.createTime, Time.currentElapsedTime());
 
         try {
             cnxn.sendResponse(hdr, rsp, "response");

src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java

@@ -37,6 +37,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import org.apache.jute.Record;
 import org.apache.jute.BinaryOutputArchive;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.CreateMode;
@@ -361,7 +362,8 @@ public class PrepRequestProcessor extends ZooKeeperCriticalThread implements
                                 Record record, boolean deserialize)
         throws KeeperException, IOException, RequestProcessorException
     {
-        request.setHdr(new TxnHeader(request.sessionId, request.cxid, zxid, zks.getTime(), type));
+        request.setHdr(new TxnHeader(request.sessionId, request.cxid, zxid,
+                Time.currentWallTime(), type));
 
         switch (type) {
             case OpCode.create: {
@@ -748,8 +750,8 @@ public class PrepRequestProcessor extends ZooKeeperCriticalThread implements
                     ByteBufferInputStream.byteBuffer2Record(request.request, multiRequest);
                 } catch(IOException e) {
                     request.setHdr(new TxnHeader(request.sessionId, request.cxid, zks.getNextZxid(),
-                                                 zks.getTime(), OpCode.multi));
-                   throw e;
+                            Time.currentWallTime(), OpCode.multi));
+                    throw e;
                 }
                 List<Txn> txns = new ArrayList<Txn>();
                 //Each op in a multi-op must have the same zxid!
@@ -807,7 +809,8 @@ public class PrepRequestProcessor extends ZooKeeperCriticalThread implements
                     txns.add(new Txn(type, bb.array()));
                 }
 
-                request.setHdr(new TxnHeader(request.sessionId, request.cxid, zxid, zks.getTime(), request.type));
+                request.setHdr(new TxnHeader(request.sessionId, request.cxid, zxid,
+                        Time.currentWallTime(), request.type));
                 request.setTxn(new MultiTxn(txns));
 
                 break;

src/java/main/org/apache/zookeeper/server/RateLogger.java

@@ -18,6 +18,7 @@
 
 package org.apache.zookeeper.server;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 
 public class RateLogger {
@@ -43,7 +44,7 @@ public class RateLogger {
     }
 
     public void rateLimitLog(String newMsg) {
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         if (newMsg.equals(msg)) {
             ++count;
             if (now - timestamp >= 100) {

src/java/main/org/apache/zookeeper/server/Request.java

@@ -24,6 +24,7 @@ import java.util.List;
 import org.apache.jute.Record;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.OpCode;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.data.Id;
 import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
 import org.apache.zookeeper.txn.TxnHeader;
@@ -75,7 +76,7 @@ public class Request {
 
     public final List<Id> authInfo;
 
-    public final long createTime = System.currentTimeMillis();
+    public final long createTime = Time.currentElapsedTime();
 
     private Object owner;
 

src/java/main/org/apache/zookeeper/server/ServerStats.java

@@ -20,6 +20,8 @@ package org.apache.zookeeper.server;
 
 
 
+import org.apache.zookeeper.common.Time;
+
 /**
  * Basic Server Statistics
  */
@@ -107,7 +109,7 @@ public class ServerStats {
     }
     // mutators
     synchronized void updateLatency(long requestCreateTime) {
-        long latency = System.currentTimeMillis() - requestCreateTime;
+        long latency = Time.currentElapsedTime() - requestCreateTime;
         totalLatency += latency;
         count++;
         if (latency < minLatency) {

src/java/main/org/apache/zookeeper/server/SessionTrackerImpl.java

@@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.SessionExpiredException;
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -81,7 +82,7 @@ public class SessionTrackerImpl extends ZooKeeperCriticalThread implements
      */
     public static long initializeNextSession(long id) {
         long nextSid;
-        nextSid = (System.currentTimeMillis() << 24) >>> 8;
+        nextSid = (Time.currentElapsedTime() << 24) >>> 8;
         nextSid =  nextSid | (id <<56);
         return nextSid;
     }

src/java/main/org/apache/zookeeper/server/WorkerService.java

@@ -26,6 +26,7 @@ import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -227,11 +228,11 @@ public class WorkerService {
 
     public void join(long shutdownTimeoutMS) {
         // Give the worker threads time to finish executing
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         long endTime = now + shutdownTimeoutMS;
         for(ExecutorService worker : workers) {
             boolean terminated = false;
-            while ((now = System.currentTimeMillis()) <= endTime) {
+            while ((now = Time.currentElapsedTime()) <= endTime) {
                 try {
                     terminated = worker.awaitTermination(
                         endTime - now, TimeUnit.MILLISECONDS);

src/java/main/org/apache/zookeeper/server/ZKDatabase.java

@@ -41,6 +41,7 @@ import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.WatcherType;
 import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.server.DataTree.ProcessTxnResult;
@@ -570,17 +571,17 @@ public class ZKDatabase {
         this.snapLog.close();
     }
 
-    public synchronized void initConfigInZKDatabase(QuorumVerifier qv) {   
-    	if (qv == null) return; // only happens during tests
+    public synchronized void initConfigInZKDatabase(QuorumVerifier qv) {
+        if (qv == null) return; // only happens during tests
         try {
-             if (this.dataTree.getNode(ZooDefs.CONFIG_NODE) == null) {
-            	 // should only happen during upgrade
-            	 LOG.warn("configuration znode missing (hould only happen during upgrade), creating the node");
-                 this.dataTree.addConfigNode();
-             }
-             this.dataTree.setData(ZooDefs.CONFIG_NODE, qv.toString().getBytes(), -1, qv.getVersion(), System.currentTimeMillis());           
+            if (this.dataTree.getNode(ZooDefs.CONFIG_NODE) == null) {
+                // should only happen during upgrade
+                LOG.warn("configuration znode missing (hould only happen during upgrade), creating the node");
+                this.dataTree.addConfigNode();
+            }
+            this.dataTree.setData(ZooDefs.CONFIG_NODE, qv.toString().getBytes(), -1, qv.getVersion(), Time.currentWallTime());
         } catch (NoNodeException e) {
-           System.out.println("configuration node missing - should not happen");         
+            System.out.println("configuration node missing - should not happen");
         }
     }
  

src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java

@@ -41,6 +41,7 @@ import javax.security.sasl.SaslException;
 import org.apache.jute.BinaryInputArchive;
 import org.apache.jute.BinaryOutputArchive;
 import org.apache.jute.Record;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.Environment;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.Code;
@@ -314,10 +315,6 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         hzxid.set(zxid);
     }
 
-    long getTime() {
-        return System.currentTimeMillis();
-    }
-
     private void close(long sessionId) {
         Request si = new Request(null, sessionId, 0, OpCode.closeSession, null, null);
         setLocalSessionFlag(si);

src/java/main/org/apache/zookeeper/server/quorum/AuthFastLeaderElection.java

@@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -417,7 +418,7 @@ public class AuthFastLeaderElection implements Election {
                 super("WorkerSender");
                 maxAttempts = attempts;
                 rand = new Random(java.lang.Thread.currentThread().getId()
-                        + System.currentTimeMillis());
+                        + Time.currentElapsedTime());
             }
 
             long genChallenge() {

src/java/main/org/apache/zookeeper/server/quorum/FastLeaderElection.java

@@ -28,6 +28,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.jmx.MBeanRegistry;
 import org.apache.zookeeper.server.ZooKeeperThread;
 import org.apache.zookeeper.server.quorum.QuorumCnxManager.Message;
@@ -871,7 +872,7 @@ public class FastLeaderElection implements Election {
             self.jmxLeaderElectionBean = null;
         }
         if (self.start_fle == 0) {
-           self.start_fle = System.currentTimeMillis();
+           self.start_fle = Time.currentElapsedTime();
         }
         try {
             HashMap<Long, Vote> recvset = new HashMap<Long, Vote>();

src/java/main/org/apache/zookeeper/server/quorum/Follower.java

@@ -24,6 +24,7 @@ import java.nio.ByteBuffer;
 
 import org.apache.jute.Record;
 import org.apache.zookeeper.ZooDefs.OpCode;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.server.Request;
 import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
 import org.apache.zookeeper.server.util.SerializeUtils;
@@ -61,7 +62,7 @@ public class Follower extends Learner{
      * @throws InterruptedException
      */
     void followLeader() throws InterruptedException {
-        self.end_fle = System.currentTimeMillis();
+        self.end_fle = Time.currentElapsedTime();
         LOG.info("FOLLOWING - LEADER ELECTION TOOK - " +
               (self.end_fle - self.start_fle));
         self.start_fle = 0;

src/java/main/org/apache/zookeeper/server/quorum/Leader.java

@@ -40,6 +40,7 @@ import java.util.concurrent.ConcurrentMap;
 
 import org.apache.jute.BinaryOutputArchive;
 import org.apache.zookeeper.ZooDefs.OpCode;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.server.FinalRequestProcessor;
 import org.apache.zookeeper.server.Request;
 import org.apache.zookeeper.server.RequestProcessor;
@@ -405,7 +406,7 @@ public class Leader {
      * @throws InterruptedException
      */
     void lead() throws IOException, InterruptedException {
-        self.end_fle = System.currentTimeMillis();
+        self.end_fle = Time.currentElapsedTime();
         LOG.info("LEADING - LEADER ELECTION TOOK - " +
               (self.end_fle - self.start_fle));
         self.start_fle = 0;
@@ -549,12 +550,12 @@ public class Leader {
 
             while (true) {
                 synchronized (this) {
-                    long start = System.currentTimeMillis();
+                    long start = Time.currentElapsedTime();
                     long cur = start;
                     long end = start + self.tickTime / 2;
                     while (cur < end) {
                         wait(end - cur);
-                        cur = System.currentTimeMillis();
+                        cur = Time.currentElapsedTime();
                     }
 
                     if (!tickSkip) {
@@ -1167,12 +1168,12 @@ public class Leader {
                 self.setAcceptedEpoch(epoch);
                 connectingFollowers.notifyAll();
             } else {
-                long start = System.currentTimeMillis();
+                long start = Time.currentElapsedTime();
                 long cur = start;
                 long end = start + self.getInitLimit()*self.getTickTime();
                 while(waitingForNewEpoch && cur < end) {
                     connectingFollowers.wait(end - cur);
-                    cur = System.currentTimeMillis();
+                    cur = Time.currentElapsedTime();
                 }
                 if (waitingForNewEpoch) {
                     throw new InterruptedException("Timeout while waiting for epoch from quorum");
@@ -1204,12 +1205,12 @@ public class Leader {
                 electionFinished = true;
                 electingFollowers.notifyAll();
             } else {
-                long start = System.currentTimeMillis();
+                long start = Time.currentElapsedTime();
                 long cur = start;
                 long end = start + self.getInitLimit()*self.getTickTime();
                 while(!electionFinished && cur < end) {
                     electingFollowers.wait(end - cur);
-                    cur = System.currentTimeMillis();
+                    cur = Time.currentElapsedTime();
                 }
                 if (!electionFinished) {
                     throw new InterruptedException("Timeout while waiting for epoch to be acked by quorum");
@@ -1310,12 +1311,12 @@ public class Leader {
                 quorumFormed = true;
                 newLeaderProposal.qvAcksetPairs.notifyAll();
             } else {
-                long start = System.currentTimeMillis();
+                long start = Time.currentElapsedTime();
                 long cur = start;
                 long end = start + self.getInitLimit() * self.getTickTime();
                 while (!quorumFormed && cur < end) {
                     newLeaderProposal.qvAcksetPairs.wait(end - cur);
-                    cur = System.currentTimeMillis();
+                    cur = Time.currentElapsedTime();
                 }
                 if (!quorumFormed) {
                     throw new InterruptedException(

src/java/main/org/apache/zookeeper/server/quorum/LearnerSnapshotThrottler.java

@@ -18,8 +18,7 @@
 
 package org.apache.zookeeper.server.quorum;
 
-import java.util.concurrent.atomic.AtomicInteger;
-
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -97,11 +96,11 @@ public class LearnerSnapshotThrottler {
             if (!essential
                 && timeoutMillis > 0
                 && snapsInProgress >= maxConcurrentSnapshots) {
-                long timestamp = System.currentTimeMillis();
+                long timestamp = Time.currentElapsedTime();
                 do {
                     snapCountSyncObject.wait(timeoutMillis);
                 } while (snapsInProgress >= maxConcurrentSnapshots
-                         && timestamp + timeoutMillis < System.currentTimeMillis());
+                         && timestamp + timeoutMillis < Time.currentElapsedTime());
             }
 
             if (essential || snapsInProgress < maxConcurrentSnapshots) {

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java

@@ -45,6 +45,7 @@ import org.apache.zookeeper.common.AtomicFileWritingIdiom;
 import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement;
 import org.apache.zookeeper.common.HostNameUtils;
 import org.apache.zookeeper.common.PathUtils;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.jmx.MBeanRegistry;
 import org.apache.zookeeper.jmx.ZKMBeanInfo;
 import org.apache.zookeeper.server.ServerCnxnFactory;
@@ -985,7 +986,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
                     }
                     break;
                 }
-                start_fle = System.currentTimeMillis();
+                start_fle = Time.currentElapsedTime();
             }
         } finally {
             LOG.warn("QuorumPeer main thread exited");

src/java/systest/org/apache/zookeeper/test/system/GenerateLoad.java

@@ -53,6 +53,8 @@ import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.common.Time;
+
 
 public class GenerateLoad {
     protected static final Logger LOG = LoggerFactory.getLogger(GenerateLoad.class);
@@ -194,7 +196,7 @@ public class GenerateLoad {
 
         public void run() {
             try {
-                currentInterval = System.currentTimeMillis() / INTERVAL;
+                currentInterval = Time.currentElapsedTime() / INTERVAL;
                 // Give things time to report;
                 Thread.sleep(INTERVAL * 2);
                 long min = 99999;
@@ -202,7 +204,7 @@ public class GenerateLoad {
                 long total = 0;
                 int number = 0;
                 while (true) {
-                    long now = System.currentTimeMillis();
+                    long now = Time.currentElapsedTime();
                     long lastInterval = currentInterval;
                     currentInterval += 1;
                     long count = remove(lastInterval);
@@ -249,13 +251,13 @@ public class GenerateLoad {
     }
 
     synchronized static void sendChange(int percentage) {
-        long now = System.currentTimeMillis();
+        long now = Time.currentElapsedTime();
         long start = now;
         ReporterThread.percentage = percentage;
         for (SlaveThread st : slaves.toArray(new SlaveThread[0])) {
             st.send(percentage);
         }
-        now = System.currentTimeMillis();
+        now = Time.currentElapsedTime();
         long delay = now - start;
         if (delay > 1000) {
             System.out.println("Delay of " + delay + " to send new percentage");
@@ -387,7 +389,7 @@ public class GenerateLoad {
                         errors++;
                     } else {
                         finished++;
-                        rlatency += System.currentTimeMillis() - (Long) ctx;
+                        rlatency += Time.currentElapsedTime() - (Long) ctx;
                         reads++;
                     }
                 }
@@ -401,7 +403,7 @@ public class GenerateLoad {
                         errors++;
                     } else {
                         finished++;
-                        wlatency += System.currentTimeMillis() - (Long) ctx;
+                        wlatency += Time.currentElapsedTime() - (Long) ctx;
                         writes++;
                     }
                 }
@@ -427,7 +429,7 @@ public class GenerateLoad {
                         if (percentage == -1 || (finished == 0 && errors == 0)) {
                             continue;
                         }
-                        String report = System.currentTimeMillis() + " "
+                        String report = Time.currentElapsedTime() + " "
                                 + percentage + " " + finished + " " + errors + " "
                                 + outstanding + "\n";
                        /* String subreport = reads + " "
@@ -543,9 +545,9 @@ public class GenerateLoad {
 
         synchronized public boolean waitConnected(long timeout)
                 throws InterruptedException {
-            long endTime = System.currentTimeMillis() + timeout;
-            while (!connected && System.currentTimeMillis() < endTime) {
-                wait(endTime - System.currentTimeMillis());
+            long endTime = Time.currentElapsedTime() + timeout;
+            while (!connected && Time.currentElapsedTime() < endTime) {
+                wait(endTime - Time.currentElapsedTime());
             }
             return connected;
         }

src/java/systest/org/apache/zookeeper/test/system/InstanceManager.java

@@ -38,6 +38,7 @@ import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.common.Time;
 
 /**
  * This class doles out assignments to InstanceContainers that are registered to
@@ -294,9 +295,9 @@ public class InstanceManager implements AsyncCallback.ChildrenCallback, Watcher
     public String getStatus(String name, long timeout) throws KeeperException, InterruptedException {
         Stat stat = new Stat();
         byte data[] = null;
-        long endTime = System.currentTimeMillis() + timeout;
+        long endTime = Time.currentElapsedTime() + timeout;
         KeeperException lastException = null;
-        for(int i = 0; i < maxTries && endTime > System.currentTimeMillis(); i++) {
+        for(int i = 0; i < maxTries && endTime > Time.currentElapsedTime(); i++) {
             try {
                 data = zk.getData(reportsNode + '/' + name, false, stat);
                 if (LOG.isDebugEnabled()) {
@@ -317,7 +318,7 @@ public class InstanceManager implements AsyncCallback.ChildrenCallback, Watcher
                             }
                         }});
                     if (eStat == null) {
-                        eventObj.wait(endTime - System.currentTimeMillis());
+                        eventObj.wait(endTime - Time.currentElapsedTime());
                     }
                 }
                 lastException = e;

src/java/systest/org/apache/zookeeper/test/system/SimpleSysTest.java

@@ -31,6 +31,7 @@ import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.data.Stat;
 import org.junit.Test;
+import org.apache.zookeeper.common.Time;
 
 /**
  * This does a basic system test. It starts up an ensemble of servers and a set of clients.
@@ -46,8 +47,8 @@ public class SimpleSysTest extends BaseSysTest implements Watcher {
     
     synchronized private boolean waitForConnect(ZooKeeper zk, long timeout) throws InterruptedException {
         connected = (zk.getState() == States.CONNECTED);
-        long end = System.currentTimeMillis() + timeout;
-        while(!connected && end > System.currentTimeMillis()) {
+        long end = Time.currentElapsedTime() + timeout;
+        while(!connected && end > Time.currentElapsedTime()) {
             wait(timeout);
             connected = (zk.getState() == States.CONNECTED);
         }

src/java/test/org/apache/zookeeper/common/TimeTest.java

@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.common;
+
+import junit.framework.Assert;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.test.ClientBase;
+import org.junit.Test;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Command line program for demonstrating robustness to clock
+ * changes.
+ * <p/>
+ * How to run:
+ * ant clean compile-test
+ * echo build/test/lib/*.jar build/lib/*.jar build/classes build/test/classes | sed -e 's/ /:/g' > cp
+ * java -cp $(cat cp) org.apache.zookeeper.common.TimeTest | tee log-without-patch
+ * <p/>
+ * After test program starts, in another window, do commands:
+ * date -s '+1hour'
+ * date -s '-1hour'
+ * <p/>
+ * As long as there isn't any expired event, the experiment is successful.
+ */
+public class TimeTest extends ClientBase {
+    private static final long mt0 = System.currentTimeMillis();
+    private static final long nt0 = Time.currentElapsedTime();
+
+    private static AtomicInteger watchCount = new AtomicInteger(0);
+
+
+    public static void main(String[] args) throws Exception {
+        System.out.printf("Starting\n");
+        final TimeTest test = new TimeTest();
+        System.out.printf("After construct\n");
+        test.setUp();
+        ZooKeeper zk = test.createClient();
+        zk.create("/ephemeral", new byte[]{1, 2, 3},
+                ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+        while (Time.currentElapsedTime() - nt0 < 100000) {
+            System.out.printf("%d\t%s\n", discrepancy(),
+                    zk.exists("/ephemeral",
+                            watchCount.get() == 0 ? createWatcher() : null) != null);
+            waitByYielding(500);
+        }
+    }
+
+    private static Watcher createWatcher() {
+        watchCount.incrementAndGet();
+        return new Watcher() {
+            @Override
+            public void process(WatchedEvent event) {
+                watchCount.decrementAndGet();
+                System.out.printf("%d event = %s\n", discrepancy(), event);
+            }
+        };
+
+    }
+
+    private static void waitByYielding(long delay) {
+        long t0 = Time.currentElapsedTime();
+        while (Time.currentElapsedTime() < t0 + delay) {
+            Thread.yield();
+        }
+    }
+
+    private static long discrepancy() {
+        return (System.currentTimeMillis() - mt0) - (Time.currentElapsedTime() - nt0);
+    }
+
+    @Test
+    public void testElapsedTimeToDate() throws Exception {
+        long walltime = Time.currentWallTime();
+        long elapsedTime = Time.currentElapsedTime();
+        Thread.sleep(200);
+
+        Calendar cal = Calendar.getInstance();
+        cal.setTime(Time.elapsedTimeToDate(elapsedTime));
+        int calculatedDate = cal.get(Calendar.HOUR_OF_DAY);
+        cal.setTime(new Date(walltime));
+        int realDate = cal.get(Calendar.HOUR_OF_DAY);
+
+        Assert.assertEquals(calculatedDate, realDate);
+    }
+}

src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java

@@ -41,6 +41,7 @@ import org.apache.zookeeper.ZooDefs.OpCode;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper.States;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.server.quorum.Leader.Proposal;
 import org.apache.zookeeper.test.ClientBase;
 import org.junit.Assert;
@@ -671,9 +672,9 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase {
         q1.start();
         // Let the notifications timeout
         Thread.sleep(30000);
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         q1.shutdown();
-        long end = System.currentTimeMillis();
+        long end = Time.currentElapsedTime();
         if ((end - start) > maxwait) {
             Assert.fail("QuorumPeer took " + (end - start) +
                     " to shutdown, expected " + maxwait);

src/java/test/org/apache/zookeeper/test/ClientBase.java

@@ -42,6 +42,7 @@ import javax.management.ObjectName;
 
 import junit.framework.TestCase;
 
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.PortAssignment;
 import org.apache.zookeeper.TestableZooKeeper;
@@ -121,11 +122,11 @@ public abstract class ClientBase extends ZKTestCase {
         synchronized public void waitForConnected(long timeout)
             throws InterruptedException, TimeoutException
         {
-            long expire = System.currentTimeMillis() + timeout;
+            long expire = Time.currentElapsedTime() + timeout;
             long left = timeout;
             while(!connected && left > 0) {
                 wait(left);
-                left = expire - System.currentTimeMillis();
+                left = expire - Time.currentElapsedTime();
             }
             if (!connected) {
                 throw new TimeoutException("Did not connect");
@@ -135,11 +136,11 @@ public abstract class ClientBase extends ZKTestCase {
         synchronized public void waitForDisconnected(long timeout)
             throws InterruptedException, TimeoutException
         {
-            long expire = System.currentTimeMillis() + timeout;
+            long expire = Time.currentElapsedTime() + timeout;
             long left = timeout;
             while(connected && left > 0) {
                 wait(left);
-                left = expire - System.currentTimeMillis();
+                left = expire - Time.currentElapsedTime();
             }
             if (connected) {
                 throw new TimeoutException("Did not disconnect");
@@ -228,7 +229,7 @@ public abstract class ClientBase extends ZKTestCase {
     }
 
     public static boolean waitForServerUp(String hp, long timeout) {
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         while (true) {
             try {
                 // if there are multiple hostports, just take the first one
@@ -243,7 +244,7 @@ public abstract class ClientBase extends ZKTestCase {
                 LOG.info("server " + hp + " not up " + e);
             }
 
-            if (System.currentTimeMillis() > start + timeout) {
+            if (Time.currentElapsedTime() > start + timeout) {
                 break;
             }
             try {
@@ -255,7 +256,7 @@ public abstract class ClientBase extends ZKTestCase {
         return false;
     }
     public static boolean waitForServerDown(String hp, long timeout) {
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         while (true) {
             try {
                 HostPort hpobj = parseHostPortList(hp).get(0);
@@ -264,7 +265,7 @@ public abstract class ClientBase extends ZKTestCase {
                 return true;
             }
 
-            if (System.currentTimeMillis() > start + timeout) {
+            if (Time.currentElapsedTime() > start + timeout) {
                 break;
             }
             try {
@@ -278,7 +279,7 @@ public abstract class ClientBase extends ZKTestCase {
 
     public static boolean waitForServerState(QuorumPeer qp, int timeout,
             String serverState) {
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         while (true) {
             try {
                 Thread.sleep(250);
@@ -287,7 +288,7 @@ public abstract class ClientBase extends ZKTestCase {
             }
             if (qp.getServerState().equals(serverState))
                 return true;
-            if (System.currentTimeMillis() > start + timeout) {
+            if (Time.currentElapsedTime() > start + timeout) {
                 return false;
             }
         }

src/java/test/org/apache/zookeeper/test/ClientHammerTest.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Date;
 import java.util.List;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.CreateMode;
@@ -124,7 +125,7 @@ public class ClientHammerTest extends ClientBase {
     {
         try {
             HammerThread[] threads = new HammerThread[threadCount];
-            long start = System.currentTimeMillis();
+            long start = Time.currentElapsedTime();
             for (int i = 0; i < threads.length; i++) {
                 ZooKeeper zk = createClient();
                 String prefix = "/test-" + i;
@@ -157,7 +158,7 @@ public class ClientHammerTest extends ClientBase {
             final int childCount = 10;
 
             HammerThread[] threads = new HammerThread[threadCount];
-            long start = System.currentTimeMillis();
+            long start = Time.currentElapsedTime();
             for (int i = 0; i < threads.length; i++) {
                 String prefix = "/test-" + i;
                 {
@@ -218,7 +219,7 @@ public class ClientHammerTest extends ClientBase {
                     * HAMMERTHREAD_LATENCY * (long)safetyFactor);
         }
         LOG.info(new Date() + " Total time "
-                + (System.currentTimeMillis() - start));
+                + (Time.currentElapsedTime() - start));
 
         ZooKeeper zk = createClient();
         try {

src/java/test/org/apache/zookeeper/test/CnxManagerTest.java

@@ -30,6 +30,7 @@ import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.net.Socket;
 
+import org.apache.zookeeper.common.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.PortAssignment;
@@ -203,9 +204,9 @@ public class CnxManagerTest extends ZKTestCase {
             LOG.error("Null listener when initializing cnx manager");
         }
 
-        long begin = System.currentTimeMillis();
+        long begin = Time.currentElapsedTime();
         cnxManager.toSend(2L, createMsg(ServerState.LOOKING.ordinal(), 1, -1, 1));
-        long end = System.currentTimeMillis();
+        long end = Time.currentElapsedTime();
 
         if((end - begin) > 6000) Assert.fail("Waited more than necessary");
         cnxManager.halt();
@@ -348,10 +349,10 @@ public class CnxManagerTest extends ZKTestCase {
 
         Socket sock = new Socket();
         sock.connect(peers.get(1L).electionAddr, 5000);
-        long begin = System.currentTimeMillis();
+        long begin = Time.currentElapsedTime();
         // Read without sending data. Verify timeout.
         cnxManager.receiveConnection(sock);
-        long end = System.currentTimeMillis();
+        long end = Time.currentElapsedTime();
         if((end - begin) > ((peer.getSyncLimit() * peer.getTickTime()) + 500)) Assert.fail("Waited more than necessary");
         cnxManager.halt();
         Assert.assertFalse(cnxManager.listener.isAlive());

src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java

@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.zookeeper.common.Time;
 import org.apache.jute.BinaryInputArchive;
 import org.apache.jute.BinaryOutputArchive;
 import org.apache.jute.Record;
@@ -236,7 +237,7 @@ public class LoadFromLogTest extends ZKTestCase implements  Watcher {
         dt.createNode("/test", new byte[0], null, 0, -1, 1, 1);
         for (count = 1; count <= 3; count++) {
             dt.createNode("/test/" + count, new byte[0], null, 0, -1, count,
-                    System.currentTimeMillis());
+                    Time.currentElapsedTime());
         }
         DataNode zk = dt.getNode("/test");
 
@@ -285,15 +286,15 @@ public class LoadFromLogTest extends ZKTestCase implements  Watcher {
         if (type == OpCode.delete) {
             txn = new DeleteTxn(path);
             txnHeader = new TxnHeader(0xabcd, 0x123, prevPzxid + 1,
-                System.currentTimeMillis(), OpCode.delete);
+                Time.currentElapsedTime(), OpCode.delete);
         } else if (type == OpCode.create) {
             txnHeader = new TxnHeader(0xabcd, 0x123, prevPzxid + 1,
-                    System.currentTimeMillis(), OpCode.create);
+                    Time.currentElapsedTime(), OpCode.create);
             txn = new CreateTxn(path, new byte[0], null, false, cversion);
         }
         else if (type == OpCode.multi) {
             txnHeader = new TxnHeader(0xabcd, 0x123, prevPzxid + 1,
-                    System.currentTimeMillis(), OpCode.create);
+                    Time.currentElapsedTime(), OpCode.create);
             txn = new CreateTxn(path, new byte[0], null, false, cversion);
             ArrayList txnList = new ArrayList();
             ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -304,7 +305,7 @@ public class LoadFromLogTest extends ZKTestCase implements  Watcher {
             txnList.add(txact);
             txn = new MultiTxn(txnList);
             txnHeader = new TxnHeader(0xabcd, 0x123, prevPzxid + 1,
-                    System.currentTimeMillis(), OpCode.multi);
+                    Time.currentElapsedTime(), OpCode.multi);
         }
         logFile.processTransaction(txnHeader, dt, null, txn);
 
@@ -331,7 +332,7 @@ public class LoadFromLogTest extends ZKTestCase implements  Watcher {
         File tmpDir = ClientBase.createTmpDir();
         FileTxnLog txnLog = new FileTxnLog(tmpDir);
         TxnHeader txnHeader = new TxnHeader(0xabcd, 0x123, 0x123,
-              System.currentTimeMillis(), OpCode.create);
+              Time.currentElapsedTime(), OpCode.create);
         Record txn = new CreateTxn("/Test", new byte[0], null, false, 1);
         txnLog.append(txnHeader, txn);
         FileInputStream in = new FileInputStream(tmpDir.getPath() + "/log." +
@@ -568,9 +569,9 @@ public class LoadFromLogTest extends ZKTestCase implements  Watcher {
     private ZooKeeper getConnectedZkClient() throws IOException {
         ZooKeeper zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this);
 
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         while (!connected) {
-            long end = System.currentTimeMillis();
+            long end = Time.currentElapsedTime();
             if (end - start > 5000) {
                 Assert.assertTrue("Could not connect with server in 5 seconds",
                         false);

src/java/test/org/apache/zookeeper/test/ReadOnlyModeTest.java

@@ -42,6 +42,7 @@ import org.apache.zookeeper.ZKTestCase;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.ZooKeeper.States;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.test.ClientBase.CountdownWatcher;
 import org.junit.After;
 import org.junit.Before;
@@ -197,13 +198,12 @@ public class ReadOnlyModeTest extends ZKTestCase {
                         states.add(event.getState());
                     }
                 }, true);
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         while (!(zk.getState() == States.CONNECTEDREADONLY)) {
             Thread.sleep(200);
             // FIXME this was originally 5 seconds, but realistically, on random/slow/virt hosts, there is no way to guarantee this
-            Assert.assertTrue("Can't connect to the server", System
-                    .currentTimeMillis()
-                    - start < 30000);
+            Assert.assertTrue("Can't connect to the server",
+                              Time.currentElapsedTime() - start < 30000);
         }
 
         // At this point states list should contain, in the given order,

src/java/test/org/apache/zookeeper/test/StaticHostProviderTest.java

@@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue;
 import org.apache.zookeeper.ZKTestCase;
 import org.apache.zookeeper.client.HostProvider;
 import org.apache.zookeeper.client.StaticHostProvider;
+import org.apache.zookeeper.common.Time;
 import org.junit.Test;
 
 import java.net.InetAddress;
@@ -55,9 +56,9 @@ public class StaticHostProviderTest extends ZKTestCase {
             hostProvider.next(0);
             --size;
         }
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         hostProvider.next(1000);
-        long stop = System.currentTimeMillis();
+        long stop = Time.currentElapsedTime();
         assertTrue(900 <= stop - start);
     }
 
@@ -69,9 +70,9 @@ public class StaticHostProviderTest extends ZKTestCase {
             hostProvider.next(0);
             --size;
         }
-        long start = System.currentTimeMillis();
+        long start = Time.currentElapsedTime();
         hostProvider.next(0);
-        long stop = System.currentTimeMillis();
+        long stop = Time.currentElapsedTime();
         assertTrue(5 > stop - start);
     }
 

src/java/test/org/apache/zookeeper/test/TestHammer.java

@@ -24,6 +24,7 @@ import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.AsyncCallback.VoidCallback;
 import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.common.Time;
 
 public class TestHammer implements VoidCallback {
 
@@ -32,7 +33,7 @@ public class TestHammer implements VoidCallback {
      */
     static int REPS = 50000;
     public static void main(String[] args) {
-            long startTime = System.currentTimeMillis();
+            long startTime = Time.currentElapsedTime();
             ZooKeeper zk = null;
             try {
                 zk = new ZooKeeper(args[0], 10000, null);
@@ -51,7 +52,7 @@ public class TestHammer implements VoidCallback {
                     e.printStackTrace();
                 }
             }
-            System.out.println("creates/sec=" + (REPS*1000/(System.currentTimeMillis()-startTime)));
+            System.out.println("creates/sec=" + (REPS*1000/(Time.currentElapsedTime()-startTime)));
     }
 
     public void processResult(int rc, String path, Object ctx) {

src/java/test/org/apache/zookeeper/test/ZooKeeperTestClient.java

@@ -33,6 +33,7 @@ import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.KeeperException.Code;
 import org.apache.zookeeper.Watcher.Event.EventType;
 import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.server.ServerCnxnFactory;
 import org.apache.zookeeper.server.ZooKeeperServer;
@@ -43,7 +44,7 @@ public class ZooKeeperTestClient extends ZKTestCase implements Watcher {
 
   protected static final String dirOnZK = "/test_dir";
 
-  protected String testDirOnZK = dirOnZK + "/" + System.currentTimeMillis();
+  protected String testDirOnZK = dirOnZK + "/" + Time.currentElapsedTime();
 
   LinkedBlockingQueue<WatchedEvent> events = new LinkedBlockingQueue<WatchedEvent>();