ZOOKEEPER-1994. Auto-backup configuration files; config version becomes part of filename (Hongchao Deng via shralex)

git-svn-id: https://svn.apache.org/repos/asf/zookeeper/trunk@1617886 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -10,6 +10,8 @@ IMPROVEMENTS:
 Release 3.5.0 - 8/4/2014
 
 NEW FEATURES:
+  ZOOKEEPER-1994. Auto-backup configuration files; config version becomes part of filename (Hongchao Deng via shralex)
+
   ZOOKEEPER-1355. Add zk.updateServerList(newServerList) (Alex Shraer, Marshall McMullen via fpj)
 
   ZOOKEEPER-1572. Add an async (Java) interface for multi request (Sijie Guo via camille)

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java

@@ -18,16 +18,12 @@
 package org.apache.zookeeper.server.quorum;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
-import java.io.OutputStreamWriter;
-import java.io.StringReader;
-import java.io.StringWriter;
 import java.io.Writer;
 import java.net.DatagramPacket;
 import java.net.DatagramSocket;
@@ -45,15 +41,12 @@ import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.common.AtomicFileWritingIdiom;
 import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement;
 import org.apache.zookeeper.common.HostNameUtils;
 import org.apache.zookeeper.common.PathUtils;
 import org.apache.zookeeper.jmx.MBeanRegistry;
 import org.apache.zookeeper.jmx.ZKMBeanInfo;
-import org.apache.zookeeper.server.DataNode;
 import org.apache.zookeeper.server.ServerCnxnFactory;
 import org.apache.zookeeper.server.ZKDatabase;
 import org.apache.zookeeper.server.ZooKeeperServer;
@@ -317,32 +310,11 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
         learnerType = p;
     }
 
-       
-    protected synchronized void setDynamicConfigFilename(String s) {
-        dynamicConfigFilename = PathUtils.normalizeFileSystemPath(s);
-    }
-
-    protected synchronized String getDynamicConfigFilename() {
-        return dynamicConfigFilename;
-    }
-
     protected synchronized void setConfigFileName(String s) {
         configFilename = s;
     }
 
-    protected synchronized void setConfigBackwardCompatibility(boolean bc) {
-        configBackwardCompatibility = bc;
-    }
-    
-    protected synchronized boolean getConfigBackwardCompatibility() {
-        return configBackwardCompatibility;
-    }
-    
-    private String dynamicConfigFilename = null;
-    
     private String configFilename = null;
-    
-    private boolean configBackwardCompatibility = false;
 
     public int getQuorumSize(){
         return getVotingView().size();
@@ -606,9 +578,9 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
             File dataLogDir, int electionType,
             long myid, int tickTime, int initLimit, int syncLimit,
             ServerCnxnFactory cnxnFactory) throws IOException {
-        this(quorumPeers, dataDir, dataLogDir, electionType, myid, tickTime, 
-                initLimit, syncLimit, false, cnxnFactory, 
-                new QuorumMaj(quorumPeers), null);
+        this(quorumPeers, dataDir, dataLogDir, electionType, myid, tickTime,
+                initLimit, syncLimit, false, cnxnFactory,
+                new QuorumMaj(quorumPeers));
     }
 
     public QuorumPeer(Map<Long, QuorumServer> quorumPeers, File dataDir,
@@ -616,7 +588,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
             long myid, int tickTime, int initLimit, int syncLimit,
             boolean quorumListenOnAllIPs,
             ServerCnxnFactory cnxnFactory,
-            QuorumVerifier quorumConfig, String memFilename) throws IOException {
+            QuorumVerifier quorumConfig) throws IOException {
         this();
         this.cnxnFactory = cnxnFactory;
         this.electionType = electionType;
@@ -627,7 +599,6 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
         this.quorumListenOnAllIPs = quorumListenOnAllIPs;
         this.logFactory = new FileTxnSnapLog(dataLogDir, dataDir);
         this.zkDb = new ZKDatabase(this.logFactory);
-        this.dynamicConfigFilename = (memFilename != null) ? memFilename : "zoo_replicated" + myid + ".dynamic";
         if(quorumConfig == null) quorumConfig = new QuorumMaj(quorumPeers);
         setQuorumVerifier(quorumConfig, false);
         adminServer = AdminServerFactory.createAdminServer();
@@ -757,7 +728,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
         this(quorumPeers, snapDir, logDir, electionAlg,
                 myid,tickTime, initLimit,syncLimit, false,
                 ServerCnxnFactory.createFactory(new InetSocketAddress(clientPort), -1),
-                new QuorumMaj(quorumPeers), null);
+                new QuorumMaj(quorumPeers));
     }
 
     /**
@@ -773,7 +744,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
         this(quorumPeers, snapDir, logDir, electionAlg,
                 myid,tickTime, initLimit,syncLimit, false,
                 ServerCnxnFactory.createFactory(new InetSocketAddress(clientPort), -1),
-                quorumConfig, null);
+                quorumConfig);
     }
 
     /**
@@ -1289,7 +1260,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
            }
         }
     }
-    
+
     public synchronized void restartLeaderElection(QuorumVerifier qvOLD, QuorumVerifier qvNEW){
         if (qvOLD == null || !qvOLD.equals(qvNEW)) {
             LOG.warn("Restarting Leader Election");
@@ -1298,6 +1269,10 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
             startLeaderElection();
         }           
     }
+
+    public String getNextDynamicConfigFilename() {
+        return configFilename + QuorumPeerConfig.nextDynamicConfigFileSuffix;
+    }
     
     public synchronized void setLastSeenQuorumVerifier(QuorumVerifier qv, boolean writeToDisk){
         if (lastSeenQuorumVerifier!=null && lastSeenQuorumVerifier.getVersion() > qv.getVersion()) {
@@ -1315,9 +1290,10 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
         connectNewPeers();
         if (writeToDisk) {
             try {
-                QuorumPeerConfig.writeDynamicConfig(dynamicConfigFilename + ".next", null, false, qv, false);
+                QuorumPeerConfig.writeDynamicConfig(
+                        getNextDynamicConfigFilename(), qv, true);
            } catch(IOException e){
-                LOG.error("Error closing file: ", e.getMessage());
+                LOG.error("Error writing next dynamic config file to disk: ", e.getMessage());
             }
         } 
 
@@ -1332,50 +1308,48 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
            return quorumVerifier;  
         }
         QuorumVerifier prevQV = quorumVerifier;
-       quorumVerifier = qv;
-       if (lastSeenQuorumVerifier == null || (qv.getVersion() > lastSeenQuorumVerifier.getVersion()))
-           lastSeenQuorumVerifier = qv;
+        quorumVerifier = qv;
+        if (lastSeenQuorumVerifier == null || (qv.getVersion() > lastSeenQuorumVerifier.getVersion()))
+            lastSeenQuorumVerifier = qv;
+
         if (writeToDisk) {
-                // we need to write the dynamic config file. Either it already exists
-                // or we have the old-style config file and we're in the backward compatibility mode,
-                // so we'll create the dynamic config file for the first time now
-                if (dynamicConfigFilename !=null || (configFilename !=null && configBackwardCompatibility)) { 
+            // some tests initialize QuorumPeer without a static config file
+            if (configFilename != null) {
                 try {
-                    if (configBackwardCompatibility) {
-                        setDynamicConfigFilename(configFilename + ".dynamic");
-                    }
-                    QuorumPeerConfig.writeDynamicConfig(dynamicConfigFilename, configFilename,
-                            configBackwardCompatibility, qv,
-                            needEraseClientInfoFromStaticConfig(prevQV, qv));
-                    configBackwardCompatibility = false;
-                } catch(IOException e){
-                    LOG.error("Error closing file: ", e.getMessage());     
+                    String dynamicConfigFilename = makeDynamicConfigFilename(
+                            qv.getVersion());
+                    QuorumPeerConfig.writeDynamicConfig(
+                            dynamicConfigFilename, qv, false);
+                    QuorumPeerConfig.editStaticConfig(configFilename,
+                            dynamicConfigFilename,
+                            needEraseClientInfoFromStaticConfig());
+                } catch (IOException e) {
+                    LOG.error("Error closing file: ", e.getMessage());
                 }
             } else {
-                LOG.error("writeToDisk == true but dynamicConfigFilename == null, configFilename "
-                          + (configFilename == null ? "== null": "!=null")
-                          + " and configBackwardCompatibility == " + configBackwardCompatibility);
+                LOG.error("writeToDisk == true but configFilename == null");
             }
         }
 
         if (qv.getVersion() == lastSeenQuorumVerifier.getVersion()){
-           QuorumPeerConfig.deleteFile(dynamicConfigFilename + ".next");
-       }
-       QuorumServer qs = qv.getAllMembers().get(getId());
-       if (qs!=null){
-           setQuorumAddress(qs.addr);
-           setElectionAddress(qs.electionAddr);
-           setClientAddress(qs.clientAddr);
-       }
-       return prevQV;
+           QuorumPeerConfig.deleteFile( getNextDynamicConfigFilename() );
+        }
+        QuorumServer qs = qv.getAllMembers().get(getId());
+        if (qs!=null){
+            setQuorumAddress(qs.addr);
+            setElectionAddress(qs.electionAddr);
+            setClientAddress(qs.clientAddr);
+        }
+        return prevQV;
+    }
+
+    private String makeDynamicConfigFilename(long version) {
+        return configFilename + ".dynamic." + Long.toHexString(version);
     }
 
-    private boolean needEraseClientInfoFromStaticConfig(QuorumVerifier oldQV,
-            QuorumVerifier newQV) {
-        QuorumServer myOldSpec = oldQV.getAllMembers().get(getId());
-        QuorumServer myNewSpec = newQV.getAllMembers().get(getId());
-        return (myNewSpec != null && myNewSpec.clientAddr != null 
-                     && (myOldSpec == null || myOldSpec.clientAddr == null));
+    private boolean needEraseClientInfoFromStaticConfig() {
+        QuorumServer server = quorumVerifier.getAllMembers().get(getId());
+        return (server != null && server.clientAddr != null);
     }
 
     /**

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java

@@ -19,22 +19,24 @@
 package org.apache.zookeeper.server.quorum;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.FileReader;
-import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.StringReader;
 import java.io.Writer;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Map.Entry;
 
+import org.apache.zookeeper.common.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
@@ -54,10 +56,11 @@ public class QuorumPeerConfig {
     private static final Logger LOG = LoggerFactory.getLogger(QuorumPeerConfig.class);
     private static boolean standaloneEnabled = true;
 
+    public static final String nextDynamicConfigFileSuffix = ".dynamic.next";
+
     protected InetSocketAddress clientPortAddress;
     protected File dataDir;
     protected File dataLogDir;
-    protected boolean configBackwardCompatibilityMode = false;
     protected String dynamicConfigFileStr = null;
     protected String configFileStr = null;
     protected int tickTime = ZooKeeperServer.DEFAULT_TICK_TIME;
@@ -136,10 +139,20 @@ public class QuorumPeerConfig {
                FileInputStream inConfig = new FileInputStream(dynamicConfigFileStr);
                try {
                    dynamicCfg.load(inConfig);
+                   if (dynamicCfg.getProperty("version") != null) {
+                       throw new ConfigException("dynamic file shouldn't have version inside");
+                   }
+
+                   String version = getVersionFromFilename(dynamicConfigFileStr);
+                   // If there isn't any version associated with the filename,
+                   // the default version is 0.
+                   if (version != null) {
+                       dynamicCfg.setProperty("version", version);
+                   }
                } finally {
                    inConfig.close();
                }
-               quorumVerifier = parseDynamicConfig(dynamicCfg, electionAlg, true, configBackwardCompatibilityMode);
+               quorumVerifier = parseDynamicConfig(dynamicCfg, electionAlg, true, false);
                checkValidity();
            
            } catch (IOException e) {
@@ -147,7 +160,7 @@ public class QuorumPeerConfig {
            } catch (IllegalArgumentException e) {
                throw new ConfigException("Error processing " + dynamicConfigFileStr, e);
            }        
-           File nextDynamicConfigFile = new File(dynamicConfigFileStr + ".next");
+           File nextDynamicConfigFile = new File(configFileStr + nextDynamicConfigFileSuffix);
            if (nextDynamicConfigFile.exists()) {
                try {           
                    Properties dynamicConfigNextCfg = new Properties();
@@ -165,7 +178,7 @@ public class QuorumPeerConfig {
                            break;
                        }
                    }
-                   lastSeenQuorumVerifier = createQuorumVerifier(dynamicConfigNextCfg, isHierarchical);    
+                   lastSeenQuorumVerifier = createQuorumVerifier(dynamicConfigNextCfg, isHierarchical);
                } catch (IOException e) {
                    LOG.warn("NextQuorumVerifier is initiated to null");
                }
@@ -173,6 +186,25 @@ public class QuorumPeerConfig {
         }
     }
 
+    // This method gets the version from the end of dynamic file name.
+    // For example, "zoo.cfg.dynamic.0" returns initial version "0".
+    // "zoo.cfg.dynamic.1001" returns version of hex number "0x1001".
+    // If a dynamic file name doesn't have any version at the end of file,
+    // e.g. "zoo.cfg.dynamic", it returns null.
+    public static String getVersionFromFilename(String filename) {
+        int i = filename.lastIndexOf('.');
+        if(i < 0 || i >= filename.length())
+            return null;
+
+        String hexVersion = filename.substring(i + 1);
+        try {
+            long version = Long.parseLong(hexVersion, 16);
+            return Long.toHexString(version);
+        } catch (NumberFormatException e) {
+            return null;
+        }
+    }
+
     /**
      * Parse config from a Properties.
      * @param zkProp Properties to parse from.
@@ -284,53 +316,83 @@ public class QuorumPeerConfig {
         }          
 
         // backward compatibility - dynamic configuration in the same file as
-        // static configuration params see writeDynamicConfig() - we change the
-        // config file to new format if reconfig happens
+        // static configuration params see writeDynamicConfig()
         if (dynamicConfigFileStr == null) {
-            configBackwardCompatibilityMode = true;
-            quorumVerifier = parseDynamicConfig(zkProp, electionAlg, true,
-                configBackwardCompatibilityMode);
+            backupOldConfig();
+            quorumVerifier = parseDynamicConfig(zkProp, electionAlg, true, true);
             checkValidity();
         }
     }
-    
+
     /**
-     * Writes dynamic configuration file, updates static config file if needed. 
-     * @param dynamicConfigFilename
-     * @param configFileStr
-     * @param configBackwardCompatibilityMode
-     * @param qv
-     * @param needEraseStaticClientInfo indicates whether we need to erase the clientPort
-     *                    and clientPortAddress from static config file.
+     * Backward compatibility -- It would backup static config file on bootup
+     * if users write dynamic configuration in "zoo.cfg".
      */
-    public static void writeDynamicConfig(String dynamicConfigFilename, String configFileStr,
-            final boolean configBackwardCompatibilityMode, final QuorumVerifier qv,
-            final boolean needEraseStaticClientInfo) throws IOException {
-
-        final String actualDynamicConfigFilename = dynamicConfigFilename;
-        new AtomicFileWritingIdiom(new File(actualDynamicConfigFilename), new OutputStreamStatement() {
+    private void backupOldConfig() throws IOException {
+        new AtomicFileWritingIdiom(new File(configFileStr + ".bak"), new OutputStreamStatement() {
             @Override
-            public void write(OutputStream outConfig) throws IOException {
-                byte b[] = qv.toString().getBytes();
-                outConfig.write(b);
+            public void write(OutputStream output) throws IOException {
+                InputStream input = null;
+                try {
+                    input = new FileInputStream(new File(configFileStr));
+                    byte[] buf = new byte[1024];
+                    int bytesRead;
+                    while ((bytesRead = input.read(buf)) > 0) {
+                        output.write(buf, 0, bytesRead);
+                    }
+                } finally {
+                    if( input != null) {
+                        input.close();
+                    }
+                }
             }
         });
-        // the following is for users who run without a dynamic config file (old config file)
-        // we create a dynamic config file, remove all the dynamic definitions from the config file and add a pointer
-        // to the config file. The dynamic config file's name will be the same as the config file's
-        // with ".dynamic" appended to it
+    }
 
-        if (!configBackwardCompatibilityMode && !needEraseStaticClientInfo)
-            return;
+    /**
+     * Writes dynamic configuration file
+     */
+    public static void writeDynamicConfig(final String dynamicConfigFilename,
+                                          final QuorumVerifier qv,
+                                          final boolean needKeepVersion)
+            throws IOException {
+
+        new AtomicFileWritingIdiom(new File(dynamicConfigFilename), new WriterStatement() {
+            @Override
+            public void write(Writer out) throws IOException {
+                Properties cfg = new Properties();
+                cfg.load( new StringReader(
+                        qv.toString()));
 
-        editStaticConfig(configFileStr, actualDynamicConfigFilename,
-                configBackwardCompatibilityMode, needEraseStaticClientInfo);
+                List<String> servers = new ArrayList<String>();
+                for (Entry<Object, Object> entry : cfg.entrySet()) {
+                    String key = entry.getKey().toString().trim();
+                    if ( !needKeepVersion && key.startsWith("version"))
+                        continue;
+
+                    String value = entry.getValue().toString().trim();
+                    servers.add(key
+                            .concat("=")
+                            .concat(value));
+                }
+
+                Collections.sort(servers);
+                out.write(StringUtils.joinStrings(servers, "\n"));
+            }
+        });
     }
 
-    private static void editStaticConfig(final String configFileStr,
-                                         final String dynamicFileStr,
-                                         final boolean backwardCompatible,
-                                         final boolean eraseClientPortAddress)
+    /**
+     * Edit static config file.
+     * If there are quorum information in static file, e.g. "server.X", "group",
+     * it will remove them.
+     * If it needs to erase client port information left by the old config,
+     * "eraseClientPortAddress" should be set true.
+     * It should also updates dynamic file pointer on reconfig.
+     */
+    public static void editStaticConfig(final String configFileStr,
+                                        final String dynamicFileStr,
+                                        final boolean eraseClientPortAddress)
             throws IOException {
         // Some tests may not have a static config file.
         if (configFileStr == null)
@@ -358,6 +420,7 @@ public class QuorumPeerConfig {
                     if (key.startsWith("server.")
                         || key.startsWith("group")
                         || key.startsWith("weight")
+                        || key.startsWith("dynamicConfigFile")
                         || (eraseClientPortAddress
                             && (key.startsWith("clientPort")
                                 || key.startsWith("clientPortAddress")))) {
@@ -369,10 +432,10 @@ public class QuorumPeerConfig {
                     out.write(key.concat("=").concat(value).concat("\n"));
                 }
 
-                if ( ! backwardCompatible )
-                    return;
-
-                out.write("dynamicConfigFile=".concat(dynamicFileStr).concat("\n"));
+                // updates the dynamic file pointer
+                out.write("dynamicConfigFile="
+                         .concat(dynamicFileStr)
+                         .concat("\n"));
             }
         });
     }
@@ -502,9 +565,6 @@ public class QuorumPeerConfig {
                    (clientPortAddress.getAddress().isAnyLocalAddress() 
                        && clientPortAddress.getPort()!=qs.clientAddr.getPort())) 
                     throw new ConfigException("client address for this server (id = " + serverId + ") in static config file is " + clientPortAddress + " is different from client address found in dynamic file: " + qs.clientAddr);
-                else {
-                    editStaticConfig(configFileStr, null, false, true);
-                }
             }
             if (qs!=null && qs.clientAddr != null) clientPortAddress = qs.clientAddr;                       
             
@@ -520,7 +580,7 @@ public class QuorumPeerConfig {
             }
            
        }
-       
+
     }
     
     public InetSocketAddress getClientPortAddress() { return clientPortAddress; }
@@ -574,19 +634,11 @@ public class QuorumPeerConfig {
     public LearnerType getPeerType() {
         return peerType;
     }
-    
-    public String getDynamicConfigFilename() {
-       return dynamicConfigFileStr;
-    }
-    
+
     public String getConfigFilename(){
         return configFileStr;
     }
     
-    public boolean getConfigBackwardCompatibility(){
-        return configBackwardCompatibilityMode;
-    }
-    
     public Boolean getQuorumListenOnAllIPs() {
         return quorumListenOnAllIPs;
     }

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java

@@ -154,9 +154,7 @@ public class QuorumPeerMain {
           quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout());
           quorumPeer.setInitLimit(config.getInitLimit());
           quorumPeer.setSyncLimit(config.getSyncLimit());
-          quorumPeer.setDynamicConfigFilename(config.getDynamicConfigFilename());
           quorumPeer.setConfigFileName(config.getConfigFilename());
-          quorumPeer.setConfigBackwardCompatibility(config.getConfigBackwardCompatibility());
           quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory()));
           quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false);
           if (config.getLastSeenQuorumVerifier()!=null) {

src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java

@@ -22,8 +22,14 @@
 package org.apache.zookeeper.server.quorum;
 
 import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -58,9 +64,8 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
     
     public static class MainThread implements Runnable {
         final File confFile;
-        final File dynamicConfigFile;
         final File tmpDir;
-        
+
         volatile TestQPMain main;
 
         public MainThread(int myid, int clientPort, String quorumCfgSection)
@@ -73,6 +78,12 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
             this(myid, clientPort, JettyAdminServer.DEFAULT_PORT, quorumCfgSection, null, writeDynamicConfigFile);
         }
 
+        public MainThread(int myid, int clientPort, String quorumCfgSection, boolean writeDynamicConfigFile,
+                          String version) throws IOException {
+            this(myid, clientPort, JettyAdminServer.DEFAULT_PORT, quorumCfgSection, null,
+                    writeDynamicConfigFile, version);
+        }
+
         public MainThread(int myid, int clientPort, String quorumCfgSection, String configs)
                 throws IOException {
             this(myid, clientPort, JettyAdminServer.DEFAULT_PORT, quorumCfgSection, configs, true);
@@ -86,6 +97,12 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
         public MainThread(int myid, int clientPort, int adminServerPort, String quorumCfgSection,
                 String configs, boolean writeDynamicConfigFile)
                 throws IOException {
+            this(myid, clientPort, adminServerPort, quorumCfgSection, configs, writeDynamicConfigFile, null);
+        }
+
+        public MainThread(int myid, int clientPort, int adminServerPort, String quorumCfgSection,
+                          String configs, boolean writeDynamicConfigFile, String version)
+                throws IOException {
             tmpDir = ClientBase.createTmpDir();
             LOG.info("id = " + myid + " tmpDir = " + tmpDir + " clientPort = "
                     + clientPort + " adminServerPort = " + adminServerPort);
@@ -96,7 +113,6 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
             }
 
             confFile = new File(tmpDir, "zoo.cfg");
-            dynamicConfigFile = new File(tmpDir, "zoo.cfg.dynamic");
 
             FileWriter fwriter = new FileWriter(confFile);
             fwriter.write("tickTime=4000\n");
@@ -116,14 +132,10 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
             fwriter.write("admin.serverPort=" + adminServerPort + "\n");
 
             if (writeDynamicConfigFile) {
-                String dynamicConfigFilename = PathUtils.normalizeFileSystemPath(dynamicConfigFile.toString());
+                String dynamicConfigFilename = createDynamicFile(quorumCfgSection, version);
                 fwriter.write("dynamicConfigFile=" + dynamicConfigFilename + "\n");
-                FileWriter fDynamicConfigWriter = new FileWriter(dynamicConfigFile);
-                fDynamicConfigWriter.write(quorumCfgSection + "\n");
-                fDynamicConfigWriter.flush();
-                fDynamicConfigWriter.close();
             } else {
-                fwriter.write(quorumCfgSection + "\n");
+                fwriter.write(quorumCfgSection);
             }
             fwriter.flush();
             fwriter.close();
@@ -135,11 +147,49 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
             fwriter.close();
         }
 
-        public void writeTempDynamicConfigFile(String nextQuorumCfgSection)
+        private String createDynamicFile(String quorumCfgSection, String version)
+                throws IOException {
+            String filename = "zoo.cfg.dynamic";
+            if( version != null ){
+                filename = filename + "." + version;
+            }
+
+            File dynamicConfigFile = new File(tmpDir, filename);
+            String dynamicConfigFilename = PathUtils.normalizeFileSystemPath(dynamicConfigFile.toString());
+
+            FileWriter fDynamicConfigWriter = new FileWriter(dynamicConfigFile);
+            fDynamicConfigWriter.write(quorumCfgSection);
+            fDynamicConfigWriter.flush();
+            fDynamicConfigWriter.close();
+
+            return dynamicConfigFilename;
+        }
+
+        public File[] getDynamicFiles() {
+            return getFilesWithPrefix("zoo.cfg.dynamic");
+        }
+
+        public File[] getFilesWithPrefix(final String prefix) {
+            return tmpDir.listFiles(new FilenameFilter() {      
+                @Override
+                public boolean accept(File dir, String name) {
+                    return name.startsWith(prefix);
+                }});
+        }
+
+        public File getFileByName(String filename) {
+            File f = new File(tmpDir.getPath(), filename);
+            return f.isFile() ? f : null;
+        }
+
+        public void writeTempDynamicConfigFile(String nextQuorumCfgSection, String version)
                 throws IOException {
-            File nextDynamicConfigFile = new File(tmpDir, "zoo.cfg.dynamic.next");
+            File nextDynamicConfigFile = new File(tmpDir,
+                    "zoo.cfg" + QuorumPeerConfig.nextDynamicConfigFileSuffix);
             FileWriter fwriter = new FileWriter(nextDynamicConfigFile);
-            fwriter.write(nextQuorumCfgSection + "\n");
+            fwriter.write(nextQuorumCfgSection
+                    + "\n"
+                    + "version=" + version);
             fwriter.flush();
             fwriter.close();
         }
@@ -193,5 +243,11 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
         public boolean isQuorumPeerRunning() {
             return main.quorumPeer != null;
         }
+
+        public String getPropFromStaticFile(String key) throws IOException {
+            Properties props = new Properties();
+            props.load(new FileReader(confFile));
+            return props.getProperty(key, "");
+        }
     }
 }

src/java/test/org/apache/zookeeper/server/quorum/ReconfigBackupTest.java

@@ -0,0 +1,345 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.quorum;
+
+import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.common.StringUtils;
+import org.apache.zookeeper.test.ClientBase;
+import org.apache.zookeeper.test.ReconfigTest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.Scanner;
+
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+
+public class ReconfigBackupTest extends QuorumPeerTestBase {
+
+    public static String getVersionFromConfigStr(String config) throws IOException {
+        Properties props = new Properties();
+        props.load(new StringReader(config));
+        return props.getProperty("version", "");
+    }
+
+    // upgrade this once we have Google-Guava or Java 7+
+    public static String getFileContent(File file) throws FileNotFoundException {
+        Scanner sc = new Scanner(file);
+        StringBuilder sb = new StringBuilder();
+        while (sc.hasNextLine()) {
+            sb.append(sc.nextLine() + "\n");
+        }
+        return sb.toString();
+    }
+
+    @Before
+    public void setup() {
+        ClientBase.setupTestEnv();
+    }
+
+    /**
+     * This test checks that it will backup static file on bootup.
+     */
+    @Test
+    public void testBackupStatic() throws Exception {
+        final int SERVER_COUNT = 3;
+        final int clientPorts[] = new int[SERVER_COUNT];
+        StringBuilder sb = new StringBuilder();
+        String server;
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            clientPorts[i] = PortAssignment.unique();
+            server = "server." + i + "=localhost:" + PortAssignment.unique()
+                    + ":" + PortAssignment.unique() + ":participant;localhost:"
+                    + clientPorts[i];
+            sb.append(server + "\n");
+        }
+        String currentQuorumCfgSection = sb.toString();
+
+        MainThread mt[] = new MainThread[SERVER_COUNT];
+        String[] staticFileContent = new String[SERVER_COUNT];
+        String[] staticBackupContent = new String[SERVER_COUNT];
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, false);
+            // check that a dynamic configuration file doesn't exist
+            Assert.assertNull("static file backup shouldn't exist before bootup",
+                    mt[i].getFileByName("zoo.cfg.bak"));
+            staticFileContent[i] = getFileContent(mt[i].confFile);
+            mt[i].start();
+        }
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            Assert.assertTrue("waiting for server " + i + " being up",
+                    ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+                            CONNECTION_TIMEOUT));
+            File backupFile = mt[i].getFileByName("zoo.cfg.bak");
+            Assert.assertNotNull("static file backup should exist", backupFile);
+            staticBackupContent[i] = getFileContent(backupFile);
+            Assert.assertEquals(staticFileContent[i], staticBackupContent[i]);
+        }
+    }
+
+    /**
+     * This test checks that on reconfig, a new dynamic file will be created with
+     * current version appended to file name. Meanwhile, the dynamic file pointer
+     * in static config file should also be changed.
+     */
+    @Test
+    public void testReconfigCreateNewVersionFile() throws Exception {
+        final int SERVER_COUNT = 3;
+        final int NEW_SERVER_COUNT = 5;
+
+        final int clientPorts[] = new int[NEW_SERVER_COUNT];
+        final int quorumPorts[] = new int[NEW_SERVER_COUNT];
+        final int electionPorts[] = new int[NEW_SERVER_COUNT];
+        final String servers[] = new String[NEW_SERVER_COUNT];
+
+        StringBuilder sb = new StringBuilder();
+        ArrayList<String> oldServers = new ArrayList<String>();
+        ArrayList<String> newServers = new ArrayList<String>();
+
+        for (int i = 0; i < NEW_SERVER_COUNT; i++) {
+            clientPorts[i] = PortAssignment.unique();
+            quorumPorts[i] = PortAssignment.unique();
+            electionPorts[i] = PortAssignment.unique();
+            servers[i] = "server." + i + "=localhost:" + quorumPorts[i]
+                    + ":" + electionPorts[i] + ":participant;localhost:"
+                    + clientPorts[i];
+
+            newServers.add(servers[i]);
+
+            if (i >= SERVER_COUNT) {
+                continue;
+            }
+            oldServers.add(servers[i]);
+            sb.append(servers[i] + "\n");
+        }
+        String quorumCfgSection = sb.toString();
+
+        MainThread mt[] = new MainThread[NEW_SERVER_COUNT];
+        ZooKeeper zk[] = new ZooKeeper[NEW_SERVER_COUNT];
+
+        // start old cluster
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            mt[i] = new MainThread(i, clientPorts[i], quorumCfgSection);
+            mt[i].start();
+        }
+
+        String firstVersion = null, secondVersion = null;
+
+        // test old cluster
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            Assert.assertTrue("waiting for server " + i + " being up",
+                    ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+                            CONNECTION_TIMEOUT));
+            zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
+                    ClientBase.CONNECTION_TIMEOUT, this);
+
+            Properties cfg = ReconfigLegacyTest.readPropertiesFromFile(mt[i].confFile);
+            String filename = cfg.getProperty("dynamicConfigFile", "");
+
+            String version = QuorumPeerConfig.getVersionFromFilename(filename);
+            Assert.assertNotNull(version);
+
+            String configStr = ReconfigTest.testServerHasConfig(
+                    zk[i], oldServers, null);
+
+            String configVersion = getVersionFromConfigStr(configStr);
+            // the version appended to filename should be the same as
+            // the one of quorum verifier.
+            Assert.assertEquals(version, configVersion);
+
+            if (i == 0) {
+                firstVersion = version;
+            } else {
+                Assert.assertEquals(firstVersion, version);
+            }
+        }
+
+        ReconfigTest.reconfig(zk[1], null, null, newServers, -1);
+
+        // start additional new servers
+        for (int i = SERVER_COUNT; i < NEW_SERVER_COUNT; i++) {
+            mt[i] = new MainThread(i, clientPorts[i], quorumCfgSection + servers[i]);
+            mt[i].start();
+        }
+
+        // wait for new servers to be up running
+        for (int i = SERVER_COUNT; i < NEW_SERVER_COUNT; i++) {
+            Assert.assertTrue("waiting for server " + i + " being up",
+                    ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+                            CONNECTION_TIMEOUT));
+            zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
+                    ClientBase.CONNECTION_TIMEOUT, this);
+        }
+
+        // test that all servers have:
+        // a different, larger version dynamic file
+        for (int i = 0; i < NEW_SERVER_COUNT; i++) {
+            Properties cfg = ReconfigLegacyTest.readPropertiesFromFile(mt[i].confFile);
+            String filename = cfg.getProperty("dynamicConfigFile", "");
+
+            String version = QuorumPeerConfig.getVersionFromFilename(filename);
+            Assert.assertNotNull(version);
+
+            String configStr = ReconfigTest.testServerHasConfig(zk[i],
+                    newServers, null);
+
+            String quorumVersion = getVersionFromConfigStr(configStr);
+            Assert.assertEquals(version, quorumVersion);
+
+            if (i == 0) {
+                secondVersion = version;
+                Assert.assertTrue(
+                        Long.parseLong(secondVersion, 16)
+                                > Long.parseLong(firstVersion, 16));
+            } else {
+                Assert.assertEquals(secondVersion, version);
+            }
+        }
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            mt[i].shutdown();
+            zk[i].close();
+        }
+    }
+
+    /**
+     * This test checks that if a version is appended to dynamic file,
+     * then peer should use that version as quorum config version.
+     * <p/>
+     * The scenario: one server has an older version of 3 servers, and
+     * four others have newer version of 5 servers. Finally, the lag-off one
+     * should have server config of 5 servers.
+     */
+    @Test
+    public void testVersionOfDynamicFilename() throws Exception {
+        final int SERVER_COUNT = 5;
+        final int oldServerCount = 3;
+        final int lagOffServerId = 0;
+        final int clientPorts[] = new int[SERVER_COUNT];
+        StringBuilder sb = new StringBuilder();
+        String server;
+        StringBuilder oldSb = new StringBuilder();
+        ArrayList<String> allServers = new ArrayList<String>();
+
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            clientPorts[i] = PortAssignment.unique();
+            server = "server." + i + "=localhost:" + PortAssignment.unique()
+                    + ":" + PortAssignment.unique() + ":participant;localhost:"
+                    + clientPorts[i];
+            sb.append(server + "\n");
+            allServers.add(server);
+
+            if (i < oldServerCount) {
+                // only take in the first 3 servers as old quorum config.
+                oldSb.append(server + "\n");
+            }
+        }
+
+        String currentQuorumCfgSection = sb.toString();
+
+        String oldQuorumCfg = oldSb.toString();
+
+        MainThread mt[] = new MainThread[SERVER_COUNT];
+
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            if (i == lagOffServerId) {
+                mt[i] = new MainThread(i, clientPorts[i], oldQuorumCfg, true, "100000000");
+            } else {
+                mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection,
+                        true, "200000000");
+            }
+
+            // before connecting to quorum, servers should have set up dynamic file
+            // version and pointer. And the lag-off server is using the older
+            // version dynamic file.
+            if (i == lagOffServerId) {
+                Assert.assertNotNull(
+                        mt[i].getFileByName("zoo.cfg.dynamic.100000000"));
+                Assert.assertNull(
+                        mt[i].getFileByName("zoo.cfg.dynamic.200000000"));
+                Assert.assertTrue(
+                        mt[i].getPropFromStaticFile("dynamicConfigFile")
+                                .endsWith(".100000000"));
+            } else {
+                Assert.assertNotNull(
+                        mt[i].getFileByName("zoo.cfg.dynamic.200000000"));
+                Assert.assertTrue(
+                        mt[i].getPropFromStaticFile("dynamicConfigFile")
+                                .endsWith(".200000000"));
+            }
+
+            mt[i].start();
+        }
+
+        String dynamicFileContent = null;
+
+        for (int i = 0; i < SERVER_COUNT; i++) {
+            Assert.assertTrue("waiting for server " + i + " being up",
+                    ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+                            CONNECTION_TIMEOUT));
+
+            ZooKeeper zk = new ZooKeeper("127.0.0.1:" + clientPorts[i],
+                    ClientBase.CONNECTION_TIMEOUT, this);
+
+            // we should see that now all servers have the same config of 5 servers
+            // including the lag-off server.
+            String configStr = ReconfigTest.testServerHasConfig(zk, allServers, null);
+            Assert.assertEquals("200000000", getVersionFromConfigStr(configStr));
+            
+            List<String> configLines = Arrays.asList(configStr.split("\n"));
+            Collections.sort(configLines);
+            String sortedConfigStr = StringUtils.joinStrings(configLines, "\n");
+            
+             File dynamicConfigFile = mt[i].getFileByName("zoo.cfg.dynamic.200000000");
+             Assert.assertNotNull(dynamicConfigFile);
+
+            // All dynamic files created with the same version should have
+            // same configs, and they should be equal to the config we get from QuorumPeer.
+            if (i == 0) {
+                dynamicFileContent = getFileContent(dynamicConfigFile);                
+                Assert.assertEquals(sortedConfigStr, dynamicFileContent + 
+                        "version=200000000");
+            } else {
+                String otherDynamicFileContent = getFileContent(dynamicConfigFile);
+                Assert.assertEquals(dynamicFileContent, otherDynamicFileContent);
+            }
+        }
+
+        // finally, we should also check that the lag-off server has updated
+        // the dynamic file pointer.
+        Assert.assertTrue(
+                mt[lagOffServerId].getPropFromStaticFile("dynamicConfigFile")
+                        .endsWith(".200000000"));
+    }
+}

src/java/test/org/apache/zookeeper/server/quorum/ReconfigLegacyTest.java

@@ -75,7 +75,7 @@ public class ReconfigLegacyTest extends QuorumPeerTestBase {
         for (int i = 0; i < SERVER_COUNT; i++) {
             mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, false);
             // check that a dynamic configuration file doesn't exist
-            Assert.assertFalse(mt[i].dynamicConfigFile.exists());
+            Assert.assertEquals( mt[i].getDynamicFiles().length, 0 );
             mt[i].start();
         }
         // Check that the servers are up, have the right config and can process operations.
@@ -86,7 +86,9 @@ public class ReconfigLegacyTest extends QuorumPeerTestBase {
                             CONNECTION_TIMEOUT));
             zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
-            Assert.assertTrue(mt[i].dynamicConfigFile.exists());
+            File[] dynamicFiles = mt[i].getDynamicFiles();
+
+            Assert.assertTrue( dynamicFiles.length== 1 );
             ReconfigTest.testServerHasConfig(zk[i], allServers, null);
             // check that static config file doesn't include membership info
             // and has a pointer to the dynamic configuration file
@@ -98,7 +100,7 @@ public class ReconfigLegacyTest extends QuorumPeerTestBase {
             Assert.assertFalse(cfg.containsKey("clientPort"));
 
             // check that the dynamic configuration file contains the membership info
-            cfg = readPropertiesFromFile(mt[i].dynamicConfigFile);
+            cfg = readPropertiesFromFile(dynamicFiles[0]);
             for (int j = 0; j < SERVER_COUNT; j++) {
                 String serverLine = cfg.getProperty("server." + j, "");
                 Assert.assertEquals(allServers.get(j), "server." + j + "="
@@ -176,8 +178,6 @@ public class ReconfigLegacyTest extends QuorumPeerTestBase {
         // Start the servers with a static config file, without a dynamic config file.
         for (int i = 0; i < SERVER_COUNT; i++) {
             mt[i] = new MainThread(i, clientPorts[i], quorumCfgSection, false);
-            // check that a dynamic configuration file doesn't exist
-            Assert.assertFalse(mt[i].dynamicConfigFile.exists());
             mt[i].start();
         }
 
@@ -224,8 +224,7 @@ public class ReconfigLegacyTest extends QuorumPeerTestBase {
         }
     }
 
-
-    private Properties readPropertiesFromFile(File file) throws IOException {
+    public static Properties readPropertiesFromFile(File file) throws IOException {
         Properties cfg = new Properties();
         FileInputStream in = new FileInputStream(file);
         try {

src/java/test/org/apache/zookeeper/server/quorum/ReconfigRecoveryTest.java

@@ -56,9 +56,8 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
             allServers.add(server);
             sb.append(server + "\n");
             if (i == 1)
-                currentQuorumCfgSection = sb.toString() + "version=100000000\n";
+                currentQuorumCfgSection = sb.toString();
         }
-        sb.append("version=200000000\n"); // version of current config is 100000000
         nextQuorumCfgSection = sb.toString();
 
         // Both servers 0 and 1 will have the .next config file, which means
@@ -67,12 +66,13 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         MainThread mt[] = new MainThread[SERVER_COUNT];
         ZooKeeper zk[] = new ZooKeeper[SERVER_COUNT];
         for (int i = 0; i < SERVER_COUNT - 1; i++) {
-            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection);
+            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection,
+                    true, "100000000");
             // note that we should run the server, shut it down and only then
             // simulate a reconfig in progress by writing the temp file, but here no
             // other server is competing with them in FLE, so we can skip this step
             // (server 2 is booted after FLE ends)
-            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection);
+            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection, "200000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -132,9 +132,8 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         StringBuilder sb = new StringBuilder();
         String server;
 
-        String currentQuorumCfg = null, currentQuorumCfgSection = null, nextQuorumCfgSection = null;
+        String currentQuorumCfg, nextQuorumCfgSection;
 
-        ArrayList<String> allServersCurrent = new ArrayList<String>();
         ArrayList<String> allServersNext = new ArrayList<String>();
 
         for (int i = 0; i < 2; i++) {
@@ -142,13 +141,10 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
             server = "server." + i + "=localhost:" + PortAssignment.unique()
                     + ":" + PortAssignment.unique() + ":participant;localhost:"
                     + oldClientPorts[i];
-            allServersCurrent.add(server);
             sb.append(server + "\n");
         }
 
         currentQuorumCfg = sb.toString();
-        sb.append("version=100000000\n");
-        currentQuorumCfgSection = sb.toString();
 
         sb = new StringBuilder();
         String role;
@@ -165,7 +161,6 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
             allServersNext.add(server);
             sb.append(server + "\n");
         }
-        sb.append("version=200000000\n"); // version of current config is 100000000
         nextQuorumCfgSection = sb.toString();
 
         MainThread mt[] = new MainThread[SERVER_COUNT];
@@ -173,8 +168,8 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
 
         // run servers 0 and 1 normally
         for (int i = 0; i < 2; i++) {
-            mt[i] = new MainThread(i, oldClientPorts[i],
-                    currentQuorumCfgSection);
+            mt[i] = new MainThread(i, oldClientPorts[i], currentQuorumCfg,
+                    true, "100000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + oldClientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -203,7 +198,7 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         }
 
         for (int i = 0; i < 2; i++) {
-            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection);
+            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection, "200000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -252,18 +247,15 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
 
         String currentQuorumCfgSection = null, nextQuorumCfgSection;
 
-        ArrayList<String> allServers = new ArrayList<String>();
         for (int i = 0; i < SERVER_COUNT; i++) {
             clientPorts[i] = PortAssignment.unique();
             server = "server." + i + "=localhost:" + PortAssignment.unique()
                     + ":" + PortAssignment.unique() + ":participant;localhost:"
                     + clientPorts[i];
-            allServers.add(server);
             sb.append(server + "\n");
             if (i == 1)
-                currentQuorumCfgSection = sb.toString() + "version=100000000\n";
+                currentQuorumCfgSection = sb.toString();
         }
-        sb.append("version=200000000\n"); // version of current config is 100000000 
         nextQuorumCfgSection = sb.toString();
 
         MainThread mt[] = new MainThread[SERVER_COUNT];
@@ -272,11 +264,12 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         // Both servers 0 and 1 will have the .next config file, which means
         // for them that a reconfiguration was in progress when they failed
         for (int i = 0; i < 2; i++) {
-            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection);
+            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection,
+                    true, "100000000");
             // note that we should run the server, shut it down and only then
             // simulate a reconfig in progress by writing the temp file, but here no
             // other server is competing with them in FLE, so we can skip this step
-            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection);
+            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection, "200000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -322,16 +315,16 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
                     + clientPorts[i];
             allServers.add(server);
             sb.append(server + "\n");
-            if (i == 1) currentQuorumCfgSection = sb.toString() + "version=100000000\n";
+            if (i == 1) currentQuorumCfgSection = sb.toString();
         }
-        sb.append("version=200000000\n"); // version of current config is 100000000
         nextQuorumCfgSection = sb.toString();
 
         // lets start servers 2, 3, 4 with the new config
         MainThread mt[] = new MainThread[SERVER_COUNT];
         ZooKeeper zk[] = new ZooKeeper[SERVER_COUNT];
         for (int i = 2; i < SERVER_COUNT; i++) {
-            mt[i] = new MainThread(i, clientPorts[i], nextQuorumCfgSection);
+            mt[i] = new MainThread(i, clientPorts[i], nextQuorumCfgSection,
+                    true, "200000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -350,8 +343,9 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         // for them that a reconfiguration was in progress when they failed
         // and the leader will complete it.
         for (int i = 0; i < 2; i++) {
-            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection);
-            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection);
+            mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection,
+                    true, "100000000");
+            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection, "200000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + clientPorts[i],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -402,7 +396,6 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         HashSet<Integer> observers = new HashSet<Integer>();
         observers.add(2);
         StringBuilder sb = generateConfig(3, ports, observers);
-        sb.append("version=100000000");
         currentQuorumCfgSection = sb.toString();
 
         // generate new config string
@@ -414,20 +407,21 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
             allServersNext.add(server);
             sb.append(server + "\n");
         }
-        sb.append("version=200000000"); // version of current config is 100000000
         nextQuorumCfgSection = sb.toString();
 
         MainThread mt[] = new MainThread[SERVER_COUNT];
         ZooKeeper zk[] = new ZooKeeper[SERVER_COUNT];
 
         // start server 2 with old config, where it is an observer
-        mt[2] = new MainThread(2, ports[2][2], currentQuorumCfgSection);
+        mt[2] = new MainThread(2, ports[2][2], currentQuorumCfgSection,
+                true, "100000000");
         mt[2].start();
         zk[2] = new ZooKeeper("127.0.0.1:" + ports[2][2],
                 ClientBase.CONNECTION_TIMEOUT, this);
 
         // start server 3 with new config
-        mt[3] = new MainThread(3, ports[3][2], nextQuorumCfgSection);
+        mt[3] = new MainThread(3, ports[3][2], nextQuorumCfgSection,
+                true, "200000000");
         mt[3].start();
         zk[3] = new ZooKeeper("127.0.0.1:" + ports[3][2],
                 ClientBase.CONNECTION_TIMEOUT, this);
@@ -439,9 +433,9 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
             ReconfigTest.testServerHasConfig(zk[i], allServersNext, null);
         }
 
-        Assert.assertEquals(nextQuorumCfgSection,
+        Assert.assertEquals(nextQuorumCfgSection + "version=200000000",
                 ReconfigTest.testServerHasConfig(zk[2], null, null));
-        Assert.assertEquals(nextQuorumCfgSection,
+        Assert.assertEquals(nextQuorumCfgSection + "version=200000000",
                 ReconfigTest.testServerHasConfig(zk[3], null, null));
         ReconfigTest.testNormalOperation(zk[2], zk[2]);
         ReconfigTest.testNormalOperation(zk[3], zk[2]);
@@ -470,7 +464,7 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
 
         final int SERVER_COUNT = 4;
         int[][] ports = generatePorts(SERVER_COUNT);
-        String currentQuorumCfg, currentQuorumCfgSection, nextQuorumCfgSection;
+        String currentQuorumCfg, nextQuorumCfgSection;
 
         // generate old config string
         HashSet<Integer> observers = new HashSet<Integer>();
@@ -478,14 +472,13 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
 
         StringBuilder sb = generateConfig(3, ports, observers);
         currentQuorumCfg = sb.toString();
-        sb.append("version=100000000");
-        currentQuorumCfgSection = sb.toString();
 
         // Run servers 0..2 for a while
         MainThread mt[] = new MainThread[SERVER_COUNT];
         ZooKeeper zk[] = new ZooKeeper[SERVER_COUNT];
         for (int i = 0; i <= 2; i++) {
-            mt[i] = new MainThread(i, ports[i][2], currentQuorumCfgSection);
+            mt[i] = new MainThread(i, ports[i][2], currentQuorumCfg
+                    , true, "100000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + ports[i][2],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -514,13 +507,12 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
             allServersNext.add(server);
             sb.append(server + "\n");
         }
-        sb.append("version=200000000"); // version of current config is 100000000
         nextQuorumCfgSection = sb.toString();
 
         // simulate reconfig in progress - servers 0..2 have a temp reconfig
         // file when they boot
         for (int i = 0; i <= 2; i++) {
-            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection);
+            mt[i].writeTempDynamicConfigFile(nextQuorumCfgSection, "200000000");
             mt[i].start();
             zk[i] = new ZooKeeper("127.0.0.1:" + ports[i][2],
                     ClientBase.CONNECTION_TIMEOUT, this);
@@ -536,15 +528,15 @@ public class ReconfigRecoveryTest extends QuorumPeerTestBase {
         for (int i = 2; i < SERVER_COUNT; i++) {
             Assert.assertTrue("waiting for server " + i + " being up",
                     ClientBase.waitForServerUp("127.0.0.1:" + ports[i][2],
-                            CONNECTION_TIMEOUT * 2));
+                            CONNECTION_TIMEOUT * 3));
             ReconfigTest.testServerHasConfig(zk[i], allServersNext, null);
         }
 
         ReconfigTest.testNormalOperation(zk[0], zk[2]);
         ReconfigTest.testNormalOperation(zk[3], zk[1]);
-        Assert.assertEquals(nextQuorumCfgSection,
+        Assert.assertEquals(nextQuorumCfgSection + "version=200000000",
                 ReconfigTest.testServerHasConfig(zk[2], null, null));
-        Assert.assertEquals(nextQuorumCfgSection,
+        Assert.assertEquals(nextQuorumCfgSection + "version=200000000",
                 ReconfigTest.testServerHasConfig(zk[3], null, null));
 
         for (int i = 0; i < SERVER_COUNT; i++) {

src/java/test/org/apache/zookeeper/test/LENonTerminateTest.java

@@ -219,7 +219,7 @@ public class LENonTerminateTest extends ZKTestCase {
             super(quorumPeers, snapDir, logDir, electionAlg,
                     myid,tickTime, initLimit,syncLimit, false,
                     ServerCnxnFactory.createFactory(clientPort, -1),
-                    new QuorumMaj(quorumPeers), null);
+                    new QuorumMaj(quorumPeers));
         }
 
         protected  Election createElectionAlgorithm(int electionAlgorithm){