Forráskód Böngészése

ZOOKEEPER-261: Reinitialized servers should not participate in leader election

Author: Brian Nixon <nixon@fb.com>

Reviewers: Michael Han <hanm@apache.org>, Edward Ribeiro <edward.ribeiro@gmail.com>, benjamin reed <breed@apache.org>

Closes #120 from enixon/ZOOKEEPER-261
Brian Nixon 8 éve
szülő
commit
8773ecb78a

+ 2 - 0
bin/zkServer-initialize.sh

@@ -113,6 +113,8 @@ initialize() {
     else
         echo "No myid provided, be sure to specify it in $ZOO_DATADIR/myid if using non-standalone"
     fi
+
+    touch "$ZOO_DATADIR/initialize"
 }
 
 eval set -- "${OPTS}"

+ 45 - 1
src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml

@@ -277,6 +277,16 @@ server.3=zoo3:2888:3888</programlisting>
           ensemble and should have a value between 1 and 255.</para>
         </listitem>
 
+        <listitem>
+          <para>Create an initialization marker file <filename>initialize</filename>
+            in the same directory as <filename>myid</filename>. This file indicates
+            that an empty data directory is expected. When present, an empty data base
+            is created and the marker file deleted. When not present, an empty data
+            directory will mean this peer will not have voting rights and it will not
+            populate the data directory until it communicates with an active leader.
+            Intended use is to only create this file when bringing up a new
+            ensemble. </para>
+        </listitem>
         <listitem>
           <para>If your configuration file is set up, you can start a
           ZooKeeper server:</para>
@@ -1408,6 +1418,35 @@ server.3=zoo3:2888:3888</programlisting>
         </para>
       </section>
 
+      <section id="sc_db_existence_validation">
+        <title>Enabling db existence validation</title>
+
+        <para><emphasis role="bold">New in 3.6.0:</emphasis> The default
+          behavior of a ZooKeeper server on startup when no data tree
+          is found is to set zxid to zero and join the quorum as a
+          voting member. This can be dangerous if some event (e.g. a
+          rogue 'rm -rf') has removed the data directory while the
+          server was down since this server may help elect a leader
+          that is missing transactions. Enabling db existence validation
+          will change the behavior on startup when no data tree is
+          found: the server joins the ensemble as a non-voting participant
+          until it is able to sync with the leader and acquire an up-to-date
+          version of the ensemble data. To indicate an empty data tree is
+          expected (ensemble creation), the user should place a file
+          'initialize' in the same directory as 'myid'. This file will
+          be detected and deleted by the server on startup.
+        </para>
+
+        <para> Initialization validation can be enabled when running
+          ZooKeeper servers directly from class files by setting
+          <emphasis role="bold">zookeeper.db.autocreate=false</emphasis>
+          on the java command line, i.e.
+          <emphasis role="bold">-Dzookeeper.db.autocreate=false</emphasis>.
+          Running <emphasis role="bold">zkServer-initialize.sh</emphasis>
+          will create the required initialization file.
+        </para>
+      </section>
+
       <section id="sc_performance_options">
         <title>Performance Tuning Options</title>
 
@@ -1943,7 +1982,7 @@ server.3=zoo3:2888:3888</programlisting>
       <section>
         <title>The Data Directory</title>
 
-        <para>This directory has two files in it:</para>
+        <para>This directory has two or three files in it:</para>
 
         <itemizedlist>
           <listitem>
@@ -1951,6 +1990,11 @@ server.3=zoo3:2888:3888</programlisting>
             human readable ASCII text that represents the server id.</para>
           </listitem>
 
+          <listitem>
+            <para><filename>initialize</filename> - presence indicates lack of
+              data tree is expected. Cleaned up once data tree is created.</para>
+          </listitem>
+
           <listitem>
             <para><filename>snapshot.&lt;zxid&gt;</filename> - holds the fuzzy
             snapshot of a data tree.</para>

+ 32 - 6
src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java

@@ -20,6 +20,7 @@ package org.apache.zookeeper.server.persistence;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -45,7 +46,7 @@ import org.slf4j.LoggerFactory;
  * classes
  */
 public class FileTxnSnapLog {
-    //the direcotry containing the
+    //the directory containing the
     //the transaction logs
     private final File dataDir;
     //the directory containing the
@@ -53,6 +54,7 @@ public class FileTxnSnapLog {
     private final File snapDir;
     private TxnLog txnLog;
     private SnapShot snapLog;
+    private final boolean autoCreateDB;
     public final static int VERSION = 2;
     public final static String version = "version-";
 
@@ -63,6 +65,10 @@ public class FileTxnSnapLog {
 
     public static final String ZOOKEEPER_DATADIR_AUTOCREATE_DEFAULT = "true";
 
+    static final String ZOOKEEPER_DB_AUTOCREATE = "zookeeper.db.autocreate";
+
+    private static final String ZOOKEEPER_DB_AUTOCREATE_DEFAULT = "true";
+
     /**
      * This listener helps
      * the external apis calling
@@ -132,6 +138,9 @@ public class FileTxnSnapLog {
 
         txnLog = new FileTxnLog(this.dataDir);
         snapLog = new FileSnap(this.snapDir);
+
+        autoCreateDB = Boolean.parseBoolean(System.getProperty(ZOOKEEPER_DB_AUTOCREATE,
+                ZOOKEEPER_DB_AUTOCREATE_DEFAULT));
     }
 
     /**
@@ -167,6 +176,14 @@ public class FileTxnSnapLog {
             PlayBackListener listener) throws IOException {
         long deserializeResult = snapLog.deserialize(dt, sessions);
         FileTxnLog txnLog = new FileTxnLog(dataDir);
+        boolean trustEmptyDB;
+        File initFile = new File(dataDir.getParent(), "initialize");
+        if (Files.deleteIfExists(initFile.toPath())) {
+            LOG.info("Initialize file found, an empty database will not block voting participation");
+            trustEmptyDB = true;
+        } else {
+            trustEmptyDB = autoCreateDB;
+        }
         if (-1L == deserializeResult) {
             /* this means that we couldn't find any snapshot, so we need to
              * initialize an empty database (reported in ZOOKEEPER-2325) */
@@ -175,11 +192,20 @@ public class FileTxnSnapLog {
                         "No snapshot found, but there are log entries. " +
                         "Something is broken!");
             }
-            /* TODO: (br33d) we should either put a ConcurrentHashMap on restore()
-             *       or use Map on save() */
-            save(dt, (ConcurrentHashMap<Long, Integer>)sessions);
-            /* return a zxid of zero, since we the database is empty */
-            return 0;
+
+            if (trustEmptyDB) {
+                /* TODO: (br33d) we should either put a ConcurrentHashMap on restore()
+                 *       or use Map on save() */
+                save(dt, (ConcurrentHashMap<Long, Integer>)sessions);
+
+                /* return a zxid of 0, since we know the database is empty */
+                return 0L;
+            } else {
+                /* return a zxid of -1, since we are possibly missing data */
+                LOG.warn("Unexpected empty data tree, setting zxid to -1");
+                dt.lastProcessedZxid = -1L;
+                return -1L;
+            }
         }
         TxnIterator itr = txnLog.read(dt.lastProcessedZxid+1);
         long highestZxid = dt.lastProcessedZxid;

+ 8 - 0
src/java/main/org/apache/zookeeper/server/quorum/FastLeaderElection.java

@@ -934,6 +934,14 @@ public class FastLeaderElection implements Election {
                      */
                     switch (n.state) {
                     case LOOKING:
+                        if (getInitLastLoggedZxid() == -1) {
+                            LOG.debug("Ignoring notification as our zxid is -1");
+                            break;
+                        }
+                        if (n.zxid == -1) {
+                            LOG.debug("Ignoring notification from member with -1 zxid" + n.sid);
+                            break;
+                        }
                         // If notification > current, replace and send messages out
                         if (n.electionEpoch > logicalclock.get()) {
                             logicalclock.set(n.electionEpoch);

+ 3 - 1
src/java/main/org/apache/zookeeper/server/quorum/Leader.java

@@ -1211,7 +1211,9 @@ public class Leader {
                                                     + leaderStateSummary.getLastZxid()
                                                     + " (last zxid)");
                 }
-                electingFollowers.add(id);
+                if (ss.getLastZxid() != -1) {
+                    electingFollowers.add(id);
+                }
             }
             QuorumVerifier verifier = self.getQuorumVerifier();
             if (electingFollowers.contains(self.getId()) && verifier.containsQuorum(electingFollowers)) {

+ 4 - 68
src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java

@@ -90,6 +90,7 @@ public class ZooKeeperServerMainTest extends ZKTestCase implements Watcher {
                 if (!logDir.mkdir()) {
                     throw new IOException("unable to mkdir " + logDir);
                 }
+                ClientBase.createInitializeFile(logDir);
             }
             
             String normalizedDataDir = PathUtils.normalizeFileSystemPath(dataDir.toString());
@@ -318,71 +319,6 @@ public class ZooKeeperServerMainTest extends ZKTestCase implements Watcher {
                         ClientBase.CONNECTION_TIMEOUT));
     }
 
-    /**
-     * Test verifies server should fail when data dir or data log dir doesn't
-     * exists. Sets "zookeeper.datadir.autocreate" to false.
-     */
-    @Test(timeout = 30000)
-    public void testWithoutAutoCreateDataLogDir() throws Exception {
-        ClientBase.setupTestEnv();
-        System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE, "false");
-        try {
-            final int CLIENT_PORT = PortAssignment.unique();
-
-            MainThread main = new MainThread(CLIENT_PORT, false, null);
-            String args[] = new String[1];
-            args[0] = main.confFile.toString();
-            main.start();
-
-            Assert.assertFalse("waiting for server being up", ClientBase
-                    .waitForServerUp("127.0.0.1:" + CLIENT_PORT,
-                            CONNECTION_TIMEOUT / 2));
-        } finally {
-            // resets "zookeeper.datadir.autocreate" flag
-            System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE,
-                    FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE_DEFAULT);
-        }
-    }
-
-    /**
-     * Test verifies the auto creation of data dir and data log dir.
-     * Sets "zookeeper.datadir.autocreate" to true.
-     */
-    @Test(timeout = 30000)
-    public void testWithAutoCreateDataLogDir() throws Exception {
-        ClientBase.setupTestEnv();
-        System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE, "true");
-        final int CLIENT_PORT = PortAssignment.unique();
-
-        MainThread main = new MainThread(CLIENT_PORT, false, null);
-        String args[] = new String[1];
-        args[0] = main.confFile.toString();
-        main.start();
-
-        Assert.assertTrue("waiting for server being up",
-                ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT,
-                        CONNECTION_TIMEOUT));
-        clientConnected = new CountDownLatch(1);
-        ZooKeeper zk = new ZooKeeper("127.0.0.1:" + CLIENT_PORT,
-                ClientBase.CONNECTION_TIMEOUT, this);
-        Assert.assertTrue("Failed to establish zkclient connection!",
-                clientConnected.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS));
-
-        zk.create("/foo", "foobar".getBytes(), Ids.OPEN_ACL_UNSAFE,
-                CreateMode.PERSISTENT);
-        Assert.assertEquals(new String(zk.getData("/foo", null, null)),
-                "foobar");
-        zk.close();
-
-        main.shutdown();
-        main.join();
-        main.deleteDirs();
-
-        Assert.assertTrue("waiting for server down", ClientBase
-                .waitForServerDown("127.0.0.1:" + CLIENT_PORT,
-                        ClientBase.CONNECTION_TIMEOUT));
-    }
-
     /**
      * Test verifies that the server shouldn't allow minsessiontimeout >
      * maxsessiontimeout
@@ -398,7 +334,7 @@ public class ZooKeeperServerMainTest extends ZKTestCase implements Watcher {
         final int maxSessionTimeout = tickTime * 2 - 100; // max is lower
         final String configs = "maxSessionTimeout=" + maxSessionTimeout + "\n"
                 + "minSessionTimeout=" + minSessionTimeout + "\n";
-        MainThread main = new MainThread(CLIENT_PORT, false, configs);
+        MainThread main = new MainThread(CLIENT_PORT, true, configs);
         String args[] = new String[1];
         args[0] = main.confFile.toString();
         try {
@@ -423,7 +359,7 @@ public class ZooKeeperServerMainTest extends ZKTestCase implements Watcher {
         final int minSessionTimeout = tickTime * 2 - 100;
         int maxSessionTimeout = 20 * tickTime;
         final String configs = "minSessionTimeout=" + minSessionTimeout + "\n";
-        MainThread main = new MainThread(CLIENT_PORT, false, configs);
+        MainThread main = new MainThread(CLIENT_PORT, true, configs);
         main.start();
 
         String HOSTPORT = "127.0.0.1:" + CLIENT_PORT;
@@ -456,7 +392,7 @@ public class ZooKeeperServerMainTest extends ZKTestCase implements Watcher {
         final int maxSessionTimeout = 20 * tickTime + 1000;
         final String configs = "maxSessionTimeout=" + maxSessionTimeout + "\n"
                 + "minSessionTimeout=" + minSessionTimeout + "\n";
-        MainThread main = new MainThread(CLIENT_PORT, false, configs);
+        MainThread main = new MainThread(CLIENT_PORT, true, configs);
         main.start();
 
         String HOSTPORT = "127.0.0.1:" + CLIENT_PORT;

+ 139 - 0
src/java/test/org/apache/zookeeper/server/persistence/FileTxnSnapLogTest.java

@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.persistence;
+
+import org.apache.jute.Record;
+import org.apache.zookeeper.server.DataTree;
+import org.apache.zookeeper.test.ClientBase;
+import org.apache.zookeeper.txn.TxnHeader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class FileTxnSnapLogTest {
+
+    /**
+     * Test verifies the auto creation of data dir and data log dir.
+     * Sets "zookeeper.datadir.autocreate" to true.
+     */
+    @Test
+    public void testWithAutoCreateDataLogDir() throws IOException {
+        File tmpDir = ClientBase.createEmptyTestDir();
+        File dataDir = new File(tmpDir, "data");
+        File snapDir = new File(tmpDir, "data_txnlog");
+        Assert.assertFalse("data directory already exists", dataDir.exists());
+        Assert.assertFalse("snapshot directory already exists", snapDir.exists());
+
+        String priorAutocreateDirValue = System.getProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE);
+        System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE, "true");
+        FileTxnSnapLog fileTxnSnapLog;
+        try {
+            fileTxnSnapLog = new FileTxnSnapLog(dataDir, snapDir);
+        } finally {
+            if (priorAutocreateDirValue == null) {
+                System.clearProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE);
+            } else {
+                System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE, priorAutocreateDirValue);
+            }
+        }
+        Assert.assertTrue(dataDir.exists());
+        Assert.assertTrue(snapDir.exists());
+        Assert.assertTrue(fileTxnSnapLog.getDataDir().exists());
+        Assert.assertTrue(fileTxnSnapLog.getSnapDir().exists());
+    }
+
+    /**
+     * Test verifies server should fail when data dir or data log dir doesn't
+     * exists. Sets "zookeeper.datadir.autocreate" to false.
+     */
+    @Test
+    public void testWithoutAutoCreateDataLogDir() throws Exception {
+        File tmpDir = ClientBase.createEmptyTestDir();
+        File dataDir = new File(tmpDir, "data");
+        File snapDir = new File(tmpDir, "data_txnlog");
+        Assert.assertFalse("data directory already exists", dataDir.exists());
+        Assert.assertFalse("snapshot directory already exists", snapDir.exists());
+
+        String priorAutocreateDirValue = System.getProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE);
+        System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE, "false");
+        try {
+            FileTxnSnapLog fileTxnSnapLog = new FileTxnSnapLog(dataDir, snapDir);
+        } catch (FileTxnSnapLog.DatadirException e) {
+            Assert.assertFalse(dataDir.exists());
+            Assert.assertFalse(snapDir.exists());
+            return;
+        } finally {
+            if (priorAutocreateDirValue == null) {
+                System.clearProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE);
+            } else {
+                System.setProperty(FileTxnSnapLog.ZOOKEEPER_DATADIR_AUTOCREATE, priorAutocreateDirValue);
+            }
+        }
+        Assert.fail("Expected exception from FileTxnSnapLog");
+    }
+
+    @Test
+    public void testAutoCreateDb() throws IOException {
+        File tmpDir = ClientBase.createEmptyTestDir();
+        File dataDir = new File(tmpDir, "data");
+        File snapDir = new File(tmpDir, "data_txnlog");
+        Assert.assertTrue("cannot create data directory", dataDir.mkdir());
+        Assert.assertTrue("cannot create snapshot directory", snapDir.mkdir());
+        File initFile = new File(dataDir, "initialize");
+        Assert.assertFalse("initialize file already exists", initFile.exists());
+
+        String priorAutocreateDbValue = System.getProperty(FileTxnSnapLog.ZOOKEEPER_DB_AUTOCREATE);
+        Map<Long, Integer> sessions = new ConcurrentHashMap<>();
+
+        attemptAutoCreateDb(dataDir, snapDir, sessions, priorAutocreateDbValue, "false", -1L);
+
+        attemptAutoCreateDb(dataDir, snapDir, sessions, priorAutocreateDbValue, "true", 0L);
+
+        Assert.assertTrue("cannot create initialize file", initFile.createNewFile());
+        attemptAutoCreateDb(dataDir, snapDir, sessions, priorAutocreateDbValue, "false", 0L);
+    }
+
+    private void attemptAutoCreateDb(File dataDir, File snapDir, Map<Long, Integer> sessions,
+                                     String priorAutocreateDbValue, String autoCreateValue,
+                                     long expectedValue) throws IOException {
+        sessions.clear();
+        System.setProperty(FileTxnSnapLog.ZOOKEEPER_DB_AUTOCREATE, autoCreateValue);
+        FileTxnSnapLog fileTxnSnapLog = new FileTxnSnapLog(dataDir, snapDir);
+
+        try {
+            long zxid = fileTxnSnapLog.restore(new DataTree(), sessions, new FileTxnSnapLog.PlayBackListener() {
+                @Override
+                public void onTxnLoaded(TxnHeader hdr, Record rec) {
+                    // empty by default
+                }
+            });
+            Assert.assertEquals("unexpected zxid", expectedValue, zxid);
+        } finally {
+            if (priorAutocreateDbValue == null) {
+                System.clearProperty(FileTxnSnapLog.ZOOKEEPER_DB_AUTOCREATE);
+            } else {
+                System.setProperty(FileTxnSnapLog.ZOOKEEPER_DB_AUTOCREATE, priorAutocreateDbValue);
+            }
+        }
+    }
+}

+ 6 - 3
src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java

@@ -186,6 +186,8 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
             fwriter.write(Integer.toString(myid));
             fwriter.flush();
             fwriter.close();
+
+            ClientBase.createInitializeFile(dataDir);
         }
 
         private String createDynamicFile(String quorumCfgSection, String version)
@@ -280,9 +282,10 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
             return t != null && t.isAlive();
         }
 
-        public void clean() {
-            ClientBase.recursiveDelete(main.quorumPeer.getTxnFactory()
-                    .getDataDir());
+        public void reinitialize() throws IOException {
+            File dataDir = main.quorumPeer.getTxnFactory().getDataDir();
+            ClientBase.recursiveDelete(dataDir);
+            ClientBase.createInitializeFile(dataDir.getParentFile());
         }
 
         public boolean isQuorumPeerRunning() {

+ 2 - 0
src/java/test/org/apache/zookeeper/server/quorum/Zab1_0Test.java

@@ -59,6 +59,7 @@ import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.apache.zookeeper.server.quorum.flexible.QuorumMaj;
 import org.apache.zookeeper.server.util.ZxidUtils;
+import org.apache.zookeeper.test.ClientBase;
 import org.apache.zookeeper.test.TestUtils;
 import org.apache.zookeeper.txn.CreateSessionTxn;
 import org.apache.zookeeper.txn.CreateTxn;
@@ -1315,6 +1316,7 @@ public class Zab1_0Test extends ZKTestCase {
         peer.setCnxnFactory(new NullServerCnxnFactory());
         File version2 = new File(tmpDir, "version-2");
         version2.mkdir();
+        ClientBase.createInitializeFile(tmpDir);
         FileOutputStream fos;
         fos = new FileOutputStream(new File(version2, "currentEpoch"));
         fos.write("0\n".getBytes());

+ 1 - 1
src/java/test/org/apache/zookeeper/test/AtomicFileOutputStreamTest.java

@@ -43,7 +43,7 @@ public class AtomicFileOutputStreamTest extends ZKTestCase {
 
     @Before
     public void setupTestDir() throws IOException {
-        testDir = ClientBase.createTmpDir();
+        testDir = ClientBase.createEmptyTestDir();
         dstFile = new File(testDir, "test.txt");
     }
     @After

+ 18 - 3
src/java/test/org/apache/zookeeper/test/ClientBase.java

@@ -325,12 +325,15 @@ public abstract class ClientBase extends ZKTestCase {
         }
     }
 
+    public static File createEmptyTestDir() throws IOException {
+        return createTmpDir(BASETEST, false);
+    }
 
     public static File createTmpDir() throws IOException {
-        return createTmpDir(BASETEST);
+        return createTmpDir(BASETEST, true);
     }
 
-    static File createTmpDir(File parentDir) throws IOException {
+    static File createTmpDir(File parentDir, boolean createInitFile) throws IOException {
         File tmpFile = File.createTempFile("test", ".junit", parentDir);
         // don't delete tmpFile - this ensures we don't attempt to create
         // a tmpDir with a duplicate name
@@ -338,9 +341,21 @@ public abstract class ClientBase extends ZKTestCase {
         Assert.assertFalse(tmpDir.exists()); // never true if tmpfile does it's job
         Assert.assertTrue(tmpDir.mkdirs());
 
+        // todo not every tmp directory needs this file
+        if (createInitFile) {
+            createInitializeFile(tmpDir);
+        }
+
         return tmpDir;
     }
 
+    public static void createInitializeFile(File dir) throws IOException {
+        File initFile = new File(dir, "initialize");
+        if (!initFile.exists()) {
+            Assert.assertTrue(initFile.createNewFile());
+        }
+    }
+
     private static int getPort(String hostPort) {
         String[] split = hostPort.split(":");
         String portstr = split[split.length-1];
@@ -455,7 +470,7 @@ public abstract class ClientBase extends ZKTestCase {
 
         setUpAll();
 
-        tmpDir = createTmpDir(BASETEST);
+        tmpDir = createTmpDir(BASETEST, true);
 
         startServer();
 

+ 7 - 5
src/java/test/org/apache/zookeeper/test/QuorumZxidSyncTest.java

@@ -19,6 +19,7 @@
 package org.apache.zookeeper.test;
 
 import java.io.File;
+import java.io.IOException;
 
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.WatchedEvent;
@@ -75,10 +76,10 @@ public class QuorumZxidSyncTest extends ZKTestCase {
         zk.create("/2", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
         zk.close();
         qb.shutdownServers();
-        deleteFiles(qb.s1dir);
-        deleteFiles(qb.s2dir);
-        deleteFiles(qb.s3dir);
-        deleteFiles(qb.s4dir);
+        cleanAndInitializeDataDir(qb.s1dir);
+        cleanAndInitializeDataDir(qb.s2dir);
+        cleanAndInitializeDataDir(qb.s3dir);
+        cleanAndInitializeDataDir(qb.s4dir);
         qb.setupServers();
         qb.s1.start();
         qb.s2.start();
@@ -91,11 +92,12 @@ public class QuorumZxidSyncTest extends ZKTestCase {
                 ClientBase.waitForServerUp(hostPort, 10000));
     }
 
-    private void deleteFiles(File f) {
+    private void cleanAndInitializeDataDir(File f) throws IOException {
         File v = new File(f, "version-2");
         for(File c: v.listFiles()) {
             c.delete();
         }
+        ClientBase.createInitializeFile(f);
     }
 
     /**