ZOOKEEPER-4730: Incorrect datadir and logdir size reported from admin and and 4lw dirs command

Reviewers: kezhuw, eolivelli, maoling
Author: li4wang
Closes #2043 from li4wang/ZOOKEEPER-4730

zookeeper-server/src/main/java/org/apache/zookeeper/server/PurgeTxnLog.java

@@ -135,7 +135,7 @@ public class PurgeTxnLog {
 
         }
         // add all non-excluded log files
-        File[] logs = txnLog.getDataDir().listFiles(new MyFileFilter(PREFIX_LOG));
+        File[] logs = txnLog.getDataLogDir().listFiles(new MyFileFilter(PREFIX_LOG));
         List<File> files = new ArrayList<>();
         if (logs != null) {
             files.addAll(Arrays.asList(logs));

zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java

@@ -388,13 +388,13 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
                 + " minSessionTimeout {} ms"
                 + " maxSessionTimeout {} ms"
                 + " clientPortListenBacklog {}"
-                + " datadir {}"
+                + " dataLogdir {}"
                 + " snapdir {}",
             tickTime,
             getMinSessionTimeout(),
             getMaxSessionTimeout(),
             getClientPortListenBacklog(),
-            txnLogFactory.getDataDir(),
+            txnLogFactory.getDataLogDir(),
             txnLogFactory.getSnapDir());
     }
 
@@ -447,7 +447,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         pwriter.print("dataDirSize=");
         pwriter.println(getDataDirSize());
         pwriter.print("dataLogDir=");
-        pwriter.println(zkDb.snapLog.getDataDir().getAbsolutePath());
+        pwriter.println(zkDb.snapLog.getDataLogDir().getAbsolutePath());
         pwriter.print("dataLogSize=");
         pwriter.println(getLogDirSize());
         pwriter.print("tickTime=");
@@ -469,7 +469,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         return new ZooKeeperServerConf(
             getClientPort(),
             zkDb.snapLog.getSnapDir().getAbsolutePath(),
-            zkDb.snapLog.getDataDir().getAbsolutePath(),
+            zkDb.snapLog.getDataLogDir().getAbsolutePath(),
             getTickTime(),
             getMaxClientCnxnsPerHost(),
             getMinSessionTimeout(),
@@ -664,7 +664,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         if (zkDb == null) {
             return 0L;
         }
-        File path = zkDb.snapLog.getDataDir();
+        File path = zkDb.snapLog.getDataLogDir();
         return getDirSize(path);
     }
 

zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java

@@ -204,11 +204,11 @@ public class FileTxnSnapLog {
     }
 
     /**
-     * get the datadir used by this filetxn
+     * get the data log dir used by this filetxn
      * snap log
-     * @return the data dir
+     * @return the data log dir
      */
-    public File getDataDir() {
+    public File getDataLogDir() {
         return this.dataDir;
     }
 

zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerMainTest.java

@@ -198,7 +198,7 @@ public class ZooKeeperServerMainTest extends ZKTestCase implements Watcher {
         // inject problem in server
         ZooKeeperServer zooKeeperServer = main.getCnxnFactory().getZooKeeperServer();
         FileTxnSnapLog snapLog = zooKeeperServer.getTxnLogFactory();
-        FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(snapLog.getDataDir(), snapLog.getSnapDir()) {
+        FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(snapLog.getDataLogDir(), snapLog.getSnapDir()) {
             @Override
             public void commit() throws IOException {
                 throw new IOException("Input/output error");

zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java

@@ -29,6 +29,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.zookeeper.PortAssignment;
 import org.apache.zookeeper.ZKTestCase;
 import org.apache.zookeeper.metrics.MetricsUtils;
 import org.apache.zookeeper.proto.ConnectRequest;
@@ -42,6 +43,45 @@ import org.junit.jupiter.api.Test;
 
 public class ZooKeeperServerTest extends ZKTestCase {
 
+    @Test
+    public void testDirSize() throws Exception {
+        ZooKeeperServer zks = null;
+        ServerCnxnFactory cnxnFactory = null;
+
+        try {
+            final File dataDir = ClientBase.createTmpDir();
+            final File logDir = ClientBase.createTmpDir();
+
+            zks = new ZooKeeperServer(dataDir, logDir, 3000);
+
+            // validate dir size before server starts
+            assertEquals(0, zks.getDataDirSize());
+            assertEquals(0, zks.getLogDirSize());
+
+            // start server
+            final String hostPort = "127.0.0.1:" + PortAssignment.unique();
+            final int port = Integer.parseInt(hostPort.split(":")[1]);
+            cnxnFactory = ServerCnxnFactory.createFactory(port, -1);
+            cnxnFactory.startup(zks);
+            assertTrue(ClientBase.waitForServerUp(hostPort, 120000));
+
+            // validate data size is greater than 0 as snapshot has been taken when server starts
+            assertTrue(zks.getDataDirSize() > 0);
+
+            // validate log size is 0 as no txn yet
+            assertEquals(0, zks.getLogDirSize());
+        } finally {
+            if (cnxnFactory != null) {
+                cnxnFactory.shutdown();
+            }
+
+            if (zks != null) {
+                zks.shutdown();
+            }
+        }
+    }
+
+
     @Test
     public void testSortDataDirAscending() {
         File[] files = new File[5];

zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/FileTxnSnapLogTest.java

@@ -177,7 +177,7 @@ public class FileTxnSnapLogTest {
 
         assertTrue(logDir.exists());
         assertTrue(snapDir.exists());
-        assertTrue(fileTxnSnapLog.getDataDir().exists());
+        assertTrue(fileTxnSnapLog.getDataLogDir().exists());
         assertTrue(fileTxnSnapLog.getSnapDir().exists());
     }
 

zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java

@@ -935,7 +935,7 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase {
 
             // Assert
             FileTxnSnapLog txnFactory = qpMain.getQuorumPeer().getTxnFactory();
-            assertEquals(Paths.get(dataLogDir.getAbsolutePath(), "version-2").toString(), txnFactory.getDataDir().getAbsolutePath());
+            assertEquals(Paths.get(dataLogDir.getAbsolutePath(), "version-2").toString(), txnFactory.getDataLogDir().getAbsolutePath());
             assertEquals(Paths.get(dataDir.getAbsolutePath(), "version-2").toString(), txnFactory.getSnapDir().getAbsolutePath());
         } finally {
             FileUtils.deleteDirectory(dataDir);

zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java

@@ -368,7 +368,7 @@ public class QuorumPeerTestBase extends ZKTestCase implements Watcher {
         }
 
         public void reinitialize() throws IOException {
-            File dataDir = main.quorumPeer.getTxnFactory().getDataDir();
+            File dataDir = main.quorumPeer.getTxnFactory().getDataLogDir();
             ClientBase.recursiveDelete(dataDir);
             ClientBase.createInitializeFile(dataDir.getParentFile());
         }

zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WatchLeakTest.java

@@ -102,7 +102,7 @@ public class WatchLeakTest {
         QuorumPeer quorumPeer = mock(QuorumPeer.class);
         FileTxnSnapLog logfactory = mock(FileTxnSnapLog.class);
         // Directories are not used but we need it to avoid NPE
-        when(logfactory.getDataDir()).thenReturn(new File(""));
+        when(logfactory.getDataLogDir()).thenReturn(new File(""));
         when(logfactory.getSnapDir()).thenReturn(new File(""));
         FollowerZooKeeperServer fzks = null;
 

zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java

@@ -575,7 +575,7 @@ public class Zab1_0Test extends ZKTestCase {
                 File tmpDir = File.createTempFile("test", "dir", testData);
                 tmpDir.delete();
                 tmpDir.mkdir();
-                File logDir = f.fzk.getTxnLogFactory().getDataDir().getParentFile();
+                File logDir = f.fzk.getTxnLogFactory().getDataLogDir().getParentFile();
                 File snapDir = f.fzk.getTxnLogFactory().getSnapDir().getParentFile();
                 //Spy on ZK so we can check if a snapshot happened or not.
                 f.zk = spy(f.zk);
@@ -709,7 +709,7 @@ public class Zab1_0Test extends ZKTestCase {
                 File tmpDir = File.createTempFile("test", "dir", testData);
                 tmpDir.delete();
                 tmpDir.mkdir();
-                File logDir = f.fzk.getTxnLogFactory().getDataDir().getParentFile();
+                File logDir = f.fzk.getTxnLogFactory().getDataLogDir().getParentFile();
                 File snapDir = f.fzk.getTxnLogFactory().getSnapDir().getParentFile();
                 //Spy on ZK so we can check if a snapshot happened or not.
                 f.zk = spy(f.zk);
@@ -940,7 +940,7 @@ public class Zab1_0Test extends ZKTestCase {
                 File tmpDir = File.createTempFile("test", "dir", testData);
                 tmpDir.delete();
                 tmpDir.mkdir();
-                File logDir = o.zk.getTxnLogFactory().getDataDir().getParentFile();
+                File logDir = o.zk.getTxnLogFactory().getDataLogDir().getParentFile();
                 File snapDir = o.zk.getTxnLogFactory().getSnapDir().getParentFile();
                 try {
                     assertEquals(0, o.self.getAcceptedEpoch());

zookeeper-server/src/test/java/org/apache/zookeeper/test/NonRecoverableErrorTest.java

@@ -93,7 +93,7 @@ public class NonRecoverableErrorTest extends QuorumPeerTestBase {
 
         // inject problem in leader
         FileTxnSnapLog snapLog = leader.getActiveServer().getTxnLogFactory();
-        FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(snapLog.getDataDir(), snapLog.getSnapDir()) {
+        FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(snapLog.getDataLogDir(), snapLog.getSnapDir()) {
             @Override
             public void commit() throws IOException {
                 throw new IOException("Input/output error");

zookeeper-server/src/test/java/org/apache/zookeeper/test/RepeatStartupTest.java

@@ -49,7 +49,7 @@ public class RepeatStartupTest extends ZKTestCase {
         QuorumBase.shutdown(qb.s4);
         QuorumBase.shutdown(qb.s5);
         String hp = qb.hostPort.split(",")[0];
-        ZooKeeperServer zks = new ZooKeeperServer(qb.s1.getTxnFactory().getSnapDir(), qb.s1.getTxnFactory().getDataDir(), 3000);
+        ZooKeeperServer zks = new ZooKeeperServer(qb.s1.getTxnFactory().getSnapDir(), qb.s1.getTxnFactory().getDataLogDir(), 3000);
         final int PORT = Integer.parseInt(hp.split(":")[1]);
         ServerCnxnFactory factory = ServerCnxnFactory.createFactory(PORT, -1);
 

zookeeper-server/src/test/java/org/apache/zookeeper/test/TruncateTest.java

@@ -121,7 +121,7 @@ public class TruncateTest extends ZKTestCase {
             append(zkdb, i);
         }
         zkdb.close();
-        File[] logs = snaplog.getDataDir().listFiles();
+        File[] logs = snaplog.getDataLogDir().listFiles();
         for (int i = 0; i < logs.length; i++) {
             LOG.debug("Deleting: {}", logs[i].getName());
             assertTrue(logs[i].delete(), "Failed to delete log file: " + logs[i].getName());