HDFS-2102. Zero-pad edits filename to make them lexically sortable. Contributed by Ivan Kelly.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1073@1141752 13f79535-47bb-0310-9956-ffa450edef68

hdfs/CHANGES.HDFS-1073.txt

@@ -62,3 +62,5 @@ HDFS-2078. NameNode should not clear directory when restoring removed storage.
 HDFS-2088. Move edits log archiving logic into FSEditLog/JournalManager (todd)
 HDFS-2093. Handle case where an entirely empty log is left during NN crash
            (todd)
+HDFS-2102. Zero-pad edits filename to make them lexically sortable. (Ivan
+           Kelly via todd)

hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java

@@ -626,7 +626,7 @@ public class NNStorage extends Storage implements Closeable {
   
   static File getStorageFile(StorageDirectory sd, NameNodeFile type, long imageTxId) {
     return new File(sd.getCurrentDir(),
-        type.getName() + "_" + imageTxId);
+                    String.format("%s_%019d", type.getName(), imageTxId));
   }
   
   /**
@@ -637,16 +637,22 @@ public class NNStorage extends Storage implements Closeable {
     return new File(sd.getCurrentDir(), type.getName());
   }
 
-  static String getCheckpointImageFileName(long txid) {
-    return NameNodeFile.IMAGE_NEW.getName() + "_" + txid;
+  @VisibleForTesting
+  public static String getCheckpointImageFileName(long txid) {
+    return String.format("%s_%019d",
+                         NameNodeFile.IMAGE_NEW.getName(), txid);
   }
 
-  static String getImageFileName(long txid) {
-    return NameNodeFile.IMAGE.getName() + "_" + txid;
+  @VisibleForTesting
+  public static String getImageFileName(long txid) {
+    return String.format("%s_%019d",
+                         NameNodeFile.IMAGE.getName(), txid);
   }
   
-  static String getInProgressEditsFileName(long startTxId) {
-    return NameNodeFile.EDITS_INPROGRESS.getName() + "_" + startTxId;
+  @VisibleForTesting
+  public static String getInProgressEditsFileName(long startTxId) {
+    return String.format("%s_%019d", NameNodeFile.EDITS_INPROGRESS.getName(),
+                         startTxId);
   }
   
   static File getInProgressEditsFile(StorageDirectory sd, long startTxId) {
@@ -664,8 +670,10 @@ public class NNStorage extends Storage implements Closeable {
         getImageFileName(txid));
   }
   
-  static String getFinalizedEditsFileName(long startTxId, long endTxId) {
-    return NameNodeFile.EDITS.getName() + "_" + startTxId + "-" + endTxId;
+  @VisibleForTesting
+  public static String getFinalizedEditsFileName(long startTxId, long endTxId) {
+    return String.format("%s_%019d-%019d", NameNodeFile.EDITS.getName(),
+                         startTxId, endTxId);
   }
   
   /**

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java

@@ -34,6 +34,9 @@ import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
+
 import org.junit.Test;
 
 import com.google.common.base.Joiner;
@@ -79,8 +82,10 @@ public class TestDFSUpgrade {
       
       assertTrue(new File(baseDir,"current").isDirectory());
       assertTrue(new File(baseDir,"current/VERSION").isFile());
-      assertTrue(new File(baseDir,"current/edits_inprogress_" + (imageTxId + 1)).isFile());
-      assertTrue(new File(baseDir,"current/fsimage_" + imageTxId).isFile());
+      assertTrue(new File(baseDir,"current/" 
+                          + getInProgressEditsFileName(imageTxId + 1)).isFile());
+      assertTrue(new File(baseDir,"current/" 
+                          + getImageFileName(imageTxId)).isFile());
       assertTrue(new File(baseDir,"current/seen_txid").isFile());
       
       File previous = new File(baseDir, "previous");

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java

@@ -799,7 +799,9 @@ public class TestCheckpoint extends TestCase {
       Collection<URI> editsDirs = cluster.getNameEditsDirs(0);
       for(URI uri : editsDirs) {
         File ed = new File(uri.getPath());
-        assertTrue(new File(ed, "current/edits_inprogress_1").length() > Integer.SIZE/Byte.SIZE);
+        assertTrue(new File(ed, "current/"
+                            + NNStorage.getInProgressEditsFileName(1))
+                   .length() > Integer.SIZE/Byte.SIZE);
       }
 
       // Saving image in safe mode should succeed
@@ -821,19 +823,23 @@ public class TestCheckpoint extends TestCase {
         System.err.println("Files in " + curDir + ":\n  " +
             Joiner.on("\n  ").join(curDir.list()));
         // Verify that the first edits file got finalized
-        File originalEdits = new File(curDir, "edits_inprogress_1");
+        File originalEdits = new File(curDir,
+                                      NNStorage.getInProgressEditsFileName(1));
         assertFalse(originalEdits.exists());
-        File finalizedEdits = new File(curDir, "edits_1-8");
+        File finalizedEdits = new File(curDir,
+            NNStorage.getFinalizedEditsFileName(1,8));
         assertTrue(finalizedEdits.exists());
         assertTrue(finalizedEdits.length() > Integer.SIZE/Byte.SIZE);
 
-        assertTrue(new File(ed, "current/edits_inprogress_9").exists());
+        assertTrue(new File(ed, "current/"
+                       + NNStorage.getInProgressEditsFileName(9)).exists());
       }
       
       Collection<URI> imageDirs = cluster.getNameDirs(0);
       for (URI uri : imageDirs) {
         File imageDir = new File(uri.getPath());
-        File savedImage = new File(imageDir, "current/fsimage_8");
+        File savedImage = new File(imageDir, "current/"
+                                   + NNStorage.getImageFileName(8));
         assertTrue("Should have saved image at " + savedImage,
             savedImage.exists());        
       }
@@ -1025,9 +1031,9 @@ public class TestCheckpoint extends TestCase {
       .getStorage().getMostRecentCheckpointTxId();
 
       File secondaryFsImageBefore = new File(secondaryCurrent,
-          "fsimage_" + expectedTxIdToDownload);
+          NNStorage.getImageFileName(expectedTxIdToDownload));
       File secondaryFsImageAfter = new File(secondaryCurrent,
-          "fsimage_" + (expectedTxIdToDownload + 2));
+          NNStorage.getImageFileName(expectedTxIdToDownload + 2));
       
       assertFalse("Secondary should start with empty current/ dir " +
           "but " + secondaryFsImageBefore + " exists",
@@ -1541,7 +1547,8 @@ public class TestCheckpoint extends TestCase {
     for (File nameDir : getNameNodeCurrentDirs(cluster)) {
       // Should have fsimage_N for the three checkpoints
       for (long checkpointTxId : txids) {
-        File image = new File(nameDir, "fsimage_" + checkpointTxId);
+        File image = new File(nameDir,
+                              NNStorage.getImageFileName(checkpointTxId));
         assertTrue("Expected non-empty " + image, image.length() > 0);
       }
     }

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java

@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.StringUtils;
@@ -194,7 +195,8 @@ public class TestEditLog extends TestCase {
       final FSEditLog editLog = fsimage.getEditLog();
       
       assertExistsInStorageDirs(
-          cluster, NameNodeDirType.EDITS, "edits_inprogress_1");
+          cluster, NameNodeDirType.EDITS, 
+          NNStorage.getInProgressEditsFileName(1));
       
 
       editLog.logSetReplication("fakefile", (short) 1);
@@ -203,9 +205,11 @@ public class TestEditLog extends TestCase {
       editLog.rollEditLog();
 
       assertExistsInStorageDirs(
-          cluster, NameNodeDirType.EDITS, "edits_1-3");
+          cluster, NameNodeDirType.EDITS,
+          NNStorage.getFinalizedEditsFileName(1,3));
       assertExistsInStorageDirs(
-          cluster, NameNodeDirType.EDITS, "edits_inprogress_4");
+          cluster, NameNodeDirType.EDITS,
+          NNStorage.getInProgressEditsFileName(4));
 
       
       editLog.logSetReplication("fakefile", (short) 2);
@@ -569,13 +573,14 @@ public class TestEditLog extends TestCase {
         File currentDir = new File(nameDir, "current");
 
         // We should see the file as in-progress
-        File editsFile = new File(currentDir, "edits_inprogress_1");
+        File editsFile = new File(currentDir,
+            NNStorage.getInProgressEditsFileName(1));
         assertTrue("Edits file " + editsFile + " should exist", editsFile.exists());        
         
         File imageFile = FSImageTestUtil.findNewestImageFile(
             currentDir.getAbsolutePath());
         assertNotNull("No image found in " + nameDir, imageFile);
-        assertEquals("fsimage_0", imageFile.getName());
+        assertEquals(NNStorage.getImageFileName(0), imageFile.getName());
         
         // Try to start a new cluster
         LOG.info("\n===========================================\n" +
@@ -598,7 +603,8 @@ public class TestEditLog extends TestCase {
         imageFile = FSImageTestUtil.findNewestImageFile(
             currentDir.getAbsolutePath());
         assertNotNull("No image found in " + nameDir, imageFile);
-        assertEquals("fsimage_" + expectedTxId, imageFile.getName());
+        assertEquals(NNStorage.getImageFileName(expectedTxId),
+                     imageFile.getName());
         
         // Started successfully
         cluster.shutdown();    
@@ -645,7 +651,7 @@ public class TestEditLog extends TestCase {
       File currentDir = new File(dir, "current");
       // We should start with only the finalized edits_1-2
       GenericTestUtils.assertGlobEquals(currentDir, "edits_.*",
-          "edits_1-2");
+          NNStorage.getFinalizedEditsFileName(1, 2));
       // Make a truncated edits_3_inprogress
       File log = new File(currentDir,
           NNStorage.getInProgressEditsFileName(3));

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java

@@ -32,6 +32,10 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
+
 import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.TransactionalLoadPlan;
@@ -55,14 +59,14 @@ public class TestFSImageStorageInspector {
     StorageDirectory mockDir = mockDirectory(
         NameNodeDirType.IMAGE_AND_EDITS,
         false,
-        "/foo/current/fsimage_123",
-        "/foo/current/edits_123-456",
-        "/foo/current/fsimage_456",
-        "/foo/current/edits_inprogress_457");
+        "/foo/current/" + getImageFileName(123),
+        "/foo/current/" + getFinalizedEditsFileName(123, 456),
+        "/foo/current/" + getImageFileName(456),
+        "/foo/current/" + getInProgressEditsFileName(457));
 
     inspector.inspectDirectory(mockDir);
     mockLogValidation(inspector,
-        "/foo/current/edits_inprogress_457", 10);
+        "/foo/current/" + getInProgressEditsFileName(457), 10);
     
     assertEquals(2, inspector.foundEditLogs.size());
     assertEquals(2, inspector.foundImages.size());
@@ -76,9 +80,10 @@ public class TestFSImageStorageInspector {
     LoadPlan plan = inspector.createLoadPlan();
     LOG.info("Plan: " + plan);
     
-    assertEquals(new File("/foo/current/fsimage_456"), plan.getImageFile());
+    assertEquals(new File("/foo/current/"+getImageFileName(456)), 
+                 plan.getImageFile());
     assertArrayEquals(new File[] {
-        new File("/foo/current/edits_inprogress_457") },
+        new File("/foo/current/" + getInProgressEditsFileName(457)) },
         plan.getEditsFiles().toArray(new File[0]));
   }
   
@@ -93,11 +98,11 @@ public class TestFSImageStorageInspector {
     StorageDirectory mockDir = mockDirectory(
         NameNodeDirType.IMAGE_AND_EDITS,
         false,
-        "/foo/current/fsimage_123",
-        "/foo/current/fsimage_456",
-        "/foo/current/edits_457-900",
-        "/foo/current/edits_901-950",
-        "/foo/current/edits_952-1000"); // <-- missing edit 951!
+        "/foo/current/" + getImageFileName(123),
+        "/foo/current/" + getImageFileName(456),
+        "/foo/current/" + getFinalizedEditsFileName(457,900),
+        "/foo/current/" + getFinalizedEditsFileName(901,950),
+        "/foo/current/" + getFinalizedEditsFileName(952,1000)); // <-- missing edit 951!
 
     inspector.inspectDirectory(mockDir);
     try {
@@ -121,24 +126,25 @@ public class TestFSImageStorageInspector {
     StorageDirectory mockDir = mockDirectory(
         NameNodeDirType.IMAGE_AND_EDITS,
         false,
-        "/foo/current/fsimage_123",
-        "/foo/current/fsimage_456",
-        "/foo/current/edits_457-900",
-        "/foo/current/edits_inprogress_901", // <-- inprogress in middle
-        "/foo/current/edits_952-1000");
+        "/foo/current/" + getImageFileName(123),
+        "/foo/current/" + getImageFileName(456),
+        "/foo/current/" + getFinalizedEditsFileName(457,900),
+        "/foo/current/" + getInProgressEditsFileName(901), // <-- inprogress in middle
+        "/foo/current/" + getFinalizedEditsFileName(952,1000));
 
     inspector.inspectDirectory(mockDir);
     mockLogValidation(inspector,
-        "/foo/current/edits_inprogress_901", 51);
+        "/foo/current/" + getInProgressEditsFileName(901), 51);
 
     LoadPlan plan = inspector.createLoadPlan();
     LOG.info("Plan: " + plan);
     
-    assertEquals(new File("/foo/current/fsimage_456"), plan.getImageFile());
+    assertEquals(new File("/foo/current/" + getImageFileName(456)), 
+                 plan.getImageFile());
     assertArrayEquals(new File[] {
-        new File("/foo/current/edits_457-900"),
-        new File("/foo/current/edits_inprogress_901"),
-        new File("/foo/current/edits_952-1000") },
+        new File("/foo/current/" + getFinalizedEditsFileName(457,900)),
+        new File("/foo/current/" + getInProgressEditsFileName(901)),
+        new File("/foo/current/" + getFinalizedEditsFileName(952,1000)) },
         plan.getEditsFiles().toArray(new File[0]));
 
   }
@@ -154,12 +160,14 @@ public class TestFSImageStorageInspector {
         new FSImageTransactionalStorageInspector();
 
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo1/current/edits_123-456"));
+        mockDirectoryWithEditLogs("/foo1/current/" 
+                                  + getFinalizedEditsFileName(123,456)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo2/current/edits_123-456"));
+        mockDirectoryWithEditLogs("/foo2/current/"
+                                  + getFinalizedEditsFileName(123,456)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo3/current/edits_123-456"));
-
+        mockDirectoryWithEditLogs("/foo3/current/"
+                                  + getFinalizedEditsFileName(123,456)));
     LogGroup lg = inspector.logGroups.get(123L);
     assertEquals(3, lg.logs.size());
     
@@ -180,15 +188,18 @@ public class TestFSImageStorageInspector {
         new FSImageTransactionalStorageInspector();
 
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo1/current/edits_123-456"));
+        mockDirectoryWithEditLogs("/foo1/current/"
+                                  + getFinalizedEditsFileName(123,456)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo2/current/edits_123-456"));
+        mockDirectoryWithEditLogs("/foo2/current/"
+                                  + getFinalizedEditsFileName(123,456)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo3/current/edits_inprogress_123"));
+        mockDirectoryWithEditLogs("/foo3/current/"
+                                  + getInProgressEditsFileName(123)));
     inspector.inspectDirectory(mockDirectory(
         NameNodeDirType.IMAGE,
         false,
-        "/foo4/current/fsimage_122"));
+        "/foo4/current/" + getImageFileName(122)));
 
     LogGroup lg = inspector.logGroups.get(123L);
     assertEquals(3, lg.logs.size());
@@ -221,9 +232,11 @@ public class TestFSImageStorageInspector {
     FSImageTransactionalStorageInspector inspector =
         new FSImageTransactionalStorageInspector();
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo1/current/edits_123-456"));
+        mockDirectoryWithEditLogs("/foo1/current/"
+                                  + getFinalizedEditsFileName(123,456)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo2/current/edits_123-678"));
+        mockDirectoryWithEditLogs("/foo2/current/"
+                                  + getFinalizedEditsFileName(123,678)));
 
     LogGroup lg = inspector.logGroups.get(123L);
     assertEquals(2, lg.logs.size());
@@ -243,9 +256,9 @@ public class TestFSImageStorageInspector {
   @Test
   public void testLogGroupRecoveryInProgress() throws IOException {
     String paths[] = new String[] {
-        "/foo1/current/edits_inprogress_123",
-        "/foo2/current/edits_inprogress_123",
-        "/foo3/current/edits_inprogress_123"
+        "/foo1/current/" + getInProgressEditsFileName(123),
+        "/foo2/current/" + getInProgressEditsFileName(123),
+        "/foo3/current/" + getInProgressEditsFileName(123)
     };
     FSImageTransactionalStorageInspector inspector =
         new FSImageTransactionalStorageInspector();
@@ -317,23 +330,23 @@ public class TestFSImageStorageInspector {
     StorageDirectory mockImageDir = mockDirectory(
         NameNodeDirType.IMAGE,
         false,
-        "/foo/current/fsimage_123");
+        "/foo/current/" + getImageFileName(123));
     StorageDirectory mockImageDir2 = mockDirectory(
         NameNodeDirType.IMAGE,
         false,
-        "/foo2/current/fsimage_456");
+        "/foo2/current/" + getImageFileName(456));
     StorageDirectory mockEditsDir = mockDirectory(
         NameNodeDirType.EDITS,
         false,
-        "/foo3/current/edits_123-456",
-        "/foo3/current/edits_inprogress_457");
+        "/foo3/current/" + getFinalizedEditsFileName(123, 456),
+        "/foo3/current/" + getInProgressEditsFileName(457));
     
     inspector.inspectDirectory(mockImageDir);
     inspector.inspectDirectory(mockEditsDir);
     inspector.inspectDirectory(mockImageDir2);
     
     mockLogValidation(inspector,
-        "/foo3/current/edits_inprogress_457", 2);
+        "/foo3/current/" + getInProgressEditsFileName(457), 2);
 
     assertEquals(2, inspector.foundEditLogs.size());
     assertEquals(2, inspector.foundImages.size());
@@ -346,9 +359,10 @@ public class TestFSImageStorageInspector {
     FoundFSImage pickedImage = plan.image;
     assertEquals(456, pickedImage.txId);
     assertSame(mockImageDir2, pickedImage.sd);
-    assertEquals(new File("/foo2/current/fsimage_456"), plan.getImageFile());
+    assertEquals(new File("/foo2/current/" + getImageFileName(456)),
+                 plan.getImageFile());
     assertArrayEquals(new File[] {
-        new File("/foo3/current/edits_inprogress_457")
+        new File("/foo3/current/" + getInProgressEditsFileName(457))
       }, plan.getEditsFiles().toArray(new File[0]));
 
     // Check log manifest
@@ -362,14 +376,20 @@ public class TestFSImageStorageInspector {
     FSImageTransactionalStorageInspector inspector =
         new FSImageTransactionalStorageInspector();
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo1/current/edits_1-1",
-                                  "/foo1/current/edits_2-200"));
+        mockDirectoryWithEditLogs("/foo1/current/" 
+                                  + getFinalizedEditsFileName(1,1),
+                                  "/foo1/current/"
+                                  + getFinalizedEditsFileName(2,200)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo2/current/edits_inprogress_1",
-                                  "/foo2/current/edits_201-400"));
+        mockDirectoryWithEditLogs("/foo2/current/" 
+                                  + getInProgressEditsFileName(1),
+                                  "/foo2/current/"
+                                  + getFinalizedEditsFileName(201, 400)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo3/current/edits_1-1",
-                                  "/foo3/current/edits_2-200"));
+        mockDirectoryWithEditLogs("/foo3/current/"
+                                  + getFinalizedEditsFileName(1, 1),
+                                  "/foo3/current/"
+                                  + getFinalizedEditsFileName(2,200)));
     
     assertEquals("[[1,1], [2,200], [201,400]]",
                  inspector.getEditLogManifest(1).toString());
@@ -391,14 +411,21 @@ public class TestFSImageStorageInspector {
     FSImageTransactionalStorageInspector inspector =
         new FSImageTransactionalStorageInspector();
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo1/current/edits_2622-2623",
-                                  "/foo1/current/edits_2624-2625",
-                                  "/foo1/current/edits_inprogress_2626"));
+        mockDirectoryWithEditLogs("/foo1/current/" 
+                                  + getFinalizedEditsFileName(2622,2623),
+                                  "/foo1/current/"
+                                  + getFinalizedEditsFileName(2624,2625),
+                                  "/foo1/current/"
+                                  + getInProgressEditsFileName(2626)));
     inspector.inspectDirectory(
-        mockDirectoryWithEditLogs("/foo2/current/edits_2622-2623",
-                                  "/foo2/current/edits_2624-2625",
-                                  "/foo2/current/edits_2626-2627",
-                                  "/foo2/current/edits_2628-2629"));
+        mockDirectoryWithEditLogs("/foo2/current/"
+                                  + getFinalizedEditsFileName(2622,2623),
+                                  "/foo2/current/"
+                                  + getFinalizedEditsFileName(2624,2625),
+                                  "/foo2/current/"
+                                  + getFinalizedEditsFileName(2626,2627),
+                                  "/foo2/current/"
+                                  + getFinalizedEditsFileName(2628,2629)));
     
     assertEquals("[[2622,2623], [2624,2625], [2626,2627], [2628,2629]]",
                  inspector.getEditLogManifest(2621).toString());

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalFunctional.java

@@ -27,12 +27,14 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
-import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
 import com.google.common.base.Joiner;
 
 import static org.apache.hadoop.test.GenericTestUtils.assertGlobEquals;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
 
 
 /**
@@ -77,18 +79,30 @@ public class TestNNStorageArchivalFunctional {
 
       doSaveNamespace(nn);
       LOG.info("After first save, images 0 and 2 should exist in both dirs");
-      assertGlobEquals(cd0, "fsimage_\\d*", "fsimage_0", "fsimage_2");
-      assertGlobEquals(cd1, "fsimage_\\d*", "fsimage_0", "fsimage_2");
-      assertGlobEquals(cd0, "edits_.*", "edits_1-2", "edits_inprogress_3");
-      assertGlobEquals(cd1, "edits_.*", "edits_1-2", "edits_inprogress_3");
+      assertGlobEquals(cd0, "fsimage_\\d*", 
+          getImageFileName(0), getImageFileName(2));
+      assertGlobEquals(cd1, "fsimage_\\d*",
+          getImageFileName(0), getImageFileName(2));
+      assertGlobEquals(cd0, "edits_.*",
+          getFinalizedEditsFileName(1, 2),
+          getInProgressEditsFileName(3));
+      assertGlobEquals(cd1, "edits_.*",
+          getFinalizedEditsFileName(1, 2),
+          getInProgressEditsFileName(3));
       
       doSaveNamespace(nn);
       LOG.info("After second save, image 0 should be archived, " +
           "and image 4 should exist in both.");
-      assertGlobEquals(cd0, "fsimage_\\d*", "fsimage_2", "fsimage_4");
-      assertGlobEquals(cd1, "fsimage_\\d*", "fsimage_2", "fsimage_4");
-      assertGlobEquals(cd0, "edits_.*", "edits_3-4", "edits_inprogress_5");
-      assertGlobEquals(cd1, "edits_.*", "edits_3-4", "edits_inprogress_5");
+      assertGlobEquals(cd0, "fsimage_\\d*",
+          getImageFileName(2), getImageFileName(4));
+      assertGlobEquals(cd1, "fsimage_\\d*",
+          getImageFileName(2), getImageFileName(4));
+      assertGlobEquals(cd0, "edits_.*",
+          getFinalizedEditsFileName(3, 4),
+          getInProgressEditsFileName(5));
+      assertGlobEquals(cd1, "edits_.*",
+          getFinalizedEditsFileName(3, 4),
+          getInProgressEditsFileName(5));
       
       LOG.info("Failing first storage dir by chmodding it");
       sd0.setExecutable(false);
@@ -97,23 +111,31 @@ public class TestNNStorageArchivalFunctional {
       sd0.setExecutable(true);
 
       LOG.info("nothing should have been archived in first storage dir");
-      assertGlobEquals(cd0, "fsimage_\\d*", "fsimage_2", "fsimage_4");
-      assertGlobEquals(cd0, "edits_.*", "edits_3-4", "edits_inprogress_5");
+      assertGlobEquals(cd0, "fsimage_\\d*",
+          getImageFileName(2), getImageFileName(4));
+      assertGlobEquals(cd0, "edits_.*",
+          getFinalizedEditsFileName(3, 4),
+          getInProgressEditsFileName(5));
 
       LOG.info("fsimage_2 should be archived in second storage dir");
-      assertGlobEquals(cd1, "fsimage_\\d*", "fsimage_4", "fsimage_6");
-      assertGlobEquals(cd1, "edits_.*", "edits_5-6", "edits_inprogress_7");
+      assertGlobEquals(cd1, "fsimage_\\d*",
+          getImageFileName(4), getImageFileName(6));
+      assertGlobEquals(cd1, "edits_.*",
+          getFinalizedEditsFileName(5, 6),
+          getInProgressEditsFileName(7));
 
       LOG.info("On next save, we should archive logs from the failed dir," +
           " but not images, since the image directory is in failed state.");
       doSaveNamespace(nn);
-      assertGlobEquals(cd1, "fsimage_\\d*", "fsimage_6", "fsimage_8");
-      assertGlobEquals(cd1, "edits_.*", "edits_7-8", "edits_inprogress_9");
-      assertGlobEquals(cd0, "fsimage_\\d*", "fsimage_2", "fsimage_4");
-      assertGlobEquals(cd0, "edits_.*", "edits_inprogress_9");
-      
-
-
+      assertGlobEquals(cd1, "fsimage_\\d*",
+          getImageFileName(6), getImageFileName(8));
+      assertGlobEquals(cd1, "edits_.*",
+          getFinalizedEditsFileName(7, 8),
+          getInProgressEditsFileName(9));
+      assertGlobEquals(cd0, "fsimage_\\d*",
+          getImageFileName(2), getImageFileName(4));
+      assertGlobEquals(cd0, "edits_.*",
+          getInProgressEditsFileName(9));
     } finally {
       sd0.setExecutable(true);
 

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java

@@ -27,6 +27,10 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
+
 import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver;
 import org.junit.Assert;
 import org.junit.Test;
@@ -51,14 +55,14 @@ public class TestNNStorageArchivalManager {
   public void testArchiveEasyCase() throws IOException {
     TestCaseDescription tc = new TestCaseDescription();
     tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
-    tc.addImage("/foo1/current/fsimage_100", true);
-    tc.addImage("/foo1/current/fsimage_200", true);
-    tc.addImage("/foo1/current/fsimage_300", false);
-    tc.addImage("/foo1/current/fsimage_400", false);
-    tc.addLog("/foo1/current/edits_101-200", true);
-    tc.addLog("/foo1/current/edits_201-300", true);
-    tc.addLog("/foo1/current/edits_301-400", false);
-    tc.addLog("/foo1/current/edits_inprogress_401", false);
+    tc.addImage("/foo1/current/" + getImageFileName(100), true);
+    tc.addImage("/foo1/current/" + getImageFileName(200), true);
+    tc.addImage("/foo1/current/" + getImageFileName(300), false);
+    tc.addImage("/foo1/current/" + getImageFileName(400), false);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101,200), true);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201,300), true);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301,400), false);
+    tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false);
     
     // Test that other files don't get archived
     tc.addLog("/foo1/current/VERSION", false);
@@ -73,17 +77,17 @@ public class TestNNStorageArchivalManager {
     TestCaseDescription tc = new TestCaseDescription();
     tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
     tc.addRoot("/foo2", NameNodeDirType.IMAGE_AND_EDITS);
-    tc.addImage("/foo1/current/fsimage_100", true);
-    tc.addImage("/foo1/current/fsimage_200", true);
-    tc.addImage("/foo2/current/fsimage_200", true);
-    tc.addImage("/foo1/current/fsimage_300", false);
-    tc.addImage("/foo1/current/fsimage_400", false);
-    tc.addLog("/foo1/current/edits_101-200", true);
-    tc.addLog("/foo1/current/edits_201-300", true);
-    tc.addLog("/foo2/current/edits_201-300", true);
-    tc.addLog("/foo1/current/edits_301-400", false);
-    tc.addLog("/foo2/current/edits_301-400", false);
-    tc.addLog("/foo1/current/edits_inprogress_401", false);
+    tc.addImage("/foo1/current/" + getImageFileName(100), true);
+    tc.addImage("/foo1/current/" + getImageFileName(200), true);
+    tc.addImage("/foo2/current/" + getImageFileName(200), true);
+    tc.addImage("/foo1/current/" + getImageFileName(300), false);
+    tc.addImage("/foo1/current/" + getImageFileName(400), false);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101, 200), true);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201, 300), true);
+    tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301, 400), false);
+    tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
+    tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false);
     runTest(tc);
   }
   
@@ -95,11 +99,11 @@ public class TestNNStorageArchivalManager {
   public void testArchiveLessThanRetention() throws IOException {
     TestCaseDescription tc = new TestCaseDescription();
     tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
-    tc.addImage("/foo1/current/fsimage_100", false);
-    tc.addLog("/foo1/current/edits_101-200", false);
-    tc.addLog("/foo1/current/edits_201-300", false);
-    tc.addLog("/foo1/current/edits_301-400", false);
-    tc.addLog("/foo1/current/edits_inprogress_401", false);
+    tc.addImage("/foo1/current/" + getImageFileName(100), false);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(101,200), false);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(201,300), false);
+    tc.addLog("/foo1/current/" + getFinalizedEditsFileName(301,400), false);
+    tc.addLog("/foo1/current/" + getInProgressEditsFileName(401), false);
     runTest(tc);
   }
 
@@ -110,10 +114,10 @@ public class TestNNStorageArchivalManager {
   public void testNoLogs() throws IOException {
     TestCaseDescription tc = new TestCaseDescription();
     tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
-    tc.addImage("/foo1/current/fsimage_100", true);
-    tc.addImage("/foo1/current/fsimage_200", true);
-    tc.addImage("/foo1/current/fsimage_300", false);
-    tc.addImage("/foo1/current/fsimage_400", false);
+    tc.addImage("/foo1/current/" + getImageFileName(100), true);
+    tc.addImage("/foo1/current/" + getImageFileName(200), true);
+    tc.addImage("/foo1/current/" + getImageFileName(300), false);
+    tc.addImage("/foo1/current/" + getImageFileName(400), false);
     runTest(tc);
   }
   
@@ -134,11 +138,11 @@ public class TestNNStorageArchivalManager {
   public void testOldInProgress() throws IOException {
     TestCaseDescription tc = new TestCaseDescription();
     tc.addRoot("/foo1", NameNodeDirType.IMAGE_AND_EDITS);
-    tc.addImage("/foo1/current/fsimage_100", true);
-    tc.addImage("/foo1/current/fsimage_200", true);
-    tc.addImage("/foo1/current/fsimage_300", false);
-    tc.addImage("/foo1/current/fsimage_400", false);
-    tc.addLog("/foo1/current/edits_inprogress_101", true);
+    tc.addImage("/foo1/current/" + getImageFileName(100), true);
+    tc.addImage("/foo1/current/" + getImageFileName(200), true);
+    tc.addImage("/foo1/current/" + getImageFileName(300), false);
+    tc.addImage("/foo1/current/" + getImageFileName(400), false);
+    tc.addLog("/foo1/current/" + getInProgressEditsFileName(101), true);
     runTest(tc);
   }
 
@@ -147,14 +151,15 @@ public class TestNNStorageArchivalManager {
     TestCaseDescription tc = new TestCaseDescription();
     tc.addRoot("/foo1", NameNodeDirType.IMAGE);
     tc.addRoot("/foo2", NameNodeDirType.EDITS);
-    tc.addImage("/foo1/current/fsimage_100", true);
-    tc.addImage("/foo1/current/fsimage_200", true);
-    tc.addImage("/foo1/current/fsimage_300", false);
-    tc.addImage("/foo1/current/fsimage_400", false);
-    tc.addLog("/foo2/current/edits_101-200", true);
-    tc.addLog("/foo2/current/edits_201-300", true);
-    tc.addLog("/foo2/current/edits_301-400", false);
-    tc.addLog("/foo2/current/edits_inprogress_401", false);
+    tc.addImage("/foo1/current/" + getImageFileName(100), true);
+    tc.addImage("/foo1/current/" + getImageFileName(200), true);
+    tc.addImage("/foo1/current/" + getImageFileName(300), false);
+    tc.addImage("/foo1/current/" + getImageFileName(400), false);
+
+    tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
+    tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), true);
+    tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
+    tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
     runTest(tc);    
   }
   

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java

@@ -455,7 +455,7 @@ public class TestStartup extends TestCase {
         cluster = null;
 
         // Corrupt the md5 file to all 0s
-        File imageFile = new File(nameDir, "current/fsimage_0");
+        File imageFile = new File(nameDir, "current/" + NNStorage.getImageFileName(0));
         MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
         
         // Try to start a new cluster

hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java

@@ -50,6 +50,9 @@ import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName;
+import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 
 
@@ -190,12 +193,12 @@ public class TestStorageRestore extends TestCase {
     // We did another edit, so the still-active directory at 'path1'
     // should now differ from the others
     FSImageTestUtil.assertFileContentsDifferent(2,
-        new File(path1, "current/edits_inprogress_1"),
-        new File(path2, "current/edits_inprogress_1"),
-        new File(path3, "current/edits_inprogress_1"));
+        new File(path1, "current/" + getInProgressEditsFileName(1)),
+        new File(path2, "current/" + getInProgressEditsFileName(1)),
+        new File(path3, "current/" + getInProgressEditsFileName(1)));
     FSImageTestUtil.assertFileContentsSame(
-        new File(path2, "current/edits_inprogress_1"),
-        new File(path3, "current/edits_inprogress_1"));
+        new File(path2, "current/" + getInProgressEditsFileName(1)),
+        new File(path3, "current/" + getInProgressEditsFileName(1)));
         
     System.out.println("****testStorageRestore: checkfiles(false) run");
     
@@ -204,34 +207,34 @@ public class TestStorageRestore extends TestCase {
     // We should have a checkpoint through txid 4 in the two image dirs
     // (txid=4 for BEGIN, mkdir, mkdir, END)
     FSImageTestUtil.assertFileContentsSame(
-        new File(path1, "current/fsimage_4"),
-        new File(path2, "current/fsimage_4"));
+        new File(path1, "current/" + getImageFileName(4)),
+        new File(path2, "current/" + getImageFileName(4)));
     assertFalse("Should not have any image in an edits-only directory",
-        new File(path3, "current/fsimage_4").exists());
+        new File(path3, "current/" + getImageFileName(4)).exists());
 
     // Should have finalized logs in the directory that didn't fail
     assertTrue("Should have finalized logs in the directory that didn't fail",
-        new File(path1, "current/edits_1-4").exists());
+        new File(path1, "current/" + getFinalizedEditsFileName(1,4)).exists());
     // Should not have finalized logs in the failed directories
     assertFalse("Should not have finalized logs in the failed directories",
-        new File(path2, "current/edits_1-4").exists());
+        new File(path2, "current/" + getFinalizedEditsFileName(1,4)).exists());
     assertFalse("Should not have finalized logs in the failed directories",
-        new File(path3, "current/edits_1-4").exists());
+        new File(path3, "current/" + getFinalizedEditsFileName(1,4)).exists());
     
     // The new log segment should be in all of the directories.
     FSImageTestUtil.assertFileContentsSame(
-        new File(path1, "current/edits_inprogress_5"),
-        new File(path2, "current/edits_inprogress_5"),
-        new File(path3, "current/edits_inprogress_5"));
+        new File(path1, "current/" + getInProgressEditsFileName(5)),
+        new File(path2, "current/" + getInProgressEditsFileName(5)),
+        new File(path3, "current/" + getInProgressEditsFileName(5)));
     String md5BeforeEdit = FSImageTestUtil.getFileMD5(
-        new File(path1, "current/edits_inprogress_5"));
+        new File(path1, "current/" + getInProgressEditsFileName(5)));
     
     // The original image should still be the previously failed image
     // directory after it got restored, since it's still useful for
     // a recovery!
     FSImageTestUtil.assertFileContentsSame(
-            new File(path1, "current/fsimage_0"),
-            new File(path2, "current/fsimage_0"));
+            new File(path1, "current/" + getImageFileName(0)),
+            new File(path2, "current/" + getImageFileName(0)));
     
     // Do another edit to verify that all the logs are active.
     path = new Path("/", "test2");
@@ -239,23 +242,23 @@ public class TestStorageRestore extends TestCase {
 
     // Logs should be changed by the edit.
     String md5AfterEdit =  FSImageTestUtil.getFileMD5(
-        new File(path1, "current/edits_inprogress_5"));
+        new File(path1, "current/" + getInProgressEditsFileName(5)));
     assertFalse(md5BeforeEdit.equals(md5AfterEdit));
 
     // And all logs should be changed.
     FSImageTestUtil.assertFileContentsSame(
-        new File(path1, "current/edits_inprogress_5"),
-        new File(path2, "current/edits_inprogress_5"),
-        new File(path3, "current/edits_inprogress_5"));
+        new File(path1, "current/" + getInProgressEditsFileName(5)),
+        new File(path2, "current/" + getInProgressEditsFileName(5)),
+        new File(path3, "current/" + getInProgressEditsFileName(5)));
 
     secondary.shutdown();
     cluster.shutdown();
     
     // All logs should be finalized by clean shutdown
     FSImageTestUtil.assertFileContentsSame(
-        new File(path1, "current/edits_5-7"),
-        new File(path2, "current/edits_5-7"),        
-        new File(path3, "current/edits_5-7"));
+        new File(path1, "current/" + getFinalizedEditsFileName(5,7)),
+        new File(path2, "current/" + getFinalizedEditsFileName(5,7)),        
+        new File(path3, "current/" + getFinalizedEditsFileName(5,7)));
   }
   
   /**