|
|
@@ -26,8 +26,6 @@ import java.io.FileOutputStream;
|
|
|
import java.io.IOException;
|
|
|
import java.nio.ByteBuffer;
|
|
|
import java.nio.channels.FileChannel;
|
|
|
-import java.util.HashMap;
|
|
|
-import java.util.Map;
|
|
|
|
|
|
import org.apache.commons.logging.Log;
|
|
|
import org.apache.commons.logging.LogFactory;
|
|
|
@@ -36,168 +34,142 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
|
|
|
import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper;
|
|
|
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer.Flags;
|
|
|
import org.apache.hadoop.test.PathUtils;
|
|
|
+import org.junit.After;
|
|
|
import org.junit.Before;
|
|
|
+import org.junit.Rule;
|
|
|
import org.junit.Test;
|
|
|
+import org.junit.rules.TemporaryFolder;
|
|
|
|
|
|
-public class TestOfflineEditsViewer {
|
|
|
- private static final Log LOG = LogFactory.getLog(TestOfflineEditsViewer.class);
|
|
|
-
|
|
|
- private static final Map<FSEditLogOpCodes, Boolean> obsoleteOpCodes =
|
|
|
- new HashMap<FSEditLogOpCodes, Boolean>();
|
|
|
-
|
|
|
- private static final Map<FSEditLogOpCodes, Boolean> missingOpCodes =
|
|
|
- new HashMap<FSEditLogOpCodes, Boolean>();
|
|
|
+import com.google.common.collect.ImmutableSet;
|
|
|
|
|
|
- static {
|
|
|
- initializeObsoleteOpCodes();
|
|
|
- initializeMissingOpCodes();
|
|
|
- }
|
|
|
-
|
|
|
- private static String buildDir =
|
|
|
- PathUtils.getTestDirName(TestOfflineEditsViewer.class);
|
|
|
+public class TestOfflineEditsViewer {
|
|
|
+ private static final Log LOG = LogFactory
|
|
|
+ .getLog(TestOfflineEditsViewer.class);
|
|
|
|
|
|
- private static String cacheDir =
|
|
|
- System.getProperty("test.cache.data", "build/test/cache");
|
|
|
+ private static String buildDir = PathUtils
|
|
|
+ .getTestDirName(TestOfflineEditsViewer.class);
|
|
|
|
|
|
// to create edits and get edits filename
|
|
|
- private static final OfflineEditsViewerHelper nnHelper
|
|
|
- = new OfflineEditsViewerHelper();
|
|
|
+ private static final OfflineEditsViewerHelper nnHelper = new OfflineEditsViewerHelper();
|
|
|
+ private static final ImmutableSet<FSEditLogOpCodes> skippedOps = skippedOps();
|
|
|
|
|
|
- /**
|
|
|
- * Initialize obsoleteOpCodes
|
|
|
- *
|
|
|
- * Reason for suppressing "deprecation" warnings:
|
|
|
- *
|
|
|
- * These are the opcodes that are not used anymore, some
|
|
|
- * are marked deprecated, we need to include them here to make
|
|
|
- * sure we exclude them when checking for completeness of testing,
|
|
|
- * that's why the "deprecation" warnings are suppressed.
|
|
|
- */
|
|
|
@SuppressWarnings("deprecation")
|
|
|
- private static void initializeObsoleteOpCodes() {
|
|
|
- obsoleteOpCodes.put(FSEditLogOpCodes.OP_DATANODE_ADD, true);
|
|
|
- obsoleteOpCodes.put(FSEditLogOpCodes.OP_DATANODE_REMOVE, true);
|
|
|
- obsoleteOpCodes.put(FSEditLogOpCodes.OP_SET_NS_QUOTA, true);
|
|
|
- obsoleteOpCodes.put(FSEditLogOpCodes.OP_CLEAR_NS_QUOTA, true);
|
|
|
+ private static ImmutableSet<FSEditLogOpCodes> skippedOps() {
|
|
|
+ ImmutableSet.Builder<FSEditLogOpCodes> b = ImmutableSet
|
|
|
+ .<FSEditLogOpCodes> builder();
|
|
|
+
|
|
|
+ // Deprecated opcodes
|
|
|
+ b.add(FSEditLogOpCodes.OP_DATANODE_ADD)
|
|
|
+ .add(FSEditLogOpCodes.OP_DATANODE_REMOVE)
|
|
|
+ .add(FSEditLogOpCodes.OP_SET_NS_QUOTA)
|
|
|
+ .add(FSEditLogOpCodes.OP_CLEAR_NS_QUOTA)
|
|
|
+ .add(FSEditLogOpCodes.OP_SET_GENSTAMP_V1);
|
|
|
+
|
|
|
+ // Cannot test delegation token related code in insecure set up
|
|
|
+ b.add(FSEditLogOpCodes.OP_GET_DELEGATION_TOKEN)
|
|
|
+ .add(FSEditLogOpCodes.OP_RENEW_DELEGATION_TOKEN)
|
|
|
+ .add(FSEditLogOpCodes.OP_CANCEL_DELEGATION_TOKEN);
|
|
|
+
|
|
|
+ // Skip invalid opcode
|
|
|
+ b.add(FSEditLogOpCodes.OP_INVALID);
|
|
|
+ return b.build();
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Initialize missingOpcodes
|
|
|
- *
|
|
|
- * Opcodes that are not available except after uprade from
|
|
|
- * an older version. We don't test these here.
|
|
|
- */
|
|
|
- private static void initializeMissingOpCodes() {
|
|
|
- obsoleteOpCodes.put(FSEditLogOpCodes.OP_SET_GENSTAMP_V1, true);
|
|
|
- }
|
|
|
+ @Rule
|
|
|
+ public final TemporaryFolder folder = new TemporaryFolder();
|
|
|
|
|
|
@Before
|
|
|
- public void setup() {
|
|
|
- new File(cacheDir).mkdirs();
|
|
|
+ public void setUp() throws IOException {
|
|
|
+ nnHelper.startCluster(buildDir + "/dfs/");
|
|
|
+ }
|
|
|
+
|
|
|
+ @After
|
|
|
+ public void tearDown() throws IOException {
|
|
|
+ nnHelper.shutdownCluster();
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
/**
|
|
|
* Test the OfflineEditsViewer
|
|
|
*/
|
|
|
@Test
|
|
|
public void testGenerated() throws IOException {
|
|
|
-
|
|
|
- LOG.info("START - testing with generated edits");
|
|
|
-
|
|
|
- nnHelper.startCluster(buildDir + "/dfs/");
|
|
|
-
|
|
|
// edits generated by nnHelper (MiniDFSCluster), should have all op codes
|
|
|
// binary, XML, reparsed binary
|
|
|
- String edits = nnHelper.generateEdits();
|
|
|
- String editsParsedXml = cacheDir + "/editsParsed.xml";
|
|
|
- String editsReparsed = cacheDir + "/editsReparsed";
|
|
|
+ String edits = nnHelper.generateEdits();
|
|
|
+ String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath();
|
|
|
+ String editsReparsed = folder.newFile("editsParsed").getAbsolutePath();
|
|
|
|
|
|
// parse to XML then back to binary
|
|
|
assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
|
|
|
assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false));
|
|
|
|
|
|
// judgment time
|
|
|
+ assertTrue("Edits " + edits + " should have all op codes",
|
|
|
+ hasAllOpCodes(edits));
|
|
|
+ LOG.info("Comparing generated file " + editsReparsed
|
|
|
+ + " with reference file " + edits);
|
|
|
assertTrue(
|
|
|
- "Edits " + edits + " should have all op codes",
|
|
|
- hasAllOpCodes(edits));
|
|
|
- LOG.info("Comparing generated file " + editsReparsed +
|
|
|
- " with reference file " + edits);
|
|
|
- assertTrue(
|
|
|
- "Generated edits and reparsed (bin to XML to bin) should be same",
|
|
|
- filesEqualIgnoreTrailingZeros(edits, editsReparsed));
|
|
|
-
|
|
|
- // removes edits so do this at the end
|
|
|
- nnHelper.shutdownCluster();
|
|
|
-
|
|
|
- LOG.info("END");
|
|
|
+ "Generated edits and reparsed (bin to XML to bin) should be same",
|
|
|
+ filesEqualIgnoreTrailingZeros(edits, editsReparsed));
|
|
|
}
|
|
|
|
|
|
@Test
|
|
|
public void testRecoveryMode() throws IOException {
|
|
|
- LOG.info("START - testing with generated edits");
|
|
|
-
|
|
|
- nnHelper.startCluster(buildDir + "/dfs/");
|
|
|
-
|
|
|
// edits generated by nnHelper (MiniDFSCluster), should have all op codes
|
|
|
// binary, XML, reparsed binary
|
|
|
- String edits = nnHelper.generateEdits();
|
|
|
-
|
|
|
+ String edits = nnHelper.generateEdits();
|
|
|
+ FileOutputStream os = new FileOutputStream(edits, true);
|
|
|
// Corrupt the file by truncating the end
|
|
|
- FileChannel editsFile = new FileOutputStream(edits, true).getChannel();
|
|
|
+ FileChannel editsFile = os.getChannel();
|
|
|
editsFile.truncate(editsFile.size() - 5);
|
|
|
-
|
|
|
- String editsParsedXml = cacheDir + "/editsRecoveredParsed.xml";
|
|
|
- String editsReparsed = cacheDir + "/editsRecoveredReparsed";
|
|
|
- String editsParsedXml2 = cacheDir + "/editsRecoveredParsed2.xml";
|
|
|
+
|
|
|
+ String editsParsedXml = folder.newFile("editsRecoveredParsed.xml")
|
|
|
+ .getAbsolutePath();
|
|
|
+ String editsReparsed = folder.newFile("editsRecoveredReparsed")
|
|
|
+ .getAbsolutePath();
|
|
|
+ String editsParsedXml2 = folder.newFile("editsRecoveredParsed2.xml")
|
|
|
+ .getAbsolutePath();
|
|
|
|
|
|
// Can't read the corrupted file without recovery mode
|
|
|
assertEquals(-1, runOev(edits, editsParsedXml, "xml", false));
|
|
|
-
|
|
|
+
|
|
|
// parse to XML then back to binary
|
|
|
assertEquals(0, runOev(edits, editsParsedXml, "xml", true));
|
|
|
- assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false));
|
|
|
+ assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false));
|
|
|
assertEquals(0, runOev(editsReparsed, editsParsedXml2, "xml", false));
|
|
|
|
|
|
// judgment time
|
|
|
assertTrue("Test round trip",
|
|
|
- filesEqualIgnoreTrailingZeros(editsParsedXml, editsParsedXml2));
|
|
|
+ filesEqualIgnoreTrailingZeros(editsParsedXml, editsParsedXml2));
|
|
|
|
|
|
- // removes edits so do this at the end
|
|
|
- nnHelper.shutdownCluster();
|
|
|
-
|
|
|
- LOG.info("END");
|
|
|
+ os.close();
|
|
|
}
|
|
|
|
|
|
@Test
|
|
|
public void testStored() throws IOException {
|
|
|
-
|
|
|
- LOG.info("START - testing with stored reference edits");
|
|
|
-
|
|
|
// reference edits stored with source code (see build.xml)
|
|
|
+ final String cacheDir = System.getProperty("test.cache.data",
|
|
|
+ "build/test/cache");
|
|
|
// binary, XML, reparsed binary
|
|
|
- String editsStored = cacheDir + "/editsStored";
|
|
|
- String editsStoredParsedXml = cacheDir + "/editsStoredParsed.xml";
|
|
|
- String editsStoredReparsed = cacheDir + "/editsStoredReparsed";
|
|
|
+ String editsStored = cacheDir + "/editsStored";
|
|
|
+ String editsStoredParsedXml = cacheDir + "/editsStoredParsed.xml";
|
|
|
+ String editsStoredReparsed = cacheDir + "/editsStoredReparsed";
|
|
|
// reference XML version of editsStored (see build.xml)
|
|
|
- String editsStoredXml = cacheDir + "/editsStored.xml";
|
|
|
-
|
|
|
+ String editsStoredXml = cacheDir + "/editsStored.xml";
|
|
|
+
|
|
|
// parse to XML then back to binary
|
|
|
assertEquals(0, runOev(editsStored, editsStoredParsedXml, "xml", false));
|
|
|
- assertEquals(0, runOev(editsStoredParsedXml, editsStoredReparsed,
|
|
|
- "binary", false));
|
|
|
+ assertEquals(0,
|
|
|
+ runOev(editsStoredParsedXml, editsStoredReparsed, "binary", false));
|
|
|
|
|
|
// judgement time
|
|
|
+ assertTrue("Edits " + editsStored + " should have all op codes",
|
|
|
+ hasAllOpCodes(editsStored));
|
|
|
+ assertTrue("Reference XML edits and parsed to XML should be same",
|
|
|
+ filesEqual(editsStoredXml, editsStoredParsedXml));
|
|
|
assertTrue(
|
|
|
- "Edits " + editsStored + " should have all op codes",
|
|
|
- hasAllOpCodes(editsStored));
|
|
|
- assertTrue(
|
|
|
- "Reference XML edits and parsed to XML should be same",
|
|
|
- filesEqual(editsStoredXml, editsStoredParsedXml));
|
|
|
- assertTrue(
|
|
|
- "Reference edits and reparsed (bin to XML to bin) should be same",
|
|
|
- filesEqualIgnoreTrailingZeros(editsStored, editsStoredReparsed));
|
|
|
-
|
|
|
- LOG.info("END");
|
|
|
+ "Reference edits and reparsed (bin to XML to bin) should be same",
|
|
|
+ filesEqualIgnoreTrailingZeros(editsStored, editsStoredReparsed));
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -233,22 +205,17 @@ public class TestOfflineEditsViewer {
|
|
|
OfflineEditsViewer oev = new OfflineEditsViewer();
|
|
|
if (oev.go(inFilename, outFilename, "stats", new Flags(), visitor) != 0)
|
|
|
return false;
|
|
|
- LOG.info("Statistics for " + inFilename + "\n" +
|
|
|
- visitor.getStatisticsString());
|
|
|
-
|
|
|
+ LOG.info("Statistics for " + inFilename + "\n"
|
|
|
+ + visitor.getStatisticsString());
|
|
|
+
|
|
|
boolean hasAllOpCodes = true;
|
|
|
- for(FSEditLogOpCodes opCode : FSEditLogOpCodes.values()) {
|
|
|
+ for (FSEditLogOpCodes opCode : FSEditLogOpCodes.values()) {
|
|
|
// don't need to test obsolete opCodes
|
|
|
- if(obsoleteOpCodes.containsKey(opCode)) {
|
|
|
+ if (skippedOps.contains(opCode))
|
|
|
continue;
|
|
|
- } else if (missingOpCodes.containsKey(opCode)) {
|
|
|
- continue;
|
|
|
- } else if (opCode == FSEditLogOpCodes.OP_INVALID) {
|
|
|
- continue;
|
|
|
- }
|
|
|
|
|
|
Long count = visitor.getStatistics().get(opCode);
|
|
|
- if((count == null) || (count == 0)) {
|
|
|
+ if ((count == null) || (count == 0)) {
|
|
|
hasAllOpCodes = false;
|
|
|
LOG.info("Opcode " + opCode + " not tested in " + inFilename);
|
|
|
}
|
|
|
@@ -257,9 +224,9 @@ public class TestOfflineEditsViewer {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * Compare two files, ignore trailing zeros at the end,
|
|
|
- * for edits log the trailing zeros do not make any difference,
|
|
|
- * throw exception is the files are not same
|
|
|
+ * Compare two files, ignore trailing zeros at the end, for edits log the
|
|
|
+ * trailing zeros do not make any difference, throw exception is the files are
|
|
|
+ * not same
|
|
|
*
|
|
|
* @param filenameSmall first file to compare (doesn't have to be smaller)
|
|
|
* @param filenameLarge second file to compare (doesn't have to be larger)
|
|
|
@@ -271,7 +238,7 @@ public class TestOfflineEditsViewer {
|
|
|
ByteBuffer large = ByteBuffer.wrap(DFSTestUtil.loadFile(filenameLarge));
|
|
|
|
|
|
// now correct if it's otherwise
|
|
|
- if(small.capacity() > large.capacity()) {
|
|
|
+ if (small.capacity() > large.capacity()) {
|
|
|
ByteBuffer tmpByteBuffer = small;
|
|
|
small = large;
|
|
|
large = tmpByteBuffer;
|
|
|
@@ -288,13 +255,15 @@ public class TestOfflineEditsViewer {
|
|
|
large.limit(small.capacity());
|
|
|
|
|
|
// compares position to limit
|
|
|
- if(!small.equals(large)) { return false; }
|
|
|
+ if (!small.equals(large)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
|
|
|
// everything after limit should be 0xFF
|
|
|
int i = large.limit();
|
|
|
large.clear();
|
|
|
- for(; i < large.capacity(); i++) {
|
|
|
- if(large.get(i) != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
|
|
|
+ for (; i < large.capacity(); i++) {
|
|
|
+ if (large.get(i) != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
|
|
|
return false;
|
|
|
}
|
|
|
}
|
Jing Zhao