|
|
@@ -20,23 +20,20 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
|
|
import static org.junit.Assert.assertEquals;
|
|
|
import static org.junit.Assert.assertFalse;
|
|
|
import static org.junit.Assert.assertTrue;
|
|
|
-import static org.junit.Assert.fail;
|
|
|
|
|
|
import java.io.BufferedReader;
|
|
|
-import java.io.DataInputStream;
|
|
|
-import java.io.DataOutputStream;
|
|
|
-import java.io.EOFException;
|
|
|
import java.io.File;
|
|
|
import java.io.FileInputStream;
|
|
|
import java.io.FileOutputStream;
|
|
|
import java.io.FileReader;
|
|
|
import java.io.IOException;
|
|
|
-import java.io.InputStream;
|
|
|
-import java.io.OutputStream;
|
|
|
+import java.io.PrintWriter;
|
|
|
+import java.io.RandomAccessFile;
|
|
|
+import java.io.StringWriter;
|
|
|
import java.util.HashMap;
|
|
|
-import java.util.LinkedList;
|
|
|
-import java.util.List;
|
|
|
import java.util.Set;
|
|
|
+import java.util.regex.Matcher;
|
|
|
+import java.util.regex.Pattern;
|
|
|
|
|
|
import org.apache.commons.logging.Log;
|
|
|
import org.apache.commons.logging.LogFactory;
|
|
|
@@ -46,27 +43,29 @@ import org.apache.hadoop.fs.FSDataOutputStream;
|
|
|
import org.apache.hadoop.fs.FileStatus;
|
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
|
import org.apache.hadoop.fs.Path;
|
|
|
+import org.apache.hadoop.fs.permission.FsPermission;
|
|
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
|
|
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
|
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
|
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
|
|
|
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
|
|
|
+import org.apache.hadoop.io.IOUtils;
|
|
|
import org.apache.hadoop.security.token.Token;
|
|
|
import org.apache.hadoop.test.PathUtils;
|
|
|
import org.junit.AfterClass;
|
|
|
import org.junit.BeforeClass;
|
|
|
+import org.junit.Rule;
|
|
|
import org.junit.Test;
|
|
|
-
|
|
|
+import org.junit.rules.TemporaryFolder;
|
|
|
|
|
|
/**
|
|
|
- * Test function of OfflineImageViewer by:
|
|
|
- * * confirming it can correctly process a valid fsimage file and that
|
|
|
- * the processing generates a correct representation of the namespace
|
|
|
- * * confirming it correctly fails to process an fsimage file with a layout
|
|
|
- * version it shouldn't be able to handle
|
|
|
- * * confirm it correctly bails on malformed image files, in particular, a
|
|
|
- * file that ends suddenly.
|
|
|
+ * Test function of OfflineImageViewer by: * confirming it can correctly process
|
|
|
+ * a valid fsimage file and that the processing generates a correct
|
|
|
+ * representation of the namespace * confirming it correctly fails to process an
|
|
|
+ * fsimage file with a layout version it shouldn't be able to handle * confirm
|
|
|
+ * it correctly bails on malformed image files, in particular, a file that ends
|
|
|
+ * suddenly.
|
|
|
*/
|
|
|
public class TestOfflineImageViewer {
|
|
|
private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
|
|
|
@@ -76,22 +75,22 @@ public class TestOfflineImageViewer {
|
|
|
private static File originalFsimage = null;
|
|
|
|
|
|
// Elements of lines of ls-file output to be compared to FileStatus instance
|
|
|
- private static class LsElements {
|
|
|
- public String perms;
|
|
|
- public int replication;
|
|
|
- public String username;
|
|
|
- public String groupname;
|
|
|
- public long filesize;
|
|
|
- public char dir; // d if dir, - otherwise
|
|
|
+ private static final class LsElements {
|
|
|
+ private String perms;
|
|
|
+ private int replication;
|
|
|
+ private String username;
|
|
|
+ private String groupname;
|
|
|
+ private long filesize;
|
|
|
+ private boolean isDir;
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
// namespace as written to dfs, to be compared with viewer's output
|
|
|
- final static HashMap<String, FileStatus> writtenFiles =
|
|
|
- new HashMap<String, FileStatus>();
|
|
|
-
|
|
|
- private static String ROOT = PathUtils.getTestDirName(TestOfflineImageViewer.class);
|
|
|
-
|
|
|
- // Create a populated namespace for later testing. Save its contents to a
|
|
|
+ final static HashMap<String, FileStatus> writtenFiles = new HashMap<String, FileStatus>();
|
|
|
+
|
|
|
+ @Rule
|
|
|
+ public TemporaryFolder folder = new TemporaryFolder();
|
|
|
+
|
|
|
+ // Create a populated namespace for later testing. Save its contents to a
|
|
|
// data structure and store its fsimage location.
|
|
|
// We only want to generate the fsimage file once and use it for
|
|
|
// multiple tests.
|
|
|
@@ -100,35 +99,39 @@ public class TestOfflineImageViewer {
|
|
|
MiniDFSCluster cluster = null;
|
|
|
try {
|
|
|
Configuration conf = new HdfsConfiguration();
|
|
|
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
|
|
|
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
|
|
|
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
|
|
|
+ conf.setLong(
|
|
|
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
|
|
|
+ conf.setLong(
|
|
|
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
|
|
|
+ conf.setBoolean(
|
|
|
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
|
|
|
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
|
|
|
"RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
|
|
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
|
|
cluster.waitActive();
|
|
|
FileSystem hdfs = cluster.getFileSystem();
|
|
|
-
|
|
|
+
|
|
|
int filesize = 256;
|
|
|
-
|
|
|
- // Create a reasonable namespace
|
|
|
- for(int i = 0; i < NUM_DIRS; i++) {
|
|
|
+
|
|
|
+ // Create a reasonable namespace
|
|
|
+ for (int i = 0; i < NUM_DIRS; i++) {
|
|
|
Path dir = new Path("/dir" + i);
|
|
|
hdfs.mkdirs(dir);
|
|
|
writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
|
|
|
- for(int j = 0; j < FILES_PER_DIR; j++) {
|
|
|
+ for (int j = 0; j < FILES_PER_DIR; j++) {
|
|
|
Path file = new Path(dir, "file" + j);
|
|
|
FSDataOutputStream o = hdfs.create(file);
|
|
|
- o.write(new byte[ filesize++ ]);
|
|
|
+ o.write(new byte[filesize++]);
|
|
|
o.close();
|
|
|
-
|
|
|
- writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString()));
|
|
|
+
|
|
|
+ writtenFiles.put(file.toString(),
|
|
|
+ pathToFileEntry(hdfs, file.toString()));
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// Get delegation tokens so we log the delegation token op
|
|
|
- Token<?>[] delegationTokens =
|
|
|
- hdfs.addDelegationTokens(TEST_RENEWER, null);
|
|
|
+ Token<?>[] delegationTokens = hdfs
|
|
|
+ .addDelegationTokens(TEST_RENEWER, null);
|
|
|
for (Token<?> t : delegationTokens) {
|
|
|
LOG.debug("got token " + t);
|
|
|
}
|
|
|
@@ -137,329 +140,113 @@ public class TestOfflineImageViewer {
|
|
|
cluster.getNameNodeRpc()
|
|
|
.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
|
|
|
cluster.getNameNodeRpc().saveNamespace();
|
|
|
-
|
|
|
+
|
|
|
// Determine location of fsimage file
|
|
|
- originalFsimage = FSImageTestUtil.findLatestImageFile(
|
|
|
- FSImageTestUtil.getFSImage(
|
|
|
- cluster.getNameNode()).getStorage().getStorageDir(0));
|
|
|
+ originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
|
|
|
+ .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
|
|
|
if (originalFsimage == null) {
|
|
|
throw new RuntimeException("Didn't generate or can't find fsimage");
|
|
|
}
|
|
|
LOG.debug("original FS image file is " + originalFsimage);
|
|
|
} finally {
|
|
|
- if(cluster != null)
|
|
|
+ if (cluster != null)
|
|
|
cluster.shutdown();
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
@AfterClass
|
|
|
public static void deleteOriginalFSImage() throws IOException {
|
|
|
- if(originalFsimage != null && originalFsimage.exists()) {
|
|
|
+ if (originalFsimage != null && originalFsimage.exists()) {
|
|
|
originalFsimage.delete();
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- // Convenience method to generate a file status from file system for
|
|
|
+
|
|
|
+ // Convenience method to generate a file status from file system for
|
|
|
// later comparison
|
|
|
- private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
|
|
|
- throws IOException {
|
|
|
+ private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
|
|
|
+ throws IOException {
|
|
|
return hdfs.getFileStatus(new Path(file));
|
|
|
}
|
|
|
-
|
|
|
- // Verify that we can correctly generate an ls-style output for a valid
|
|
|
+
|
|
|
+ // Verify that we can correctly generate an ls-style output for a valid
|
|
|
// fsimage
|
|
|
@Test
|
|
|
public void outputOfLSVisitor() throws IOException {
|
|
|
- File testFile = new File(ROOT, "/basicCheck");
|
|
|
- File outputFile = new File(ROOT, "/basicCheckOutput");
|
|
|
-
|
|
|
- try {
|
|
|
- DFSTestUtil.copyFile(originalFsimage, testFile);
|
|
|
-
|
|
|
- ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
|
|
|
- OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
|
|
|
-
|
|
|
- oiv.go();
|
|
|
-
|
|
|
- HashMap<String, LsElements> fileOutput = readLsfile(outputFile);
|
|
|
-
|
|
|
- compareNamespaces(writtenFiles, fileOutput);
|
|
|
- } finally {
|
|
|
- if(testFile.exists()) testFile.delete();
|
|
|
- if(outputFile.exists()) outputFile.delete();
|
|
|
- }
|
|
|
- LOG.debug("Correctly generated ls-style output.");
|
|
|
- }
|
|
|
-
|
|
|
- // Confirm that attempting to read an fsimage file with an unsupported
|
|
|
- // layout results in an error
|
|
|
- @Test
|
|
|
- public void unsupportedFSLayoutVersion() throws IOException {
|
|
|
- File testFile = new File(ROOT, "/invalidLayoutVersion");
|
|
|
- File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
|
|
|
-
|
|
|
- try {
|
|
|
- int badVersionNum = -432;
|
|
|
- changeLayoutVersion(originalFsimage, testFile, badVersionNum);
|
|
|
- ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
|
|
|
- OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
|
|
|
-
|
|
|
- try {
|
|
|
- oiv.go();
|
|
|
- fail("Shouldn't be able to read invalid laytout version");
|
|
|
- } catch(IOException e) {
|
|
|
- if(!e.getMessage().contains(Integer.toString(badVersionNum)))
|
|
|
- throw e; // wasn't error we were expecting
|
|
|
- LOG.debug("Correctly failed at reading bad image version.");
|
|
|
+ StringWriter output = new StringWriter();
|
|
|
+ PrintWriter out = new PrintWriter(output);
|
|
|
+ LsrPBImage v = new LsrPBImage(new Configuration(), out);
|
|
|
+ v.visit(new RandomAccessFile(originalFsimage, "r"));
|
|
|
+ out.close();
|
|
|
+ Pattern pattern = Pattern
|
|
|
+ .compile("([d\\-])([rwx\\-]{9})\\s*(-|\\d+)\\s*(\\w+)\\s*(\\w+)\\s*(\\d+)\\s*(\\d+)\\s*([\b/]+)");
|
|
|
+ int count = 0;
|
|
|
+ for (String s : output.toString().split("\n")) {
|
|
|
+ Matcher m = pattern.matcher(s);
|
|
|
+ assertTrue(m.find());
|
|
|
+ LsElements e = new LsElements();
|
|
|
+ e.isDir = m.group(1).equals("d");
|
|
|
+ e.perms = m.group(2);
|
|
|
+ e.replication = m.group(3).equals("-") ? 0 : Integer.parseInt(m.group(3));
|
|
|
+ e.username = m.group(4);
|
|
|
+ e.groupname = m.group(5);
|
|
|
+ e.filesize = Long.parseLong(m.group(7));
|
|
|
+ String path = m.group(8);
|
|
|
+ if (!path.equals("/")) {
|
|
|
+ compareFiles(writtenFiles.get(path), e);
|
|
|
}
|
|
|
- } finally {
|
|
|
- if(testFile.exists()) testFile.delete();
|
|
|
- if(outputFile.exists()) outputFile.delete();
|
|
|
+ ++count;
|
|
|
}
|
|
|
+ assertEquals(writtenFiles.size() + 1, count);
|
|
|
}
|
|
|
-
|
|
|
- // Verify that image viewer will bail on a file that ends unexpectedly
|
|
|
- @Test
|
|
|
- public void truncatedFSImage() throws IOException {
|
|
|
- File testFile = new File(ROOT, "/truncatedFSImage");
|
|
|
- File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
|
|
|
- try {
|
|
|
- copyPartOfFile(originalFsimage, testFile);
|
|
|
- assertTrue("Created truncated fsimage", testFile.exists());
|
|
|
-
|
|
|
- ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
|
|
|
- OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
|
|
|
-
|
|
|
- try {
|
|
|
- oiv.go();
|
|
|
- fail("Managed to process a truncated fsimage file");
|
|
|
- } catch (EOFException e) {
|
|
|
- LOG.debug("Correctly handled EOF");
|
|
|
- }
|
|
|
|
|
|
- } finally {
|
|
|
- if(testFile.exists()) testFile.delete();
|
|
|
- if(outputFile.exists()) outputFile.delete();
|
|
|
- }
|
|
|
+ @Test(expected = IOException.class)
|
|
|
+ public void testTruncatedFSImage() throws IOException {
|
|
|
+ File truncatedFile = folder.newFile();
|
|
|
+ StringWriter output = new StringWriter();
|
|
|
+ copyPartOfFile(originalFsimage, truncatedFile);
|
|
|
+ new FileDistributionCalculator(new Configuration(), 0, 0, new PrintWriter(
|
|
|
+ output)).visit(new RandomAccessFile(truncatedFile, "r"));
|
|
|
}
|
|
|
-
|
|
|
- // Test that our ls file has all the same compenents of the original namespace
|
|
|
- private void compareNamespaces(HashMap<String, FileStatus> written,
|
|
|
- HashMap<String, LsElements> fileOutput) {
|
|
|
- assertEquals( "Should be the same number of files in both, plus one for root"
|
|
|
- + " in fileoutput", fileOutput.keySet().size(),
|
|
|
- written.keySet().size() + 1);
|
|
|
- Set<String> inFile = fileOutput.keySet();
|
|
|
|
|
|
- // For each line in the output file, verify that the namespace had a
|
|
|
- // filestatus counterpart
|
|
|
- for (String path : inFile) {
|
|
|
- if (path.equals("/")) // root's not included in output from system call
|
|
|
- continue;
|
|
|
-
|
|
|
- assertTrue("Path in file (" + path + ") was written to fs", written
|
|
|
- .containsKey(path));
|
|
|
-
|
|
|
- compareFiles(written.get(path), fileOutput.get(path));
|
|
|
-
|
|
|
- written.remove(path);
|
|
|
- }
|
|
|
-
|
|
|
- assertEquals("No more files were written to fs", 0, written.size());
|
|
|
- }
|
|
|
-
|
|
|
// Compare two files as listed in the original namespace FileStatus and
|
|
|
// the output of the ls file from the image processor
|
|
|
private void compareFiles(FileStatus fs, LsElements elements) {
|
|
|
- assertEquals("directory listed as such",
|
|
|
- fs.isDirectory() ? 'd' : '-', elements.dir);
|
|
|
- assertEquals("perms string equal",
|
|
|
- fs.getPermission().toString(), elements.perms);
|
|
|
+ assertEquals("directory listed as such", fs.isDirectory(), elements.isDir);
|
|
|
+ assertEquals("perms string equal", fs.getPermission().toString(),
|
|
|
+ elements.perms);
|
|
|
assertEquals("replication equal", fs.getReplication(), elements.replication);
|
|
|
assertEquals("owner equal", fs.getOwner(), elements.username);
|
|
|
assertEquals("group equal", fs.getGroup(), elements.groupname);
|
|
|
assertEquals("lengths equal", fs.getLen(), elements.filesize);
|
|
|
}
|
|
|
|
|
|
- // Read the contents of the file created by the Ls processor
|
|
|
- private HashMap<String, LsElements> readLsfile(File lsFile) throws IOException {
|
|
|
- BufferedReader br = new BufferedReader(new FileReader(lsFile));
|
|
|
- String line = null;
|
|
|
- HashMap<String, LsElements> fileContents = new HashMap<String, LsElements>();
|
|
|
-
|
|
|
- while((line = br.readLine()) != null)
|
|
|
- readLsLine(line, fileContents);
|
|
|
-
|
|
|
- br.close();
|
|
|
- return fileContents;
|
|
|
- }
|
|
|
-
|
|
|
- // Parse a line from the ls output. Store permissions, replication,
|
|
|
- // username, groupname and filesize in hashmap keyed to the path name
|
|
|
- private void readLsLine(String line, HashMap<String, LsElements> fileContents) {
|
|
|
- String elements [] = line.split("\\s+");
|
|
|
-
|
|
|
- assertEquals("Not enough elements in ls output", 8, elements.length);
|
|
|
-
|
|
|
- LsElements lsLine = new LsElements();
|
|
|
-
|
|
|
- lsLine.dir = elements[0].charAt(0);
|
|
|
- lsLine.perms = elements[0].substring(1);
|
|
|
- lsLine.replication = elements[1].equals("-")
|
|
|
- ? 0 : Integer.valueOf(elements[1]);
|
|
|
- lsLine.username = elements[2];
|
|
|
- lsLine.groupname = elements[3];
|
|
|
- lsLine.filesize = Long.valueOf(elements[4]);
|
|
|
- // skipping date and time
|
|
|
-
|
|
|
- String path = elements[7];
|
|
|
-
|
|
|
- // Check that each file in the ls output was listed once
|
|
|
- assertFalse("LS file had duplicate file entries",
|
|
|
- fileContents.containsKey(path));
|
|
|
-
|
|
|
- fileContents.put(path, lsLine);
|
|
|
- }
|
|
|
-
|
|
|
- // Copy one fsimage to another, changing the layout version in the process
|
|
|
- private void changeLayoutVersion(File src, File dest, int newVersion)
|
|
|
- throws IOException {
|
|
|
- DataInputStream in = null;
|
|
|
- DataOutputStream out = null;
|
|
|
-
|
|
|
- try {
|
|
|
- in = new DataInputStream(new FileInputStream(src));
|
|
|
- out = new DataOutputStream(new FileOutputStream(dest));
|
|
|
-
|
|
|
- in.readInt();
|
|
|
- out.writeInt(newVersion);
|
|
|
-
|
|
|
- byte [] b = new byte[1024];
|
|
|
- while( in.read(b) > 0 ) {
|
|
|
- out.write(b);
|
|
|
- }
|
|
|
- } finally {
|
|
|
- if(in != null) in.close();
|
|
|
- if(out != null) out.close();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // Only copy part of file into the other. Used for testing truncated fsimage
|
|
|
private void copyPartOfFile(File src, File dest) throws IOException {
|
|
|
- InputStream in = null;
|
|
|
- OutputStream out = null;
|
|
|
-
|
|
|
- byte [] b = new byte[256];
|
|
|
- int bytesWritten = 0;
|
|
|
- int count;
|
|
|
- int maxBytes = 700;
|
|
|
-
|
|
|
+ FileInputStream in = null;
|
|
|
+ FileOutputStream out = null;
|
|
|
+ final int MAX_BYTES = 700;
|
|
|
try {
|
|
|
in = new FileInputStream(src);
|
|
|
out = new FileOutputStream(dest);
|
|
|
-
|
|
|
- while( (count = in.read(b)) > 0 && bytesWritten < maxBytes ) {
|
|
|
- out.write(b);
|
|
|
- bytesWritten += count;
|
|
|
- }
|
|
|
+ in.getChannel().transferTo(0, MAX_BYTES, out.getChannel());
|
|
|
} finally {
|
|
|
- if(in != null) in.close();
|
|
|
- if(out != null) out.close();
|
|
|
+ IOUtils.cleanup(null, in);
|
|
|
+ IOUtils.cleanup(null, out);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@Test
|
|
|
- public void outputOfFileDistributionVisitor() throws IOException {
|
|
|
- File testFile = new File(ROOT, "/basicCheck");
|
|
|
- File outputFile = new File(ROOT, "/fileDistributionCheckOutput");
|
|
|
-
|
|
|
- int totalFiles = 0;
|
|
|
- BufferedReader reader = null;
|
|
|
- try {
|
|
|
- DFSTestUtil.copyFile(originalFsimage, testFile);
|
|
|
- ImageVisitor v = new FileDistributionVisitor(outputFile.getPath(), 0, 0);
|
|
|
- OfflineImageViewer oiv =
|
|
|
- new OfflineImageViewer(testFile.getPath(), v, false);
|
|
|
-
|
|
|
- oiv.go();
|
|
|
-
|
|
|
- reader = new BufferedReader(new FileReader(outputFile));
|
|
|
- String line = reader.readLine();
|
|
|
- assertEquals(line, "Size\tNumFiles");
|
|
|
- while((line = reader.readLine()) != null) {
|
|
|
- String[] row = line.split("\t");
|
|
|
- assertEquals(row.length, 2);
|
|
|
- totalFiles += Integer.parseInt(row[1]);
|
|
|
- }
|
|
|
- } finally {
|
|
|
- if (reader != null) {
|
|
|
- reader.close();
|
|
|
- }
|
|
|
- if(testFile.exists()) testFile.delete();
|
|
|
- if(outputFile.exists()) outputFile.delete();
|
|
|
- }
|
|
|
+ public void testFileDistributionVisitor() throws IOException {
|
|
|
+ StringWriter output = new StringWriter();
|
|
|
+ PrintWriter o = new PrintWriter(output);
|
|
|
+ new FileDistributionCalculator(new Configuration(), 0, 0, o)
|
|
|
+ .visit(new RandomAccessFile(originalFsimage, "r"));
|
|
|
+ o.close();
|
|
|
+
|
|
|
+ Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
|
|
|
+ Matcher matcher = p.matcher(output.getBuffer());
|
|
|
+
|
|
|
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
|
|
|
+ int totalFiles = Integer.parseInt(matcher.group(1));
|
|
|
assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR);
|
|
|
}
|
|
|
-
|
|
|
- private static class TestImageVisitor extends ImageVisitor {
|
|
|
- private List<String> delegationTokenRenewers = new LinkedList<String>();
|
|
|
- TestImageVisitor() {
|
|
|
- }
|
|
|
-
|
|
|
- List<String> getDelegationTokenRenewers() {
|
|
|
- return delegationTokenRenewers;
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void start() throws IOException {
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void finish() throws IOException {
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void finishAbnormally() throws IOException {
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void visit(ImageElement element, String value) throws IOException {
|
|
|
- if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) {
|
|
|
- delegationTokenRenewers.add(value);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void visitEnclosingElement(ImageElement element) throws IOException {
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void visitEnclosingElement(ImageElement element, ImageElement key,
|
|
|
- String value) throws IOException {
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- void leaveEnclosingElement() throws IOException {
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Test
|
|
|
- public void outputOfTestVisitor() throws IOException {
|
|
|
- File testFile = new File(ROOT, "/basicCheck");
|
|
|
-
|
|
|
- try {
|
|
|
- DFSTestUtil.copyFile(originalFsimage, testFile);
|
|
|
- TestImageVisitor v = new TestImageVisitor();
|
|
|
- OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true);
|
|
|
- oiv.go();
|
|
|
-
|
|
|
- // Validated stored delegation token identifiers.
|
|
|
- List<String> dtrs = v.getDelegationTokenRenewers();
|
|
|
- assertEquals(1, dtrs.size());
|
|
|
- assertEquals(TEST_RENEWER, dtrs.get(0));
|
|
|
- } finally {
|
|
|
- if(testFile.exists()) testFile.delete();
|
|
|
- }
|
|
|
- LOG.debug("Passed TestVisitor validation.");
|
|
|
- }
|
|
|
}
|
Jing Zhao