|
|
@@ -16,611 +16,354 @@
|
|
|
* limitations under the License.
|
|
|
*/
|
|
|
|
|
|
-import org.apache.hadoop.hdfs.*;
|
|
|
-import junit.framework.TestCase;
|
|
|
import java.io.*;
|
|
|
+import java.net.URI;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.concurrent.atomic.*;
|
|
|
+
|
|
|
+import org.apache.log4j.Level;
|
|
|
+import org.apache.commons.logging.Log;
|
|
|
+import org.apache.commons.logging.LogFactory;
|
|
|
+import org.apache.commons.logging.impl.Log4JLogger;
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
|
import org.apache.hadoop.fs.*;
|
|
|
import org.apache.hadoop.fs.permission.*;
|
|
|
-import java.net.*;
|
|
|
+import org.apache.hadoop.hdfs.*;
|
|
|
+import org.apache.hadoop.io.IOUtils;
|
|
|
+import org.apache.hadoop.util.StringUtils;
|
|
|
|
|
|
-/**
|
|
|
- * This class tests that the Fuse module for DFS can mount properly
|
|
|
- * and does a few simple commands:
|
|
|
- * mkdir
|
|
|
- * rmdir
|
|
|
- * ls
|
|
|
- * cat
|
|
|
- *
|
|
|
- * cp and touch are purposely not tested because they won't work with the current module
|
|
|
+import org.junit.Test;
|
|
|
+import org.junit.BeforeClass;
|
|
|
+import org.junit.AfterClass;
|
|
|
+import static org.junit.Assert.*;
|
|
|
|
|
|
- *
|
|
|
+/**
|
|
|
+ * Basic functional tests on a fuse-dfs mount.
|
|
|
*/
|
|
|
-public class TestFuseDFS extends TestCase {
|
|
|
-
|
|
|
- /**
|
|
|
- * mount the fuse file system using assumed fuse module library installed in /usr/local/lib or somewhere else on your
|
|
|
- * pre-existing LD_LIBRARY_PATH
|
|
|
- *
|
|
|
- */
|
|
|
-
|
|
|
- static Process fuse_process;
|
|
|
- static String fuse_cmd;
|
|
|
- static private void mount(String mountpoint, URI dfs) throws IOException, InterruptedException {
|
|
|
-
|
|
|
- String cp = System.getProperty("java.class.path");
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- fuse_cmd = System.getProperty("build.test") + "/../fuse_dfs";
|
|
|
- String libhdfs = System.getProperty("build.test") + "/../../../libhdfs/";
|
|
|
- String arch = System.getProperty("os.arch");
|
|
|
- String jvm = System.getProperty("java.home") + "/lib/" + arch + "/server";
|
|
|
- String lp = System.getProperty("LD_LIBRARY_PATH") + ":" + "/usr/local/lib:" + libhdfs + ":" + jvm;
|
|
|
- System.err.println("LD_LIBRARY_PATH=" + lp);
|
|
|
- String cmd[] = { fuse_cmd, "dfs://" + dfs.getHost() + ":" + String.valueOf(dfs.getPort()),
|
|
|
- mountpoint, "-obig_writes", "-odebug", "-oentry_timeout=0.1", "-oattribute_timeout=0.1", "-ousetrash", "rw", "-oinitchecks",
|
|
|
- "-ordbuffer=32768"};
|
|
|
- final String [] envp = {
|
|
|
- "CLASSPATH="+ cp,
|
|
|
- "LD_LIBRARY_PATH=" + lp,
|
|
|
- "PATH=" + "/usr/bin:/bin"
|
|
|
-
|
|
|
- };
|
|
|
-
|
|
|
- // ensure the mount point is not currently mounted
|
|
|
- Process p = r.exec("fusermount -u " + mountpoint);
|
|
|
- p.waitFor();
|
|
|
+public class TestFuseDFS {
|
|
|
|
|
|
- // clean up the mount point
|
|
|
- p = r.exec("rm -rf " + mountpoint);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
+ private static MiniDFSCluster cluster;
|
|
|
+ private static FileSystem fs;
|
|
|
+ private static Runtime r;
|
|
|
+ private static String mountPoint;
|
|
|
|
|
|
- // make the mount point if needed
|
|
|
- p = r.exec("mkdir -p " + mountpoint);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // mount fuse to the mount point
|
|
|
- fuse_process = r.exec(cmd, envp);
|
|
|
-
|
|
|
- // give DFS a chance to come up
|
|
|
- try { Thread.sleep(3000); } catch(Exception e) { }
|
|
|
+ private static final Log LOG = LogFactory.getLog(TestFuseDFS.class);
|
|
|
+ {
|
|
|
+ ((Log4JLogger)LOG).getLogger().setLevel(Level.ALL);
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * unmounts fuse for before shutting down.
|
|
|
- */
|
|
|
- static private void umount(String mpoint) throws IOException, InterruptedException {
|
|
|
- Runtime r= Runtime.getRuntime();
|
|
|
- Process p = r.exec("fusermount -u " + mpoint);
|
|
|
- p.waitFor();
|
|
|
+ /** Dump the given intput stream to stderr */
|
|
|
+ private static void dumpInputStream(InputStream is) throws IOException {
|
|
|
+ int len;
|
|
|
+ do {
|
|
|
+ byte b[] = new byte[is.available()];
|
|
|
+ len = is.read(b);
|
|
|
+ System.out.println("Read "+len+" bytes");
|
|
|
+ System.out.write(b, 0, b.length);
|
|
|
+ } while (len > 0);
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Set things up - create mini dfs cluster and mount the fuse filesystem.
|
|
|
+ /**
|
|
|
+ * Wait for the given process to return and check that it exited
|
|
|
+ * as required. Log if the process failed.
|
|
|
*/
|
|
|
- public TestFuseDFS() throws IOException,InterruptedException {
|
|
|
- }
|
|
|
-
|
|
|
- static private MiniDFSCluster cluster;
|
|
|
- static private DistributedFileSystem fileSys;
|
|
|
- final static private String mpoint;
|
|
|
-
|
|
|
- static {
|
|
|
- mpoint = System.getProperty("build.test") + "/mnt";
|
|
|
- System.runFinalizersOnExit(true);
|
|
|
- startStuff();
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- static public void startStuff() {
|
|
|
+ private static void checkProcessRet(Process p, boolean expectPass)
|
|
|
+ throws IOException {
|
|
|
try {
|
|
|
- Configuration conf = new HdfsConfiguration();
|
|
|
- conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,false);
|
|
|
- cluster = new MiniDFSCluster.Builder(conf).build();
|
|
|
- fileSys = (DistributedFileSystem)cluster.getFileSystem();
|
|
|
- assertTrue(fileSys.getFileStatus(new Path("/")).isDir());
|
|
|
- mount(mpoint, fileSys.getUri());
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+ int ret = p.waitFor();
|
|
|
+ if (ret != 0) {
|
|
|
+ dumpInputStream(p.getErrorStream());
|
|
|
+ }
|
|
|
+ if (expectPass) {
|
|
|
+ assertEquals(0, ret);
|
|
|
+ } else {
|
|
|
+ assertTrue(ret != 0);
|
|
|
+ }
|
|
|
+ } catch (InterruptedException ie) {
|
|
|
+ fail("Process interrupted: "+ie.getMessage());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- public void setUp() {
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * use shell to create a dir and then use filesys to see it exists.
|
|
|
- */
|
|
|
- public void testMkdir() throws IOException,InterruptedException, Exception {
|
|
|
+ /** Exec the given command and assert it executed successfully */
|
|
|
+ private static void execWaitRet(String cmd) throws IOException {
|
|
|
+ LOG.debug("EXEC "+cmd);
|
|
|
+ Process p = r.exec(cmd);
|
|
|
try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Path path = new Path("/foo");
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- String cmd = "mkdir -p " + mpoint + path.toString();
|
|
|
- Process p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // check it is there
|
|
|
- assertTrue(fileSys.getFileStatus(path).isDir());
|
|
|
-
|
|
|
- // check again through the shell
|
|
|
- String lsCmd = "ls " + mpoint + path.toString();
|
|
|
- p = r.exec(lsCmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- throw e;
|
|
|
+ p.waitFor();
|
|
|
+ } catch (InterruptedException ie) {
|
|
|
+ fail("Process interrupted: "+ie.getMessage());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /** Exec the given command and assert it executed successfully */
|
|
|
+ private static void execIgnoreRet(String cmd) throws IOException {
|
|
|
+ LOG.debug("EXEC "+cmd);
|
|
|
+ r.exec(cmd);
|
|
|
+ }
|
|
|
|
|
|
- /**
|
|
|
- * use shell to create a dir and then use filesys to see it exists.
|
|
|
- */
|
|
|
- public void testWrites() throws IOException,InterruptedException {
|
|
|
- try {
|
|
|
-
|
|
|
- // write a hello file
|
|
|
- File file = new File(mpoint, "hello.txt");
|
|
|
- FileOutputStream f = new FileOutputStream(file);
|
|
|
- String s = "hello ";
|
|
|
- f.write(s.getBytes());
|
|
|
- s = "world";
|
|
|
- f.write(s.getBytes());
|
|
|
- f.flush();
|
|
|
- f.close();
|
|
|
-
|
|
|
+ /** Exec the given command and assert it executed successfully */
|
|
|
+ private static void execAssertSucceeds(String cmd) throws IOException {
|
|
|
+ LOG.debug("EXEC "+cmd);
|
|
|
+ checkProcessRet(r.exec(cmd), true);
|
|
|
+ }
|
|
|
|
|
|
- try {
|
|
|
- Thread.sleep(1000);
|
|
|
- } catch(Exception e) {
|
|
|
- }
|
|
|
+ /** Exec the given command, assert it returned an error code */
|
|
|
+ private static void execAssertFails(String cmd) throws IOException {
|
|
|
+ LOG.debug("EXEC "+cmd);
|
|
|
+ checkProcessRet(r.exec(cmd), false);
|
|
|
+ }
|
|
|
|
|
|
- // check the file exists.
|
|
|
- Path myPath = new Path("/hello.txt");
|
|
|
- assertTrue(fileSys.exists(myPath));
|
|
|
-
|
|
|
- // check the data is ok
|
|
|
- FileInputStream fi = new FileInputStream(new File(mpoint, "hello.txt"));
|
|
|
- byte b[] = new byte[12];
|
|
|
- int length = fi.read(b,0,12);
|
|
|
- assertTrue(length > 0);
|
|
|
- String s2 = new String( b, 0, length);
|
|
|
- assertEquals("hello world", s2);
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- } finally {
|
|
|
- }
|
|
|
+ /** Create and write the given file */
|
|
|
+ private static void createFile(File f, String s) throws IOException {
|
|
|
+ InputStream is = new ByteArrayInputStream(s.getBytes());
|
|
|
+ FileOutputStream fos = new FileOutputStream(f);
|
|
|
+ IOUtils.copyBytes(is, fos, s.length(), true);
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Test ls for dir already created in testMkdDir also tests bad ls
|
|
|
- */
|
|
|
- public void testLs() throws IOException,InterruptedException {
|
|
|
+ /** Check that the given file exists with the given contents */
|
|
|
+ private static void checkFile(File f, String expectedContents)
|
|
|
+ throws IOException {
|
|
|
+ FileInputStream fi = new FileInputStream(f);
|
|
|
+ int len = expectedContents.length();
|
|
|
+ byte[] b = new byte[len];
|
|
|
try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
-
|
|
|
- // mkdir
|
|
|
- Process p = r.exec("mkdir -p " + mpoint + "/test/mkdirs");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // ls
|
|
|
- p = r.exec("ls " + mpoint + "/test/mkdirs");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // ls non-existant directory
|
|
|
- p = r.exec("ls " + mpoint + "/test/mkdirsNotThere");
|
|
|
- int res = p.waitFor();
|
|
|
- assertFalse(res == 0);
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+ IOUtils.readFully(fi, b, 0, len);
|
|
|
+ } catch (IOException ie) {
|
|
|
+ fail("Reading "+f.getName()+" failed with "+ie.getMessage());
|
|
|
+ } finally {
|
|
|
+ fi.close(); // NB: leaving f unclosed prevents unmount
|
|
|
}
|
|
|
-
|
|
|
+ String s = new String(b, 0, len);
|
|
|
+ assertEquals("File content differs", expectedContents, s);
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Remove a dir using the shell and use filesys to see it no longer exists.
|
|
|
- */
|
|
|
- public void testRmdir() throws IOException,InterruptedException {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
-
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- Process p = r.exec("mkdir -p " + mpoint + "/test/rmdir");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- Path myPath = new Path("/test/rmdir");
|
|
|
- assertTrue(fileSys.exists(myPath));
|
|
|
-
|
|
|
- // remove it
|
|
|
- p = r.exec("rmdir " + mpoint + "/test/rmdir");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // check it is not there
|
|
|
- assertFalse(fileSys.exists(myPath));
|
|
|
-
|
|
|
- Path trashPath = new Path("/user/root/.Trash/Current/test/rmdir");
|
|
|
- assertTrue(fileSys.exists(trashPath));
|
|
|
-
|
|
|
- // make it again to test trashing same thing twice
|
|
|
- p = r.exec("mkdir -p " + mpoint + "/test/rmdir");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
+ /** Run a fuse-dfs process to mount the given DFS */
|
|
|
+ private static void establishMount(URI uri) throws IOException {
|
|
|
+ Runtime r = Runtime.getRuntime();
|
|
|
+ String cp = System.getProperty("java.class.path");
|
|
|
|
|
|
- assertTrue(fileSys.exists(myPath));
|
|
|
+ String buildTestDir = System.getProperty("build.test");
|
|
|
+ String fuseCmd = buildTestDir + "/../fuse_dfs";
|
|
|
+ String libHdfs = buildTestDir + "/../../../c++/lib";
|
|
|
|
|
|
- // remove it
|
|
|
- p = r.exec("rmdir " + mpoint + "/test/rmdir");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
+ String arch = System.getProperty("os.arch");
|
|
|
+ String jvm = System.getProperty("java.home") + "/lib/" + arch + "/server";
|
|
|
+ String lp = System.getProperty("LD_LIBRARY_PATH")+":"+libHdfs+":"+jvm;
|
|
|
+ LOG.debug("LD_LIBRARY_PATH=" + lp);
|
|
|
+
|
|
|
+ String nameNode =
|
|
|
+ "dfs://" + uri.getHost() + ":" + String.valueOf(uri.getPort());
|
|
|
+
|
|
|
+ // NB: We're mounting via an unprivileged user, therefore
|
|
|
+ // user_allow_other needs to be set in /etc/fuse.conf, which also
|
|
|
+ // needs to be world readable.
|
|
|
+ String mountCmd[] = {
|
|
|
+ fuseCmd, nameNode, mountPoint,
|
|
|
+ // "-odebug", // Don't daemonize
|
|
|
+ "-obig_writes", // Allow >4kb writes
|
|
|
+ "-oentry_timeout=0.1", // Don't cache dents long
|
|
|
+ "-oattribute_timeout=0.1", // Don't cache attributes long
|
|
|
+ "-ordbuffer=32768", // Read buffer size in kb
|
|
|
+ "rw"
|
|
|
+ };
|
|
|
|
|
|
- // check it is not there
|
|
|
- assertFalse(fileSys.exists(myPath));
|
|
|
+ String [] env = {
|
|
|
+ "CLASSPATH="+cp,
|
|
|
+ "LD_LIBRARY_PATH="+lp,
|
|
|
+ "PATH=/usr/bin:/bin"
|
|
|
+ };
|
|
|
|
|
|
- trashPath = new Path("/user/root/.Trash/Current/test/rmdir.1");
|
|
|
- assertTrue(fileSys.exists(trashPath));
|
|
|
+ execWaitRet("fusermount -u " + mountPoint);
|
|
|
+ execAssertSucceeds("rm -rf " + mountPoint);
|
|
|
+ execAssertSucceeds("mkdir -p " + mountPoint);
|
|
|
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+ // Mount the mini cluster
|
|
|
+ try {
|
|
|
+ Process fuseProcess = r.exec(mountCmd, env);
|
|
|
+ assertEquals(0, fuseProcess.waitFor());
|
|
|
+ } catch (InterruptedException ie) {
|
|
|
+ fail("Failed to mount");
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * use shell to create a dir and then use filesys to see it exists.
|
|
|
- */
|
|
|
- public void testDF() throws IOException,InterruptedException, Exception {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Path path = new Path("/foo");
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- String cmd = "mkdir -p " + mpoint + path.toString();
|
|
|
- Process p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
- File f = new File(mpoint + "/foo");
|
|
|
-
|
|
|
- DistributedFileSystem.DiskStatus d = fileSys.getDiskStatus();
|
|
|
-
|
|
|
- long fileUsedBlocks = (f.getTotalSpace() - f.getFreeSpace())/(64 * 1024 * 1024);
|
|
|
- long dfsUsedBlocks = (long)Math.ceil((double)d.getDfsUsed()/(64 * 1024 * 1024));
|
|
|
+ /** Tear down the fuse-dfs process and mount */
|
|
|
+ private static void teardownMount() throws IOException {
|
|
|
+ execWaitRet("fusermount -u " + mountPoint);
|
|
|
+ }
|
|
|
|
|
|
- assertTrue(fileUsedBlocks == dfsUsedBlocks);
|
|
|
- assertTrue(d.getCapacity() == f.getTotalSpace());
|
|
|
+ @BeforeClass
|
|
|
+ public static void startUp() throws IOException {
|
|
|
+ Configuration conf = new HdfsConfiguration();
|
|
|
+ r = Runtime.getRuntime();
|
|
|
+ mountPoint = System.getProperty("build.test") + "/mnt";
|
|
|
+ conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
|
|
|
+ cluster = new MiniDFSCluster.Builder(conf).build();
|
|
|
+ cluster.waitClusterUp();
|
|
|
+ fs = cluster.getFileSystem();
|
|
|
+ establishMount(fs.getUri());
|
|
|
+ }
|
|
|
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- throw e;
|
|
|
+ @AfterClass
|
|
|
+ public static void tearDown() throws IOException {
|
|
|
+ // Unmount before taking down the mini cluster
|
|
|
+ // so no outstanding operations hang.
|
|
|
+ teardownMount();
|
|
|
+ if (fs != null) {
|
|
|
+ fs.close();
|
|
|
+ }
|
|
|
+ if (cluster != null) {
|
|
|
+ cluster.shutdown();
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * use shell to create a dir and then use filesys to see it exists.
|
|
|
- */
|
|
|
- public void testChown() throws IOException,InterruptedException, Exception {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Path path = new Path("/foo");
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- String cmd = "mkdir -p " + mpoint + path.toString();
|
|
|
- Process p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
+ /** Test basic directory creation, access, removal */
|
|
|
+ @Test
|
|
|
+ public void testBasicDir() throws IOException {
|
|
|
+ File d = new File(mountPoint, "dir1");
|
|
|
|
|
|
- // check it is there
|
|
|
- assertTrue(fileSys.getFileStatus(path).isDir());
|
|
|
+ // Mkdir, access and rm via the mount
|
|
|
+ execAssertSucceeds("mkdir " + d.getAbsolutePath());
|
|
|
+ execAssertSucceeds("ls " + d.getAbsolutePath());
|
|
|
+ execAssertSucceeds("rmdir " + d.getAbsolutePath());
|
|
|
|
|
|
- FileStatus foo = fileSys.getFileStatus(path);
|
|
|
- System.err.println("DEBUG:owner=" + foo.getOwner());
|
|
|
+ // The dir should no longer exist
|
|
|
+ execAssertFails("ls " + d.getAbsolutePath());
|
|
|
+ }
|
|
|
|
|
|
- cmd = "chown nobody " + mpoint + path.toString();
|
|
|
- p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
+ /** Test basic file creation and writing */
|
|
|
+ @Test
|
|
|
+ public void testCreate() throws IOException {
|
|
|
+ final String contents = "hello world";
|
|
|
+ File f = new File(mountPoint, "file1");
|
|
|
|
|
|
- // cmd = "chgrp nobody " + mpoint + path.toString();
|
|
|
- // p = r.exec(cmd);
|
|
|
- // assertTrue(p.waitFor() == 0);
|
|
|
+ // Create and access via the mount
|
|
|
+ createFile(f, contents);
|
|
|
|
|
|
- foo = fileSys.getFileStatus(path);
|
|
|
+ // XX avoids premature EOF
|
|
|
+ try {
|
|
|
+ Thread.sleep(1000);
|
|
|
+ } catch (InterruptedException ie) { }
|
|
|
|
|
|
- System.err.println("DEBUG:owner=" + foo.getOwner());
|
|
|
+ checkFile(f, contents);
|
|
|
|
|
|
- assertTrue(foo.getOwner().equals("nobody"));
|
|
|
- assertTrue(foo.getGroup().equals("nobody"));
|
|
|
+ // Cat, stat and delete via the mount
|
|
|
+ execAssertSucceeds("cat " + f.getAbsolutePath());
|
|
|
+ execAssertSucceeds("stat " + f.getAbsolutePath());
|
|
|
+ execAssertSucceeds("rm " + f.getAbsolutePath());
|
|
|
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- throw e;
|
|
|
- }
|
|
|
+ // The file should no longer exist
|
|
|
+ execAssertFails("ls " + f.getAbsolutePath());
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * use shell to create a dir and then use filesys to see it exists.
|
|
|
- */
|
|
|
- public void testChmod() throws IOException,InterruptedException, Exception {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Path path = new Path("/foo");
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- String cmd = "mkdir -p " + mpoint + path.toString();
|
|
|
- Process p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // check it is there
|
|
|
- assertTrue(fileSys.getFileStatus(path).isDir());
|
|
|
-
|
|
|
- cmd = "chmod 777 " + mpoint + path.toString();
|
|
|
- p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- FileStatus foo = fileSys.getFileStatus(path);
|
|
|
- FsPermission perm = foo.getPermission();
|
|
|
- assertTrue(perm.toShort() == 0777);
|
|
|
-
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- throw e;
|
|
|
- }
|
|
|
+ /** Test creating a file via touch */
|
|
|
+ @Test
|
|
|
+ public void testTouch() throws IOException {
|
|
|
+ File f = new File(mountPoint, "file1");
|
|
|
+ execAssertSucceeds("touch " + f.getAbsolutePath());
|
|
|
+ execAssertSucceeds("rm " + f.getAbsolutePath());
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * use shell to create a dir and then use filesys to see it exists.
|
|
|
- */
|
|
|
- public void testUtimes() throws IOException,InterruptedException, Exception {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Path path = new Path("/utimetest");
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- String cmd = "touch " + mpoint + path.toString();
|
|
|
- Process p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // check it is there
|
|
|
- assertTrue(fileSys.exists(path));
|
|
|
+ /** Test random access to a file */
|
|
|
+ @Test
|
|
|
+ public void testRandomAccess() throws IOException {
|
|
|
+ final String contents = "hello world";
|
|
|
+ File f = new File(mountPoint, "file1");
|
|
|
|
|
|
- FileStatus foo = fileSys.getFileStatus(path);
|
|
|
- long oldTime = foo.getModificationTime();
|
|
|
- try { Thread.sleep(1000); } catch(Exception e) {}
|
|
|
+ createFile(f, contents);
|
|
|
|
|
|
- cmd = "touch " + mpoint + path.toString();
|
|
|
- p = r.exec(cmd);
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- try { Thread.sleep(1000); } catch(Exception e) {}
|
|
|
- foo = fileSys.getFileStatus(path);
|
|
|
- long newTime = foo.getModificationTime();
|
|
|
-
|
|
|
- assertTrue(newTime > oldTime);
|
|
|
-
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- throw e;
|
|
|
+ RandomAccessFile raf = new RandomAccessFile(f, "rw");
|
|
|
+ raf.seek(f.length());
|
|
|
+ try {
|
|
|
+ raf.write('b');
|
|
|
+ } catch (IOException e) {
|
|
|
+ // Expected: fuse-dfs not yet support append
|
|
|
+ assertEquals("Operation not supported", e.getMessage());
|
|
|
} finally {
|
|
|
+ raf.close();
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
- /**
|
|
|
- *
|
|
|
- * Test dfs_read on a file size that will trigger multiple internal reads.
|
|
|
- * First, just check raw size reading is ok and then check with smaller reads
|
|
|
- * including checking the validity of the data read.
|
|
|
- *
|
|
|
- */
|
|
|
- public void testReads() throws IOException,InterruptedException {
|
|
|
+ raf = new RandomAccessFile(f, "rw");
|
|
|
+ raf.seek(0);
|
|
|
try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- Process p;
|
|
|
-
|
|
|
- // create the file
|
|
|
- Path myPath = new Path("/test/hello.reads");
|
|
|
- FSDataOutputStream s = fileSys.create(myPath);
|
|
|
- String hello = "hello world!";
|
|
|
- int written = 0;
|
|
|
- int mycount = 0;
|
|
|
- while(written < 1024 * 9) {
|
|
|
- s.writeUTF(hello);
|
|
|
- s.writeInt(mycount++);
|
|
|
- written += hello.length() + 4;
|
|
|
- }
|
|
|
- s.close();
|
|
|
-
|
|
|
- // check it exists
|
|
|
- assertTrue(fileSys.exists(myPath));
|
|
|
- FileStatus foo = fileSys.getFileStatus(myPath);
|
|
|
- assertTrue(foo.getLen() >= 9 * 1024);
|
|
|
-
|
|
|
-
|
|
|
- {
|
|
|
- // cat the file
|
|
|
- DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello.reads"));
|
|
|
- byte buf [] = new byte[4096];
|
|
|
- // test reading 0 length
|
|
|
- assertTrue(is.read(buf, 0, 0) == 0);
|
|
|
-
|
|
|
- // test real reads
|
|
|
- assertTrue(is.read(buf, 0, 1024) == 1024);
|
|
|
- assertTrue(is.read(buf, 0, 4096) == 4096);
|
|
|
- assertTrue(is.read(buf, 0, 4096) == 4096);
|
|
|
- is.close();
|
|
|
- }
|
|
|
-
|
|
|
- {
|
|
|
- DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello.reads"));
|
|
|
- int read = 0;
|
|
|
- int counter = 0;
|
|
|
- try {
|
|
|
- while(true) {
|
|
|
- String s2 = DataInputStream.readUTF(is);
|
|
|
- int s3 = is.readInt();
|
|
|
- assertTrue(s2.equals(hello));
|
|
|
- assertTrue(s3 == counter++);
|
|
|
- read += hello.length() + 4;
|
|
|
- }
|
|
|
- } catch(EOFException e) {
|
|
|
- assertTrue(read >= 9 * 1024);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // check reading an empty file for EOF
|
|
|
- {
|
|
|
- // create the file
|
|
|
- myPath = new Path("/test/hello.reads2");
|
|
|
- s = fileSys.create(myPath);
|
|
|
- s.close();
|
|
|
-
|
|
|
- FSDataInputStream fs = fileSys.open(myPath);
|
|
|
- assertEquals(-1, fs.read());
|
|
|
-
|
|
|
- FileInputStream f = new FileInputStream(mpoint + "/test/hello.reads2");
|
|
|
- assertEquals(-1, f.read());
|
|
|
- }
|
|
|
-
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+ raf.write('b');
|
|
|
+ fail("Over-wrote existing bytes");
|
|
|
+ } catch (IOException e) {
|
|
|
+ // Expected: can-not overwrite a file
|
|
|
+ assertEquals("Invalid argument", e.getMessage());
|
|
|
} finally {
|
|
|
+ raf.close();
|
|
|
}
|
|
|
+ execAssertSucceeds("rm " + f.getAbsolutePath());
|
|
|
}
|
|
|
|
|
|
-
|
|
|
- /**
|
|
|
- * Use filesys to create the hello world! file and then cat it and see its contents are correct.
|
|
|
- */
|
|
|
- public void testCat() throws IOException,InterruptedException {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
- Process p = r.exec("rm -rf " + mpoint + "/test/hello");
|
|
|
- assertTrue(p.waitFor() == 0);
|
|
|
-
|
|
|
- // create the file
|
|
|
- Path myPath = new Path("/test/hello");
|
|
|
- FSDataOutputStream s = fileSys.create(myPath);
|
|
|
- String hello = "hello world!";
|
|
|
- s.writeUTF(hello);
|
|
|
- s.writeInt(1033);
|
|
|
- s.close();
|
|
|
-
|
|
|
- // check it exists
|
|
|
- assertTrue(fileSys.exists(myPath));
|
|
|
-
|
|
|
- // cat the file
|
|
|
- DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello"));
|
|
|
- String s2 = DataInputStream.readUTF(is);
|
|
|
- int s3 = is.readInt();
|
|
|
- assertTrue(s2.equals(hello));
|
|
|
- assertTrue(s3 == 1033);
|
|
|
-
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- } finally {
|
|
|
+ /** Test copying a set of files from the mount to itself */
|
|
|
+ @Test
|
|
|
+ public void testCopyFiles() throws IOException {
|
|
|
+ final String contents = "hello world";
|
|
|
+ File d1 = new File(mountPoint, "dir1");
|
|
|
+ File d2 = new File(mountPoint, "dir2");
|
|
|
+
|
|
|
+ // Create and populate dir1 via the mount
|
|
|
+ execAssertSucceeds("mkdir " + d1.getAbsolutePath());
|
|
|
+ for (int i = 0; i < 5; i++) {
|
|
|
+ createFile(new File(d1, "file"+i), contents);
|
|
|
}
|
|
|
+ assertEquals(5, d1.listFiles().length);
|
|
|
+
|
|
|
+ // Copy dir from the mount to the mount
|
|
|
+ execAssertSucceeds("cp -r " + d1.getAbsolutePath() +
|
|
|
+ " " + d2.getAbsolutePath());
|
|
|
+ assertEquals(5, d2.listFiles().length);
|
|
|
+
|
|
|
+ // Access all the files in the dirs and remove them
|
|
|
+ execAssertSucceeds("find " + d1.getAbsolutePath());
|
|
|
+ execAssertSucceeds("find " + d2.getAbsolutePath());
|
|
|
+ execAssertSucceeds("rm -r " + d1.getAbsolutePath());
|
|
|
+ execAssertSucceeds("rm -r " + d2.getAbsolutePath());
|
|
|
}
|
|
|
|
|
|
+ /** Test concurrent creation and access of the mount */
|
|
|
+ @Test
|
|
|
+ public void testMultipleThreads() throws IOException {
|
|
|
+ ArrayList<Thread> threads = new ArrayList<Thread>();
|
|
|
+ final AtomicReference<String> errorMessage = new AtomicReference<String>();
|
|
|
+
|
|
|
+ for (int i = 0; i < 10; i++) {
|
|
|
+ Thread t = new Thread() {
|
|
|
+ public void run() {
|
|
|
+ try {
|
|
|
+ File d = new File(mountPoint, "dir"+getId());
|
|
|
+ execWaitRet("mkdir " + d.getAbsolutePath());
|
|
|
+ for (int j = 0; j < 10; j++) {
|
|
|
+ File f = new File(d, "file"+j);
|
|
|
+ final String contents = "thread "+getId()+" "+j;
|
|
|
+ createFile(f, contents);
|
|
|
+ }
|
|
|
+ for (int j = 0; j < 10; j++) {
|
|
|
+ File f = new File(d, "file"+j);
|
|
|
+ execWaitRet("cat " + f.getAbsolutePath());
|
|
|
+ execWaitRet("rm " + f.getAbsolutePath());
|
|
|
+ }
|
|
|
+ execWaitRet("rmdir " + d.getAbsolutePath());
|
|
|
+ } catch (IOException ie) {
|
|
|
+ errorMessage.set(
|
|
|
+ String.format("Exception %s",
|
|
|
+ StringUtils.stringifyException(ie)));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ };
|
|
|
+ t.start();
|
|
|
+ threads.add(t);
|
|
|
+ }
|
|
|
|
|
|
-
|
|
|
- /**
|
|
|
- * Use filesys to create the hello world! file and then cat it and see its contents are correct.
|
|
|
- */
|
|
|
- public void testAppends() throws IOException,InterruptedException {
|
|
|
- try {
|
|
|
- // First create a new directory with mkdirs
|
|
|
- Runtime r = Runtime.getRuntime();
|
|
|
-
|
|
|
- {
|
|
|
- FileOutputStream os = new FileOutputStream(mpoint + "/appends");
|
|
|
- String hello = "hello";
|
|
|
- os.write(hello.getBytes());
|
|
|
- os.flush();
|
|
|
- os.close();
|
|
|
- }
|
|
|
-
|
|
|
- // check it exists
|
|
|
- Path myPath = new Path("/appends");
|
|
|
- assertTrue(fileSys.exists(myPath));
|
|
|
-
|
|
|
+ for (Thread t : threads) {
|
|
|
try {
|
|
|
- Thread.sleep(1000);
|
|
|
- } catch(Exception e) {
|
|
|
+ t.join();
|
|
|
+ } catch (InterruptedException ie) {
|
|
|
+ fail("Thread interrupted: "+ie.getMessage());
|
|
|
}
|
|
|
-
|
|
|
- FileStatus foo = fileSys.getFileStatus(myPath);
|
|
|
-
|
|
|
- File f = new File(mpoint + "/appends");
|
|
|
- assertTrue(f.length() > 0);
|
|
|
-
|
|
|
- {
|
|
|
- FileOutputStream os = new FileOutputStream(mpoint + "/appends", true);
|
|
|
- String hello = " world!";
|
|
|
- os.write(hello.getBytes());
|
|
|
- os.flush();
|
|
|
- os.close();
|
|
|
- }
|
|
|
-
|
|
|
- // cat the file
|
|
|
- FileInputStream is = new FileInputStream(mpoint + "/appends");
|
|
|
- byte b[] = new byte[1024];
|
|
|
- int len = is.read(b);
|
|
|
- assertTrue(len > 0);
|
|
|
- String s2 = new String(b,0,len);
|
|
|
- assertTrue(s2.equals("hello world!"));
|
|
|
-
|
|
|
- } catch(Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- } finally {
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- public void testDone() throws IOException {
|
|
|
- close();
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Unmount and close
|
|
|
- */
|
|
|
- protected void tearDown() throws Exception {
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Unmount and close
|
|
|
- */
|
|
|
- protected void finalize() throws Throwable {
|
|
|
- close();
|
|
|
- }
|
|
|
-
|
|
|
- public void close() {
|
|
|
- try {
|
|
|
- int length;
|
|
|
-
|
|
|
- // print out the fuse debug output
|
|
|
- {
|
|
|
- do {
|
|
|
- InputStream i = fuse_process.getInputStream();
|
|
|
- byte b[] = new byte[i.available()];
|
|
|
- length = i.read(b);
|
|
|
- System.err.println("read x bytes: " + length);
|
|
|
- System.err.write(b,0,b.length);
|
|
|
- } while(length > 0) ;
|
|
|
- }
|
|
|
-
|
|
|
- do {
|
|
|
- InputStream i = fuse_process.getErrorStream();
|
|
|
- byte b[] = new byte[i.available()];
|
|
|
- length = i.read(b);
|
|
|
- System.err.println("read x bytes: " + length);
|
|
|
- System.err.write(b,0,b.length);
|
|
|
- } while(length > 0) ;
|
|
|
-
|
|
|
- umount(mpoint);
|
|
|
-
|
|
|
- fuse_process.destroy();
|
|
|
- fuse_process = null;
|
|
|
- if(fileSys != null) {
|
|
|
- fileSys.close();
|
|
|
- fileSys = null;
|
|
|
- }
|
|
|
- if(cluster != null) {
|
|
|
- cluster.shutdown();
|
|
|
- cluster = null;
|
|
|
- }
|
|
|
- } catch(Exception e) { }
|
|
|
+ assertNull(errorMessage.get(), errorMessage.get());
|
|
|
}
|
|
|
-};
|
|
|
+}
|
Suresh Srinivas