|
@@ -19,27 +19,17 @@ package org.apache.hadoop.fs.contract;
|
|
|
|
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.IOException;
|
|
import java.io.IOException;
|
|
-import java.nio.ByteBuffer;
|
|
|
|
-import java.util.Arrays;
|
|
|
|
|
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
|
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
|
import org.apache.hadoop.fs.FSDataInputStream;
|
|
import org.apache.hadoop.fs.FSDataInputStream;
|
|
import org.apache.hadoop.fs.FileStatus;
|
|
import org.apache.hadoop.fs.FileStatus;
|
|
-import org.apache.hadoop.fs.InvalidPathHandleException;
|
|
|
|
-import org.apache.hadoop.fs.Options.HandleOpt;
|
|
|
|
import org.apache.hadoop.fs.Path;
|
|
import org.apache.hadoop.fs.Path;
|
|
-import org.apache.hadoop.fs.PathHandle;
|
|
|
|
-import org.apache.hadoop.fs.RawPathHandle;
|
|
|
|
import org.apache.hadoop.io.IOUtils;
|
|
import org.apache.hadoop.io.IOUtils;
|
|
|
|
|
|
-import static org.apache.hadoop.fs.contract.ContractTestUtils.appendFile;
|
|
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
|
|
-import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
|
|
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
|
|
-import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyRead;
|
|
|
|
-import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyFileContents;
|
|
|
|
|
|
|
|
import org.junit.Test;
|
|
import org.junit.Test;
|
|
|
|
|
|
@@ -173,252 +163,4 @@ public abstract class AbstractContractOpenTest
|
|
instream.close();
|
|
instream.close();
|
|
}
|
|
}
|
|
|
|
|
|
- /**
|
|
|
|
- * Skip a test case if the FS doesn't support file references.
|
|
|
|
- * The feature is assumed to be unsupported unless stated otherwise.
|
|
|
|
- */
|
|
|
|
- protected void assumeSupportsFileReference() throws IOException {
|
|
|
|
- if (getContract().isSupported(SUPPORTS_FILE_REFERENCE, false)) {
|
|
|
|
- return;
|
|
|
|
- }
|
|
|
|
- skip("Skipping as unsupported feature: " + SUPPORTS_FILE_REFERENCE);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Skip a test case if the FS doesn't support content validation.
|
|
|
|
- * The feature is assumed to be unsupported unless stated otherwise.
|
|
|
|
- */
|
|
|
|
- protected void assumeSupportsContentCheck() throws IOException {
|
|
|
|
- if (getContract().isSupported(SUPPORTS_CONTENT_CHECK, false)) {
|
|
|
|
- return;
|
|
|
|
- }
|
|
|
|
- skip("Skipping as unsupported feature: " + SUPPORTS_CONTENT_CHECK);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- private PathHandle getHandleOrSkip(FileStatus stat, HandleOpt... opts) {
|
|
|
|
- try {
|
|
|
|
- return getFileSystem().getPathHandle(stat, opts);
|
|
|
|
- } catch (UnsupportedOperationException e) {
|
|
|
|
- skip("FileSystem does not support " + Arrays.toString(opts));
|
|
|
|
- }
|
|
|
|
- // unreachable
|
|
|
|
- return null;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Verify {@link HandleOpt#exact()} handle semantics.
|
|
|
|
- * @throws Throwable on error
|
|
|
|
- */
|
|
|
|
- @Test
|
|
|
|
- public void testOpenFileByExact() throws Throwable {
|
|
|
|
- describe("verify open(getPathHandle(FileStatus, exact())) operations" +
|
|
|
|
- "detect changes");
|
|
|
|
- assumeSupportsContentCheck();
|
|
|
|
- assumeSupportsFileReference();
|
|
|
|
- Path path1 = path("testopenfilebyexact1");
|
|
|
|
- Path path2 = path("testopenfilebyexact2");
|
|
|
|
- byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
- FileStatus stat1 = getFileSystem().getFileStatus(path1);
|
|
|
|
- assertNotNull(stat1);
|
|
|
|
- assertEquals(path1, stat1.getPath());
|
|
|
|
- ContractTestUtils.rename(getFileSystem(), path1, path2);
|
|
|
|
- FileStatus stat2 = getFileSystem().getFileStatus(path2);
|
|
|
|
- assertNotNull(stat2);
|
|
|
|
- assertEquals(path2, stat2.getPath());
|
|
|
|
- // create identical file at same location, orig still exists at path2
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
-
|
|
|
|
- PathHandle fd1 = getHandleOrSkip(stat1, HandleOpt.exact());
|
|
|
|
- PathHandle fd2 = getHandleOrSkip(stat2, HandleOpt.exact());
|
|
|
|
-
|
|
|
|
- // verify path1, path2 contents identical
|
|
|
|
- verifyFileContents(getFileSystem(), path1, file1);
|
|
|
|
- verifyFileContents(getFileSystem(), path2, file1);
|
|
|
|
- try {
|
|
|
|
- // the PathHandle will not resolve, even though
|
|
|
|
- // the original entity exists, it has not been modified, and an
|
|
|
|
- // identical file exists at the old path. The handle would also
|
|
|
|
- // fail to resolve if path1 had been modified
|
|
|
|
- instream = getFileSystem().open(fd1);
|
|
|
|
- fail("Expected an exception");
|
|
|
|
- } catch (InvalidPathHandleException e) {
|
|
|
|
- // expected
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- // verify unchanged resolves
|
|
|
|
- instream = getFileSystem().open(fd2);
|
|
|
|
- verifyRead(instream, file1, 0, TEST_FILE_LEN);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Verify {@link HandleOpt#content()} handle semantics.
|
|
|
|
- * @throws Throwable on error
|
|
|
|
- */
|
|
|
|
- @Test
|
|
|
|
- public void testOpenFileByContent() throws Throwable {
|
|
|
|
- describe("verify open(getPathHandle(FileStatus, content())) operations" +
|
|
|
|
- "follow relocation");
|
|
|
|
- assumeSupportsContentCheck();
|
|
|
|
- assumeSupportsFileReference();
|
|
|
|
- Path path1 = path("testopenfilebycontent1");
|
|
|
|
- Path path2 = path("testopenfilebycontent2");
|
|
|
|
- byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
- FileStatus stat = getFileSystem().getFileStatus(path1);
|
|
|
|
- assertNotNull(stat);
|
|
|
|
- assertEquals(path1, stat.getPath());
|
|
|
|
- // rename the file after obtaining FileStatus
|
|
|
|
- ContractTestUtils.rename(getFileSystem(), path1, path2);
|
|
|
|
-
|
|
|
|
- // obtain handle to entity from #getFileStatus call
|
|
|
|
- PathHandle fd = getHandleOrSkip(stat, HandleOpt.content());
|
|
|
|
-
|
|
|
|
- try (FSDataInputStream in = getFileSystem().open(fd)) {
|
|
|
|
- // verify read of consistent content at new location
|
|
|
|
- verifyRead(in, file1, 0, TEST_FILE_LEN);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- // modify the file at its new location by appending data
|
|
|
|
- byte[] file1a = dataset(TEST_FILE_LEN, 44, 255);
|
|
|
|
- appendFile(getFileSystem(), path2, file1a);
|
|
|
|
- byte[] file1x = Arrays.copyOf(file1, file1.length + file1a.length);
|
|
|
|
- System.arraycopy(file1a, 0, file1x, file1.length, file1a.length);
|
|
|
|
- // verify fd entity contains contents of file1 + appended bytes
|
|
|
|
- verifyFileContents(getFileSystem(), path2, file1x);
|
|
|
|
-
|
|
|
|
- try {
|
|
|
|
- // handle should not resolve when content changed
|
|
|
|
- instream = getFileSystem().open(fd);
|
|
|
|
- fail("Failed to detect change to content");
|
|
|
|
- } catch (InvalidPathHandleException e) {
|
|
|
|
- // expected
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Verify {@link HandleOpt#path()} handle semantics.
|
|
|
|
- * @throws Throwable on error
|
|
|
|
- */
|
|
|
|
- @Test
|
|
|
|
- public void testOpenFileByPath() throws Throwable {
|
|
|
|
- describe("verify open(getPathHandle(FileStatus, path())) operations" +
|
|
|
|
- "detect changes");
|
|
|
|
- assumeSupportsContentCheck();
|
|
|
|
- Path path1 = path("testopenfilebypath1");
|
|
|
|
- Path path2 = path("testopenfilebypath2");
|
|
|
|
-
|
|
|
|
- byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
- FileStatus stat1 = getFileSystem().getFileStatus(path1);
|
|
|
|
- assertNotNull(stat1);
|
|
|
|
- assertEquals(path1, stat1.getPath());
|
|
|
|
- ContractTestUtils.rename(getFileSystem(), path1, path2);
|
|
|
|
- FileStatus stat2 = getFileSystem().getFileStatus(path2);
|
|
|
|
- assertNotNull(stat2);
|
|
|
|
- assertEquals(path2, stat2.getPath());
|
|
|
|
- // create identical file at same location, orig still exists at path2
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
-
|
|
|
|
- PathHandle fd1 = getHandleOrSkip(stat1, HandleOpt.path());
|
|
|
|
- PathHandle fd2 = getHandleOrSkip(stat2, HandleOpt.path());
|
|
|
|
-
|
|
|
|
- // verify path1, path2 contents identical
|
|
|
|
- verifyFileContents(getFileSystem(), path1, file1);
|
|
|
|
- verifyFileContents(getFileSystem(), path2, file1);
|
|
|
|
- try {
|
|
|
|
- // verify attempt to resolve the handle fails
|
|
|
|
- instream = getFileSystem().open(fd1);
|
|
|
|
- fail("Expected an exception");
|
|
|
|
- } catch (InvalidPathHandleException e) {
|
|
|
|
- // expected
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- // verify content change OK
|
|
|
|
- byte[] file2a = dataset(TEST_FILE_LEN, 44, 255);
|
|
|
|
- ContractTestUtils.appendFile(getFileSystem(), path2, file2a);
|
|
|
|
- byte[] file2x = Arrays.copyOf(file1, file1.length + file2a.length);
|
|
|
|
- System.arraycopy(file2a, 0, file2x, file1.length, file2a.length);
|
|
|
|
- // verify path2 contains contents of orig + appended bytes
|
|
|
|
- verifyFileContents(getFileSystem(), path2, file2x);
|
|
|
|
- // verify open by fd succeeds
|
|
|
|
- instream = getFileSystem().open(fd2);
|
|
|
|
- verifyRead(instream, file2x, 0, 2 * TEST_FILE_LEN);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Verify {@link HandleOpt#reference()} handle semantics.
|
|
|
|
- * @throws Throwable on error
|
|
|
|
- */
|
|
|
|
- @Test
|
|
|
|
- public void testOpenFileByReference() throws Throwable {
|
|
|
|
- describe("verify open(getPathHandle(FileStatus, reference())) operations" +
|
|
|
|
- " are independent of rename");
|
|
|
|
- assumeSupportsFileReference();
|
|
|
|
- Path path1 = path("testopenfilebyref1");
|
|
|
|
- Path path2 = path("testopenfilebyref2");
|
|
|
|
-
|
|
|
|
- byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
- FileStatus stat = getFileSystem().getFileStatus(path1);
|
|
|
|
- assertNotNull(stat);
|
|
|
|
- assertEquals(path1, stat.getPath());
|
|
|
|
- ContractTestUtils.rename(getFileSystem(), path1, path2);
|
|
|
|
-
|
|
|
|
- byte[] file2 = dataset(TEST_FILE_LEN, 44, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file2);
|
|
|
|
- byte[] file1a = dataset(TEST_FILE_LEN, 42, 255);
|
|
|
|
- appendFile(getFileSystem(), path2, file1a);
|
|
|
|
- byte[] file1x = Arrays.copyOf(file1, file1.length + file1a.length);
|
|
|
|
- System.arraycopy(file1a, 0, file1x, file1.length, file1a.length);
|
|
|
|
-
|
|
|
|
- PathHandle fd = getHandleOrSkip(stat, HandleOpt.reference());
|
|
|
|
-
|
|
|
|
- // verify path2 contains contents of file1 + appended bytes
|
|
|
|
- verifyFileContents(getFileSystem(), path2, file1x);
|
|
|
|
- // verify path1 contents contents of file2
|
|
|
|
- verifyFileContents(getFileSystem(), path1, file2);
|
|
|
|
-
|
|
|
|
- // verify fd contains contents of file1 + appended bytes
|
|
|
|
- instream = getFileSystem().open(fd);
|
|
|
|
- verifyRead(instream, file1x, 0, 2 * TEST_FILE_LEN);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Verify {@link PathHandle} may be serialized and restored.
|
|
|
|
- * @throws Throwable on error
|
|
|
|
- */
|
|
|
|
- @Test
|
|
|
|
- public void testOpenFileBySerializedReference() throws Throwable {
|
|
|
|
- describe("verify PathHandle supports generic serialization");
|
|
|
|
- assumeSupportsFileReference();
|
|
|
|
- Path path1 = path("testopenfilebyref1");
|
|
|
|
- Path path2 = path("testopenfilebyref2");
|
|
|
|
-
|
|
|
|
- byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file1);
|
|
|
|
- FileStatus stat = getFileSystem().getFileStatus(path1);
|
|
|
|
- assertNotNull(stat);
|
|
|
|
- assertEquals(path1, stat.getPath());
|
|
|
|
- ContractTestUtils.rename(getFileSystem(), path1, path2);
|
|
|
|
-
|
|
|
|
- byte[] file2 = dataset(TEST_FILE_LEN, 44, 255);
|
|
|
|
- createFile(getFileSystem(), path1, false, file2);
|
|
|
|
-
|
|
|
|
- PathHandle fd = getHandleOrSkip(stat, HandleOpt.reference());
|
|
|
|
-
|
|
|
|
- // serialize PathHandle
|
|
|
|
- ByteBuffer sb = fd.bytes();
|
|
|
|
- PathHandle fdb = new RawPathHandle(sb);
|
|
|
|
-
|
|
|
|
- instream = getFileSystem().open(fdb);
|
|
|
|
- // verify stat contains contents of file1
|
|
|
|
- verifyRead(instream, file1, 0, TEST_FILE_LEN);
|
|
|
|
- // verify path2 contains contents of file1
|
|
|
|
- verifyFileContents(getFileSystem(), path2, file1);
|
|
|
|
- // verify path1 contents contents of file2
|
|
|
|
- verifyFileContents(getFileSystem(), path1, file2);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
}
|
|
}
|