|
@@ -47,6 +47,8 @@ public class TestErasureCodingPolicies {
|
|
private MiniDFSCluster cluster;
|
|
private MiniDFSCluster cluster;
|
|
private DistributedFileSystem fs;
|
|
private DistributedFileSystem fs;
|
|
private static final int BLOCK_SIZE = 1024;
|
|
private static final int BLOCK_SIZE = 1024;
|
|
|
|
+ private static final ErasureCodingPolicy EC_POLICY =
|
|
|
|
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
|
|
private FSNamesystem namesystem;
|
|
private FSNamesystem namesystem;
|
|
|
|
|
|
@Before
|
|
@Before
|
|
@@ -80,7 +82,8 @@ public class TestErasureCodingPolicies {
|
|
DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
|
|
DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
|
|
|
|
|
|
// set ec policy on dir
|
|
// set ec policy on dir
|
|
- fs.setErasureCodingPolicy(dir, null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(dir,
|
|
|
|
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
|
|
// create a file which should be using ec
|
|
// create a file which should be using ec
|
|
final Path ecSubDir = new Path(dir, "ecSubDir");
|
|
final Path ecSubDir = new Path(dir, "ecSubDir");
|
|
final Path ecFile = new Path(ecSubDir, "ecFile");
|
|
final Path ecFile = new Path(ecSubDir, "ecFile");
|
|
@@ -132,7 +135,7 @@ public class TestErasureCodingPolicies {
|
|
fs.mkdir(testDir, FsPermission.getDirDefault());
|
|
fs.mkdir(testDir, FsPermission.getDirDefault());
|
|
|
|
|
|
/* Normal creation of an erasure coding directory */
|
|
/* Normal creation of an erasure coding directory */
|
|
- fs.getClient().setErasureCodingPolicy(testDir.toString(), null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(testDir, EC_POLICY.getName());
|
|
|
|
|
|
/* Verify files under the directory are striped */
|
|
/* Verify files under the directory are striped */
|
|
final Path ECFilePath = new Path(testDir, "foo");
|
|
final Path ECFilePath = new Path(testDir, "foo");
|
|
@@ -148,7 +151,7 @@ public class TestErasureCodingPolicies {
|
|
fs.mkdir(notEmpty, FsPermission.getDirDefault());
|
|
fs.mkdir(notEmpty, FsPermission.getDirDefault());
|
|
final Path oldFile = new Path(notEmpty, "old");
|
|
final Path oldFile = new Path(notEmpty, "old");
|
|
fs.create(oldFile);
|
|
fs.create(oldFile);
|
|
- fs.getClient().setErasureCodingPolicy(notEmpty.toString(), null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName());
|
|
final Path newFile = new Path(notEmpty, "new");
|
|
final Path newFile = new Path(notEmpty, "new");
|
|
fs.create(newFile);
|
|
fs.create(newFile);
|
|
INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
|
|
INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
|
|
@@ -160,10 +163,10 @@ public class TestErasureCodingPolicies {
|
|
final Path dir1 = new Path("/dir1");
|
|
final Path dir1 = new Path("/dir1");
|
|
final Path dir2 = new Path(dir1, "dir2");
|
|
final Path dir2 = new Path(dir1, "dir2");
|
|
fs.mkdir(dir1, FsPermission.getDirDefault());
|
|
fs.mkdir(dir1, FsPermission.getDirDefault());
|
|
- fs.getClient().setErasureCodingPolicy(dir1.toString(), null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(dir1, EC_POLICY.getName());
|
|
fs.mkdir(dir2, FsPermission.getDirDefault());
|
|
fs.mkdir(dir2, FsPermission.getDirDefault());
|
|
try {
|
|
try {
|
|
- fs.getClient().setErasureCodingPolicy(dir2.toString(), null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(dir2, EC_POLICY.getName());
|
|
} catch (IOException e) {
|
|
} catch (IOException e) {
|
|
fail("Nested erasure coding policies are supported");
|
|
fail("Nested erasure coding policies are supported");
|
|
}
|
|
}
|
|
@@ -172,7 +175,7 @@ public class TestErasureCodingPolicies {
|
|
final Path fPath = new Path("/file");
|
|
final Path fPath = new Path("/file");
|
|
fs.create(fPath);
|
|
fs.create(fPath);
|
|
try {
|
|
try {
|
|
- fs.getClient().setErasureCodingPolicy(fPath.toString(), null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(fPath, EC_POLICY.getName());
|
|
fail("Erasure coding policy on file");
|
|
fail("Erasure coding policy on file");
|
|
} catch (IOException e) {
|
|
} catch (IOException e) {
|
|
assertExceptionContains("erasure coding policy for a file", e);
|
|
assertExceptionContains("erasure coding policy for a file", e);
|
|
@@ -185,8 +188,8 @@ public class TestErasureCodingPolicies {
|
|
final Path dstECDir = new Path("/dstEC");
|
|
final Path dstECDir = new Path("/dstEC");
|
|
fs.mkdir(srcECDir, FsPermission.getDirDefault());
|
|
fs.mkdir(srcECDir, FsPermission.getDirDefault());
|
|
fs.mkdir(dstECDir, FsPermission.getDirDefault());
|
|
fs.mkdir(dstECDir, FsPermission.getDirDefault());
|
|
- fs.getClient().setErasureCodingPolicy(srcECDir.toString(), null);
|
|
|
|
- fs.getClient().setErasureCodingPolicy(dstECDir.toString(), null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName());
|
|
|
|
+ fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName());
|
|
final Path srcFile = new Path(srcECDir, "foo");
|
|
final Path srcFile = new Path(srcECDir, "foo");
|
|
fs.create(srcFile);
|
|
fs.create(srcFile);
|
|
|
|
|
|
@@ -220,7 +223,8 @@ public class TestErasureCodingPolicies {
|
|
public void testReplication() throws IOException {
|
|
public void testReplication() throws IOException {
|
|
final Path testDir = new Path("/ec");
|
|
final Path testDir = new Path("/ec");
|
|
fs.mkdir(testDir, FsPermission.getDirDefault());
|
|
fs.mkdir(testDir, FsPermission.getDirDefault());
|
|
- fs.setErasureCodingPolicy(testDir, null);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(testDir,
|
|
|
|
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
|
|
final Path fooFile = new Path(testDir, "foo");
|
|
final Path fooFile = new Path(testDir, "foo");
|
|
// create ec file with replication=0
|
|
// create ec file with replication=0
|
|
fs.create(fooFile, FsPermission.getFileDefault(), true,
|
|
fs.create(fooFile, FsPermission.getFileDefault(), true,
|
|
@@ -241,8 +245,9 @@ public class TestErasureCodingPolicies {
|
|
// dir EC policy should be null
|
|
// dir EC policy should be null
|
|
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
|
|
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
|
|
// dir EC policy after setting
|
|
// dir EC policy after setting
|
|
- fs.getClient().setErasureCodingPolicy(src, null); //Default one will be used.
|
|
|
|
- ErasureCodingPolicy sysDefaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
|
|
|
|
|
|
+ ErasureCodingPolicy sysDefaultECPolicy =
|
|
|
|
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
|
|
|
|
+ fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName());
|
|
verifyErasureCodingInfo(src, sysDefaultECPolicy);
|
|
verifyErasureCodingInfo(src, sysDefaultECPolicy);
|
|
fs.create(new Path(ecDir, "child1")).close();
|
|
fs.create(new Path(ecDir, "child1")).close();
|
|
// verify for the files in ec dir
|
|
// verify for the files in ec dir
|
|
@@ -263,7 +268,7 @@ public class TestErasureCodingPolicies {
|
|
// dir ECInfo before being set
|
|
// dir ECInfo before being set
|
|
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
|
|
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
|
|
// dir ECInfo after set
|
|
// dir ECInfo after set
|
|
- fs.getClient().setErasureCodingPolicy(src, usingECPolicy);
|
|
|
|
|
|
+ fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName());
|
|
verifyErasureCodingInfo(src, usingECPolicy);
|
|
verifyErasureCodingInfo(src, usingECPolicy);
|
|
fs.create(new Path(ecDir, "child1")).close();
|
|
fs.create(new Path(ecDir, "child1")).close();
|
|
// verify for the files in ec dir
|
|
// verify for the files in ec dir
|
|
@@ -291,12 +296,12 @@ public class TestErasureCodingPolicies {
|
|
final Path ecDir = new Path(src);
|
|
final Path ecDir = new Path(src);
|
|
try {
|
|
try {
|
|
fs.mkdir(ecDir, FsPermission.getDirDefault());
|
|
fs.mkdir(ecDir, FsPermission.getDirDefault());
|
|
- fs.getClient().setErasureCodingPolicy(src, ecPolicy);
|
|
|
|
|
|
+ fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName());
|
|
fail("HadoopIllegalArgumentException should be thrown for"
|
|
fail("HadoopIllegalArgumentException should be thrown for"
|
|
+ "setting an invalid erasure coding policy");
|
|
+ "setting an invalid erasure coding policy");
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
- assertExceptionContains("Policy [ RS-4-2-128k ] does not match " +
|
|
|
|
- "any of the supported policies",e);
|
|
|
|
|
|
+ assertExceptionContains("Policy 'RS-4-2-128k' does not match " +
|
|
|
|
+ "any supported erasure coding policies",e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -338,7 +343,7 @@ public class TestErasureCodingPolicies {
|
|
for (ErasureCodingPolicy policy : sysPolicies) {
|
|
for (ErasureCodingPolicy policy : sysPolicies) {
|
|
Path dir = new Path("/policy_" + policy.getId());
|
|
Path dir = new Path("/policy_" + policy.getId());
|
|
fs.mkdir(dir, FsPermission.getDefault());
|
|
fs.mkdir(dir, FsPermission.getDefault());
|
|
- fs.setErasureCodingPolicy(dir, policy);
|
|
|
|
|
|
+ fs.setErasureCodingPolicy(dir, policy.getName());
|
|
Path file = new Path(dir, "child");
|
|
Path file = new Path(dir, "child");
|
|
fs.create(file).close();
|
|
fs.create(file).close();
|
|
assertEquals(policy, fs.getErasureCodingPolicy(file));
|
|
assertEquals(policy, fs.getErasureCodingPolicy(file));
|