|
@@ -685,13 +685,12 @@ public class TestFsck {
|
|
final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
|
|
final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
|
|
int blockSize = 2 * cellSize;
|
|
int blockSize = 2 * cellSize;
|
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
|
- conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
|
|
- ecPolicy.getName());
|
|
|
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
|
numAllUnits + 1).build();
|
|
numAllUnits + 1).build();
|
|
String topDir = "/myDir";
|
|
String topDir = "/myDir";
|
|
cluster.waitActive();
|
|
cluster.waitActive();
|
|
DistributedFileSystem fs = cluster.getFileSystem();
|
|
DistributedFileSystem fs = cluster.getFileSystem();
|
|
|
|
+ fs.enableErasureCodingPolicy(ecPolicy.getName());
|
|
util.createFiles(fs, topDir);
|
|
util.createFiles(fs, topDir);
|
|
// set topDir to EC when it has replicated files
|
|
// set topDir to EC when it has replicated files
|
|
cluster.getFileSystem().getClient().setErasureCodingPolicy(
|
|
cluster.getFileSystem().getClient().setErasureCodingPolicy(
|
|
@@ -1999,19 +1998,19 @@ public class TestFsck {
|
|
|
|
|
|
@Test
|
|
@Test
|
|
public void testECFsck() throws Exception {
|
|
public void testECFsck() throws Exception {
|
|
- FileSystem fs = null;
|
|
|
|
|
|
+ DistributedFileSystem fs = null;
|
|
final long precision = 1L;
|
|
final long precision = 1L;
|
|
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
|
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
|
precision);
|
|
precision);
|
|
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
|
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
|
- conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
|
|
- StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
|
|
int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
|
|
int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
|
|
int parityBlocks =
|
|
int parityBlocks =
|
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
|
int totalSize = dataBlocks + parityBlocks;
|
|
int totalSize = dataBlocks + parityBlocks;
|
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
|
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
|
|
fs = cluster.getFileSystem();
|
|
fs = cluster.getFileSystem();
|
|
|
|
+ fs.enableErasureCodingPolicy(
|
|
|
|
+ StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
|
|
|
|
// create a contiguous file
|
|
// create a contiguous file
|
|
Path replDirPath = new Path("/replicated");
|
|
Path replDirPath = new Path("/replicated");
|
|
@@ -2301,11 +2300,11 @@ public class TestFsck {
|
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
|
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
|
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
|
int totalSize = dataBlocks + parityBlocks;
|
|
int totalSize = dataBlocks + parityBlocks;
|
|
- conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
|
|
- StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
|
|
cluster = new MiniDFSCluster.Builder(conf)
|
|
cluster = new MiniDFSCluster.Builder(conf)
|
|
.numDataNodes(totalSize).build();
|
|
.numDataNodes(totalSize).build();
|
|
fs = cluster.getFileSystem();
|
|
fs = cluster.getFileSystem();
|
|
|
|
+ fs.enableErasureCodingPolicy(
|
|
|
|
+ StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
Map<Integer, Integer> dnIndices = new HashMap<>();
|
|
Map<Integer, Integer> dnIndices = new HashMap<>();
|
|
ArrayList<DataNode> dnList = cluster.getDataNodes();
|
|
ArrayList<DataNode> dnList = cluster.getDataNodes();
|
|
for (int i = 0; i < totalSize; i++) {
|
|
for (int i = 0; i < totalSize; i++) {
|
|
@@ -2372,11 +2371,11 @@ public class TestFsck {
|
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
|
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
|
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
|
int totalSize = dataBlocks + parityBlocks;
|
|
int totalSize = dataBlocks + parityBlocks;
|
|
- conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
|
|
- StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
|
|
cluster = new MiniDFSCluster.Builder(conf)
|
|
cluster = new MiniDFSCluster.Builder(conf)
|
|
.numDataNodes(totalSize).build();
|
|
.numDataNodes(totalSize).build();
|
|
fs = cluster.getFileSystem();
|
|
fs = cluster.getFileSystem();
|
|
|
|
+ fs.enableErasureCodingPolicy(
|
|
|
|
+ StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
|
|
|
|
// create file
|
|
// create file
|
|
Path ecDirPath = new Path("/striped");
|
|
Path ecDirPath = new Path("/striped");
|