|
@@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
|
|
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
|
|
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
|
|
import org.apache.hadoop.ozone.OzoneConsts;
|
|
import org.apache.hadoop.ozone.OzoneConsts;
|
|
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
|
|
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
|
|
-import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage;
|
|
|
|
|
|
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
|
|
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
|
|
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
|
|
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
|
|
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
|
|
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
|
|
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
|
|
@@ -103,6 +103,10 @@ public class TestChunkManagerImpl {
|
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ private DispatcherContext getDispatcherContext() {
|
|
|
|
+ return new DispatcherContext.Builder().build();
|
|
|
|
+ }
|
|
|
|
+
|
|
@Test
|
|
@Test
|
|
public void testWriteChunkStageWriteAndCommit() throws Exception {
|
|
public void testWriteChunkStageWriteAndCommit() throws Exception {
|
|
//As in Setup, we try to create container, these paths should exist.
|
|
//As in Setup, we try to create container, these paths should exist.
|
|
@@ -115,16 +119,20 @@ public class TestChunkManagerImpl {
|
|
// As no chunks are written to the volume writeBytes should be 0
|
|
// As no chunks are written to the volume writeBytes should be 0
|
|
checkWriteIOStats(0, 0);
|
|
checkWriteIOStats(0, 0);
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.WRITE_DATA);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), new DispatcherContext.Builder()
|
|
|
|
+ .setStage(DispatcherContext.WriteChunkStage.WRITE_DATA).build());
|
|
// Now a chunk file is being written with Stage WRITE_DATA, so it should
|
|
// Now a chunk file is being written with Stage WRITE_DATA, so it should
|
|
// create a temporary chunk file.
|
|
// create a temporary chunk file.
|
|
assertTrue(chunksPath.listFiles().length == 1);
|
|
assertTrue(chunksPath.listFiles().length == 1);
|
|
|
|
|
|
|
|
+ long term = 0;
|
|
|
|
+ long index = 0;
|
|
File chunkFile = ChunkUtils.getChunkFile(keyValueContainerData, chunkInfo);
|
|
File chunkFile = ChunkUtils.getChunkFile(keyValueContainerData, chunkInfo);
|
|
File tempChunkFile = new File(chunkFile.getParent(),
|
|
File tempChunkFile = new File(chunkFile.getParent(),
|
|
- chunkFile.getName() +
|
|
|
|
- OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
|
|
|
|
- OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
|
|
|
|
|
|
+ chunkFile.getName() + OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER
|
|
|
|
+ + OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX
|
|
|
|
+ + OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER + term
|
|
|
|
+ + OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER + index);
|
|
|
|
|
|
// As chunk write stage is WRITE_DATA, temp chunk file will be created.
|
|
// As chunk write stage is WRITE_DATA, temp chunk file will be created.
|
|
assertTrue(tempChunkFile.exists());
|
|
assertTrue(tempChunkFile.exists());
|
|
@@ -132,7 +140,8 @@ public class TestChunkManagerImpl {
|
|
checkWriteIOStats(data.length, 1);
|
|
checkWriteIOStats(data.length, 1);
|
|
|
|
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.COMMIT_DATA);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), new DispatcherContext.Builder()
|
|
|
|
+ .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA).build());
|
|
|
|
|
|
checkWriteIOStats(data.length, 1);
|
|
checkWriteIOStats(data.length, 1);
|
|
|
|
|
|
@@ -152,7 +161,7 @@ public class TestChunkManagerImpl {
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
.getLocalID(), 0), 0, randomLength);
|
|
.getLocalID(), 0), 0, randomLength);
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.WRITE_DATA);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), getDispatcherContext());
|
|
fail("testWriteChunkIncorrectLength failed");
|
|
fail("testWriteChunkIncorrectLength failed");
|
|
} catch (StorageContainerException ex) {
|
|
} catch (StorageContainerException ex) {
|
|
// As we got an exception, writeBytes should be 0.
|
|
// As we got an exception, writeBytes should be 0.
|
|
@@ -173,7 +182,7 @@ public class TestChunkManagerImpl {
|
|
assertTrue(chunksPath.listFiles().length == 0);
|
|
assertTrue(chunksPath.listFiles().length == 0);
|
|
checkWriteIOStats(0, 0);
|
|
checkWriteIOStats(0, 0);
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), getDispatcherContext());
|
|
// Now a chunk file is being written with Stage COMBINED_DATA, so it should
|
|
// Now a chunk file is being written with Stage COMBINED_DATA, so it should
|
|
// create a chunk file.
|
|
// create a chunk file.
|
|
assertTrue(chunksPath.listFiles().length == 1);
|
|
assertTrue(chunksPath.listFiles().length == 1);
|
|
@@ -186,11 +195,11 @@ public class TestChunkManagerImpl {
|
|
public void testReadChunk() throws Exception {
|
|
public void testReadChunk() throws Exception {
|
|
checkWriteIOStats(0, 0);
|
|
checkWriteIOStats(0, 0);
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), getDispatcherContext());
|
|
checkWriteIOStats(data.length, 1);
|
|
checkWriteIOStats(data.length, 1);
|
|
checkReadIOStats(0, 0);
|
|
checkReadIOStats(0, 0);
|
|
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
|
|
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
|
|
- chunkInfo, false);
|
|
|
|
|
|
+ chunkInfo, getDispatcherContext());
|
|
assertEquals(expectedData.length, data.length);
|
|
assertEquals(expectedData.length, data.length);
|
|
assertTrue(Arrays.equals(expectedData, data));
|
|
assertTrue(Arrays.equals(expectedData, data));
|
|
checkReadIOStats(data.length, 1);
|
|
checkReadIOStats(data.length, 1);
|
|
@@ -200,7 +209,7 @@ public class TestChunkManagerImpl {
|
|
public void testDeleteChunk() throws Exception {
|
|
public void testDeleteChunk() throws Exception {
|
|
File chunksPath = new File(keyValueContainerData.getChunksPath());
|
|
File chunksPath = new File(keyValueContainerData.getChunksPath());
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), getDispatcherContext());
|
|
assertTrue(chunksPath.listFiles().length == 1);
|
|
assertTrue(chunksPath.listFiles().length == 1);
|
|
chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo);
|
|
chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo);
|
|
assertTrue(chunksPath.listFiles().length == 0);
|
|
assertTrue(chunksPath.listFiles().length == 0);
|
|
@@ -210,7 +219,7 @@ public class TestChunkManagerImpl {
|
|
public void testDeleteChunkUnsupportedRequest() throws Exception {
|
|
public void testDeleteChunkUnsupportedRequest() throws Exception {
|
|
try {
|
|
try {
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), getDispatcherContext());
|
|
long randomLength = 200L;
|
|
long randomLength = 200L;
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
.getLocalID(), 0), 0, randomLength);
|
|
.getLocalID(), 0), 0, randomLength);
|
|
@@ -227,7 +236,7 @@ public class TestChunkManagerImpl {
|
|
try {
|
|
try {
|
|
// trying to read a chunk, where chunk file does not exist
|
|
// trying to read a chunk, where chunk file does not exist
|
|
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
|
|
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
|
|
- chunkInfo, false);
|
|
|
|
|
|
+ chunkInfo, getDispatcherContext());
|
|
fail("testReadChunkFileNotExists failed");
|
|
fail("testReadChunkFileNotExists failed");
|
|
} catch (StorageContainerException ex) {
|
|
} catch (StorageContainerException ex) {
|
|
GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
|
|
GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
|
|
@@ -242,7 +251,7 @@ public class TestChunkManagerImpl {
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
.getLocalID(), i), 0, data.length);
|
|
.getLocalID(), i), 0, data.length);
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
|
|
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
|
|
|
|
|
|
+ ByteBuffer.wrap(data), getDispatcherContext());
|
|
}
|
|
}
|
|
checkWriteIOStats(data.length*100, 100);
|
|
checkWriteIOStats(data.length*100, 100);
|
|
assertTrue(hddsVolume.getVolumeIOStats().getWriteTime() > 0);
|
|
assertTrue(hddsVolume.getVolumeIOStats().getWriteTime() > 0);
|
|
@@ -250,7 +259,8 @@ public class TestChunkManagerImpl {
|
|
for (int i=0; i<100; i++) {
|
|
for (int i=0; i<100; i++) {
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
|
|
.getLocalID(), i), 0, data.length);
|
|
.getLocalID(), i), 0, data.length);
|
|
- chunkManager.readChunk(keyValueContainer, blockID, chunkInfo, false);
|
|
|
|
|
|
+ chunkManager.readChunk(keyValueContainer, blockID, chunkInfo,
|
|
|
|
+ getDispatcherContext());
|
|
}
|
|
}
|
|
checkReadIOStats(data.length*100, 100);
|
|
checkReadIOStats(data.length*100, 100);
|
|
assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0);
|
|
assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0);
|