|
@@ -21,23 +21,21 @@ package org.apache.hadoop.ozone.client.rpc;
|
|
|
import org.apache.commons.lang3.RandomStringUtils;
|
|
|
import org.apache.commons.lang3.RandomUtils;
|
|
|
import org.apache.hadoop.fs.StorageType;
|
|
|
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
|
|
|
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
|
|
|
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
|
|
|
-import org.apache.hadoop.ozone.MiniOzoneCluster;
|
|
|
-import org.apache.hadoop.ozone.OzoneAcl;
|
|
|
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
|
|
|
+import org.apache.hadoop.ozone.*;
|
|
|
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
|
|
|
-import org.apache.hadoop.ozone.client.BucketArgs;
|
|
|
-import org.apache.hadoop.ozone.client.ObjectStore;
|
|
|
-import org.apache.hadoop.ozone.client.OzoneBucket;
|
|
|
-import org.apache.hadoop.ozone.client.OzoneClient;
|
|
|
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
|
|
|
-import org.apache.hadoop.ozone.client.OzoneKey;
|
|
|
+import org.apache.hadoop.ozone.client.*;
|
|
|
import org.apache.hadoop.hdds.client.OzoneQuota;
|
|
|
-import org.apache.hadoop.ozone.client.OzoneVolume;
|
|
|
import org.apache.hadoop.hdds.client.ReplicationFactor;
|
|
|
import org.apache.hadoop.hdds.client.ReplicationType;
|
|
|
-import org.apache.hadoop.ozone.client.VolumeArgs;
|
|
|
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
|
|
|
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
|
|
|
+import org.apache.hadoop.ozone.container.common.helpers.KeyData;
|
|
|
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator;
|
|
|
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
|
|
|
import org.apache.hadoop.ozone.om.OzoneManager;
|
|
|
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
|
|
|
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
|
|
@@ -56,6 +54,7 @@ import org.junit.Rule;
|
|
|
import org.junit.Test;
|
|
|
import org.junit.rules.ExpectedException;
|
|
|
|
|
|
+import java.io.File;
|
|
|
import java.io.IOException;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.Iterator;
|
|
@@ -535,6 +534,88 @@ public class TestOzoneRpcClient {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ @Test
|
|
|
+ public void testGetKeyDetails() throws IOException, OzoneException {
|
|
|
+ String volumeName = UUID.randomUUID().toString();
|
|
|
+ String bucketName = UUID.randomUUID().toString();
|
|
|
+
|
|
|
+ store.createVolume(volumeName);
|
|
|
+ OzoneVolume volume = store.getVolume(volumeName);
|
|
|
+ volume.createBucket(bucketName);
|
|
|
+ OzoneBucket bucket = volume.getBucket(bucketName);
|
|
|
+ String keyName = UUID.randomUUID().toString();
|
|
|
+ String keyValue = RandomStringUtils.random(128);
|
|
|
+ //String keyValue = "this is a test value.glx";
|
|
|
+ // create the initial key with size 0, write will allocate the first block.
|
|
|
+ OzoneOutputStream out = bucket.createKey(keyName,
|
|
|
+ keyValue.getBytes().length, ReplicationType.STAND_ALONE,
|
|
|
+ ReplicationFactor.ONE);
|
|
|
+ out.write(keyValue.getBytes());
|
|
|
+ out.close();
|
|
|
+
|
|
|
+ OzoneInputStream is = bucket.readKey(keyName);
|
|
|
+ byte[] fileContent = new byte[32];
|
|
|
+ is.read(fileContent);
|
|
|
+
|
|
|
+ // First, confirm the key info from the client matches the info in OM.
|
|
|
+ OmKeyArgs.Builder builder = new OmKeyArgs.Builder();
|
|
|
+ builder.setVolumeName(volumeName).setBucketName(bucketName)
|
|
|
+ .setKeyName(keyName);
|
|
|
+ OmKeyLocationInfo keyInfo = ozoneManager.lookupKey(builder.build()).
|
|
|
+ getKeyLocationVersions().get(0).getBlocksLatestVersionOnly().get(0);
|
|
|
+ long containerID = keyInfo.getContainerID();
|
|
|
+ long localID = keyInfo.getLocalID();
|
|
|
+ OzoneKeyDetails keyDetails = (OzoneKeyDetails)bucket.getKey(keyName);
|
|
|
+ Assert.assertEquals(keyName, keyDetails.getName());
|
|
|
+
|
|
|
+ List<OzoneKeyLocation> keyLocations = keyDetails.getOzoneKeyLocations();
|
|
|
+ Assert.assertEquals(1, keyLocations.size());
|
|
|
+ Assert.assertEquals(containerID, keyLocations.get(0).getContainerID());
|
|
|
+ Assert.assertEquals(localID, keyLocations.get(0).getLocalID());
|
|
|
+
|
|
|
+ // Make sure that the data size matched.
|
|
|
+ Assert.assertEquals(keyValue.getBytes().length,
|
|
|
+ keyLocations.get(0).getLength());
|
|
|
+
|
|
|
+ // Second, sum the data size from chunks in Container via containerID
|
|
|
+ // and localID, make sure the size equals to the size from keyDetails.
|
|
|
+ Pipeline pipeline = cluster.getStorageContainerManager()
|
|
|
+ .getScmContainerManager().getContainerWithPipeline(containerID)
|
|
|
+ .getPipeline();
|
|
|
+ List<DatanodeDetails> datanodes = pipeline.getMachines();
|
|
|
+ Assert.assertEquals(datanodes.size(), 1);
|
|
|
+
|
|
|
+ DatanodeDetails datanodeDetails = datanodes.get(0);
|
|
|
+ Assert.assertNotNull(datanodeDetails);
|
|
|
+ HddsDatanodeService datanodeService = null;
|
|
|
+ for (HddsDatanodeService datanodeServiceItr : cluster.getHddsDatanodes()) {
|
|
|
+ if (datanodeDetails.equals(datanodeServiceItr.getDatanodeDetails())) {
|
|
|
+ datanodeService = datanodeServiceItr;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ KeyValueContainerData containerData =
|
|
|
+ (KeyValueContainerData)(datanodeService.getDatanodeStateMachine()
|
|
|
+ .getContainer().getContainerSet().getContainer(containerID)
|
|
|
+ .getContainerData());
|
|
|
+ String containerPath = new File(containerData.getMetadataPath())
|
|
|
+ .getParent();
|
|
|
+ KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator(
|
|
|
+ containerID, new File(containerPath));
|
|
|
+ while (keyValueBlockIterator.hasNext()) {
|
|
|
+ KeyData keyData = keyValueBlockIterator.nextBlock();
|
|
|
+ if (keyData.getBlockID().getLocalID() == localID) {
|
|
|
+ long length = 0;
|
|
|
+ List<ContainerProtos.ChunkInfo> chunks = keyData.getChunks();
|
|
|
+ for (ContainerProtos.ChunkInfo chunk : chunks) {
|
|
|
+ length += chunk.getLen();
|
|
|
+ }
|
|
|
+ Assert.assertEquals(length, keyValue.getBytes().length);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
@Test
|
|
|
public void testDeleteKey()
|
|
|
throws IOException, OzoneException {
|