Pārlūkot izejas kodu

HDDS-798. Storage-class is showing incorrectly. Contributed by Bharat Viswanadham.

Márton Elek 6 gadi atpakaļ
vecāks
revīzija
aed5046f8d

+ 15 - 1
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.hdds.client.ReplicationType;
+
 /**
  * A class that encapsulates OzoneKey.
  */
@@ -48,19 +50,22 @@ public class OzoneKey {
    */
   private long modificationTime;
 
+  private ReplicationType replicationType;
+
   /**
    * Constructs OzoneKey from OmKeyInfo.
    *
    */
   public OzoneKey(String volumeName, String bucketName,
                   String keyName, long size, long creationTime,
-                  long modificationTime) {
+                  long modificationTime, ReplicationType type) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.name = keyName;
     this.dataSize = size;
     this.creationTime = creationTime;
     this.modificationTime = modificationTime;
+    this.replicationType = type;
   }
 
   /**
@@ -116,4 +121,13 @@ public class OzoneKey {
   public long getModificationTime() {
     return modificationTime;
   }
+
+  /**
+   * Returns the replication type of the key.
+   *
+   * @return replicationType
+   */
+  public ReplicationType getReplicationType() {
+    return replicationType;
+  }
 }

+ 6 - 3
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.hdds.client.ReplicationType;
+
 import java.util.List;
 
 /**
@@ -34,10 +36,11 @@ public class OzoneKeyDetails extends OzoneKey {
    * Constructs OzoneKeyDetails from OmKeyInfo.
    */
   public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
-                  long size, long creationTime, long modificationTime,
-                         List<OzoneKeyLocation> ozoneKeyLocations) {
+                         long size, long creationTime, long modificationTime,
+                         List<OzoneKeyLocation> ozoneKeyLocations,
+                         ReplicationType type) {
     super(volumeName, bucketName, keyName, size, creationTime,
-        modificationTime);
+        modificationTime, type);
     this.ozoneKeyLocations = ozoneKeyLocations;
   }
 

+ 4 - 2
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java

@@ -777,7 +777,8 @@ public class RestClient implements ClientProtocol {
           LOG.warn("Parse exception in getting creation time for volume", e);
         }
         return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(),
-            keyInfo.getSize(), creationTime, modificationTime);
+            keyInfo.getSize(), creationTime, modificationTime,
+            ReplicationType.valueOf(keyInfo.getType().toString()));
       }).collect(Collectors.toList());
     } catch (URISyntaxException e) {
       throw new IOException(e);
@@ -812,7 +813,8 @@ public class RestClient implements ClientProtocol {
           keyInfo.getSize(),
           HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()),
           HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()),
-          ozoneKeyLocations);
+          ozoneKeyLocations, ReplicationType.valueOf(
+              keyInfo.getType().toString()));
       EntityUtils.consume(response);
       return key;
     } catch (URISyntaxException | ParseException e) {

+ 6 - 8
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -537,7 +537,8 @@ public class RpcClient implements ClientProtocol {
         key.getKeyName(),
         key.getDataSize(),
         key.getCreationTime(),
-        key.getModificationTime()))
+        key.getModificationTime(),
+        ReplicationType.valueOf(key.getType().toString())))
         .collect(Collectors.toList());
   }
 
@@ -559,13 +560,10 @@ public class RpcClient implements ClientProtocol {
     keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach(
         (a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(),
             a.getLocalID(), a.getLength(), a.getOffset())));
-    return new OzoneKeyDetails(keyInfo.getVolumeName(),
-                        keyInfo.getBucketName(),
-                        keyInfo.getKeyName(),
-                        keyInfo.getDataSize(),
-                        keyInfo.getCreationTime(),
-                        keyInfo.getModificationTime(),
-                        ozoneKeyLocations);
+    return new OzoneKeyDetails(keyInfo.getVolumeName(), keyInfo.getBucketName(),
+        keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
+        keyInfo.getModificationTime(), ozoneKeyLocations, ReplicationType
+        .valueOf(keyInfo.getType().toString()));
   }
 
   @Override

+ 20 - 0
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java

@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.client.ReplicationType;
 
 /**
  * KeyInfo class is used used for parsing json response
@@ -40,6 +41,25 @@ public class KeyInfo implements Comparable<KeyInfo> {
   private String modifiedOn;
   private long size;
   private String keyName;
+  private ReplicationType type;
+
+  /**
+   * Return replication type of the key.
+   *
+   * @return replication type
+   */
+  public ReplicationType getType() {
+    return type;
+  }
+
+  /**
+   * Set replication type of the key.
+   *
+   * @param replicationType
+   */
+  public void setType(ReplicationType replicationType) {
+    this.type = replicationType;
+  }
 
   /**
    * When this key was created.

+ 21 - 0
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.web.utils.JsonUtils;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -75,6 +76,26 @@ public class KeyInfo implements Comparable<KeyInfo> {
 
   private String dataFileName;
 
+  private ReplicationType type;
+
+  /**
+   * Return replication type of the key.
+   *
+   * @return replication type
+   */
+  public ReplicationType getType() {
+    return type;
+  }
+
+  /**
+   * Set replication type of the key.
+   *
+   * @param replicationType
+   */
+  public void setType(ReplicationType replicationType) {
+    this.type = replicationType;
+  }
+
   /**
    * When this key was created.
    *

+ 4 - 0
hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot

@@ -36,9 +36,13 @@ Delete file with multi delete
                         Should contain             ${result}         multidelete/f1
                         Should contain             ${result}         multidelete/f2
                         Should contain             ${result}         multidelete/f3
+                        Should contain             ${result}         REDUCED_REDUNDANCY
+                        Should not contain         ${result}         STANDARD
     ${result} =         Execute AWSS3APICli        delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]'
                         Should not contain         ${result}         Error
     ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${BUCKET} --prefix multidelete/
                         Should not contain         ${result}         multidelete/f1
                         Should not contain         ${result}         multidelete/f2
                         Should contain             ${result}         multidelete/f3
+                        Should contain             ${result}         REDUCED_REDUNDANCY
+                        Should not contain         ${result}         STANDARD

+ 8 - 1
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -37,6 +37,7 @@ import java.io.InputStream;
 import java.time.Instant;
 import java.util.Iterator;
 
+import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneKey;
 import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata;
@@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.s3.util.S3StorageType;
 import org.apache.hadoop.ozone.s3.util.S3utils;
 import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
@@ -296,7 +298,12 @@ public class BucketEndpoint extends EndpointBase {
     keyMetadata.setKey(next.getName());
     keyMetadata.setSize(next.getDataSize());
     keyMetadata.setETag("" + next.getModificationTime());
-    keyMetadata.setStorageClass("STANDARD");
+    if (next.getReplicationType().toString().equals(ReplicationType
+        .STAND_ALONE.toString())) {
+      keyMetadata.setStorageClass(S3StorageType.REDUCED_REDUNDANCY.toString());
+    } else {
+      keyMetadata.setStorageClass(S3StorageType.STANDARD.toString());
+    }
     keyMetadata.setLastModified(Instant.ofEpochMilli(
         next.getModificationTime()));
     response.addKey(keyMetadata);

+ 1 - 1
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java

@@ -93,7 +93,7 @@ public class OzoneBucketStub extends OzoneBucket {
                 size,
                 System.currentTimeMillis(),
                 System.currentTimeMillis(),
-                new ArrayList<>()
+                new ArrayList<>(), type
             ));
             super.close();
           }