HDFS-12230. Ozone: KSM: Add creation time field in bucket info. Contributed by Yiqun Lin.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java

@@ -54,6 +54,10 @@ public final class KsmBucketInfo {
    * [RAM_DISK, SSD, DISK, ARCHIVE]
    */
   private StorageType storageType;
+  /**
+   * Creation time of bucket.
+   */
+  private final long creationTime;
 
   /**
    * Private constructor, constructed via builder.
@@ -62,15 +66,17 @@ public final class KsmBucketInfo {
    * @param acls - list of ACLs.
    * @param isVersionEnabled - Bucket version flag.
    * @param storageType - Storage type to be used.
+   * @param creationTime - Bucket creation time.
    */
   private KsmBucketInfo(String volumeName, String bucketName,
                         List<OzoneAcl> acls, boolean isVersionEnabled,
-                        StorageType storageType) {
+                        StorageType storageType, long creationTime) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.acls = acls;
     this.isVersionEnabled = isVersionEnabled;
     this.storageType = storageType;
+    this.creationTime = creationTime;
   }
 
   /**
@@ -113,6 +119,15 @@ public final class KsmBucketInfo {
     return storageType;
   }
 
+  /**
+   * Returns creation time.
+   *
+   * @return long
+   */
+  public long getCreationTime() {
+    return creationTime;
+  }
+
   /**
    * Returns new builder class that builds a KsmBucketInfo.
    *
@@ -131,6 +146,7 @@ public final class KsmBucketInfo {
     private List<OzoneAcl> acls;
     private Boolean isVersionEnabled;
     private StorageType storageType;
+    private long creationTime;
 
     Builder() {
       //Default values
@@ -164,6 +180,11 @@ public final class KsmBucketInfo {
       return this;
     }
 
+    public Builder setCreationTime(long createdOn) {
+      this.creationTime = createdOn;
+      return this;
+    }
+
     /**
      * Constructs the KsmBucketInfo.
      * @return instance of KsmBucketInfo.
@@ -174,8 +195,9 @@ public final class KsmBucketInfo {
       Preconditions.checkNotNull(acls);
       Preconditions.checkNotNull(isVersionEnabled);
       Preconditions.checkNotNull(storageType);
+
       return new KsmBucketInfo(volumeName, bucketName, acls,
-          isVersionEnabled, storageType);
+          isVersionEnabled, storageType, creationTime);
     }
   }
 
@@ -191,6 +213,7 @@ public final class KsmBucketInfo {
         .setIsVersionEnabled(isVersionEnabled)
         .setStorageType(PBHelperClient.convertStorageType(
             storageType))
+        .setCreationTime(creationTime)
         .build();
   }
 
@@ -207,6 +230,6 @@ public final class KsmBucketInfo {
             KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
         bucketInfo.getIsVersionEnabled(),
         PBHelperClient.convertStorageType(
-            bucketInfo.getStorageType()));
+            bucketInfo.getStorageType()), bucketInfo.getCreationTime());
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto

@@ -162,6 +162,7 @@ message BucketInfo {
     repeated OzoneAclInfo acls = 3;
     required bool isVersionEnabled = 4 [default = false];
     required StorageTypeProto storageType = 5 [default = DISK];
+    required uint64 creationTime = 6;
 }
 
 message BucketArgs {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java

@@ -202,6 +202,7 @@ public class BucketManagerImpl implements BucketManager {
         bucketInfoBuilder.setIsVersionEnabled(
             oldBucketInfo.getIsVersionEnabled());
       }
+      bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
 
       metadataManager.put(bucketKey, bucketInfoBuilder.build()
           .getProtobuf().toByteArray());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java

@@ -180,6 +180,7 @@ public class VolumeManagerImpl implements VolumeManager {
               .setAdminName(volumeArgs.getAdminName())
               .setOwnerName(owner)
               .setQuotaInBytes(volumeArgs.getQuotaInBytes())
+              .setCreationTime(volumeArgs.getCreationTime())
               .build();
 
       VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
@@ -221,6 +222,7 @@ public class VolumeManagerImpl implements VolumeManager {
               .setAdminName(volumeArgs.getAdminName())
               .setOwnerName(volumeArgs.getOwnerName())
               .setQuotaInBytes(quota)
+              .setCreationTime(volumeArgs.getCreationTime())
               .build();
 
       VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java

@@ -146,6 +146,15 @@ public class OzoneBucket {
     return bucketInfo.getStorageType();
   }
 
+  /**
+   * Gets the creation time of the bucket.
+   *
+   * @return String
+   */
+  public String getCreatedOn() {
+    return bucketInfo.getCreatedOn();
+  }
+
   /**
    * Puts an Object in Ozone bucket.
    *

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java

@@ -62,6 +62,7 @@ public class BucketInfo implements Comparable<BucketInfo> {
 
   private String volumeName;
   private String bucketName;
+  private String createdOn;
   private List<OzoneAcl> acls;
   private OzoneConsts.Versioning versioning;
   private StorageType storageType;
@@ -173,6 +174,24 @@ public class BucketInfo implements Comparable<BucketInfo> {
     this.bucketName = bucketName;
   }
 
+  /**
+   * Sets creation time of the bucket.
+   *
+   * @param creationTime - Date String
+   */
+  public void setCreatedOn(String creationTime) {
+    this.createdOn = creationTime;
+  }
+
+  /**
+   * Returns creation time.
+   *
+   * @return creation time of bucket.
+   */
+  public String getCreatedOn() {
+    return createdOn;
+  }
+
   /**
    * Returns a JSON string of this object.
    * After stripping out bytesUsed and keyCount

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java

@@ -239,6 +239,7 @@ public final class DistributedStorageHandler implements StorageHandler {
       builder.setIsVersionEnabled(getBucketVersioningProtobuf(
           args.getVersioning()));
     }
+    builder.setCreationTime(Time.now());
     keySpaceManagerClient.createBucket(builder.build());
   }
 
@@ -349,6 +350,7 @@ public final class DistributedStorageHandler implements StorageHandler {
         bk.setBucketName(bucketInfo.getBucketName());
         bk.setStorageType(bucketInfo.getStorageType());
         bk.setAcls(bucketInfo.getAcls());
+        bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime()));
         result.addBucket(bk);
       }
       return result;
@@ -375,6 +377,8 @@ public final class DistributedStorageHandler implements StorageHandler {
     }
     bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
     bucketInfo.setAcls(ksmBucketInfo.getAcls());
+    bucketInfo.setCreatedOn(
+        OzoneUtils.formatTime(ksmBucketInfo.getCreationTime()));
     return bucketInfo;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java

@@ -408,7 +408,11 @@ public class TestOzoneShell {
     String[] args = new String[] {"-infoBucket",
         url + "/" + vol.getVolumeName() + "/" + bucketName};
     assertEquals(0, ToolRunner.run(shell, args));
-    assertTrue(out.toString().contains(bucketName));
+
+    String output = out.toString();
+    assertTrue(output.contains(bucketName));
+    assertTrue(output.contains("createdOn")
+        && output.contains(OzoneConsts.OZONE_TIME_ZONE));
 
     // test get info from a non-exist bucket
     args = new String[] {"-infoBucket",
@@ -429,6 +433,9 @@ public class TestOzoneShell {
         url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl",
         "user:frodo:rw,group:samwise:r"};
     assertEquals(0, ToolRunner.run(shell, args));
+    String output = out.toString();
+    assertTrue(output.contains("createdOn")
+        && output.contains(OzoneConsts.OZONE_TIME_ZONE));
 
     bucket = vol.getBucket(bucketName);
     assertEquals(2, bucket.getAcls().size());
@@ -480,8 +487,11 @@ public class TestOzoneShell {
 
     List<String> volumes = getValueLines("volumeName", out.toString());
     List<String> buckets = getValueLines("bucketName", out.toString());
+    List<String> creationTimes = getValueLines("createdOn", out.toString());
     assertEquals(11, volumes.size());
     assertEquals(11, buckets.size());
+    assertEquals(11, creationTimes.size());
+
     // sort bucket names since the return buckets isn't in created order
     Collections.sort(bucketNames);
     // return bucket names should be [test-bucket0, test-bucket1,
@@ -489,6 +499,7 @@ public class TestOzoneShell {
     for (int i = 0; i < buckets.size(); i++) {
       assertTrue(buckets.get(i).contains(bucketNames.get(i)));
       assertTrue(volumes.get(i).contains(vol.getVolumeName()));
+      assertTrue(creationTimes.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
     }
 
     out.reset();

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.request.OzoneQuota;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -35,10 +36,12 @@ import org.junit.rules.Timeout;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.text.ParseException;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class TestBuckets {
@@ -90,12 +93,12 @@ public class TestBuckets {
   }
 
   @Test
-  public void testCreateBucket() throws OzoneException, IOException {
+  public void testCreateBucket() throws Exception {
     runTestCreateBucket(ozoneRestClient);
   }
 
   static void runTestCreateBucket(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth("hdfs");
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@@ -103,10 +106,15 @@ public class TestBuckets {
 
     // create 10 buckets under same volume
     for (int x = 0; x < 10; x++) {
+      long currentTime = Time.now();
       String bucketName = OzoneUtils.getRequestID().toLowerCase();
       OzoneBucket bucket =
           vol.createBucket(bucketName, acls, StorageType.DEFAULT);
       assertEquals(bucket.getBucketName(), bucketName);
+
+      // verify the bucket creation time
+      assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
+          / 1000) >= (currentTime / 1000));
     }
     client.close();
 
@@ -118,12 +126,12 @@ public class TestBuckets {
   }
 
   @Test
-  public void testAddBucketAcls() throws OzoneException, IOException {
+  public void testAddBucketAcls() throws Exception {
     runTestAddBucketAcls(ozoneRestClient);
   }
 
   static void runTestAddBucketAcls(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth("hdfs");
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@@ -133,16 +141,19 @@ public class TestBuckets {
     vol.addAcls(bucketName, acls);
     OzoneBucket updatedBucket = vol.getBucket(bucketName);
     assertEquals(updatedBucket.getAcls().size(), 2);
+    // verify if the creation time is missing after update operation
+    assertTrue(
+        (OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
     client.close();
   }
 
   @Test
-  public void testRemoveBucketAcls() throws OzoneException, IOException {
+  public void testRemoveBucketAcls() throws Exception {
     runTestRemoveBucketAcls(ozoneRestClient);
   }
 
   static void runTestRemoveBucketAcls(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth("hdfs");
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@@ -155,6 +166,9 @@ public class TestBuckets {
 
     // We removed all acls
     assertEquals(updatedBucket.getAcls().size(), 0);
+    // verify if the creation time is missing after update operation
+    assertTrue(
+        (OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
     client.close();
   }
 
@@ -183,16 +197,18 @@ public class TestBuckets {
   }
 
   @Test
-  public void testListBucket() throws OzoneException, IOException {
+  public void testListBucket() throws Exception {
     runTestListBucket(ozoneRestClient);
   }
 
   static void runTestListBucket(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth("hdfs");
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
     String[] acls = {"user:frodo:rw", "user:samwise:rw"};
+
+    long currentTime = Time.now();
     for (int x = 0; x < 10; x++) {
       String bucketName = "listbucket-test-" + x;
       vol.createBucket(bucketName, acls);
@@ -200,6 +216,11 @@ public class TestBuckets {
     List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
     assertEquals(bucketList.size(), 10);
 
+    for (OzoneBucket bucket : bucketList) {
+      assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
+          / 1000) >= (currentTime / 1000));
+    }
+
     bucketList = vol.listBuckets("3", null, null);
     assertEquals(bucketList.size(), 3);
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java

@@ -49,17 +49,17 @@ public class TestBucketsRatis {
   }
 
   @Test
-  public void testCreateBucket() throws OzoneException, IOException {
+  public void testCreateBucket() throws Exception {
     TestBuckets.runTestCreateBucket(ozoneRestClient);
   }
 
   @Test
-  public void testAddBucketAcls() throws OzoneException, IOException {
+  public void testAddBucketAcls() throws Exception {
     TestBuckets.runTestAddBucketAcls(ozoneRestClient);
   }
 
   @Test
-  public void testRemoveBucketAcls() throws OzoneException, IOException {
+  public void testRemoveBucketAcls() throws Exception {
     TestBuckets.runTestRemoveBucketAcls(ozoneRestClient);
   }
 
@@ -69,7 +69,7 @@ public class TestBucketsRatis {
   }
 
   @Test
-  public void testListBucket() throws OzoneException, IOException {
+  public void testListBucket() throws Exception {
     TestBuckets.runTestListBucket(ozoneRestClient);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java

@@ -160,27 +160,29 @@ public class TestVolume {
   }
 
   @Test
-  public void testChangeOwnerOnVolume() throws OzoneException {
+  public void testChangeOwnerOnVolume() throws Exception {
     runTestChangeOwnerOnVolume(ozoneRestClient);
   }
 
   static void runTestChangeOwnerOnVolume(OzoneRestClient client)
-      throws OzoneException {
+      throws OzoneException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
     client.setVolumeOwner(volumeName, "frodo");
     OzoneVolume newVol = client.getVolume(volumeName);
     assertEquals(newVol.getOwnerName(), "frodo");
+    // verify if the creation time is missing after setting owner operation
+    assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
   }
 
   @Test
-  public void testChangeQuotaOnVolume() throws OzoneException, IOException {
+  public void testChangeQuotaOnVolume() throws Exception {
     runTestChangeQuotaOnVolume(ozoneRestClient);
   }
 
   static void runTestChangeQuotaOnVolume(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@@ -188,6 +190,8 @@ public class TestVolume {
     OzoneVolume newVol = client.getVolume(volumeName);
     assertEquals(newVol.getQuota().getSize(), 1000);
     assertEquals(newVol.getQuota().getUnit(), OzoneQuota.Units.MB);
+    // verify if the creation time is missing after setting quota operation
+    assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
   }
 
   @Test

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java

@@ -62,12 +62,12 @@ public class TestVolumeRatis {
   }
 
   @Test
-  public void testChangeOwnerOnVolume() throws OzoneException {
+  public void testChangeOwnerOnVolume() throws Exception {
     TestVolume.runTestChangeOwnerOnVolume(ozoneClient);
   }
 
   @Test
-  public void testChangeQuotaOnVolume() throws OzoneException, IOException {
+  public void testChangeQuotaOnVolume() throws Exception {
     TestVolume.runTestChangeQuotaOnVolume(ozoneClient);
   }