HDFS-12231. Ozone: KSM: Add creation time field in volume info. Contributed by Yiqun Lin.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java

@@ -39,6 +39,7 @@ public final class KsmVolumeArgs {
   private final String adminName;
   private final String ownerName;
   private final String volume;
+  private final long creationTime;
   private final long quotaInBytes;
   private final Map<String, String> keyValueMap;
   private final KsmOzoneAclMap aclMap;
@@ -51,16 +52,18 @@ public final class KsmVolumeArgs {
    * @param quotaInBytes - Volume Quota in bytes.
    * @param keyValueMap - keyValue map.
    * @param aclMap - User to access rights map.
+   * @param creationTime - Volume creation time.
    */
   private KsmVolumeArgs(String adminName, String ownerName, String volume,
                         long quotaInBytes, Map<String, String> keyValueMap,
-                        KsmOzoneAclMap aclMap) {
+                        KsmOzoneAclMap aclMap, long creationTime) {
     this.adminName = adminName;
     this.ownerName = ownerName;
     this.volume = volume;
     this.quotaInBytes = quotaInBytes;
     this.keyValueMap = keyValueMap;
     this.aclMap = aclMap;
+    this.creationTime = creationTime;
   }
 
   /**
@@ -87,6 +90,14 @@ public final class KsmVolumeArgs {
     return volume;
   }
 
+  /**
+   * Returns creation time.
+   * @return long
+   */
+  public long getCreationTime() {
+    return creationTime;
+  }
+
   /**
    * Returns Quota in Bytes.
    * @return long, Quota in bytes.
@@ -118,6 +129,7 @@ public final class KsmVolumeArgs {
     private String adminName;
     private String ownerName;
     private String volume;
+    private long creationTime;
     private long quotaInBytes;
     private Map<String, String> keyValueMap;
     private KsmOzoneAclMap aclMap;
@@ -145,6 +157,11 @@ public final class KsmVolumeArgs {
       return this;
     }
 
+    public Builder setCreationTime(long createdOn) {
+      this.creationTime = createdOn;
+      return this;
+    }
+
     public Builder setQuotaInBytes(long quotaInBytes) {
       this.quotaInBytes = quotaInBytes;
       return this;
@@ -169,7 +186,7 @@ public final class KsmVolumeArgs {
       Preconditions.checkNotNull(ownerName);
       Preconditions.checkNotNull(volume);
       return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
-          keyValueMap, aclMap);
+          keyValueMap, aclMap, creationTime);
     }
   }
 
@@ -188,6 +205,7 @@ public final class KsmVolumeArgs {
         .setQuotaInBytes(quotaInBytes)
         .addAllMetadata(metadataList)
         .addAllVolumeAcls(aclList)
+        .setCreationTime(creationTime)
         .build();
   }
 
@@ -199,6 +217,7 @@ public final class KsmVolumeArgs {
         KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
 
     return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(),
-        volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap);
+        volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap,
+        volInfo.getCreationTime());
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto

@@ -62,6 +62,7 @@ message VolumeInfo {
     optional uint64 quotaInBytes = 4;
     repeated KeyValue metadata = 5;
     repeated OzoneAclInfo volumeAcls = 6;
+    required uint64 creationTime = 7;
 }
 
 /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java

@@ -131,6 +131,15 @@ public class OzoneVolume {
     return volumeInfo.getQuota();
   }
 
+  /**
+   * Returns creation time of Volume.
+   *
+   * @return String
+   */
+  public String getCreatedOn() {
+    return volumeInfo.getCreatedOn();
+  }
+
   /**
    * Returns a Http header from the Last Volume related call.
    *

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java

@@ -42,6 +42,7 @@ import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.scm.XceiverClientManager;
 import org.apache.hadoop.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.handlers.BucketArgs;
 import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@@ -124,7 +125,8 @@ public final class DistributedStorageHandler implements StorageHandler {
         .setOwnerName(args.getUserName())
         .setVolume(args.getVolumeName())
         .setQuotaInBytes(quota)
-        .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
+        .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl))
+        .setCreationTime(Time.now());
     if (args.getGroups() != null) {
       for (String group : args.getGroups()) {
         OzoneAcl groupAcl =
@@ -183,7 +185,7 @@ public final class DistributedStorageHandler implements StorageHandler {
           args.getMaxKeys());
     }
 
-    // TODO Add missing fields createdOn, createdBy, bucketCount and bytesUsed
+    // TODO Add missing fields createdBy, bucketCount and bytesUsed
     ListVolumes result = new ListVolumes();
     for (KsmVolumeArgs volumeArgs : listResult) {
       VolumeInfo info = new VolumeInfo();
@@ -192,6 +194,7 @@ public final class DistributedStorageHandler implements StorageHandler {
       info.setOwner(new VolumeOwner(infoProto.getOwnerName()));
       info.setQuota(OzoneQuota.getOzoneQuota(infoProto.getQuotaInBytes()));
       info.setVolumeName(infoProto.getVolume());
+      info.setCreatedOn(OzoneUtils.formatTime(infoProto.getCreationTime()));
       result.addVolume(info);
     }
 
@@ -215,6 +218,7 @@ public final class DistributedStorageHandler implements StorageHandler {
             volumeArgs.getAdminName());
     volInfo.setOwner(new VolumeOwner(volumeArgs.getOwnerName()));
     volInfo.setQuota(OzoneQuota.getOzoneQuota(volumeArgs.getQuotaInBytes()));
+    volInfo.setCreatedOn(OzoneUtils.formatTime(volumeArgs.getCreationTime()));
     return volInfo;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java

@@ -191,7 +191,11 @@ public class TestOzoneShell {
     String[] args = new String[] {"-infoVolume", url + "/" + volumeName,
         "-root"};
     assertEquals(0, ToolRunner.run(shell, args));
-    assertTrue(out.toString().contains(volumeName));
+
+    String output = out.toString();
+    assertTrue(output.contains(volumeName));
+    assertTrue(output.contains("createdOn")
+        && output.contains(OzoneConsts.OZONE_TIME_ZONE));
 
     // get info for non-exist volume
     args = new String[] {"-infoVolume", url + "/invalid-volume", "-root"};
@@ -270,12 +274,19 @@ public class TestOzoneShell {
 
     List<String> names = getValueLines("name", out.toString());
     List<String> volumes = getValueLines("volumeName", out.toString());
+    List<String> creationTimes = getValueLines("createdOn", out.toString());
     assertEquals(10, volumes.size());
     assertEquals(10, names.size());
+    assertEquals(10, creationTimes.size());
+
     for (String user : names) {
       assertTrue(user.contains(user1));
     }
 
+    for (String time : creationTimes) {
+      assertTrue(time.contains(OzoneConsts.OZONE_TIME_ZONE));
+    }
+
     out.reset();
     args = new String[] {"-listVolume", url + "/", "-user",
         user1, "-length", "2"};

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.request.OzoneQuota;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.log4j.Level;
@@ -41,10 +42,10 @@ import org.mockito.Mockito;
 
 import java.io.File;
 import java.io.IOException;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -99,15 +100,16 @@ public class TestVolume {
   }
 
   @Test
-  public void testCreateVolume() throws OzoneException, IOException {
+  public void testCreateVolume() throws Exception {
     runTestCreateVolume(ozoneRestClient);
   }
 
   static void runTestCreateVolume(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     String volumeName = OzoneUtils.getRequestID().toLowerCase();
     client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
 
+    long currentTime = Time.now();
     OzoneRestClient mockClient = Mockito.spy(client);
     List<CloseableHttpClient> mockedClients = mockHttpClients(mockClient);
     OzoneVolume vol = mockClient.createVolume(volumeName, "bilbo", "100TB");
@@ -119,6 +121,10 @@ public class TestVolume {
     assertEquals(vol.getOwnerName(), "bilbo");
     assertEquals(vol.getQuota().getUnit(), OzoneQuota.Units.TB);
     assertEquals(vol.getQuota().getSize(), 100);
+
+    // verify the key creation time
+    assertTrue((OzoneUtils.formatDate(vol.getCreatedOn())
+        / 1000) >= (currentTime / 1000));
   }
 
   @Test
@@ -270,16 +276,17 @@ public class TestVolume {
   }
 
   @Test
-  public void testListVolumes() throws OzoneException, IOException {
+  public void testListVolumes() throws Exception {
     runTestListVolumes(ozoneRestClient);
   }
 
   static void runTestListVolumes(OzoneRestClient client)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     final int volCount = 20;
     final String user1 = "test-user-a";
     final String user2 = "test-user-b";
 
+    long currentTime = Time.now();
     client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
     // Create 20 volumes, 10 for user1 and another 10 for user2.
     for (int x = 0; x < volCount; x++) {
@@ -303,9 +310,12 @@ public class TestVolume {
     List<OzoneVolume> volumeList = client.listVolumes(user1,
         null, 100, StringUtils.EMPTY);
     assertEquals(10, volumeList.size());
-    volumeList.stream()
-        .filter(item -> item.getOwnerName().equals(user1))
-        .collect(Collectors.toList());
+    // verify the owner name and creation time of volume
+    for (OzoneVolume vol : volumeList) {
+      assertTrue(vol.getOwnerName().equals(user1));
+      assertTrue((OzoneUtils.formatDate(vol.getCreatedOn())
+          / 1000) >= (currentTime / 1000));
+    }
 
     // test max key parameter of listing volumes
     volumeList = client.listVolumes(user1, null, 2, StringUtils.EMPTY);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java

@@ -47,7 +47,7 @@ public class TestVolumeRatis {
   }
 
   @Test
-  public void testCreateVolume() throws OzoneException, IOException {
+  public void testCreateVolume() throws Exception {
     TestVolume.runTestCreateVolume(ozoneClient);
   }
 
@@ -88,12 +88,12 @@ public class TestVolumeRatis {
   // TODO: remove @Ignore below once the problem has been resolved.
   @Ignore("See TestVolume.testListAllVolumes()")
   @Test
-  public void testListAllVolumes() throws OzoneException, IOException {
+  public void testListAllVolumes() throws Exception {
     TestVolume.runTestListAllVolumes(ozoneClient);
   }
 
   @Test
-  public void testListVolumes() throws OzoneException, IOException {
+  public void testListVolumes() throws Exception {
     TestVolume.runTestListVolumes(ozoneClient);
   }
 }