HDDS-658. Implement s3 bucket list backend call and use it from rest endpoint. Contributed by Bharat Viswanadham.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -144,6 +144,8 @@ public final class OzoneConsts {
   public static final String OM_KEY_PREFIX = "/";
   public static final String OM_USER_PREFIX = "$";
   public static final String OM_S3_PREFIX ="S3:";
+  public static final String OM_S3_VOLUME_PREFIX = "s3";
+
 
   /**
    * Max OM Quota size of 1024 PB.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -18,17 +18,20 @@
 
 package org.apache.hadoop.ozone.client;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.security.UserGroupInformation;
 
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 
 /**
  * ObjectStore class is responsible for the client operations that can be
@@ -150,6 +153,36 @@ public class ObjectStore {
     return volume;
   }
 
+  /**
+   * Returns Iterator to iterate over all buckets for a user.
+   * The result can be restricted using bucket prefix, will return all
+   * buckets if bucket prefix is null.
+   *
+   * @param userName user name
+   * @param bucketPrefix Bucket prefix to match
+   * @return {@code Iterator<OzoneBucket>}
+   */
+  public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+                                                       String bucketPrefix) {
+    return listS3Buckets(userName, bucketPrefix, null);
+  }
+
+  /**
+   * Returns Iterator to iterate over all buckets after prevBucket for a
+   * specific user. If prevBucket is null it returns an iterator to iterate over
+   * all the buckets of a user. The result can be restricted using bucket
+   * prefix, will return all buckets if bucket prefix is null.
+   *
+   * @param userName user name
+   * @param bucketPrefix Bucket prefix to match
+   * @param prevBucket Buckets are listed after this bucket
+   * @return {@code Iterator<OzoneBucket>}
+   */
+  public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+                                                       String bucketPrefix,
+                                                       String prevBucket) {
+    return new S3BucketIterator(userName, bucketPrefix, prevBucket);
+  }
 
   /**
    * Returns Iterator to iterate over all the volumes in object store.
@@ -270,4 +303,72 @@ public class ObjectStore {
     }
   }
 
+  /**
+   * An Iterator to iterate over {@link OzoneBucket} list.
+   */
+  public class S3BucketIterator implements Iterator<OzoneBucket> {
+
+    private String bucketPrefix = null;
+    private String userName;
+
+    private Iterator<OzoneBucket> currentIterator;
+    private OzoneBucket currentValue;
+
+
+    /**
+     * Creates an Iterator to iterate over all buckets after prevBucket for
+     * a user. If prevBucket is null it returns an iterator which list all
+     * the buckets of the user.
+     * The returned buckets match bucket prefix.
+     * @param user
+     * @param bucketPrefix
+     * @param prevBucket
+     */
+    public S3BucketIterator(String user, String bucketPrefix, String
+        prevBucket) {
+      Objects.requireNonNull(user);
+      this.userName = user;
+      this.bucketPrefix = bucketPrefix;
+      this.currentValue = null;
+      this.currentIterator = getNextListOfS3Buckets(prevBucket).iterator();
+    }
+
+    @Override
+    public boolean hasNext() {
+      if(!currentIterator.hasNext()) {
+        currentIterator = getNextListOfS3Buckets(
+            currentValue != null ? currentValue.getName() : null)
+            .iterator();
+      }
+      return currentIterator.hasNext();
+    }
+
+    @Override
+    public OzoneBucket next() {
+      if(hasNext()) {
+        currentValue = currentIterator.next();
+        return currentValue;
+      }
+      throw new NoSuchElementException();
+    }
+
+    /**
+     * Gets the next set of bucket list using proxy.
+     * @param prevBucket
+     * @return {@code List<OzoneVolume>}
+     */
+    private List<OzoneBucket> getNextListOfS3Buckets(String prevBucket) {
+      try {
+        return proxy.listS3Buckets(userName, bucketPrefix, prevBucket,
+            listCacheSize);
+      } catch (IOException e) {
+        if (e.getMessage().contains("VOLUME_NOT_FOUND")) {
+          return new ArrayList<OzoneBucket>();
+        } else {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+  }
+
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -364,6 +364,22 @@ public interface ClientProtocol {
    */
   String getOzoneBucketName(String s3BucketName) throws IOException;
 
+  /**
+   * Returns Iterator to iterate over all buckets after prevBucket for a
+   * specific user. If prevBucket is null it returns an iterator to iterate over
+   * all the buckets of a user. The result can be restricted using bucket
+   * prefix, will return all buckets if bucket prefix is null.
+   *
+   * @param userName user name
+   * @param bucketPrefix Bucket prefix to match
+   * @param prevBucket Buckets are listed after this bucket
+   * @return {@code Iterator<OzoneBucket>}
+   * @throws IOException
+   */
+  List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
+                                String prevBucket, int maxListResult)
+      throws IOException;
+
   /**
    * Close and release the resources.
    */

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java

@@ -854,6 +854,14 @@ public class RestClient implements ClientProtocol {
         "support this operation.");
   }
 
+  @Override
+  public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
+                                  String prevBucket, int maxListResult)
+      throws IOException {
+    throw new UnsupportedOperationException("Ozone REST protocol does not " +
+        "support this operation.");
+  }
+
   /**
    * Adds Ozone headers to http request.
    *

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -622,6 +622,25 @@ public class RpcClient implements ClientProtocol {
     return mapping.split("/")[1];
   }
 
+  @Override
+  public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
+                                         String prevBucket, int maxListResult)
+      throws IOException {
+    List<OmBucketInfo> buckets = ozoneManagerClient.listS3Buckets(
+        userName, prevBucket, bucketPrefix, maxListResult);
+
+    return buckets.stream().map(bucket -> new OzoneBucket(
+        conf,
+        this,
+        bucket.getVolumeName(),
+        bucket.getBucketName(),
+        bucket.getAcls(),
+        bucket.getStorageType(),
+        bucket.getIsVersionEnabled(),
+        bucket.getCreationTime()))
+        .collect(Collectors.toList());
+  }
+
   @Override
   public void close() throws IOException {
     IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java

@@ -45,7 +45,8 @@ public enum OMAction implements AuditAction {
   LIST_KEYS("LIST_KEYS"),
   READ_VOLUME("READ_VOLUME"),
   READ_BUCKET("READ_BUCKET"),
-  READ_KEY("READ_BUCKET");
+  READ_KEY("READ_BUCKET"),
+  LIST_S3BUCKETS("LIST_S3BUCKETS");
 
   private String action;
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -281,5 +281,28 @@ public interface OzoneManagerProtocol {
    */
   String getOzoneBucketMapping(String s3BucketName) throws IOException;
 
+  /**
+   * Returns a list of buckets represented by {@link OmBucketInfo}
+   * for the given user. Argument username is required, others
+   * are optional.
+   *
+   * @param userName
+   *   user Name.
+   * @param startBucketName
+   *   the start bucket name, only the buckets whose name is
+   *   after this value will be included in the result.
+   * @param bucketPrefix
+   *   bucket name prefix, only the buckets whose name has
+   *   this prefix will be included in the result.
+   * @param maxNumOfBuckets
+   *   the maximum number of buckets to return. It ensures
+   *   the size of the result will not exceed this limit.
+   * @return a list of buckets.
+   * @throws IOException
+   */
+  List<OmBucketInfo> listS3Buckets(String userName, String startBucketName,
+                                   String bucketPrefix, int maxNumOfBuckets)
+      throws IOException;
+
 }
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -126,6 +126,10 @@ import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.S3BucketInfoRequest;
 import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.S3BucketInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .S3ListBucketsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .S3ListBucketsResponse;
 
 
 
@@ -837,6 +841,40 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
     return resp.getOzoneMapping();
   }
 
+  @Override
+  public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
+                                          String prefix, int count)
+      throws IOException {
+    List<OmBucketInfo> buckets = new ArrayList<>();
+    S3ListBucketsRequest.Builder reqBuilder = S3ListBucketsRequest.newBuilder();
+    reqBuilder.setUserName(userName);
+    reqBuilder.setCount(count);
+    if (startKey != null) {
+      reqBuilder.setStartKey(startKey);
+    }
+    if (prefix != null) {
+      reqBuilder.setPrefix(prefix);
+    }
+    S3ListBucketsRequest request = reqBuilder.build();
+    final S3ListBucketsResponse resp;
+    try {
+      resp = rpcProxy.listS3Buckets(NULL_RPC_CONTROLLER, request);
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+
+    if (resp.getStatus() == Status.OK) {
+      buckets.addAll(
+          resp.getBucketInfoList().stream()
+              .map(OmBucketInfo::getFromProtobuf)
+              .collect(Collectors.toList()));
+      return buckets;
+    } else {
+      throw new IOException("List S3 Buckets failed, error: "
+          + resp.getStatus());
+    }
+  }
+
   /**
    * Return the proxy object underlying this protocol translator.
    *

hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto

@@ -393,6 +393,18 @@ message S3DeleteBucketResponse {
     required Status status = 1;
 }
 
+message S3ListBucketsRequest {
+    required string userName = 1;
+    optional string startKey = 2;
+    optional string prefix = 3;
+    optional int32 count = 4;
+}
+
+message S3ListBucketsResponse {
+    required Status status = 1;
+    repeated BucketInfo bucketInfo = 2;
+}
+
 
 /**
  The OM service that takes care of Ozone namespace.
@@ -526,4 +538,7 @@ service OzoneManagerService {
     */
     rpc getS3Bucketinfo(S3BucketInfoRequest)
     returns(S3BucketInfoResponse);
+
+    rpc listS3Buckets(S3ListBucketsRequest)
+    returns(S3ListBucketsResponse);
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java

@@ -68,6 +68,10 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.either;
+import static org.junit.Assert.assertThat;
+
 /**
  * This class is to test all the public facing APIs of Ozone Client.
  */
@@ -222,6 +226,36 @@ public class TestOzoneRpcClient {
     Assert.assertTrue(volume.getCreationTime() >= currentTime);
   }
 
+
+  @Test
+  public void testListS3Buckets()
+      throws IOException, OzoneException {
+    String userName = "ozone100";
+    String bucketName1 = UUID.randomUUID().toString();
+    String bucketName2 = UUID.randomUUID().toString();
+    store.createS3Bucket(userName, bucketName1);
+    store.createS3Bucket(userName, bucketName2);
+    Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
+        null);
+
+    while (iterator.hasNext()) {
+      assertThat(iterator.next().getName(), either(containsString(bucketName1))
+          .or(containsString(bucketName2)));
+    }
+
+  }
+
+  @Test
+  public void testListS3BucketsFail()
+      throws IOException, OzoneException {
+    String userName = "randomUser";
+    Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
+        null);
+
+    Assert.assertFalse(iterator.hasNext());
+
+  }
+
   @Test
   public void testDeleteS3Bucket()
       throws IOException, OzoneException {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java

@@ -60,6 +60,8 @@ public class OMMetrics {
   private @Metric MutableCounterLong numKeyCommits;
   private @Metric MutableCounterLong numAllocateBlockCalls;
   private @Metric MutableCounterLong numGetServiceLists;
+  private @Metric MutableCounterLong numListS3Buckets;
+
 
   // Failure Metrics
   private @Metric MutableCounterLong numVolumeCreateFails;
@@ -81,6 +83,7 @@ public class OMMetrics {
   private @Metric MutableCounterLong numKeyCommitFails;
   private @Metric MutableCounterLong numBlockAllocateCallFails;
   private @Metric MutableCounterLong numGetServiceListFails;
+  private @Metric MutableCounterLong numListS3BucketsFails;
 
   public OMMetrics() {
   }
@@ -152,6 +155,16 @@ public class OMMetrics {
     numVolumeLists.incr();
   }
 
+  public void incNumListS3Buckets() {
+    numBucketOps.incr();
+    numListS3Buckets.incr();
+  }
+
+  public void incNumListS3BucketsFails() {
+    numBucketOps.incr();
+    numListS3BucketsFails.incr();
+  }
+
   public void incNumGetServiceLists() {
     numGetServiceLists.incr();
   }
@@ -452,6 +465,16 @@ public class OMMetrics {
     return numGetServiceListFails.value();
   }
 
+  @VisibleForTesting
+  public long getNumListS3Buckets() {
+    return numListS3Buckets.value();
+  }
+
+  @VisibleForTesting
+  public long getNumListS3BucketsFails() {
+    return numListS3BucketsFails.value();
+  }
+
   public void unRegister() {
     MetricsSystem ms = DefaultMetricsSystem.instance();
     ms.unregisterSource(SOURCE_NAME);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -1156,6 +1156,36 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     return s3BucketManager.getOzoneBucketMapping(s3BucketName);
   }
 
+  @Override
+  public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
+                                          String prefix, int maxNumOfBuckets)
+      throws IOException {
+    boolean auditSuccess = true;
+    Map<String, String> auditMap = buildAuditMap(userName);
+    auditMap.put(OzoneConsts.START_KEY, startKey);
+    auditMap.put(OzoneConsts.PREFIX, prefix);
+    auditMap.put(OzoneConsts.MAX_NUM_OF_BUCKETS,
+        String.valueOf(maxNumOfBuckets));
+    try {
+      metrics.incNumListS3Buckets();
+      String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
+      return bucketManager.listBuckets(volumeName, startKey, prefix,
+          maxNumOfBuckets);
+    } catch (IOException ex) {
+      metrics.incNumListS3BucketsFails();
+      auditSuccess = false;
+      AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_S3BUCKETS,
+          auditMap, ex));
+      throw ex;
+    } finally {
+      if(auditSuccess){
+        AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction
+                .LIST_S3BUCKETS, auditMap));
+      }
+    }
+  }
+
+
 
 
   /**

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java

@@ -63,4 +63,10 @@ public interface S3BucketManager {
    * @throws IOException - in case of failure to retrieve mapping.
    */
   String getOzoneBucketName(String s3BucketName) throws IOException;
+
+  /**
+   * Returns volume Name for a user.
+   * @param userName
+   */
+  String getOzoneVolumeNameForUser(String userName) throws IOException;
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java

@@ -31,7 +31,9 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Objects;
 
+import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FAILED_VOLUME_ALREADY_EXISTS;
 
 /**
@@ -152,7 +154,7 @@ public class S3BucketManagerImpl implements S3BucketManager {
   }
 
   private String formatOzoneVolumeName(String userName) {
-    return String.format("s3%s", userName);
+    return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
   }
 
   private void createOzoneVolumeIfNeeded(String userName, String volumeName)
@@ -227,4 +229,10 @@ public class S3BucketManagerImpl implements S3BucketManager {
     return mapping.split("/")[1];
   }
 
+  @Override
+  public String getOzoneVolumeNameForUser(String userName) throws IOException {
+    Objects.requireNonNull(userName, "UserName cannot be null");
+    return formatOzoneVolumeName(userName);
+  }
+
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java

@@ -101,6 +101,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
     .S3DeleteBucketRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
     .S3DeleteBucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .S3ListBucketsResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .S3ListBucketsRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
     .ServiceListRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -627,4 +631,24 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
     }
     return resp.build();
   }
+
+  @Override
+  public S3ListBucketsResponse listS3Buckets(RpcController controller,
+                                             S3ListBucketsRequest request) {
+    S3ListBucketsResponse.Builder resp = S3ListBucketsResponse.newBuilder();
+    try {
+      List<OmBucketInfo> buckets = impl.listS3Buckets(
+          request.getUserName(),
+          request.getStartKey(),
+          request.getPrefix(),
+          request.getCount());
+      for(OmBucketInfo bucket : buckets) {
+        resp.addBucketInfo(bucket.getProtobuf());
+      }
+      resp.setStatus(Status.OK);
+    } catch (IOException e) {
+      resp.setStatus(exceptionToResponseStatus(e));
+    }
+    return resp.build();
+  }
 }

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.om;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -75,6 +77,29 @@ public class TestS3BucketManager {
 
   }
 
+  @Test
+  public void testOzoneVolumeNameForUser() throws IOException {
+    S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
+        volumeManager, bucketManager);
+    String userName = "ozone";
+    String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
+    assertEquals(OzoneConsts.OM_S3_VOLUME_PREFIX + userName, volumeName);
+  }
+
+  @Test
+  public void testOzoneVolumeNameForUserFails() throws IOException {
+    S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
+        volumeManager, bucketManager);
+    String userName = null;
+    try {
+      String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
+      fail("testOzoneVolumeNameForUserFails failed");
+    } catch (NullPointerException ex) {
+      GenericTestUtils.assertExceptionContains("UserName cannot be null", ex);
+    }
+
+  }
+
   @Test
   public void testDeleteS3Bucket() throws IOException {
     S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.s3.endpoint;
 import javax.inject.Inject;
 import javax.ws.rs.NotFoundException;
 import java.io.IOException;
+import java.util.Iterator;
 
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
@@ -172,6 +173,37 @@ public class EndpointBase {
     return client.getObjectStore().getOzoneBucketName(s3BucketName);
   }
 
+  /**
+   * Returns Iterator to iterate over all buckets for a specific user.
+   * The result can be restricted using bucket prefix, will return all
+   * buckets if bucket prefix is null.
+   *
+   * @param userName
+   * @param prefix
+   * @return {@code Iterator<OzoneBucket>}
+   */
+  public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+                                                       String prefix)  {
+    return client.getObjectStore().listS3Buckets(userName, prefix);
+  }
+
+  /**
+   * Returns Iterator to iterate over all buckets after prevBucket for a
+   * specific user. If prevBucket is null it returns an iterator to iterate
+   * over all buckets for this user. The result can be restricted using
+   * bucket prefix, will return all buckets if bucket prefix is null.
+   *
+   * @param prefix Bucket prefix to match
+   * @param previousBucket Buckets are listed after this bucket
+   * @return {@code Iterator<OzoneBucket>}
+   */
+  public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+                                                       String prefix,
+                                                       String previousBucket)  {
+    return client.getObjectStore().listS3Buckets(userName, prefix,
+        previousBucket);
+  }
+
   public AuthenticationHeaderParser getAuthenticationHeaderParser() {
     return authenticationHeaderParser;
   }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.ozone.s3.endpoint;
 
 import javax.ws.rs.GET;
-import javax.ws.rs.NotFoundException;
 import javax.ws.rs.Path;
 import java.io.IOException;
 import java.time.Instant;
@@ -53,25 +52,16 @@ public class RootEndpoint extends EndpointBase {
     OzoneVolume volume;
     ListBucketResponse response = new ListBucketResponse();
 
-    String volumeName = "s3" + getAuthenticationHeaderParser().getAccessKeyID();
-    try {
-      //TODO: we need a specific s3bucketlist endpoint instead
-      // of reimplement the naming convention here
-      volume = getVolume(volumeName);
-    } catch (NotFoundException ex) {
-      return response;
-    } catch (IOException e) {
-      throw e;
-    }
-
-    Iterator<? extends OzoneBucket> volABucketIter = volume.listBuckets(null);
+    String userName = getAuthenticationHeaderParser().getAccessKeyID();
+    Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName,
+        null);
 
-    while (volABucketIter.hasNext()) {
-      OzoneBucket next = volABucketIter.next();
+    while (bucketIterator.hasNext()) {
+      OzoneBucket next = bucketIterator.next();
       BucketMetadata bucketMetadata = new BucketMetadata();
       bucketMetadata.setName(next.getName());
-      bucketMetadata.setCreationDate(
-          Instant.ofEpochMilli(next.getCreationTime()));
+      bucketMetadata.setCreationDate(Instant.ofEpochMilli(next
+          .getCreationTime()));
       response.addBucket(bucketMetadata);
     }
 

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java

@@ -20,10 +20,7 @@
 package org.apache.hadoop.ozone.client;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -38,6 +35,7 @@ public class ObjectStoreStub extends ObjectStore {
   private Map<String, OzoneVolumeStub> volumes = new HashMap<>();
   private Map<String, String> bucketVolumeMap = new HashMap<>();
   private Map<String, Boolean> bucketEmptyStatus = new HashMap<>();
+  private Map<String, List<OzoneBucket>> userBuckets = new HashMap<>();
 
   @Override
   public void createVolume(String volumeName) throws IOException {
@@ -113,8 +111,8 @@ public class ObjectStoreStub extends ObjectStore {
   @Override
   public void createS3Bucket(String userName, String s3BucketName) throws
       IOException {
+    String volumeName = "s3" + userName;
     if (bucketVolumeMap.get(s3BucketName) == null) {
-      String volumeName = "s3"+userName;
       bucketVolumeMap.put(s3BucketName, volumeName + "/" + s3BucketName);
       bucketEmptyStatus.put(s3BucketName, true);
       createVolume(volumeName);
@@ -122,6 +120,70 @@ public class ObjectStoreStub extends ObjectStore {
     } else {
       throw new IOException("BUCKET_ALREADY_EXISTS");
     }
+
+    if (userBuckets.get(userName) == null) {
+      List<OzoneBucket> ozoneBuckets = new ArrayList<>();
+      ozoneBuckets.add(volumes.get(volumeName).getBucket(s3BucketName));
+      userBuckets.put(userName, ozoneBuckets);
+    } else {
+      userBuckets.get(userName).add(volumes.get(volumeName).getBucket(
+          s3BucketName));
+    }
+  }
+
+  public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+                                                       String bucketPrefix) {
+    if (userBuckets.get(userName) == null) {
+      return new ArrayList<OzoneBucket>().iterator();
+    } else {
+      return userBuckets.get(userName).parallelStream()
+          .filter(ozoneBucket -> {
+            if (bucketPrefix != null) {
+              return ozoneBucket.getName().startsWith(bucketPrefix);
+            } else {
+              return true;
+            }
+          }).collect(Collectors.toList())
+          .iterator();
+    }
+  }
+
+  public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+                                                       String bucketPrefix,
+                                                       String prevBucket) {
+
+    if (userBuckets.get(userName) == null) {
+      return new ArrayList<OzoneBucket>().iterator();
+    } else {
+      //Sort buckets lexicographically
+      userBuckets.get(userName).sort(
+          (bucket1, bucket2) -> {
+            int compare = bucket1.getName().compareTo(bucket2.getName());
+            if (compare < 0) {
+              return -1;
+            } else if (compare == 0) {
+              return 0;
+            } else {
+              return 1;
+            }
+          });
+      return userBuckets.get(userName).stream()
+          .filter(ozoneBucket -> {
+            if (prevBucket != null) {
+              return ozoneBucket.getName().compareTo(prevBucket) > 0;
+            } else {
+              return true;
+            }
+          })
+          .filter(ozoneBucket -> {
+            if (bucketPrefix != null) {
+              return ozoneBucket.getName().startsWith(bucketPrefix);
+            } else {
+              return true;
+            }
+          }).collect(Collectors.toList())
+          .iterator();
+    }
   }
 
   @Override

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java

@@ -51,7 +51,6 @@ public class TestObjectGet {
     client.getObjectStore().createS3Bucket("bilbo", "b1");
     String volumeName = client.getObjectStore().getOzoneVolumeName("b1");
     OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
-    volume.createBucket("b1");
     OzoneBucket bucket =
         volume.getBucket("b1");
     OzoneOutputStream keyStream =

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java

@@ -20,15 +20,12 @@
 
 package org.apache.hadoop.ozone.s3.endpoint;
 
-
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneClientStub;
-import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
 
 import org.apache.commons.lang3.RandomStringUtils;
 import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -39,7 +36,6 @@ public class TestRootList {
 
   private OzoneClientStub clientStub;
   private ObjectStore objectStoreStub;
-  private OzoneVolume volumeStub;
   private RootEndpoint rootEndpoint;
   private String userName = "ozone";
 
@@ -49,9 +45,6 @@ public class TestRootList {
     //Create client stub and object store stub.
     clientStub = new OzoneClientStub();
     objectStoreStub = clientStub.getObjectStore();
-    String volumeName = "s3" + userName;
-    objectStoreStub.createVolume(volumeName);
-    volumeStub = objectStoreStub.getVolume(volumeName);
 
     // Create HeadBucket and setClient to OzoneClientStub
     rootEndpoint = new RootEndpoint();
@@ -71,7 +64,7 @@ public class TestRootList {
 
     String bucketBaseName = "bucket-";
     for(int i = 0; i < 10; i++) {
-      volumeStub.createBucket(
+      objectStoreStub.createS3Bucket(userName,
           bucketBaseName + RandomStringUtils.randomNumeric(3));
     }
     response = rootEndpoint.get();