HDFS-5163. Miscellaneous cache pool RPC fixes (Contributed by Colin Patrick McCabe)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-4949@1520665 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchedRemoteIterator.java

@@ -28,13 +28,16 @@ public abstract class BatchedRemoteIterator<K, E> implements RemoteIterator<E> {
   public interface BatchedEntries<E> {
     public E get(int i);
     public int size();
+    public boolean hasMore();
   }
 
   public static class BatchedListEntries<E> implements BatchedEntries<E> {
     private final List<E> entries;
+    private final boolean hasMore;
 
-    public BatchedListEntries(List<E> entries) {
+    public BatchedListEntries(List<E> entries, boolean hasMore) {
       this.entries = entries;
+      this.hasMore = hasMore;
     }
 
     public E get(int i) {
@@ -44,16 +47,18 @@ public abstract class BatchedRemoteIterator<K, E> implements RemoteIterator<E> {
     public int size() {
       return entries.size();
     }
+
+    public boolean hasMore() {
+      return hasMore;
+    }
   }
 
   private K prevKey;
-  private final int maxRepliesPerRequest;
   private BatchedEntries<E> entries;
   private int idx;
 
-  public BatchedRemoteIterator(K prevKey, int maxRepliesPerRequest) {
+  public BatchedRemoteIterator(K prevKey) {
     this.prevKey = prevKey;
-    this.maxRepliesPerRequest = maxRepliesPerRequest;
     this.entries = null;
     this.idx = -1;
   }
@@ -62,21 +67,14 @@ public abstract class BatchedRemoteIterator<K, E> implements RemoteIterator<E> {
    * Perform the actual remote request.
    *
    * @param key                    The key to send.
-   * @param maxRepliesPerRequest   The maximum number of replies to allow.
    * @return                       A list of replies.
    */
-  public abstract BatchedEntries<E> makeRequest(K prevKey,
-      int maxRepliesPerRequest) throws IOException;
+  public abstract BatchedEntries<E> makeRequest(K prevKey) throws IOException;
 
   private void makeRequest() throws IOException {
     idx = 0;
     entries = null;
-    entries = makeRequest(prevKey, maxRepliesPerRequest);
-    if (entries.size() > maxRepliesPerRequest) {
-      throw new IOException("invalid number of replies returned: got " +
-          entries.size() + ", expected " + maxRepliesPerRequest +
-          " at most.");
-    }
+    entries = makeRequest(prevKey);
     if (entries.size() == 0) {
       entries = null;
     }
@@ -86,7 +84,7 @@ public abstract class BatchedRemoteIterator<K, E> implements RemoteIterator<E> {
     if (idx == -1) {
       makeRequest();
     } else if ((entries != null) && (idx >= entries.size())) {
-      if (entries.size() < maxRepliesPerRequest) {
+      if (!entries.hasMore()) {
         // Last time, we got fewer entries than requested.
         // So we should be at the end.
         entries = null;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java

@@ -303,6 +303,13 @@ public class FsPermission implements Writable {
     return new FsPermission((short)00666);
   }
 
+  /**
+   * Get the default permission for cache pools.
+   */
+  public static FsPermission getCachePoolDefault() {
+    return new FsPermission((short)00755);
+  }
+
   /**
    * Create a FsPermission from a Unix symbolic permission string
    * @param unixSymbolicPermission e.g. "-rw-rw-rw-"

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -1253,6 +1253,14 @@ public class UserGroupInformation {
     return null;
   }
 
+  public String getPrimaryGroupName() throws IOException {
+    String[] groups = getGroupNames();
+    if (groups.length == 0) {
+      throw new IOException("There is no primary group for UGI " + this);
+    }
+    return groups[0];
+  }
+
   /**
    * Get the user's full principal name.
    * @return the user's full principal name.

hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-4949.txt

@@ -24,6 +24,9 @@ HDFS-4949 (Unreleased)
     HDFS-5121. Add RPCs for creating and manipulating cache pools.
     (Contributed by Colin Patrick McCabe)
 
+    HDFS-5163. Miscellaneous cache pool RPC fixes.  (Contributed by Colin
+    Patrick McCabe)
+
 
   OPTIMIZATIONS
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -195,6 +195,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive";
   public static final int     DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 1000;
   
+  public static final String  DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES =
+      "dfs.namenode.list.cache.pools.num.responses";
+  public static final int     DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES_DEFAULT = 100;
+  public static final String  DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES =
+      "dfs.namenode.list.cache.directives.num.responses";
+  public static final int     DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES_DEFAULT = 100;
+
   // Whether to enable datanode's stale state detection and usage for reads
   public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode";
   public static final boolean DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/AddPathCacheDirectiveException.java

@@ -56,12 +56,12 @@ public abstract class AddPathCacheDirectiveException extends IOException {
     }
   }
 
-  public static class InvalidPoolError
+  public static class InvalidPoolNameError
       extends AddPathCacheDirectiveException {
     private static final long serialVersionUID = 1L;
 
-    public InvalidPoolError(PathCacheDirective directive) {
-      super("invalid pool id " + directive.getPoolId(), directive);
+    public InvalidPoolNameError(PathCacheDirective directive) {
+      super("invalid pool name '" + directive.getPool() + "'", directive);
     }
   }
 
@@ -70,7 +70,7 @@ public abstract class AddPathCacheDirectiveException extends IOException {
     private static final long serialVersionUID = 1L;
 
     public PoolWritePermissionDeniedError(PathCacheDirective directive) {
-      super("write permission denied for pool id " + directive.getPoolId(),
+      super("write permission denied for pool '" + directive.getPool() + "'",
             directive);
     }
   }
@@ -82,9 +82,7 @@ public abstract class AddPathCacheDirectiveException extends IOException {
     public UnexpectedAddPathCacheDirectiveException(
         PathCacheDirective directive) {
       super("encountered an unexpected error when trying to " +
-          "add path cache directive to pool id " + directive.getPoolId() +
-          " " + directive,
-          directive);
+          "add path cache directive " + directive, directive);
     }
   }
 };

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java

@@ -18,45 +18,38 @@
 
 package org.apache.hadoop.hdfs.protocol;
 
+import javax.annotation.Nullable;
+
 import org.apache.commons.lang.builder.EqualsBuilder;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.FsPermission;
 
-import com.google.common.base.Preconditions;
-
 /**
  * Information about a cache pool.
- * 
- * CachePoolInfo permissions roughly map to Unix file permissions.
- * Write permissions allow addition and removal of a {@link PathCacheEntry} from
- * the pool. Execute permissions allow listing of PathCacheEntries in a pool.
- * Read permissions have no associated meaning.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class CachePoolInfo {
+  final String poolName;
+
+  @Nullable
+  String ownerName;
+
+  @Nullable
+  String groupName;
+
+  @Nullable
+  FsPermission mode;
+
+  @Nullable
+  Integer weight;
 
-  private String poolName;
-  private String ownerName;
-  private String groupName;
-  private FsPermission mode;
-  private Integer weight;
-
-  /**
-   * For Builder use
-   */
-  private CachePoolInfo() {}
-
-  /**
-   * Use a CachePoolInfo {@link Builder} to create a new CachePoolInfo with
-   * more parameters
-   */
   public CachePoolInfo(String poolName) {
     this.poolName = poolName;
   }
-
+  
   public String getPoolName() {
     return poolName;
   }
@@ -65,103 +58,73 @@ public class CachePoolInfo {
     return ownerName;
   }
 
+  public CachePoolInfo setOwnerName(String ownerName) {
+    this.ownerName = ownerName;
+    return this;
+  }
+
   public String getGroupName() {
     return groupName;
   }
 
+  public CachePoolInfo setGroupName(String groupName) {
+    this.groupName = groupName;
+    return this;
+  }
+  
   public FsPermission getMode() {
     return mode;
   }
 
+  public CachePoolInfo setMode(FsPermission mode) {
+    this.mode = mode;
+    return this;
+  }
+
   public Integer getWeight() {
     return weight;
   }
 
-  public String toString() {
-    return new StringBuilder().
-        append("{ ").append("poolName:").append(poolName).
-        append(", ownerName:").append(ownerName).
-        append(", groupName:").append(groupName).
-        append(", mode:").append(mode).
-        append(", weight:").append(weight).
-        append(" }").toString();
+  public CachePoolInfo setWeight(Integer weight) {
+    this.weight = weight;
+    return this;
   }
 
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder().append(poolName).append(ownerName)
-        .append(groupName).append(mode.toShort()).append(weight).hashCode();
+  public String toString() {
+    return new StringBuilder().append("{").
+      append("poolName:").append(poolName).
+      append(", ownerName:").append(ownerName).
+      append(", groupName:").append(groupName).
+      append(", mode:").append((mode == null) ? "null" :
+          String.format("0%03o", mode)).
+      append(", weight:").append(weight).
+      append("}").toString();
   }
-
+  
   @Override
-  public boolean equals(Object obj) {
-    if (obj == null) { return false; }
-    if (obj == this) { return true; }
-    if (obj.getClass() != getClass()) {
+  public boolean equals(Object o) {
+    try {
+      CachePoolInfo other = (CachePoolInfo)o;
+      return new EqualsBuilder().
+          append(poolName, other.poolName).
+          append(ownerName, other.ownerName).
+          append(groupName, other.groupName).
+          append(mode, other.mode).
+          append(weight, other.weight).
+          isEquals();
+    } catch (ClassCastException e) {
       return false;
     }
-    CachePoolInfo rhs = (CachePoolInfo)obj;
-    return new EqualsBuilder()
-      .append(poolName, rhs.poolName)
-      .append(ownerName, rhs.ownerName)
-      .append(groupName, rhs.groupName)
-      .append(mode, rhs.mode)
-      .append(weight, rhs.weight)
-      .isEquals();
-  }
-
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  public static Builder newBuilder(CachePoolInfo info) {
-    return new Builder(info);
   }
 
-  /**
-   * CachePoolInfo Builder
-   */
-  public static class Builder {
-    private CachePoolInfo info;
-
-    public Builder() {
-      this.info = new CachePoolInfo();
-    }
-
-    public Builder(CachePoolInfo info) {
-      this.info = info;
-    }
-
-    public CachePoolInfo build() {
-      Preconditions.checkNotNull(info.poolName,
-          "Cannot create a CachePoolInfo without a pool name");
-      return info;
-    }
-
-    public Builder setPoolName(String poolName) {
-      info.poolName = poolName;
-      return this;
-    }
-
-    public Builder setOwnerName(String ownerName) {
-      info.ownerName = ownerName;
-      return this;
-    }
-
-    public Builder setGroupName(String groupName) {
-      info.groupName = groupName;
-      return this;
-    }
-
-    public Builder setMode(FsPermission mode) {
-      info.mode = mode;
-      return this;
-    }
-
-    public Builder setWeight(Integer weight) {
-      info.weight = weight;
-      return this;
-    }
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder().
+        append(poolName).
+        append(ownerName).
+        append(groupName).
+        append(mode).
+        append(weight).
+        hashCode();
   }
-
-}
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -38,7 +38,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
-import org.apache.hadoop.hdfs.server.namenode.CachePool;
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.io.EnumSetWritable;
@@ -1107,8 +1106,9 @@ public interface ClientProtocol {
    *         could not be added.
    */
   @AtMostOnce
-  public List<Fallible<PathCacheEntry>> addPathCacheDirectives(
-      List<PathCacheDirective> directives) throws IOException;
+  public List<Fallible<PathCacheEntry>>
+    addPathCacheDirectives(List<PathCacheDirective> directives)
+      throws IOException;
 
   /**
    * Remove some path cache entries from the CacheManager.
@@ -1117,7 +1117,7 @@ public interface ClientProtocol {
    * @return A Fallible list where each element is either a successfully removed
    *         ID, or an IOException describing why the ID could not be removed.
    */
-  @Idempotent
+  @AtMostOnce
   public List<Fallible<Long>> removePathCacheEntries(List<Long> ids)
       throws IOException;
 
@@ -1127,15 +1127,13 @@ public interface ClientProtocol {
    * 
    * @param prevId The last listed entry ID, or -1 if this is the first call to
    *          listPathCacheEntries.
-   * @param pool The cache pool to list, or -1 to list all pools
-   * @param maxRepliesPerRequest The maximum number of entries to return per
-   *          request
+   * @param pool The cache pool to list, or the empty string to list all pools
    * @return A RemoteIterator which returns PathCacheEntry objects.
    */
   @Idempotent
   public RemoteIterator<PathCacheEntry> listPathCacheEntries(long prevId,
-      long poolId, int maxRepliesPerRequest) throws IOException;
-
+      String pool) throws IOException;
+  
   /**
    * Add a new cache pool.
    * 
@@ -1143,39 +1141,37 @@ public interface ClientProtocol {
    * @throws IOException If the request could not be completed.
    */
   @AtMostOnce
-  public CachePool addCachePool(CachePoolInfo info) throws IOException;
+  public void addCachePool(CachePoolInfo info) throws IOException;
 
   /**
-   * Modify a cache pool, e.g. pool name, permissions, owner, group.
-   * 
-   * @param poolId ID of the cache pool to modify
-   * @param info New metadata for the cache pool
-   * @throws IOException If the request could not be completed.
+   * Modify a cache pool.
+   *
+   * @param req
+   *          The request to modify a cache pool.
+   * @throws IOException 
+   *          If the request could not be completed.
    */
   @AtMostOnce
-  public void modifyCachePool(long poolId, CachePoolInfo info)
-      throws IOException;
-
+  public void modifyCachePool(CachePoolInfo req) throws IOException;
+  
   /**
    * Remove a cache pool.
    * 
-   * @param poolId ID of the cache pool to remove.
+   * @param pool name of the cache pool to remove.
    * @throws IOException if the cache pool did not exist, or could not be
    *           removed.
    */
-  @Idempotent
-  public void removeCachePool(long poolId) throws IOException;
+  @AtMostOnce
+  public void removeCachePool(String pool) throws IOException;
 
   /**
    * List the set of cache pools. Incrementally fetches results from the server.
    * 
-   * @param prevPoolId ID of the last pool listed, or -1 if this is the first
-   *          invocation of listCachePools
-   * @param maxRepliesPerRequest Maximum number of cache pools to return per
-   *          server request.
+   * @param prevPool name of the last pool listed, or the empty string if this is
+   *          the first invocation of listCachePools
    * @return A RemoteIterator which returns CachePool objects.
    */
   @Idempotent
-  public RemoteIterator<CachePool> listCachePools(long prevPoolId,
-      int maxRepliesPerRequest) throws IOException;
+  public RemoteIterator<CachePoolInfo> listCachePools(String prevPool)
+      throws IOException;
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/PathCacheDirective.java

@@ -25,7 +25,7 @@ import com.google.common.collect.ComparisonChain;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.EmptyPathError;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolError;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolNameError;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPathNameError;
 
 /**
@@ -33,13 +33,14 @@ import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPat
  */
 public class PathCacheDirective implements Comparable<PathCacheDirective> {
   private final String path;
-  private final long poolId;
 
-  public PathCacheDirective(String path, long poolId) {
+  private final String pool;
+
+  public PathCacheDirective(String path, String pool) {
     Preconditions.checkNotNull(path);
-    Preconditions.checkArgument(poolId > 0);
+    Preconditions.checkNotNull(pool);
     this.path = path;
-    this.poolId = poolId;
+    this.pool = pool;
   }
 
   /**
@@ -52,8 +53,8 @@ public class PathCacheDirective implements Comparable<PathCacheDirective> {
   /**
    * @return The pool used in this request.
    */
-  public long getPoolId() {
-    return poolId;
+  public String getPool() {
+    return pool;
   }
 
   /**
@@ -69,22 +70,22 @@ public class PathCacheDirective implements Comparable<PathCacheDirective> {
     if (!DFSUtil.isValidName(path)) {
       throw new InvalidPathNameError(this);
     }
-    if (poolId <= 0) {
-      throw new InvalidPoolError(this);
+    if (pool.isEmpty()) {
+      throw new InvalidPoolNameError(this);
     }
   }
 
   @Override
   public int compareTo(PathCacheDirective rhs) {
     return ComparisonChain.start().
-        compare(poolId, rhs.getPoolId()).
+        compare(pool, rhs.getPool()).
         compare(path, rhs.getPath()).
         result();
   }
 
   @Override
   public int hashCode() {
-    return new HashCodeBuilder().append(path).append(poolId).hashCode();
+    return new HashCodeBuilder().append(path).append(pool).hashCode();
   }
 
   @Override
@@ -101,7 +102,7 @@ public class PathCacheDirective implements Comparable<PathCacheDirective> {
   public String toString() {
     StringBuilder builder = new StringBuilder();
     builder.append("{ path:").append(path).
-      append(", poolId:").append(poolId).
+      append(", pool:").append(pool).
       append(" }");
     return builder.toString();
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java

@@ -27,9 +27,11 @@ import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.EmptyPathError;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPathNameError;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolError;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolNameError;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -112,6 +114,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCa
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCachePoolsResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCorruptFileBlocksRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCorruptFileBlocksResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesElementProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MetaSaveRequestProto;
@@ -171,6 +174,7 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.namenode.CachePool;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
 import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
@@ -1035,19 +1039,16 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   }
 
   @Override
-  public AddPathCacheDirectivesResponseProto addPathCacheDirectives(
-      RpcController controller, AddPathCacheDirectivesRequestProto request)
-          throws ServiceException {
+  public AddPathCacheDirectivesResponseProto addPathCacheDirectives(RpcController controller,
+      AddPathCacheDirectivesRequestProto request) throws ServiceException {
     try {
       ArrayList<PathCacheDirective> input =
           new ArrayList<PathCacheDirective>(request.getElementsCount());
       for (int i = 0; i < request.getElementsCount(); i++) {
         PathCacheDirectiveProto proto = request.getElements(i);
-        input.add(new PathCacheDirective(proto.getPath(),
-            proto.getPool().getId()));
+        input.add(new PathCacheDirective(proto.getPath(), proto.getPool()));
       }
-      List<Fallible<PathCacheEntry>> output = server
-          .addPathCacheDirectives(input);
+      List<Fallible<PathCacheEntry>> output = server.addPathCacheDirectives(input);
       AddPathCacheDirectivesResponseProto.Builder builder =
          AddPathCacheDirectivesResponseProto.newBuilder();
       for (int idx = 0; idx < output.size(); idx++) {
@@ -1060,7 +1061,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
         } catch (InvalidPathNameError ioe) {
           builder.addResults(AddPathCacheDirectiveErrorProto.
               INVALID_PATH_NAME_ERROR_VALUE);
-        } catch (InvalidPoolError ioe) {
+        } catch (InvalidPoolNameError ioe) {
           builder.addResults(AddPathCacheDirectiveErrorProto.
               INVALID_POOL_NAME_ERROR_VALUE);
         } catch (IOException ioe) {
@@ -1108,21 +1109,20 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   }
 
   @Override
-  public ListPathCacheEntriesResponseProto listPathCacheEntries(
-      RpcController controller, ListPathCacheEntriesRequestProto request)
-      throws ServiceException {
+  public ListPathCacheEntriesResponseProto listPathCacheEntries(RpcController controller,
+      ListPathCacheEntriesRequestProto request) throws ServiceException {
     try {
-      CachePool pool = PBHelper.convert(request.getPool());
       RemoteIterator<PathCacheEntry> iter =
-         server.listPathCacheEntries(
-             PBHelper.convert(request.getPrevEntry()).getEntryId(),
-             pool.getId(),
-             request.getMaxReplies());
+         server.listPathCacheEntries(request.getPrevId(), request.getPool());
       ListPathCacheEntriesResponseProto.Builder builder =
           ListPathCacheEntriesResponseProto.newBuilder();
       while (iter.hasNext()) {
         PathCacheEntry entry = iter.next();
-        builder.addEntries(PBHelper.convert(entry));
+        builder.addElements(
+            ListPathCacheEntriesElementProto.newBuilder().
+              setId(entry.getEntryId()).
+              setPath(entry.getDirective().getPath()).
+              setPool(entry.getDirective().getPool()));
       }
       return builder.build();
     } catch (IOException e) {
@@ -1134,20 +1134,46 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   public AddCachePoolResponseProto addCachePool(RpcController controller,
       AddCachePoolRequestProto request) throws ServiceException {
     try {
-      server.addCachePool(PBHelper.convert(request.getInfo()));
+      CachePoolInfo info =
+          new CachePoolInfo(request.getPoolName());
+      if (request.hasOwnerName()) {
+        info.setOwnerName(request.getOwnerName());
+      }
+      if (request.hasGroupName()) {
+        info.setGroupName(request.getGroupName());
+      }
+      if (request.hasMode()) {
+        info.setMode(new FsPermission((short)request.getMode()));
+      }
+      if (request.hasWeight()) {
+        info.setWeight(request.getWeight());
+      }
+      server.addCachePool(info);
       return AddCachePoolResponseProto.newBuilder().build();
     } catch (IOException e) {
       throw new ServiceException(e);
     }
   }
-
+  
   @Override
   public ModifyCachePoolResponseProto modifyCachePool(RpcController controller,
       ModifyCachePoolRequestProto request) throws ServiceException {
     try {
-      server.modifyCachePool(
-          PBHelper.convert(request.getPool()).getId(),
-          PBHelper.convert(request.getInfo()));
+      CachePoolInfo info =
+          new CachePoolInfo(request.getPoolName());
+      if (request.hasOwnerName()) {
+        info.setOwnerName(request.getOwnerName());
+      }
+      if (request.hasGroupName()) {
+        info.setGroupName(request.getGroupName());
+      }
+      if (request.hasMode()) {
+        info.setMode(new FsPermission((short)request.getMode()));
+      }
+      if (request.hasWeight()) {
+        info.setWeight(request.getWeight());
+      }
+      server.modifyCachePool(info);
       return ModifyCachePoolResponseProto.newBuilder().build();
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -1158,7 +1184,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   public RemoveCachePoolResponseProto removeCachePool(RpcController controller,
       RemoveCachePoolRequestProto request) throws ServiceException {
     try {
-      server.removeCachePool(PBHelper.convert(request.getPool()).getId());
+      server.removeCachePool(request.getPoolName());
       return RemoveCachePoolResponseProto.newBuilder().build();
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -1169,16 +1195,27 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   public ListCachePoolsResponseProto listCachePools(RpcController controller,
       ListCachePoolsRequestProto request) throws ServiceException {
     try {
-      RemoteIterator<CachePool> iter =
-        server.listCachePools(PBHelper.convert(request.getPrevPool()).getId(),
-            request.getMaxReplies());
+      RemoteIterator<CachePoolInfo> iter =
+        server.listCachePools(request.getPrevPoolName());
       ListCachePoolsResponseProto.Builder responseBuilder =
         ListCachePoolsResponseProto.newBuilder();
       while (iter.hasNext()) {
-        CachePool pool = iter.next();
-        ListCachePoolsResponseElementProto.Builder elemBuilder =
+        CachePoolInfo pool = iter.next();
+        ListCachePoolsResponseElementProto.Builder elemBuilder = 
             ListCachePoolsResponseElementProto.newBuilder();
-        elemBuilder.setPool(PBHelper.convert(pool));
+        elemBuilder.setPoolName(pool.getPoolName());
+        if (pool.getOwnerName() != null) {
+          elemBuilder.setOwnerName(pool.getOwnerName());
+        }
+        if (pool.getGroupName() != null) {
+          elemBuilder.setGroupName(pool.getGroupName());
+        }
+        if (pool.getMode() != null) {
+          elemBuilder.setMode(pool.getMode().toShort());
+        }
+        if (pool.getWeight() != null) {
+          elemBuilder.setWeight(pool.getWeight());
+        }
         responseBuilder.addElements(elemBuilder.build());
       }
       return responseBuilder.build();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java

@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.NoSuchElementException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -37,12 +38,17 @@ import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
+import org.apache.hadoop.hdfs.protocol.PathCacheDirective;
+import org.apache.hadoop.hdfs.protocol.PathCacheEntry;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.EmptyPathError;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPathNameError;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolError;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolNameError;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.UnexpectedAddPathCacheDirectiveException;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.InvalidIdException;
+import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.NoSuchIdException;
+import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.RemovePermissionDeniedException;
+import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.UnexpectedRemovePathCacheEntryException;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
@@ -55,18 +61,14 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.PathCacheDirective;
-import org.apache.hadoop.hdfs.protocol.PathCacheEntry;
-import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.InvalidIdException;
-import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.NoSuchIdException;
-import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.RemovePermissionDeniedException;
-import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.UnexpectedRemovePathCacheEntryException;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AbandonBlockRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddCachePoolRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathCacheDirectiveProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddPathCacheDirectiveErrorProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddPathCacheDirectivesRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddPathCacheDirectivesResponseProto;
@@ -107,23 +109,23 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSna
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.IsFileClosedRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesElementProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCachePoolsRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCachePoolsResponseElementProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCachePoolsResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCorruptFileBlocksRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListPathCacheEntriesResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MetaSaveRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MkdirsRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCachePoolRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathCacheDirectiveProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathCacheEntryProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RecoverLeaseRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RefreshNodesRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemovePathCacheEntriesRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemovePathCacheEntriesResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemovePathCacheEntryErrorProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameSnapshotRequestProto;
@@ -144,7 +146,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.CachePool;
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.io.EnumSetWritable;
@@ -1026,7 +1027,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
       return new InvalidPathNameError(directive);
     } else if (code == AddPathCacheDirectiveErrorProto.
         INVALID_POOL_NAME_ERROR_VALUE) {
-      return new InvalidPoolError(directive);
+      return new InvalidPoolNameError(directive);
     } else {
       return new UnexpectedAddPathCacheDirectiveException(directive);
     }
@@ -1041,7 +1042,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
       for (PathCacheDirective directive : directives) {
         builder.addElements(PathCacheDirectiveProto.newBuilder().
             setPath(directive.getPath()).
-            setPool(PBHelper.convert(new CachePool(directive.getPoolId()))).
+            setPool(directive.getPool()).
             build());
       }
       AddPathCacheDirectivesResponseProto result = 
@@ -1120,40 +1121,45 @@ public class ClientNamenodeProtocolTranslatorPB implements
 
     @Override
     public PathCacheEntry get(int i) {
-      PathCacheEntryProto entryProto = response.getEntries(i);
-      return PBHelper.convert(entryProto);
+      ListPathCacheEntriesElementProto elementProto =
+        response.getElements(i);
+      return new PathCacheEntry(elementProto.getId(), 
+          new PathCacheDirective(elementProto.getPath(),
+              elementProto.getPool()));
     }
 
     @Override
     public int size() {
-      return response.getEntriesCount();
+      return response.getElementsCount();
+    }
+    
+    @Override
+    public boolean hasMore() {
+      return response.getHasMore();
     }
   }
 
   private class PathCacheEntriesIterator
       extends BatchedRemoteIterator<Long, PathCacheEntry> {
-    private final long poolId;
+    private final String pool;
 
-    public PathCacheEntriesIterator(long prevKey, int maxRepliesPerRequest,
-        long poolId) {
-      super(prevKey, maxRepliesPerRequest);
-      this.poolId = poolId;
+    public PathCacheEntriesIterator(long prevKey, String pool) {
+      super(prevKey);
+      this.pool = pool;
     }
 
     @Override
     public BatchedEntries<PathCacheEntry> makeRequest(
-        Long prevEntryId, int maxRepliesPerRequest) throws IOException {
+        Long nextKey) throws IOException {
       ListPathCacheEntriesResponseProto response;
       try {
         ListPathCacheEntriesRequestProto req =
             ListPathCacheEntriesRequestProto.newBuilder().
-              setPrevEntry(
-                  PBHelper.convert(new PathCacheEntry(prevEntryId, null))).
-              setPool(PBHelper.convert(new CachePool(poolId))).
-              setMaxReplies(maxRepliesPerRequest).
+              setPrevId(nextKey).
+              setPool(pool).
               build();
         response = rpcProxy.listPathCacheEntries(null, req);
-        if (response.getEntriesCount() == 0) {
+        if (response.getElementsCount() == 0) {
           response = null;
         }
       } catch (ServiceException e) {
@@ -1170,30 +1176,51 @@ public class ClientNamenodeProtocolTranslatorPB implements
 
   @Override
   public RemoteIterator<PathCacheEntry> listPathCacheEntries(long prevId,
-      long poolId, int repliesPerRequest) throws IOException {
-    return new PathCacheEntriesIterator(prevId, repliesPerRequest, poolId);
+      String pool) throws IOException {
+    return new PathCacheEntriesIterator(prevId, pool);
   }
 
   @Override
-  public CachePool addCachePool(CachePoolInfo info) throws IOException {
-    AddCachePoolRequestProto.Builder builder =
+  public void addCachePool(CachePoolInfo info) throws IOException {
+    AddCachePoolRequestProto.Builder builder = 
         AddCachePoolRequestProto.newBuilder();
-    builder.setInfo(PBHelper.convert(info));
+    builder.setPoolName(info.getPoolName());
+    if (info.getOwnerName() != null) {
+      builder.setOwnerName(info.getOwnerName());
+    }
+    if (info.getGroupName() != null) {
+      builder.setGroupName(info.getGroupName());
+    }
+    if (info.getMode() != null) {
+      builder.setMode(info.getMode().toShort());
+    }
+    if (info.getWeight() != null) {
+      builder.setWeight(info.getWeight());
+    }
     try {
-      return PBHelper.convert(
-          rpcProxy.addCachePool(null, builder.build()).getPool());
+      rpcProxy.addCachePool(null, builder.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
   }
 
   @Override
-  public void modifyCachePool(long poolId, CachePoolInfo info)
-      throws IOException {
-    ModifyCachePoolRequestProto.Builder builder =
-        ModifyCachePoolRequestProto.newBuilder()
-        .setPool(PBHelper.convert(new CachePool(poolId)))
-        .setInfo(PBHelper.convert(info));
+  public void modifyCachePool(CachePoolInfo req) throws IOException {
+    ModifyCachePoolRequestProto.Builder builder = 
+        ModifyCachePoolRequestProto.newBuilder();
+    builder.setPoolName(req.getPoolName());
+    if (req.getOwnerName() != null) {
+      builder.setOwnerName(req.getOwnerName());
+    }
+    if (req.getGroupName() != null) {
+      builder.setGroupName(req.getGroupName());
+    }
+    if (req.getMode() != null) {
+      builder.setMode(req.getMode().toShort());
+    }
+    if (req.getWeight() != null) {
+      builder.setWeight(req.getWeight());
+    }
     try {
       rpcProxy.modifyCachePool(null, builder.build());
     } catch (ServiceException e) {
@@ -1202,69 +1229,74 @@ public class ClientNamenodeProtocolTranslatorPB implements
   }
 
   @Override
-  public void removeCachePool(long poolId) throws IOException {
+  public void removeCachePool(String cachePoolName) throws IOException {
     try {
-      rpcProxy.removeCachePool(null,
+      rpcProxy.removeCachePool(null, 
           RemoveCachePoolRequestProto.newBuilder().
-          setPool(PBHelper.convert(new CachePool(poolId))).
-          build());
+            setPoolName(cachePoolName).build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
   }
 
   private static class BatchedPathDirectiveEntries
-      implements BatchedEntries<CachePool> {
-
+      implements BatchedEntries<CachePoolInfo> {
     private final ListCachePoolsResponseProto proto;
-
+    
     public BatchedPathDirectiveEntries(ListCachePoolsResponseProto proto) {
       this.proto = proto;
     }
-
+      
     @Override
-    public CachePool get(int i) {
+    public CachePoolInfo get(int i) {
       ListCachePoolsResponseElementProto elem = proto.getElements(i);
-      return PBHelper.convert(elem.getPool());
+      return new CachePoolInfo(elem.getPoolName()).
+          setOwnerName(elem.getOwnerName()).
+          setGroupName(elem.getGroupName()).
+          setMode(new FsPermission((short)elem.getMode())).
+          setWeight(elem.getWeight());
     }
 
     @Override
     public int size() {
       return proto.getElementsCount();
     }
+    
+    @Override
+    public boolean hasMore() {
+      return proto.getHasMore();
+    }
   }
+  
+  private class CachePoolIterator 
+      extends BatchedRemoteIterator<String, CachePoolInfo> {
 
-  private class CachePoolIterator
-      extends BatchedRemoteIterator<Long, CachePool> {
-
-    public CachePoolIterator(Long prevKey, int maxRepliesPerRequest) {
-      super(prevKey, maxRepliesPerRequest);
+    public CachePoolIterator(String prevKey) {
+      super(prevKey);
     }
 
     @Override
-    public BatchedEntries<CachePool> makeRequest(Long prevKey,
-        int maxRepliesPerRequest) throws IOException {
+    public BatchedEntries<CachePoolInfo> makeRequest(String prevKey)
+        throws IOException {
       try {
         return new BatchedPathDirectiveEntries(
-            rpcProxy.listCachePools(null,
+            rpcProxy.listCachePools(null, 
               ListCachePoolsRequestProto.newBuilder().
-                setPrevPool(PBHelper.convert(new CachePool(prevKey))).
-                setMaxReplies(maxRepliesPerRequest).
-                build()));
+                setPrevPoolName(prevKey).build()));
       } catch (ServiceException e) {
         throw ProtobufHelper.getRemoteException(e);
       }
     }
 
     @Override
-    public Long elementToPrevKey(CachePool element) {
-      return element.getId();
+    public String elementToPrevKey(CachePoolInfo element) {
+      return element.getPoolName();
     }
   }
 
   @Override
-  public RemoteIterator<CachePool> listCachePools(long prevPoolId,
-      int maxRepliesPerRequest) throws IOException {
-    return new CachePoolIterator(prevPoolId, maxRepliesPerRequest);
+  public RemoteIterator<CachePoolInfo> listCachePools(String prevKey)
+      throws IOException {
+    return new CachePoolIterator(prevKey);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java

@@ -32,13 +32,10 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.PathCacheDirective;
-import org.apache.hadoop.hdfs.protocol.PathCacheEntry;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -53,15 +50,9 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddCachePoolRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCachePoolRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathCacheDirectiveProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathCacheEntryProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto;
 import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto;
 import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
@@ -123,7 +114,6 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
-import org.apache.hadoop.hdfs.server.namenode.CachePool;
 import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
 import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
@@ -1503,74 +1493,6 @@ public class PBHelper {
     return HdfsProtos.ChecksumTypeProto.valueOf(type.id);
   }
 
-  public static PathCacheDirective convert(
-      PathCacheDirectiveProto directiveProto) {
-    CachePool pool = convert(directiveProto.getPool());
-    return new PathCacheDirective(directiveProto.getPath(), pool.getId());
-  }
-
-  public static PathCacheDirectiveProto convert(PathCacheDirective directive) {
-    PathCacheDirectiveProto.Builder builder = 
-        PathCacheDirectiveProto.newBuilder()
-        .setPath(directive.getPath())
-        .setPool(PBHelper.convert(new CachePool(directive.getPoolId())));
-    return builder.build();
-  }
-
-  public static PathCacheEntry convert(PathCacheEntryProto entryProto) {
-    long entryId = entryProto.getId();
-    PathCacheDirective directive = convert(entryProto.getDirective());
-    return new PathCacheEntry(entryId, directive);
-  }
-
-  public static PathCacheEntryProto convert(PathCacheEntry entry) {
-    PathCacheEntryProto.Builder builder = PathCacheEntryProto.newBuilder()
-        .setId(entry.getEntryId())
-        .setDirective(PBHelper.convert(entry.getDirective()));
-    return builder.build();
-  }
-
-  public static CachePoolInfo convert(CachePoolInfoProto infoProto) {
-    CachePoolInfo.Builder builder =
-        CachePoolInfo.newBuilder().setPoolName(infoProto.getPoolName());
-    if (infoProto.hasOwnerName()) {
-      builder.setOwnerName(infoProto.getOwnerName());
-    }
-    if (infoProto.hasGroupName()) {
-      builder.setGroupName(infoProto.getGroupName());
-    }
-    if (infoProto.hasMode()) {
-      builder.setMode(new FsPermission((short) infoProto.getMode()));
-    }
-    if (infoProto.hasWeight()) {
-      builder.setWeight(infoProto.getWeight());
-    }
-    return builder.build();
-  }
-
-  public static CachePoolInfoProto convert(CachePoolInfo info) {
-    CachePoolInfoProto.Builder builder = CachePoolInfoProto.newBuilder()
-        .setPoolName(info.getPoolName())
-        .setOwnerName(info.getOwnerName())
-        .setGroupName(info.getGroupName())
-        .setMode(info.getMode().toShort())
-        .setWeight(info.getWeight());
-    return builder.build();
-  }
-
-  public static CachePool convert(CachePoolProto poolProto) {
-    CachePoolInfo info = convert(poolProto.getInfo());
-    CachePool pool = new CachePool(poolProto.getId(), info);
-    return pool;
-  }
-
-  public static CachePoolProto convert(CachePool pool) {
-    CachePoolProto.Builder builder = CachePoolProto.newBuilder()
-        .setId(pool.getId())
-        .setInfo(convert(pool.getInfo()));
-    return builder.build();
-  }
-
   public static InputStream vintPrefixed(final InputStream input)
       throws IOException {
     final int firstByte = input.read();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java

@@ -17,28 +17,34 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES_DEFAULT;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
 import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolError;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.PoolWritePermissionDeniedError;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.UnexpectedAddPathCacheDirectiveException;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.PathCacheDirective;
 import org.apache.hadoop.hdfs.protocol.PathCacheEntry;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolNameError;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.UnexpectedAddPathCacheDirectiveException;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.PoolWritePermissionDeniedError;
 import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.InvalidIdException;
 import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.NoSuchIdException;
-import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.RemovePermissionDeniedException;
 import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.UnexpectedRemovePathCacheEntryException;
+import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.RemovePermissionDeniedException;
 import org.apache.hadoop.util.Fallible;
 
 /**
@@ -65,62 +71,58 @@ final class CacheManager {
   /**
    * Cache pools, sorted by name.
    */
-  private final TreeMap<String, CachePool> cachePoolsByName =
+  private final TreeMap<String, CachePool> cachePools =
       new TreeMap<String, CachePool>();
 
   /**
-   * Cache pools, sorted by ID
+   * The entry ID to use for a new entry.
    */
-  private final TreeMap<Long, CachePool> cachePoolsById =
-      new TreeMap<Long, CachePool>();
+  private long nextEntryId;
 
   /**
-   * The entry ID to use for a new entry.
+   * Maximum number of cache pools to list in one operation.
    */
-  private long nextEntryId;
+  private final int maxListCachePoolsResponses;
 
   /**
-   * The pool ID to use for a new pool.
+   * Maximum number of cache pool directives to list in one operation.
    */
-  private long nextPoolId;
+  private final int maxListCacheDirectivesResponses;
 
   CacheManager(FSDirectory dir, Configuration conf) {
     // TODO: support loading and storing of the CacheManager state
     clear();
+    maxListCachePoolsResponses = conf.getInt(
+        DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES,
+        DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES_DEFAULT);
+    maxListCacheDirectivesResponses = conf.getInt(
+        DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES,
+        DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES_DEFAULT);
   }
 
   synchronized void clear() {
     entriesById.clear();
     entriesByDirective.clear();
-    cachePoolsByName.clear();
-    cachePoolsById.clear();
+    cachePools.clear();
     nextEntryId = 1;
-    nextPoolId = 1;
   }
 
   synchronized long getNextEntryId() throws IOException {
     if (nextEntryId == Long.MAX_VALUE) {
-      throw new IOException("no more available entry IDs");
+      throw new IOException("no more available IDs");
     }
     return nextEntryId++;
   }
 
-  synchronized long getNextPoolId() throws IOException {
-    if (nextPoolId == Long.MAX_VALUE) {
-      throw new IOException("no more available pool IDs");
-    }
-    return nextPoolId++;
-  }
-
   private synchronized Fallible<PathCacheEntry> addDirective(
-        FSPermissionChecker pc, PathCacheDirective directive) {
-    CachePool pool = cachePoolsById.get(directive.getPoolId());
+        PathCacheDirective directive, FSPermissionChecker pc) {
+    CachePool pool = cachePools.get(directive.getPool());
     if (pool == null) {
       LOG.info("addDirective " + directive + ": pool not found.");
       return new Fallible<PathCacheEntry>(
-          new InvalidPoolError(directive));
+          new InvalidPoolNameError(directive));
     }
-    if (!pc.checkPermission(pool, FsAction.WRITE)) {
+    if ((pc != null) && (!pc.checkPermission(pool, FsAction.WRITE))) {
       LOG.info("addDirective " + directive + ": write permission denied.");
       return new Fallible<PathCacheEntry>(
           new PoolWritePermissionDeniedError(directive));
@@ -155,17 +157,17 @@ final class CacheManager {
   }
 
   public synchronized List<Fallible<PathCacheEntry>> addDirectives(
-      FSPermissionChecker pc, List<PathCacheDirective> directives) {
+      List<PathCacheDirective> directives, FSPermissionChecker pc) {
     ArrayList<Fallible<PathCacheEntry>> results = 
         new ArrayList<Fallible<PathCacheEntry>>(directives.size());
     for (PathCacheDirective directive: directives) {
-      results.add(addDirective(pc, directive));
+      results.add(addDirective(directive, pc));
     }
     return results;
   }
 
-  private synchronized Fallible<Long> removeEntry(FSPermissionChecker pc,
-        long entryId) {
+  private synchronized Fallible<Long> removeEntry(long entryId,
+        FSPermissionChecker pc) {
     // Check for invalid IDs.
     if (entryId <= 0) {
       LOG.info("removeEntry " + entryId + ": invalid non-positive entry ID.");
@@ -177,20 +179,20 @@ final class CacheManager {
       LOG.info("removeEntry " + entryId + ": entry not found.");
       return new Fallible<Long>(new NoSuchIdException(entryId));
     }
-    CachePool pool = cachePoolsById.get(existing.getDirective().getPoolId());
+    CachePool pool = cachePools.get(existing.getDirective().getPool());
     if (pool == null) {
       LOG.info("removeEntry " + entryId + ": pool not found for directive " +
         existing.getDirective());
       return new Fallible<Long>(
           new UnexpectedRemovePathCacheEntryException(entryId));
     }
-    if (!pc.checkPermission(pool, FsAction.WRITE)) {
+    if ((pc != null) && (!pc.checkPermission(pool, FsAction.WRITE))) {
       LOG.info("removeEntry " + entryId + ": write permission denied to " +
           "pool " + pool + " for entry " + existing);
       return new Fallible<Long>(
           new RemovePermissionDeniedException(entryId));
     }
-
+    
     // Remove the corresponding entry in entriesByDirective.
     if (entriesByDirective.remove(existing.getDirective()) == null) {
       LOG.warn("removeEntry " + entryId + ": failed to find existing entry " +
@@ -202,41 +204,43 @@ final class CacheManager {
     return new Fallible<Long>(entryId);
   }
 
-  public synchronized List<Fallible<Long>> removeEntries(FSPermissionChecker pc,
-      List<Long> entryIds) {
+  public synchronized List<Fallible<Long>> removeEntries(List<Long> entryIds,
+      FSPermissionChecker pc) {
     ArrayList<Fallible<Long>> results = 
         new ArrayList<Fallible<Long>>(entryIds.size());
     for (Long entryId : entryIds) {
-      results.add(removeEntry(pc, entryId));
+      results.add(removeEntry(entryId, pc));
     }
     return results;
   }
 
-  public synchronized List<PathCacheEntry> listPathCacheEntries(
-      FSPermissionChecker pc, long prevId, Long poolId, int maxReplies) {
-    final int MAX_PRE_ALLOCATED_ENTRIES = 16;
-    ArrayList<PathCacheEntry> replies = new ArrayList<PathCacheEntry>(
-        Math.min(MAX_PRE_ALLOCATED_ENTRIES, maxReplies));
+  public synchronized BatchedListEntries<PathCacheEntry> 
+        listPathCacheEntries(long prevId, String filterPool, FSPermissionChecker pc) {
+    final int NUM_PRE_ALLOCATED_ENTRIES = 16;
+    ArrayList<PathCacheEntry> replies =
+        new ArrayList<PathCacheEntry>(NUM_PRE_ALLOCATED_ENTRIES);
     int numReplies = 0;
     SortedMap<Long, PathCacheEntry> tailMap = entriesById.tailMap(prevId + 1);
-    for (PathCacheEntry entry : tailMap.values()) {
-      if (numReplies >= maxReplies) {
-        return replies;
+    for (Entry<Long, PathCacheEntry> cur : tailMap.entrySet()) {
+      if (numReplies >= maxListCacheDirectivesResponses) {
+        return new BatchedListEntries<PathCacheEntry>(replies, true);
       }
-      long entryPoolId = entry.getDirective().getPoolId();
-      if (poolId == null || poolId <= 0 || entryPoolId == poolId) {
-        if (pc.checkPermission(
-            cachePoolsById.get(entryPoolId), FsAction.EXECUTE)) {
-          replies.add(entry);
-          numReplies++;
-        }
+      PathCacheEntry curEntry = cur.getValue();
+      if (!filterPool.isEmpty() && 
+          !cur.getValue().getDirective().getPool().equals(filterPool)) {
+        continue;
+      }
+      CachePool pool = cachePools.get(curEntry.getDirective().getPool());
+      if (pool == null) {
+        LOG.error("invalid pool for PathCacheEntry " + curEntry);
+        continue;
+      }
+      if (pc.checkPermission(pool, FsAction.EXECUTE)) {
+        replies.add(cur.getValue());
+        numReplies++;
       }
     }
-    return replies;
-  }
-
-  synchronized CachePool getCachePool(long id) {
-    return cachePoolsById.get(id);
+    return new BatchedListEntries<PathCacheEntry>(replies, false);
   }
 
   /**
@@ -246,24 +250,22 @@ final class CacheManager {
    *
    * @param info
    *          The info for the cache pool to create.
-   * @return created CachePool
    */
-  public synchronized CachePool addCachePool(CachePoolInfo info)
+  public synchronized void addCachePool(CachePoolInfo info)
       throws IOException {
     String poolName = info.getPoolName();
-    if (poolName == null || poolName.isEmpty()) {
+    if (poolName.isEmpty()) {
       throw new IOException("invalid empty cache pool name");
     }
-    if (cachePoolsByName.containsKey(poolName)) {
+    CachePool pool = cachePools.get(poolName);
+    if (pool != null) {
       throw new IOException("cache pool " + poolName + " already exists.");
     }
-    CachePool cachePool = new CachePool(getNextPoolId(), poolName,
+    CachePool cachePool = new CachePool(poolName,
       info.getOwnerName(), info.getGroupName(), info.getMode(),
       info.getWeight());
-    cachePoolsById.put(cachePool.getId(), cachePool);
-    cachePoolsByName.put(poolName, cachePool);
+    cachePools.put(poolName, cachePool);
     LOG.info("created new cache pool " + cachePool);
-    return cachePool;
   }
 
   /**
@@ -274,62 +276,46 @@ final class CacheManager {
    * @param info
    *          The info for the cache pool to modify.
    */
-  public synchronized void modifyCachePool(long poolId, CachePoolInfo info)
+  public synchronized void modifyCachePool(CachePoolInfo info)
       throws IOException {
-    if (poolId <= 0) {
-      throw new IOException("invalid pool id " + poolId);
+    String poolName = info.getPoolName();
+    if (poolName.isEmpty()) {
+      throw new IOException("invalid empty cache pool name");
     }
-    if (!cachePoolsById.containsKey(poolId)) {
-      throw new IOException("cache pool id " + poolId + " does not exist.");
+    CachePool pool = cachePools.get(poolName);
+    if (pool == null) {
+      throw new IOException("cache pool " + poolName + " does not exist.");
     }
-    CachePool pool = cachePoolsById.get(poolId);
-    // Remove the old CachePoolInfo
-    removeCachePool(poolId);
-    // Build up the new CachePoolInfo
-    CachePoolInfo.Builder newInfo = CachePoolInfo.newBuilder(pool.getInfo());
     StringBuilder bld = new StringBuilder();
     String prefix = "";
-    if (info.getPoolName() != null) {
-      newInfo.setPoolName(info.getPoolName());
-      bld.append(prefix).
-      append("set name to ").append(info.getOwnerName());
-      prefix = "; ";
-    }
     if (info.getOwnerName() != null) {
-      newInfo.setOwnerName(info.getOwnerName());
+      pool.setOwnerName(info.getOwnerName());
       bld.append(prefix).
         append("set owner to ").append(info.getOwnerName());
       prefix = "; ";
     }
     if (info.getGroupName() != null) {
-      newInfo.setGroupName(info.getGroupName());
+      pool.setGroupName(info.getGroupName());
       bld.append(prefix).
         append("set group to ").append(info.getGroupName());
       prefix = "; ";
     }
     if (info.getMode() != null) {
-      newInfo.setMode(info.getMode());
+      pool.setMode(info.getMode());
       bld.append(prefix).
-        append(String.format("set mode to ", info.getMode()));
+        append(String.format("set mode to 0%3o", info.getMode()));
       prefix = "; ";
     }
     if (info.getWeight() != null) {
-      newInfo.setWeight(info.getWeight());
+      pool.setWeight(info.getWeight());
       bld.append(prefix).
         append("set weight to ").append(info.getWeight());
       prefix = "; ";
     }
     if (prefix.isEmpty()) {
       bld.append("no changes.");
-    } else {
-      pool.setInfo(newInfo.build());
     }
-    // Put the newly modified info back in
-    cachePoolsById.put(poolId, pool);
-    cachePoolsByName.put(info.getPoolName(), pool);
-    LOG.info("modified pool id " + pool.getId()
-        + " (" + pool.getInfo().getPoolName() + "); "
-        + bld.toString());
+    LOG.info("modified " + poolName + "; " + bld.toString());
   }
 
   /**
@@ -337,39 +323,47 @@ final class CacheManager {
    * 
    * Only the superuser should be able to call this function.
    *
-   * @param poolId
-   *          The id of the cache pool to remove.
+   * @param poolName
+   *          The name for the cache pool to remove.
    */
-  public synchronized void removeCachePool(long poolId) throws IOException {
-    if (!cachePoolsById.containsKey(poolId)) {
-      throw new IOException("can't remove nonexistent cache pool id " + poolId);
+  public synchronized void removeCachePool(String poolName)
+      throws IOException {
+    CachePool pool = cachePools.remove(poolName);
+    if (pool == null) {
+      throw new IOException("can't remove nonexistent cache pool " + poolName);
     }
-    // Remove all the entries associated with the pool
-    Iterator<Map.Entry<Long, PathCacheEntry>> it =
-        entriesById.entrySet().iterator();
-    while (it.hasNext()) {
-      Map.Entry<Long, PathCacheEntry> entry = it.next();
-      if (entry.getValue().getDirective().getPoolId() == poolId) {
-        it.remove();
-        entriesByDirective.remove(entry.getValue().getDirective());
+    
+    // Remove entries using this pool
+    // TODO: could optimize this somewhat to avoid the need to iterate
+    // over all entries in entriesByDirective
+    Iterator<Entry<PathCacheDirective, PathCacheEntry>> iter = 
+        entriesByDirective.entrySet().iterator();
+    while (iter.hasNext()) {
+      Entry<PathCacheDirective, PathCacheEntry> entry = iter.next();
+      if (entry.getKey().getPool().equals(poolName)) {
+        entriesById.remove(entry.getValue().getEntryId());
+        iter.remove();
       }
     }
-    // Remove the pool
-    CachePool pool = cachePoolsById.remove(poolId);
-    cachePoolsByName.remove(pool.getInfo().getPoolName());
   }
 
-  public synchronized List<CachePool> listCachePools(Long prevKey,
-      int maxRepliesPerRequest) {
-    final int MAX_PREALLOCATED_REPLIES = 16;
-    ArrayList<CachePool> results = 
-        new ArrayList<CachePool>(Math.min(MAX_PREALLOCATED_REPLIES,
-            maxRepliesPerRequest));
-    SortedMap<Long, CachePool> tailMap =
-        cachePoolsById.tailMap(prevKey, false);
-    for (CachePool pool : tailMap.values()) {
-      results.add(pool);
+  public synchronized BatchedListEntries<CachePoolInfo>
+      listCachePools(FSPermissionChecker pc, String prevKey) {
+    final int NUM_PRE_ALLOCATED_ENTRIES = 16;
+    ArrayList<CachePoolInfo> results = 
+        new ArrayList<CachePoolInfo>(NUM_PRE_ALLOCATED_ENTRIES);
+    SortedMap<String, CachePool> tailMap = cachePools.tailMap(prevKey, false);
+    int numListed = 0;
+    for (Entry<String, CachePool> cur : tailMap.entrySet()) {
+      if (numListed++ >= maxListCachePoolsResponses) {
+        return new BatchedListEntries<CachePoolInfo>(results, true);
+      }
+      if (pc == null) {
+        results.add(cur.getValue().getInfo(true));
+      } else {
+        results.add(cur.getValue().getInfo(pc));
+      }
     }
-    return results;
+    return new BatchedListEntries<CachePoolInfo>(results, false);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java

@@ -19,119 +19,137 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
 
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import javax.annotation.Nonnull;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo.Builder;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 
 /**
  * A CachePool describes a set of cache resources being managed by the NameNode.
  * User caching requests are billed to the cache pool specified in the request.
  *
- * CachePools are uniquely identified by a numeric id as well as the
- * {@link CachePoolInfo} pool name. Mutable metadata is contained in
- * CachePoolInfo, including pool name, owner, group, and permissions.
- * See this class for more details.
+ * This is an internal class, only used on the NameNode.  For identifying or
+ * describing a cache pool to clients, please use CachePoolInfo.
  */
+@InterfaceAudience.Private
 public final class CachePool {
   public static final Log LOG = LogFactory.getLog(CachePool.class);
 
-  private final long id;
-
-  private CachePoolInfo info;
+  @Nonnull
+  private final String poolName;
 
-  public CachePool(long id) {
-    this.id = id;
-    this.info = null;
-  }
+  @Nonnull
+  private String ownerName;
 
-  CachePool(long id, String poolName, String ownerName, String groupName,
+  @Nonnull
+  private String groupName;
+  
+  @Nonnull
+  private FsPermission mode;
+  
+  private int weight;
+  
+  public CachePool(String poolName, String ownerName, String groupName,
       FsPermission mode, Integer weight) throws IOException {
-    this.id = id;
-    // Set CachePoolInfo default fields if null
-    if (poolName == null || poolName.isEmpty()) {
-      throw new IOException("invalid empty cache pool name");
-    }
+    this.poolName = poolName;
     UserGroupInformation ugi = null;
     if (ownerName == null) {
-      ugi = NameNode.getRemoteUser();
-      ownerName = ugi.getShortUserName();
+      if (ugi == null) {
+        ugi = NameNode.getRemoteUser();
+      }
+      this.ownerName = ugi.getShortUserName();
+    } else {
+      this.ownerName = ownerName;
     }
     if (groupName == null) {
       if (ugi == null) {
         ugi = NameNode.getRemoteUser();
       }
-      String[] groups = ugi.getGroupNames();
-      if (groups.length == 0) {
-        throw new IOException("failed to get group names from UGI " + ugi);
-      }
-      groupName = groups[0];
-    }
-    if (mode == null) {
-      mode = FsPermission.getDirDefault();
-    }
-    if (weight == null) {
-      weight = 100;
+      this.groupName = ugi.getPrimaryGroupName();
+    } else {
+      this.groupName = ownerName;
     }
-    CachePoolInfo.Builder builder = CachePoolInfo.newBuilder();
-    builder.setPoolName(poolName).setOwnerName(ownerName)
-        .setGroupName(groupName).setMode(mode).setWeight(weight);
-    this.info = builder.build();
+    this.mode = mode != null ? 
+        new FsPermission(mode): FsPermission.getCachePoolDefault();
+    this.weight = weight != null ? weight : 100;
   }
 
-  public CachePool(long id, CachePoolInfo info) {
-    this.id = id;
-    this.info = info;
+  public String getName() {
+    return poolName;
   }
 
-  /**
-   * @return id of the pool
-   */
-  public long getId() {
-    return id;
+  public String getOwnerName() {
+    return ownerName;
+  }
+
+  public CachePool setOwnerName(String ownerName) {
+    this.ownerName = ownerName;
+    return this;
   }
 
+  public String getGroupName() {
+    return groupName;
+  }
+
+  public CachePool setGroupName(String groupName) {
+    this.groupName = groupName;
+    return this;
+  }
+
+  public FsPermission getMode() {
+    return mode;
+  }
+
+  public CachePool setMode(FsPermission mode) {
+    this.mode = new FsPermission(mode);
+    return this;
+  }
+  
+  public int getWeight() {
+    return weight;
+  }
+
+  public CachePool setWeight(int weight) {
+    this.weight = weight;
+    return this;
+  }
+  
   /**
    * Get information about this cache pool.
    *
+   * @param fullInfo
+   *          If true, only the name will be returned (i.e., what you 
+   *          would get if you didn't have read permission for this pool.)
    * @return
    *          Cache pool information.
    */
-  public CachePoolInfo getInfo() {
-    return info;
+  public CachePoolInfo getInfo(boolean fullInfo) {
+    CachePoolInfo info = new CachePoolInfo(poolName);
+    if (!fullInfo) {
+      return info;
+    }
+    return info.setOwnerName(ownerName).
+        setGroupName(groupName).
+        setMode(new FsPermission(mode)).
+        setWeight(weight);
   }
 
-  void setInfo(CachePoolInfo info) {
-    this.info = info;
+  public CachePoolInfo getInfo(FSPermissionChecker pc) {
+    return getInfo(pc.checkPermission(this, FsAction.READ)); 
   }
 
   public String toString() {
     return new StringBuilder().
-        append("{ ").append("id:").append(id).
-        append(", info:").append(info.toString()).
+        append("{ ").append("poolName:").append(poolName).
+        append(", ownerName:").append(ownerName).
+        append(", groupName:").append(groupName).
+        append(", mode:").append(mode).
+        append(", weight:").append(weight).
         append(" }").toString();
   }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder().append(id).append(info).hashCode();
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == null) { return false; }
-    if (obj == this) { return true; }
-    if (obj.getClass() != getClass()) {
-      return false;
-    }
-    CachePool rhs = (CachePool)obj;
-    return new EqualsBuilder()
-      .append(id, rhs.id)
-      .append(info, rhs.info)
-      .isEquals();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -119,6 +119,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -6756,9 +6757,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
     if (retryCacheEntry != null && retryCacheEntry.isSuccess()) {
       return (List<Fallible<PathCacheEntry>>) retryCacheEntry.getPayload();
     }
-    final FSPermissionChecker pc = getPermissionChecker();
+    final FSPermissionChecker pc = isPermissionEnabled ?
+        getPermissionChecker() : null;
     boolean success = false;
     List<Fallible<PathCacheEntry>> results = null;
+    checkOperation(OperationCategory.WRITE);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
@@ -6766,7 +6769,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
         throw new SafeModeException(
             "Cannot add path cache directive", safeMode);
       }
-      results = cacheManager.addDirectives(pc, directives);
+      results = cacheManager.addDirectives(directives, pc);
       //getEditLog().logAddPathCacheDirectives(results); FIXME: HDFS-5119
       success = true;
     } finally {
@@ -6774,7 +6777,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       if (success) {
         getEditLog().logSync();
       }
-      if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+      if (isAuditEnabled() && isExternalInvocation()) {
         logAuditEvent(success, "addPathCacheDirectives", null, null, null);
       }
       RetryCache.setState(retryCacheEntry, success, results);
@@ -6783,147 +6786,175 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   }
 
   @SuppressWarnings("unchecked")
-  List<Fallible<Long>> removePathCacheEntries(List<Long> ids)
-      throws IOException {
-    final FSPermissionChecker pc = getPermissionChecker();
+  List<Fallible<Long>> removePathCacheEntries(List<Long> ids) throws IOException {
+    CacheEntryWithPayload retryCacheEntry =
+        RetryCache.waitForCompletion(retryCache, null);
+    if (retryCacheEntry != null && retryCacheEntry.isSuccess()) {
+      return (List<Fallible<Long>>) retryCacheEntry.getPayload();
+    }
+    final FSPermissionChecker pc = isPermissionEnabled ?
+        getPermissionChecker() : null;
     boolean success = false;
     List<Fallible<Long>> results = null;
+    checkOperation(OperationCategory.WRITE);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       if (isInSafeMode()) {
         throw new SafeModeException(
-            "Cannot add path cache directive", safeMode);
+            "Cannot remove path cache directives", safeMode);
       }
-      results = cacheManager.removeEntries(pc, ids);
+      results = cacheManager.removeEntries(ids, pc);
       //getEditLog().logRemovePathCacheEntries(results); FIXME: HDFS-5119
       success = true;
     } finally {
       writeUnlock();
-      if (success) {
-        getEditLog().logSync();
-      }
-      if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+      if (isAuditEnabled() && isExternalInvocation()) {
         logAuditEvent(success, "removePathCacheEntries", null, null, null);
       }
+      RetryCache.setState(retryCacheEntry, success, results);
     }
+    getEditLog().logSync();
     return results;
   }
 
-  List<PathCacheEntry> listPathCacheEntries(long startId,
-      Long poolId, int maxReplies) throws IOException {
-    LOG.info("listPathCacheEntries with " + startId + " " + poolId);
-    final FSPermissionChecker pc = getPermissionChecker();
-    return cacheManager.listPathCacheEntries(pc, startId, poolId, maxReplies);
+  BatchedListEntries<PathCacheEntry> listPathCacheEntries(long startId,
+      String pool) throws IOException {
+    final FSPermissionChecker pc = isPermissionEnabled ?
+        getPermissionChecker() : null;
+    BatchedListEntries<PathCacheEntry> results;
+    checkOperation(OperationCategory.READ);
+    readLock();
+    boolean success = false;
+    try {
+      checkOperation(OperationCategory.READ);
+      results = cacheManager.listPathCacheEntries(startId, pool, pc);
+      success = true;
+    } finally {
+      readUnlock();
+      if (isAuditEnabled() && isExternalInvocation()) {
+        logAuditEvent(success, "listPathCacheEntries", null, null, null);
+      }
+    }
+    return results;
   }
 
-  public CachePool addCachePool(CachePoolInfo req) throws IOException {
-    final FSPermissionChecker pc = getPermissionChecker();
-    CacheEntryWithPayload cacheEntry =
-        RetryCache.waitForCompletion(retryCache, null);
+  public void addCachePool(CachePoolInfo req) throws IOException {
+    final FSPermissionChecker pc = isPermissionEnabled ?
+        getPermissionChecker() : null;
+    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
-      return (CachePool)cacheEntry.getPayload(); // Return previous response
+      return; // Return previous response
     }
+    checkOperation(OperationCategory.WRITE);
     writeLock();
-    CachePool pool = null;
+    boolean success = false;
     try {
       checkOperation(OperationCategory.WRITE);
-      if (!pc.isSuperUser()) {
-        throw new AccessControlException("Non-super users cannot " +
-            "add cache pools.");
-      }
       if (isInSafeMode()) {
         throw new SafeModeException(
             "Cannot add cache pool " + req.getPoolName(), safeMode);
       }
-      pool = cacheManager.addCachePool(req);
-      RetryCache.setState(cacheEntry, true);
+      if (pc != null) {
+        pc.checkSuperuserPrivilege();
+      }
+      cacheManager.addCachePool(req);
       //getEditLog().logAddCachePool(req); // FIXME: HDFS-5119
+      success = true;
     } finally {
       writeUnlock();
+      if (isAuditEnabled() && isExternalInvocation()) {
+        logAuditEvent(success, "addCachePool", req.getPoolName(), null, null);
+      }
+      RetryCache.setState(cacheEntry, success);
     }
-
+    
     getEditLog().logSync();
-    if (auditLog.isInfoEnabled() && isExternalInvocation()) {
-      logAuditEvent(true, "addCachePool", req.getPoolName(), null, null);
-    }
-    return pool;
   }
 
-  public void modifyCachePool(long poolId, CachePoolInfo info)
-      throws IOException {
-    final FSPermissionChecker pc = getPermissionChecker();
+  public void modifyCachePool(CachePoolInfo req) throws IOException {
+    final FSPermissionChecker pc =
+        isPermissionEnabled ? getPermissionChecker() : null;
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
+    checkOperation(OperationCategory.WRITE);
     writeLock();
+    boolean success = false;
     try {
       checkOperation(OperationCategory.WRITE);
-      if (!pc.isSuperUser()) {
-        throw new AccessControlException("Non-super users cannot " +
-            "modify cache pools.");
-      }
       if (isInSafeMode()) {
         throw new SafeModeException(
-            "Cannot modify cache pool " + info.getPoolName(), safeMode);
+            "Cannot modify cache pool " + req.getPoolName(), safeMode);
+      }
+      if (pc != null) {
+        pc.checkSuperuserPrivilege();
       }
-      cacheManager.modifyCachePool(poolId, info);
-      RetryCache.setState(cacheEntry, true);
+      cacheManager.modifyCachePool(req);
       //getEditLog().logModifyCachePool(req); // FIXME: HDFS-5119
+      success = true;
     } finally {
       writeUnlock();
+      if (isAuditEnabled() && isExternalInvocation()) {
+        logAuditEvent(success, "modifyCachePool", req.getPoolName(), null, null);
+      }
+      RetryCache.setState(cacheEntry, success);
     }
 
     getEditLog().logSync();
-    if (auditLog.isInfoEnabled() && isExternalInvocation()) {
-      logAuditEvent(true, "modifyCachePool", info.getPoolName(), null, null);
-    }
   }
 
-  public void removeCachePool(long poolId) throws IOException {
-    final FSPermissionChecker pc = getPermissionChecker();
+  public void removeCachePool(String cachePoolName) throws IOException {
+    final FSPermissionChecker pc =
+        isPermissionEnabled ? getPermissionChecker() : null;
+    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    if (cacheEntry != null && cacheEntry.isSuccess()) {
+      return; // Return previous response
+    }
+    checkOperation(OperationCategory.WRITE);
     writeLock();
-    CachePool pool;
+    boolean success = false;
     try {
       checkOperation(OperationCategory.WRITE);
-      if (!pc.isSuperUser()) {
-        throw new AccessControlException("Non-super users cannot " +
-            "remove cache pools.");
-      }
-      pool = cacheManager.getCachePool(poolId);
       if (isInSafeMode()) {
-        String identifier;
-        if (pool == null) {
-          identifier = "with id " + Long.toString(poolId);
-        } else {
-          identifier = pool.getInfo().getPoolName();
-        }
         throw new SafeModeException(
-            "Cannot remove cache pool " + identifier, safeMode);
+            "Cannot remove cache pool " + cachePoolName, safeMode);
       }
-      cacheManager.removeCachePool(poolId);
+      if (pc != null) {
+        pc.checkSuperuserPrivilege();
+      }
+      cacheManager.removeCachePool(cachePoolName);
       //getEditLog().logRemoveCachePool(req); // FIXME: HDFS-5119
+      success = true;
     } finally {
       writeUnlock();
+      if (isAuditEnabled() && isExternalInvocation()) {
+        logAuditEvent(success, "removeCachePool", cachePoolName, null, null);
+      }
+      RetryCache.setState(cacheEntry, success);
     }
-
+    
     getEditLog().logSync();
-    if (auditLog.isInfoEnabled() && isExternalInvocation()) {
-      logAuditEvent(true, "removeCachePool", pool.getInfo().getPoolName(),
-          null, null);
-    }
   }
 
-  public List<CachePool> listCachePools(long prevKey,
-      int maxRepliesPerRequest) throws IOException {
-    List<CachePool> results;
+  public BatchedListEntries<CachePoolInfo> listCachePools(String prevKey)
+      throws IOException {
+    final FSPermissionChecker pc =
+        isPermissionEnabled ? getPermissionChecker() : null;
+    BatchedListEntries<CachePoolInfo> results;
+    checkOperation(OperationCategory.READ);
+    boolean success = false;
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      results = cacheManager.listCachePools(prevKey, maxRepliesPerRequest);
+      results = cacheManager.listCachePools(pc, prevKey);
+      success = true;
     } finally {
       readUnlock();
+      if (isAuditEnabled() && isExternalInvocation()) {
+        logAuditEvent(success, "listCachePools", null, null, null);
+      }
     }
     return results;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -28,7 +29,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -264,16 +264,15 @@ class FSPermissionChecker {
    * @return if the pool can be accessed
    */
   public boolean checkPermission(CachePool pool, FsAction access) {
-    CachePoolInfo info = pool.getInfo();
-    FsPermission mode = info.getMode();
+    FsPermission mode = pool.getMode();
     if (isSuperUser()) {
       return true;
     }
-    if (user.equals(info.getOwnerName())
+    if (user.equals(pool.getOwnerName())
         && mode.getUserAction().implies(access)) {
       return true;
     }
-    if (groups.contains(info.getGroupName())
+    if (groups.contains(pool.getGroupName())
         && mode.getGroupAction().implies(access)) {
       return true;
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java

@@ -31,11 +31,13 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
+import java.util.NoSuchElementException;
 
 import org.apache.commons.logging.Log;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BatchedRemoteIterator;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
@@ -60,9 +62,9 @@ import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HDFSPolicyProvider;
 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.PathCacheDirective;
 import org.apache.hadoop.hdfs.protocol.PathCacheEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -1223,20 +1225,17 @@ class NameNodeRpcServer implements NamenodeProtocols {
   private class ServerSidePathCacheEntriesIterator
       extends BatchedRemoteIterator<Long, PathCacheEntry> {
 
-    private final Long poolId;
+    private final String pool;
 
-    public ServerSidePathCacheEntriesIterator(Long firstKey,
-        int maxRepliesPerRequest, Long poolId) {
-      super(firstKey, maxRepliesPerRequest);
-      this.poolId = poolId;
+    public ServerSidePathCacheEntriesIterator(Long firstKey, String pool) {
+      super(firstKey);
+      this.pool = pool;
     }
 
     @Override
     public BatchedEntries<PathCacheEntry> makeRequest(
-        Long prevKey, int maxRepliesPerRequest) throws IOException {
-      return new BatchedListEntries<PathCacheEntry>(
-          namesystem.listPathCacheEntries(prevKey, poolId,
-              maxRepliesPerRequest));
+        Long nextKey) throws IOException {
+      return namesystem.listPathCacheEntries(nextKey, pool);
     }
 
     @Override
@@ -1244,52 +1243,50 @@ class NameNodeRpcServer implements NamenodeProtocols {
       return entry.getEntryId();
     }
   }
-
+  
   @Override
   public RemoteIterator<PathCacheEntry> listPathCacheEntries(long prevId,
-      long poolId, int maxReplies) throws IOException {
-    return new ServerSidePathCacheEntriesIterator(prevId, maxReplies, poolId);
+      String pool) throws IOException {
+    return new ServerSidePathCacheEntriesIterator(prevId, pool);
   }
 
   @Override
-  public CachePool addCachePool(CachePoolInfo info) throws IOException {
-    return namesystem.addCachePool(info);
+  public void addCachePool(CachePoolInfo info) throws IOException {
+    namesystem.addCachePool(info);
   }
 
   @Override
-  public void modifyCachePool(long poolId, CachePoolInfo info)
-      throws IOException {
-    namesystem.modifyCachePool(poolId, info);
+  public void modifyCachePool(CachePoolInfo info) throws IOException {
+    namesystem.modifyCachePool(info);
   }
 
   @Override
-  public void removeCachePool(long poolId) throws IOException {
-    namesystem.removeCachePool(poolId);
+  public void removeCachePool(String cachePoolName) throws IOException {
+    namesystem.removeCachePool(cachePoolName);
   }
 
   private class ServerSideCachePoolIterator 
-      extends BatchedRemoteIterator<Long, CachePool> {
+      extends BatchedRemoteIterator<String, CachePoolInfo> {
 
-    public ServerSideCachePoolIterator(long prevId, int maxRepliesPerRequest) {
-      super(prevId, maxRepliesPerRequest);
+    public ServerSideCachePoolIterator(String prevKey) {
+      super(prevKey);
     }
 
     @Override
-    public BatchedEntries<CachePool> makeRequest(Long prevId,
-        int maxRepliesPerRequest) throws IOException {
-      return new BatchedListEntries<CachePool>(
-          namesystem.listCachePools(prevId, maxRepliesPerRequest));
+    public BatchedEntries<CachePoolInfo> makeRequest(String prevKey)
+        throws IOException {
+      return namesystem.listCachePools(prevKey);
     }
 
     @Override
-    public Long elementToPrevKey(CachePool element) {
-      return element.getId();
+    public String elementToPrevKey(CachePoolInfo element) {
+      return element.getPoolName();
     }
   }
 
   @Override
-  public RemoteIterator<CachePool> listCachePools(long prevPoolId,
-      int maxRepliesPerRequest) throws IOException {
-    return new ServerSideCachePoolIterator(prevPoolId, maxRepliesPerRequest);
+  public RemoteIterator<CachePoolInfo> listCachePools(String prevKey)
+      throws IOException {
+    return new ServerSideCachePoolIterator(prevKey);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto

@@ -363,27 +363,9 @@ message IsFileClosedResponseProto {
   required bool result = 1;
 }
 
-message CachePoolInfoProto {
-  optional string poolName = 1;
-  optional string ownerName = 2;
-  optional string groupName = 3;
-  optional int32 mode = 4;
-  optional int32 weight = 5;
-}
-
-message CachePoolProto {
-  optional int64 id = 1;
-  optional CachePoolInfoProto info = 2;
-}
-
 message PathCacheDirectiveProto {
   required string path = 1;
-  required CachePoolProto pool = 2;
-}
-
-message PathCacheEntryProto {
-  required int64 id = 1;
-  optional PathCacheDirectiveProto directive = 2;
+  required string pool = 2;
 }
 
 message AddPathCacheDirectivesRequestProto {
@@ -417,42 +399,52 @@ enum RemovePathCacheEntryErrorProto {
 }
 
 message ListPathCacheEntriesRequestProto {
-  required PathCacheEntryProto prevEntry = 1;
-  required CachePoolProto pool = 2;
-  optional int32 maxReplies = 3;
+  required int64 prevId = 1;
+  required string pool = 2;
+}
+
+message ListPathCacheEntriesElementProto {
+  required int64 id = 1;
+  required string path = 2;
+  required string pool = 3;
 }
 
 message ListPathCacheEntriesResponseProto {
-  repeated PathCacheEntryProto entries = 1;
+  repeated ListPathCacheEntriesElementProto elements = 1;
   required bool hasMore = 2;
 }
 
 message AddCachePoolRequestProto {
-  required CachePoolInfoProto info = 1;
+  required string poolName = 1;
+  optional string ownerName = 2;
+  optional string groupName = 3;
+  optional int32 mode = 4;
+  optional int32 weight = 5;
 }
 
-message AddCachePoolResponseProto {
-  required CachePoolProto pool = 1;
+message AddCachePoolResponseProto { // void response
 }
 
 message ModifyCachePoolRequestProto {
-  required CachePoolProto pool = 1;
-  required CachePoolInfoProto info = 2;
+  required string poolName = 1;
+  optional string ownerName = 2;
+  optional string groupName = 3;
+  optional int32 mode = 4;
+  optional int32 weight = 5;
 }
 
 message ModifyCachePoolResponseProto { // void response
 }
 
 message RemoveCachePoolRequestProto {
-  required CachePoolProto pool = 1;
+  required string poolName = 1;
 }
 
 message RemoveCachePoolResponseProto { // void response
 }
 
 message ListCachePoolsRequestProto {
-  required CachePoolProto prevPool = 1;
-  required int32 maxReplies = 2;
+  required string prevPoolName = 1;
 }
 
 message ListCachePoolsResponseProto {
@@ -461,7 +453,11 @@ message ListCachePoolsResponseProto {
 }
 
 message ListCachePoolsResponseElementProto {
-  required CachePoolProto pool = 1;
+  required string poolName = 1;
+  required string ownerName = 2;
+  required string groupName = 3;
+  required int32 mode = 4;
+  required int32 weight = 5;
 }
 
 message GetFileLinkInfoRequestProto {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPathCacheRequests.java

@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
@@ -31,64 +31,58 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.EmptyPathError;
+import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolNameError;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPathNameError;
-import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.InvalidPoolError;
 import org.apache.hadoop.hdfs.protocol.AddPathCacheDirectiveException.PoolWritePermissionDeniedError;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.InvalidIdException;
 import org.apache.hadoop.hdfs.protocol.PathCacheDirective;
 import org.apache.hadoop.hdfs.protocol.PathCacheEntry;
-import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.InvalidIdException;
 import org.apache.hadoop.hdfs.protocol.RemovePathCacheEntryException.NoSuchIdException;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Fallible;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
 public class TestPathCacheRequests {
   static final Log LOG = LogFactory.getLog(TestPathCacheRequests.class);
 
-  private static Configuration conf = new HdfsConfiguration();
-  private static MiniDFSCluster cluster = null;
-  private static NamenodeProtocols proto = null;
-
-  @Before
-  public void setUp() throws Exception {
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-    cluster.waitActive();
-    proto = cluster.getNameNodeRpc();
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    if (cluster != null) {
-      cluster.shutdown();
-    }
-  }
+  private static final UserGroupInformation unprivilegedUser =
+      UserGroupInformation.createRemoteUser("unprivilegedUser");
 
   @Test
   public void testCreateAndRemovePools() throws Exception {
-    CachePoolInfo req =
-        CachePoolInfo.newBuilder().setPoolName("pool1").setOwnerName("bob")
-            .setGroupName("bobgroup").setMode(new FsPermission((short) 0755))
-            .setWeight(150).build();
-    CachePool pool = proto.addCachePool(req);
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = null;
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
+    NamenodeProtocols proto = cluster.getNameNodeRpc();
+    CachePoolInfo req = new CachePoolInfo("pool1").
+        setOwnerName("bob").setGroupName("bobgroup").
+        setMode(new FsPermission((short)0755)).setWeight(150);
+    proto.addCachePool(req);
     try {
-      proto.removeCachePool(909);
+      proto.removeCachePool("pool99");
       Assert.fail("expected to get an exception when " +
           "removing a non-existent pool.");
     } catch (IOException ioe) {
+      GenericTestUtils.assertExceptionContains("can't remove " +
+          "nonexistent cache pool", ioe);
     }
-    proto.removeCachePool(pool.getId());
+    proto.removeCachePool("pool1");
     try {
-      proto.removeCachePool(pool.getId());
+      proto.removeCachePool("pool1");
       Assert.fail("expected to get an exception when " +
           "removing a non-existent pool.");
     } catch (IOException ioe) {
+      GenericTestUtils.assertExceptionContains("can't remove " +
+          "nonexistent cache pool", ioe);
     }
     req = new CachePoolInfo("pool2");
     proto.addCachePool(req);
@@ -96,42 +90,36 @@ public class TestPathCacheRequests {
 
   @Test
   public void testCreateAndModifyPools() throws Exception {
-    // Create a new pool
-    CachePoolInfo info = CachePoolInfo.newBuilder().
-        setPoolName("pool1").
-        setOwnerName("abc").
-        setGroupName("123").
-        setMode(new FsPermission((short)0755)).
-        setWeight(150).
-        build();
-    CachePool pool = proto.addCachePool(info);
-    CachePoolInfo actualInfo = pool.getInfo();
-    assertEquals("Expected info to match create time settings",
-        info, actualInfo);
-    // Modify the pool
-    info = CachePoolInfo.newBuilder().
-        setPoolName("pool2").
-        setOwnerName("def").
-        setGroupName("456").
-        setMode(new FsPermission((short)0644)).
-        setWeight(200).
-        build();
-    proto.modifyCachePool(pool.getId(), info);
-    // Check via listing this time
-    RemoteIterator<CachePool> iter = proto.listCachePools(0, 1);
-    CachePool listedPool = iter.next();
-    actualInfo = listedPool.getInfo();
-    assertEquals("Expected info to match modified settings", info, actualInfo);
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = null;
+    // set low limits here for testing purposes
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES, 2);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES, 2);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
+    NamenodeProtocols proto = cluster.getNameNodeRpc();
+    proto.addCachePool(new CachePoolInfo("pool1").
+        setOwnerName("abc").setGroupName("123").
+        setMode(new FsPermission((short)0755)).setWeight(150));
+    proto.modifyCachePool(new CachePoolInfo("pool1").
+        setOwnerName("def").setGroupName("456"));
+    RemoteIterator<CachePoolInfo> iter = proto.listCachePools("");
+    CachePoolInfo info = iter.next();
+    assertEquals("pool1", info.getPoolName());
+    assertEquals("def", info.getOwnerName());
+    assertEquals("456", info.getGroupName());
+    assertEquals(new FsPermission((short)0755), info.getMode());
+    assertEquals(Integer.valueOf(150), info.getWeight());
 
     try {
-      proto.removeCachePool(808);
+      proto.removeCachePool("pool99");
       Assert.fail("expected to get an exception when " +
           "removing a non-existent pool.");
     } catch (IOException ioe) {
     }
-    proto.removeCachePool(pool.getId());
+    proto.removeCachePool("pool1");
     try {
-      proto.removeCachePool(pool.getId());
+      proto.removeCachePool("pool1");
       Assert.fail("expected to get an exception when " +
           "removing a non-existent pool.");
     } catch (IOException ioe) {
@@ -142,13 +130,13 @@ public class TestPathCacheRequests {
       RemoteIterator<PathCacheEntry> iter,
       long id0, long id1, long id2) throws Exception {
     Assert.assertEquals(new PathCacheEntry(id0,
-        new PathCacheDirective("/alpha", 1)),
+        new PathCacheDirective("/alpha", "pool1")),
         iter.next());
     Assert.assertEquals(new PathCacheEntry(id1,
-        new PathCacheDirective("/beta", 2)),
+        new PathCacheDirective("/beta", "pool2")),
         iter.next());
     Assert.assertEquals(new PathCacheEntry(id2,
-        new PathCacheDirective("/gamma", 1)),
+        new PathCacheDirective("/gamma", "pool1")),
         iter.next());
     Assert.assertFalse(iter.hasNext());
   }
@@ -161,36 +149,34 @@ public class TestPathCacheRequests {
     try {
       cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
       cluster.waitActive();
-      final CachePool pool1 = proto.addCachePool(new CachePoolInfo("pool1"));
-      final CachePool pool2 = proto.addCachePool(new CachePoolInfo("pool2"));
-      final CachePool pool3 = proto.addCachePool(new CachePoolInfo("pool3"));
-      final CachePool pool4 = proto.addCachePool(CachePoolInfo.newBuilder()
-          .setPoolName("pool4")
-          .setMode(new FsPermission((short)0)).build());
-      UserGroupInformation testUgi = UserGroupInformation
-          .createUserForTesting("myuser", new String[]{"mygroup"});
-      List<Fallible<PathCacheEntry>> addResults1 = testUgi.doAs(
-          new PrivilegedExceptionAction<List<Fallible<PathCacheEntry>>>() {
-            @Override
-            public List<Fallible<PathCacheEntry>> run() throws IOException {
-              List<Fallible<PathCacheEntry>> entries;
-              entries = proto.addPathCacheDirectives(
-                  Arrays.asList(new PathCacheDirective[] {
-                      new PathCacheDirective("/alpha", pool1.getId()),
-                      new PathCacheDirective("/beta", pool2.getId()),
-                      new PathCacheDirective("", pool3.getId()),
-                      new PathCacheDirective("/zeta", 404),
-                      new PathCacheDirective("/zeta", pool4.getId())
-                  }));
-              return entries;
+      final NamenodeProtocols proto = cluster.getNameNodeRpc();
+      proto.addCachePool(new CachePoolInfo("pool1").
+          setMode(new FsPermission((short)0777)));
+      proto.addCachePool(new CachePoolInfo("pool2").
+          setMode(new FsPermission((short)0777)));
+      proto.addCachePool(new CachePoolInfo("pool3").
+          setMode(new FsPermission((short)0777)));
+      proto.addCachePool(new CachePoolInfo("pool4").
+          setMode(new FsPermission((short)0)));
+
+      List<Fallible<PathCacheEntry>> addResults1 = 
+        unprivilegedUser.doAs(new PrivilegedExceptionAction<
+            List<Fallible<PathCacheEntry>>>() {
+          @Override
+          public List<Fallible<PathCacheEntry>> run() throws IOException {
+            return proto.addPathCacheDirectives(Arrays.asList(
+              new PathCacheDirective[] {
+                new PathCacheDirective("/alpha", "pool1"),
+                new PathCacheDirective("/beta", "pool2"),
+                new PathCacheDirective("", "pool3"),
+                new PathCacheDirective("/zeta", "nonexistent_pool"),
+                new PathCacheDirective("/zeta", "pool4")
+              }));
             }
-      });
-      // Save the successful additions
+          });
       long ids1[] = new long[2];
-      for (int i=0; i<2; i++) {
-        ids1[i] = addResults1.get(i).get().getEntryId();
-      }
-      // Verify that the unsuccessful additions failed properly
+      ids1[0] = addResults1.get(0).get().getEntryId();
+      ids1[1] = addResults1.get(1).get().getEntryId();
       try {
         addResults1.get(2).get();
         Assert.fail("expected an error when adding an empty path");
@@ -201,7 +187,7 @@ public class TestPathCacheRequests {
         addResults1.get(3).get();
         Assert.fail("expected an error when adding to a nonexistent pool.");
       } catch (IOException ioe) {
-        Assert.assertTrue(ioe.getCause() instanceof InvalidPoolError);
+        Assert.assertTrue(ioe.getCause() instanceof InvalidPoolNameError);
       }
       try {
         addResults1.get(4).get();
@@ -215,10 +201,10 @@ public class TestPathCacheRequests {
       List<Fallible<PathCacheEntry>> addResults2 = 
           proto.addPathCacheDirectives(Arrays.asList(
             new PathCacheDirective[] {
-        new PathCacheDirective("/alpha", pool1.getId()),
-        new PathCacheDirective("/theta", 404),
-        new PathCacheDirective("bogus", pool1.getId()),
-        new PathCacheDirective("/gamma", pool1.getId())
+        new PathCacheDirective("/alpha", "pool1"),
+        new PathCacheDirective("/theta", ""),
+        new PathCacheDirective("bogus", "pool1"),
+        new PathCacheDirective("/gamma", "pool1")
       }));
       long id = addResults2.get(0).get().getEntryId();
       Assert.assertEquals("expected to get back the same ID as last time " +
@@ -228,7 +214,7 @@ public class TestPathCacheRequests {
         Assert.fail("expected an error when adding a path cache " +
             "directive with an empty pool name.");
       } catch (IOException ioe) {
-        Assert.assertTrue(ioe.getCause() instanceof InvalidPoolError);
+        Assert.assertTrue(ioe.getCause() instanceof InvalidPoolNameError);
       }
       try {
         addResults2.get(2).get();
@@ -240,16 +226,14 @@ public class TestPathCacheRequests {
       long ids2[] = new long[1];
       ids2[0] = addResults2.get(3).get().getEntryId();
 
-      // Validate listing all entries
       RemoteIterator<PathCacheEntry> iter =
-          proto.listPathCacheEntries(-1l, -1l, 100);
+          proto.listPathCacheEntries(0, "");
       validateListAll(iter, ids1[0], ids1[1], ids2[0]);
-      iter = proto.listPathCacheEntries(-1l, -1l, 1);
+      iter = proto.listPathCacheEntries(0, "");
       validateListAll(iter, ids1[0], ids1[1], ids2[0]);
-      // Validate listing certain pools
-      iter = proto.listPathCacheEntries(0, pool3.getId(), 1);
+      iter = proto.listPathCacheEntries(0, "pool3");
       Assert.assertFalse(iter.hasNext());
-      iter = proto.listPathCacheEntries(0, pool2.getId(), 4444);
+      iter = proto.listPathCacheEntries(0, "pool2");
       Assert.assertEquals(addResults1.get(1).get(),
           iter.next());
       Assert.assertFalse(iter.hasNext());
@@ -271,7 +255,7 @@ public class TestPathCacheRequests {
       } catch (IOException ioe) {
         Assert.assertTrue(ioe.getCause() instanceof NoSuchIdException);
       }
-      iter = proto.listPathCacheEntries(0, pool2.getId(), 4444);
+      iter = proto.listPathCacheEntries(0, "pool2");
       Assert.assertFalse(iter.hasNext());
     } finally {
       if (cluster != null) { cluster.shutdown(); }