HDFS-14490. RBF: Remove unnecessary quota checks. Contributed by Ayush Saxena.

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java

@@ -140,7 +140,7 @@ public class ErasureCoding {
     rpcServer.checkOperation(OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod remoteMethod = new RemoteMethod("getErasureCodingPolicy",
         new Class<?>[] {String.class}, new RemoteParam());
     ErasureCodingPolicy ret = rpcClient.invokeSequential(
@@ -153,7 +153,7 @@ public class ErasureCoding {
     rpcServer.checkOperation(OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod remoteMethod = new RemoteMethod("setErasureCodingPolicy",
         new Class<?>[] {String.class, String.class},
         new RemoteParam(), ecPolicyName);
@@ -168,7 +168,7 @@ public class ErasureCoding {
     rpcServer.checkOperation(OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod remoteMethod = new RemoteMethod("unsetErasureCodingPolicy",
         new Class<?>[] {String.class}, new RemoteParam());
     if (rpcServer.isInvokeConcurrent(src)) {

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java

@@ -207,7 +207,8 @@ public class RouterClientProtocol implements ClientProtocol {
       final long length) throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
-    List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, false);
+    List<RemoteLocation> locations =
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
         new Class<?>[] {String.class, long.class, long.class},
         new RemoteParam(), offset, length);
@@ -351,7 +352,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("recoverLease",
         new Class<?>[] {String.class, String.class}, new RemoteParam(),
         clientName);
@@ -395,7 +396,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("setPermission",
         new Class<?>[] {String.class, FsPermission.class},
         new RemoteParam(), permissions);
@@ -412,7 +413,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("setOwner",
         new Class<?>[] {String.class, String.class, String.class},
         new RemoteParam(), username, groupname);
@@ -531,7 +532,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
         new Class<?>[] {String.class}, new RemoteParam());
     return rpcClient.invokeSequential(locations, method, Long.class, null);
@@ -724,7 +725,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // Locate the dir and fetch the listing
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("getListing",
         new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
         new RemoteParam(), startAfter, needLocation);
@@ -821,7 +822,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("getFileInfo",
         new Class<?>[] {String.class}, new RemoteParam());
 
@@ -859,7 +860,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("isFileClosed",
         new Class<?>[] {String.class}, new RemoteParam());
     return rpcClient.invokeSequential(locations, method, Boolean.class,
@@ -871,7 +872,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("getFileLinkInfo",
         new Class<?>[] {String.class}, new RemoteParam());
     return rpcClient.invokeSequential(locations, method, HdfsFileStatus.class,
@@ -883,7 +884,7 @@ public class RouterClientProtocol implements ClientProtocol {
       boolean needBlockToken) throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("getLocatedFileInfo",
         new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
         needBlockToken);
@@ -1092,7 +1093,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(path, false);
+        rpcServer.getLocationsForPath(path, false, false);
     RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
         new Class<?>[] {String.class, String.class},
         new RemoteParam(), cookie);
@@ -1117,7 +1118,7 @@ public class RouterClientProtocol implements ClientProtocol {
     // Get the summaries from regular files
     final Collection<ContentSummary> summaries = new ArrayList<>();
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(path, false);
+        rpcServer.getLocationsForPath(path, false, false);
     final RemoteMethod method = new RemoteMethod("getContentSummary",
         new Class<?>[] {String.class}, new RemoteParam());
     final List<RemoteResult<RemoteLocation, ContentSummary>> results =
@@ -1169,7 +1170,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("fsync",
         new Class<?>[] {String.class, long.class, String.class, long.class },
         new RemoteParam(), fileId, clientName, lastBlockLength);
@@ -1181,7 +1182,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("setTimes",
         new Class<?>[] {String.class, long.class, long.class},
         new RemoteParam(), mtime, atime);
@@ -1211,7 +1212,7 @@ public class RouterClientProtocol implements ClientProtocol {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(path, true);
+        rpcServer.getLocationsForPath(path, true, false);
     RemoteMethod method = new RemoteMethod("getLinkTarget",
         new Class<?>[] {String.class}, new RemoteParam());
     return rpcClient.invokeSequential(locations, method, String.class, null);
@@ -1309,7 +1310,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("modifyAclEntries",
         new Class<?>[] {String.class, List.class},
         new RemoteParam(), aclSpec);
@@ -1327,7 +1328,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("removeAclEntries",
         new Class<?>[] {String.class, List.class},
         new RemoteParam(), aclSpec);
@@ -1344,7 +1345,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("removeDefaultAcl",
         new Class<?>[] {String.class}, new RemoteParam());
     if (rpcServer.isInvokeConcurrent(src)) {
@@ -1360,7 +1361,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("removeAcl",
         new Class<?>[] {String.class}, new RemoteParam());
     if (rpcServer.isInvokeConcurrent(src)) {
@@ -1376,7 +1377,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod(
         "setAcl", new Class<?>[] {String.class, List.class},
         new RemoteParam(), aclSpec);
@@ -1393,7 +1394,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("getAclStatus",
         new Class<?>[] {String.class}, new RemoteParam());
     return rpcClient.invokeSequential(locations, method, AclStatus.class, null);
@@ -1406,7 +1407,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("createEncryptionZone",
         new Class<?>[] {String.class, String.class},
         new RemoteParam(), keyName);
@@ -1419,7 +1420,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("getEZForPath",
         new Class<?>[] {String.class}, new RemoteParam());
     return rpcClient.invokeSequential(
@@ -1453,7 +1454,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("setXAttr",
         new Class<?>[] {String.class, XAttr.class, EnumSet.class},
         new RemoteParam(), xAttr, flag);
@@ -1472,7 +1473,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("getXAttrs",
         new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
     return (List<XAttr>) rpcClient.invokeSequential(
@@ -1486,7 +1487,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false);
+        rpcServer.getLocationsForPath(src, false, false);
     RemoteMethod method = new RemoteMethod("listXAttrs",
         new Class<?>[] {String.class}, new RemoteParam());
     return (List<XAttr>) rpcClient.invokeSequential(
@@ -1499,7 +1500,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, true);
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("removeXAttr",
         new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
     if (rpcServer.isInvokeConcurrent(src)) {
@@ -1515,7 +1516,7 @@ public class RouterClientProtocol implements ClientProtocol {
 
     // TODO handle virtual directories
     final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(path, true);
+        rpcServer.getLocationsForPath(path, true, false);
     RemoteMethod method = new RemoteMethod("checkAccess",
         new Class<?>[] {String.class, FsAction.class},
         new RemoteParam(), mode);
@@ -1735,7 +1736,7 @@ public class RouterClientProtocol implements ClientProtocol {
       throws IOException {
 
     final List<RemoteLocation> dstLocations =
-        rpcServer.getLocationsForPath(dst, true);
+        rpcServer.getLocationsForPath(dst, true, false);
     final Map<RemoteLocation, String> dstMap = new HashMap<>();
 
     Iterator<RemoteLocation> iterator = srcLocations.iterator();
@@ -2026,7 +2027,7 @@ public class RouterClientProtocol implements ClientProtocol {
     try {
       if (rpcServer.isPathAll(src)) {
         List<RemoteLocation> locations;
-        locations = rpcServer.getLocationsForPath(src, false);
+        locations = rpcServer.getLocationsForPath(src, false, false);
         RemoteMethod method = new RemoteMethod("getFileInfo",
             new Class<?>[] {String.class}, new RemoteParam());
         HdfsFileStatus fileStatus = rpcClient.invokeSequential(locations,

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java

@@ -45,7 +45,8 @@ public class RouterStoragePolicy {
       throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
-    List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
+    List<RemoteLocation> locations =
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("setStoragePolicy",
         new Class<?>[] {String.class, String.class},
         new RemoteParam(),
@@ -67,7 +68,8 @@ public class RouterStoragePolicy {
   public void unsetStoragePolicy(String src) throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE, true);
 
-    List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
+    List<RemoteLocation> locations =
+        rpcServer.getLocationsForPath(src, true, false);
     RemoteMethod method = new RemoteMethod("unsetStoragePolicy",
         new Class<?>[] {String.class},
         new RemoteParam());
@@ -82,7 +84,8 @@ public class RouterStoragePolicy {
       throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
 
-    List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, false);
+    List<RemoteLocation> locations =
+        rpcServer.getLocationsForPath(path, false, false);
     RemoteMethod method = new RemoteMethod("getStoragePolicy",
         new Class<?>[] {String.class},
         new RemoteParam());
@@ -92,7 +95,8 @@ public class RouterStoragePolicy {
   public void satisfyStoragePolicy(String path) throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
 
-    List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, true);
+    List<RemoteLocation> locations =
+        rpcServer.getLocationsForPath(path, true, false);
     RemoteMethod method = new RemoteMethod("satisfyStoragePolicy",
         new Class<?>[] {String.class},
         new RemoteParam());

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.router;
 
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNull;
@@ -34,7 +35,9 @@ import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -792,4 +795,34 @@ public class TestRouterQuota {
     assertEquals(nsQuota, quota1.getQuota());
     assertEquals(ssQuota, quota1.getSpaceQuota());
   }
+
+  @Test
+  public void testNoQuotaaExceptionForUnrelatedOperations() throws Exception {
+    FileSystem nnFs = nnContext1.getFileSystem();
+    DistributedFileSystem routerFs =
+        (DistributedFileSystem) routerContext.getFileSystem();
+    Path path = new Path("/quota");
+    nnFs.mkdirs(new Path("/dir"));
+    MountTable mountTable1 = MountTable.newInstance("/quota",
+        Collections.singletonMap("ns0", "/dir"));
+    mountTable1.setQuota(new RouterQuotaUsage.Builder().quota(0).build());
+    addMountTable(mountTable1);
+    routerFs.mkdirs(new Path("/quota/1"));
+    routerContext.getRouter().getQuotaCacheUpdateService().periodicInvoke();
+
+    // Quota check for related operation.
+    intercept(NSQuotaExceededException.class,
+        "The NameSpace quota (directories and files) is exceeded",
+        () -> routerFs.mkdirs(new Path("/quota/2")));
+
+    //Quotas shouldn't be checked for unrelated operations.
+    routerFs.setStoragePolicy(path, "COLD");
+    routerFs.setErasureCodingPolicy(path, "RS-6-3-1024k");
+    routerFs.unsetErasureCodingPolicy(path);
+    routerFs.setPermission(path, new FsPermission((short) 01777));
+    routerFs.setOwner(path, "user", "group");
+    routerFs.setTimes(path, 1L, 1L);
+    routerFs.listStatus(path);
+    routerFs.getContentSummary(path);
+  }
 }