HDFS-13583. RBF: Router admin clrQuota is not synchronized with nameservice. Contributed by Dibyendu Karmakar.

(cherry picked from commit 17a87977f29ced49724f561a68565217c8cb4e94)

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java

@@ -162,6 +162,8 @@ public class Quota {
   private QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
     long nsCount = 0;
     long ssCount = 0;
+    long nsQuota = HdfsConstants.QUOTA_RESET;
+    long ssQuota = HdfsConstants.QUOTA_RESET;
     boolean hasQuotaUnSet = false;
 
     for (Map.Entry<RemoteLocation, QuotaUsage> entry : results.entrySet()) {
@@ -173,6 +175,8 @@ public class Quota {
         if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) {
           hasQuotaUnSet = true;
         }
+        nsQuota = usage.getQuota();
+        ssQuota = usage.getSpaceQuota();
 
         nsCount += usage.getFileAndDirectoryCount();
         ssCount += usage.getSpaceConsumed();
@@ -187,7 +191,10 @@ public class Quota {
     QuotaUsage.Builder builder = new QuotaUsage.Builder()
         .fileAndDirectoryCount(nsCount).spaceConsumed(ssCount);
     if (hasQuotaUnSet) {
-      builder.quota(HdfsConstants.QUOTA_DONT_SET);
+      builder.quota(HdfsConstants.QUOTA_RESET)
+          .spaceQuota(HdfsConstants.QUOTA_RESET);
+    } else {
+      builder.quota(nsQuota).spaceQuota(ssQuota);
     }
 
     return builder.build();

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java

@@ -28,6 +28,7 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
 import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
 import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB;
@@ -253,8 +254,11 @@ public class RouterAdminServer extends AbstractService
 
     if (nsQuota != HdfsConstants.QUOTA_DONT_SET
         || ssQuota != HdfsConstants.QUOTA_DONT_SET) {
-      this.router.getRpcServer().getQuotaModule().setQuota(path, nsQuota,
-          ssQuota, null);
+      HdfsFileStatus ret = this.router.getRpcServer().getFileInfo(path);
+      if (ret != null) {
+        this.router.getRpcServer().getQuotaModule().setQuota(path, nsQuota,
+            ssQuota, null);
+      }
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java

@@ -161,8 +161,8 @@ public class RouterQuotaManager {
       long ssQuota = quota.getSpaceQuota();
 
       // once nsQuota or ssQuota was set, this mount table is quota set
-      if (nsQuota != HdfsConstants.QUOTA_DONT_SET
-          || ssQuota != HdfsConstants.QUOTA_DONT_SET) {
+      if (nsQuota != HdfsConstants.QUOTA_RESET
+          || ssQuota != HdfsConstants.QUOTA_RESET) {
         return true;
       }
     }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java

@@ -111,7 +111,7 @@ public class RouterQuotaUpdateService extends PeriodicService {
 
         // If quota is not set in some subclusters under federation path,
         // set quota for this path.
-        if (currentQuotaUsage.getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+        if (currentQuotaUsage.getQuota() == HdfsConstants.QUOTA_RESET) {
           try {
             this.rpcServer.setQuota(src, nsQuota, ssQuota, null);
           } catch (IOException ioe) {

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java

@@ -96,14 +96,14 @@ public final class RouterQuotaUsage extends QuotaUsage {
   public String toString() {
     String nsQuota = String.valueOf(getQuota());
     String nsCount = String.valueOf(getFileAndDirectoryCount());
-    if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+    if (getQuota() == HdfsConstants.QUOTA_RESET) {
       nsQuota = "-";
       nsCount = "-";
     }
 
     String ssQuota = StringUtils.byteDesc(getSpaceQuota());
     String ssCount = StringUtils.byteDesc(getSpaceConsumed());
-    if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
+    if (getSpaceQuota() == HdfsConstants.QUOTA_RESET) {
       ssQuota = "-";
       ssCount = "-";
     }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java

@@ -153,9 +153,9 @@ public abstract class MountTable extends BaseRecord {
     // Set quota for mount table
     RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
         .fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
-        .quota(HdfsConstants.QUOTA_DONT_SET)
+        .quota(HdfsConstants.QUOTA_RESET)
         .spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
-        .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
+        .spaceQuota(HdfsConstants.QUOTA_RESET).build();
     record.setQuota(quota);
 
     // Validate

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java

@@ -257,9 +257,9 @@ public class MountTablePBImpl extends MountTable implements PBRecord {
   public RouterQuotaUsage getQuota() {
     MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
 
-    long nsQuota = HdfsConstants.QUOTA_DONT_SET;
+    long nsQuota = HdfsConstants.QUOTA_RESET;
     long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
-    long ssQuota = HdfsConstants.QUOTA_DONT_SET;
+    long ssQuota = HdfsConstants.QUOTA_RESET;
     long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
     if (proto.hasQuota()) {
       QuotaUsageProto quotaProto = proto.getQuota();

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java

@@ -632,8 +632,8 @@ public class RouterAdmin extends Configured implements Tool {
    * @throws IOException Error clearing the mount point.
    */
   private boolean clrQuota(String mount) throws IOException {
-    return updateQuota(mount, HdfsConstants.QUOTA_DONT_SET,
-        HdfsConstants.QUOTA_DONT_SET);
+    return updateQuota(mount, HdfsConstants.QUOTA_RESET,
+        HdfsConstants.QUOTA_RESET);
   }
 
   /**
@@ -668,8 +668,8 @@ public class RouterAdmin extends Configured implements Tool {
       long nsCount = existingEntry.getQuota().getFileAndDirectoryCount();
       long ssCount = existingEntry.getQuota().getSpaceConsumed();
       // If nsQuota and ssQuota were unset, clear nsQuota and ssQuota.
-      if (nsQuota == HdfsConstants.QUOTA_DONT_SET &&
-          ssQuota == HdfsConstants.QUOTA_DONT_SET) {
+      if (nsQuota == HdfsConstants.QUOTA_RESET &&
+          ssQuota == HdfsConstants.QUOTA_RESET) {
         nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
         ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
       } else {

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java

@@ -64,6 +64,8 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
 
 /**
  * The administrator interface of the {@link Router} implemented by
@@ -101,6 +103,12 @@ public class TestRouterAdmin {
     membership.registerNamenode(
         createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE));
     stateStore.refreshCaches(true);
+
+    RouterRpcServer spyRpcServer =
+        Mockito.spy(routerContext.getRouter().createRpcServer());
+    Whitebox
+        .setInternalState(routerContext.getRouter(), "rpcServer", spyRpcServer);
+    Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
   }
 
   @AfterClass

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java

@@ -115,6 +115,14 @@ public class TestRouterAdminCLI {
         Mockito.anyLong(), Mockito.anyLong(), Mockito.any());
     Whitebox.setInternalState(
         routerContext.getRouter().getRpcServer(), "quotaCall", quota);
+
+    RouterRpcServer spyRpcServer =
+        Mockito.spy(routerContext.getRouter().createRpcServer());
+    Whitebox
+        .setInternalState(routerContext.getRouter(), "rpcServer", spyRpcServer);
+
+    Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
+
   }
 
   @AfterClass
@@ -447,10 +455,10 @@ public class TestRouterAdminCLI {
     // verify the default quota set
     assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
         quotaUsage.getFileAndDirectoryCount());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
     assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
         quotaUsage.getSpaceConsumed());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
 
     long nsQuota = 50;
     long ssQuota = 100;
@@ -494,8 +502,8 @@ public class TestRouterAdminCLI {
     quotaUsage = mountTable.getQuota();
 
     // verify if quota unset successfully
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
   }
 
   @Test

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java

@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
 import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
@@ -460,8 +461,10 @@ public class TestRouterQuota {
   public void testQuotaSynchronization() throws IOException {
     long updateNsQuota = 3;
     long updateSsQuota = 4;
+    FileSystem nnFs = nnContext1.getFileSystem();
+    nnFs.mkdirs(new Path("/testsync"));
     MountTable mountTable = MountTable.newInstance("/quotaSync",
-        Collections.singletonMap("ns0", "/"), Time.now(), Time.now());
+        Collections.singletonMap("ns0", "/testsync"), Time.now(), Time.now());
     mountTable.setQuota(new RouterQuotaUsage.Builder().quota(1)
         .spaceQuota(2).build());
     // Add new mount table
@@ -469,7 +472,7 @@ public class TestRouterQuota {
 
     // ensure the quota is not set as updated value
     QuotaUsage realQuota = nnContext1.getFileSystem()
-        .getQuotaUsage(new Path("/"));
+        .getQuotaUsage(new Path("/testsync"));
     assertNotEquals(updateNsQuota, realQuota.getQuota());
     assertNotEquals(updateSsQuota, realQuota.getSpaceQuota());
 
@@ -489,9 +492,26 @@ public class TestRouterQuota {
 
     // verify if the quota is updated in real path
     realQuota = nnContext1.getFileSystem().getQuotaUsage(
-        new Path("/"));
+        new Path("/testsync"));
     assertEquals(updateNsQuota, realQuota.getQuota());
     assertEquals(updateSsQuota, realQuota.getSpaceQuota());
+
+    // Clear the quota
+    mountTable.setQuota(new RouterQuotaUsage.Builder()
+        .quota(HdfsConstants.QUOTA_RESET)
+        .spaceQuota(HdfsConstants.QUOTA_RESET).build());
+
+    updateRequest = UpdateMountTableEntryRequest
+        .newInstance(mountTable);
+    client = routerContext.getAdminClient();
+    mountTableManager = client.getMountTableManager();
+    mountTableManager.updateMountTableEntry(updateRequest);
+
+    // verify if the quota is updated in real path
+    realQuota = nnContext1.getFileSystem().getQuotaUsage(
+        new Path("/testsync"));
+    assertEquals(HdfsConstants.QUOTA_RESET, realQuota.getQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, realQuota.getSpaceQuota());
   }
 
   @Test

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java

@@ -81,8 +81,8 @@ public class TestRouterQuotaManager {
 
     // test case2: get quota from an no-quota set path
     RouterQuotaUsage.Builder quota = new RouterQuotaUsage.Builder()
-        .quota(HdfsConstants.QUOTA_DONT_SET)
-        .spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+        .quota(HdfsConstants.QUOTA_RESET)
+        .spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/noQuotaSet", quota.build());
     quotaGet = manager.getQuotaUsage("/noQuotaSet");
     // it should return null
@@ -90,36 +90,36 @@ public class TestRouterQuotaManager {
 
     // test case3: get quota from an quota-set path
     quota.quota(1);
-    quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+    quota.spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/hasQuotaSet", quota.build());
     quotaGet = manager.getQuotaUsage("/hasQuotaSet");
     assertEquals(1, quotaGet.getQuota());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
 
     // test case4: get quota with an non-exist child path
     quotaGet = manager.getQuotaUsage("/hasQuotaSet/file");
     // it will return the nearest ancestor which quota was set
     assertEquals(1, quotaGet.getQuota());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
 
     // test case5: get quota with an child path which its parent
     // wasn't quota set
-    quota.quota(HdfsConstants.QUOTA_DONT_SET);
-    quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+    quota.quota(HdfsConstants.QUOTA_RESET);
+    quota.spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/hasQuotaSet/noQuotaSet", quota.build());
     // here should returns the quota of path /hasQuotaSet
     // (the nearest ancestor which quota was set)
     quotaGet = manager.getQuotaUsage("/hasQuotaSet/noQuotaSet/file");
     assertEquals(1, quotaGet.getQuota());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
 
     // test case6: get quota with an child path which its parent was quota set
     quota.quota(2);
-    quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+    quota.spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/hasQuotaSet/hasQuotaSet", quota.build());
     // here should return the quota of path /hasQuotaSet/hasQuotaSet
     quotaGet = manager.getQuotaUsage("/hasQuotaSet/hasQuotaSet/file");
     assertEquals(2, quotaGet.getQuota());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
   }
 }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java

@@ -84,9 +84,9 @@ public class TestMountTable {
 
     RouterQuotaUsage quota = record.getQuota();
     assertEquals(0, quota.getFileAndDirectoryCount());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quota.getQuota());
     assertEquals(0, quota.getSpaceConsumed());
-    assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getSpaceQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, quota.getSpaceQuota());
 
     MountTable record2 =
         MountTable.newInstance(SRC, DST_MAP, DATE_CREATED, DATE_MOD);