HDFS-14547. Improve memory efficiency of quotas when storage type quotas are not set. Contributed by Jinglun.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java

@@ -18,53 +18,119 @@
 
 package org.apache.hadoop.hdfs.server.namenode;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.util.ConstEnumCounters;
 import org.apache.hadoop.hdfs.util.EnumCounters;
+import org.apache.hadoop.hdfs.util.ConstEnumCounters.ConstEnumException;
+
+import java.util.function.Consumer;
 
 /**
  * Counters for namespace, storage space and storage type space quota and usage.
  */
 public class QuotaCounts {
+
+  /**
+   * We pre-define 4 most common used EnumCounters objects. When the nsSsCounts
+   * and tsCounts are set to the 4 most common used value, we just point them to
+   * the pre-defined const EnumCounters objects instead of constructing many
+   * objects with the same value. See HDFS-14547.
+   */
+  final static EnumCounters<Quota> QUOTA_RESET =
+      new ConstEnumCounters<>(Quota.class, HdfsConstants.QUOTA_RESET);
+  final static EnumCounters<Quota> QUOTA_DEFAULT =
+      new ConstEnumCounters<>(Quota.class, 0);
+  final static EnumCounters<StorageType> STORAGE_TYPE_RESET =
+      new ConstEnumCounters<>(StorageType.class, HdfsConstants.QUOTA_RESET);
+  final static EnumCounters<StorageType> STORAGE_TYPE_DEFAULT =
+      new ConstEnumCounters<>(StorageType.class, 0);
+
+  /**
+   * Modify counter with action. If the counter is ConstEnumCounters, copy all
+   * the values of it to a new EnumCounters object, and modify the new obj.
+   *
+   * @param counter the EnumCounters to be modified.
+   * @param action the modifying action on counter.
+   * @return the modified counter.
+   */
+  static <T extends Enum<T>> EnumCounters<T> modify(EnumCounters<T> counter,
+      Consumer<EnumCounters<T>> action) {
+    try {
+      action.accept(counter);
+    } catch (ConstEnumException cee) {
+      // We don't call clone here because ConstEnumCounters.clone() will return
+      // an object of class ConstEnumCounters. We want EnumCounters.
+      counter = counter.deepCopyEnumCounter();
+      action.accept(counter);
+    }
+    return counter;
+  }
+
   // Name space and storage space counts (HDFS-7775 refactors the original disk
   // space count to storage space counts)
-  private EnumCounters<Quota> nsSsCounts;
+  @VisibleForTesting
+  EnumCounters<Quota> nsSsCounts;
   // Storage type space counts
-  private EnumCounters<StorageType> tsCounts;
+  @VisibleForTesting
+  EnumCounters<StorageType> tsCounts;
 
   public static class Builder {
     private EnumCounters<Quota> nsSsCounts;
     private EnumCounters<StorageType> tsCounts;
 
     public Builder() {
-      this.nsSsCounts = new EnumCounters<Quota>(Quota.class);
-      this.tsCounts = new EnumCounters<StorageType>(StorageType.class);
+      this.nsSsCounts = QUOTA_DEFAULT;
+      this.tsCounts = STORAGE_TYPE_DEFAULT;
     }
 
     public Builder nameSpace(long val) {
-      this.nsSsCounts.set(Quota.NAMESPACE, val);
+      nsSsCounts =
+          setQuotaCounter(nsSsCounts, Quota.NAMESPACE, Quota.STORAGESPACE, val);
       return this;
     }
 
     public Builder storageSpace(long val) {
-      this.nsSsCounts.set(Quota.STORAGESPACE, val);
+      nsSsCounts =
+          setQuotaCounter(nsSsCounts, Quota.STORAGESPACE, Quota.NAMESPACE, val);
       return this;
     }
 
     public Builder typeSpaces(EnumCounters<StorageType> val) {
       if (val != null) {
-        this.tsCounts.set(val);
+        if (val == STORAGE_TYPE_DEFAULT || val == STORAGE_TYPE_RESET) {
+          tsCounts = val;
+        } else {
+          tsCounts = modify(tsCounts, ec -> ec.set(val));
+        }
       }
       return this;
     }
 
     public Builder typeSpaces(long val) {
-      this.tsCounts.reset(val);
+      if (val == HdfsConstants.QUOTA_RESET) {
+        tsCounts = STORAGE_TYPE_RESET;
+      } else if (val == 0) {
+        tsCounts = STORAGE_TYPE_DEFAULT;
+      } else {
+        tsCounts = modify(tsCounts, ec -> ec.reset(val));
+      }
       return this;
     }
 
     public Builder quotaCount(QuotaCounts that) {
-      this.nsSsCounts.set(that.nsSsCounts);
-      this.tsCounts.set(that.tsCounts);
+      if (that.nsSsCounts == QUOTA_DEFAULT || that.nsSsCounts == QUOTA_RESET) {
+        nsSsCounts = that.nsSsCounts;
+      } else {
+        nsSsCounts = modify(nsSsCounts, ec -> ec.set(that.nsSsCounts));
+      }
+      if (that.tsCounts == STORAGE_TYPE_DEFAULT
+          || that.tsCounts == STORAGE_TYPE_RESET) {
+        tsCounts = that.tsCounts;
+      } else {
+        tsCounts = modify(tsCounts, ec -> ec.set(that.tsCounts));
+      }
       return this;
     }
 
@@ -79,14 +145,14 @@ public class QuotaCounts {
   }
 
   public QuotaCounts add(QuotaCounts that) {
-    this.nsSsCounts.add(that.nsSsCounts);
-    this.tsCounts.add(that.tsCounts);
+    nsSsCounts = modify(nsSsCounts, ec -> ec.add(that.nsSsCounts));
+    tsCounts = modify(tsCounts, ec -> ec.add(that.tsCounts));
     return this;
   }
 
   public QuotaCounts subtract(QuotaCounts that) {
-    this.nsSsCounts.subtract(that.nsSsCounts);
-    this.tsCounts.subtract(that.tsCounts);
+    nsSsCounts = modify(nsSsCounts, ec -> ec.subtract(that.nsSsCounts));
+    tsCounts = modify(tsCounts, ec -> ec.subtract(that.tsCounts));
     return this;
   }
 
@@ -97,8 +163,8 @@ public class QuotaCounts {
    */
   public QuotaCounts negation() {
     QuotaCounts ret = new QuotaCounts.Builder().quotaCount(this).build();
-    ret.nsSsCounts.negation();
-    ret.tsCounts.negation();
+    ret.nsSsCounts = modify(ret.nsSsCounts, ec -> ec.negation());
+    ret.tsCounts = modify(ret.tsCounts, ec -> ec.negation());
     return ret;
   }
 
@@ -107,11 +173,13 @@ public class QuotaCounts {
   }
 
   public void setNameSpace(long nameSpaceCount) {
-    this.nsSsCounts.set(Quota.NAMESPACE, nameSpaceCount);
+    nsSsCounts =
+        setQuotaCounter(nsSsCounts, Quota.NAMESPACE, Quota.STORAGESPACE,
+            nameSpaceCount);
   }
 
   public void addNameSpace(long nsDelta) {
-    this.nsSsCounts.add(Quota.NAMESPACE, nsDelta);
+    nsSsCounts = modify(nsSsCounts, ec -> ec.add(Quota.NAMESPACE, nsDelta));
   }
 
   public long getStorageSpace(){
@@ -119,11 +187,13 @@ public class QuotaCounts {
   }
 
   public void setStorageSpace(long spaceCount) {
-    this.nsSsCounts.set(Quota.STORAGESPACE, spaceCount);
+    nsSsCounts =
+        setQuotaCounter(nsSsCounts, Quota.STORAGESPACE, Quota.NAMESPACE,
+            spaceCount);
   }
 
   public void addStorageSpace(long dsDelta) {
-    this.nsSsCounts.add(Quota.STORAGESPACE, dsDelta);
+    nsSsCounts = modify(nsSsCounts, ec -> ec.add(Quota.STORAGESPACE, dsDelta));
   }
 
   public EnumCounters<StorageType> getTypeSpaces() {
@@ -134,8 +204,10 @@ public class QuotaCounts {
   }
 
   void setTypeSpaces(EnumCounters<StorageType> that) {
-    if (that != null) {
-      this.tsCounts.set(that);
+    if (that == STORAGE_TYPE_DEFAULT || that == STORAGE_TYPE_RESET) {
+      tsCounts = that;
+    } else if (that != null) {
+      tsCounts = modify(tsCounts, ec -> ec.set(that));
     }
   }
 
@@ -144,21 +216,54 @@ public class QuotaCounts {
   }
 
   void setTypeSpace(StorageType type, long spaceCount) {
-    this.tsCounts.set(type, spaceCount);
+    tsCounts = modify(tsCounts, ec -> ec.set(type, spaceCount));
   }
 
   public void addTypeSpace(StorageType type, long delta) {
-    this.tsCounts.add(type, delta);
+    tsCounts = modify(tsCounts, ec -> ec.add(type, delta));
   }
 
   public boolean anyNsSsCountGreaterOrEqual(long val) {
+  if (nsSsCounts == QUOTA_DEFAULT) {
+      return val <= 0;
+    } else if (nsSsCounts == QUOTA_RESET) {
+      return val <= HdfsConstants.QUOTA_RESET;
+    }
     return nsSsCounts.anyGreaterOrEqual(val);
   }
 
   public boolean anyTypeSpaceCountGreaterOrEqual(long val) {
+    if (tsCounts == STORAGE_TYPE_DEFAULT) {
+      return val <= 0;
+    } else if (tsCounts == STORAGE_TYPE_RESET) {
+      return val <= HdfsConstants.QUOTA_RESET;
+    }
     return tsCounts.anyGreaterOrEqual(val);
   }
 
+  /**
+   * Set inputCounts' value of Quota type quotaToSet to val.
+   * inputCounts should be the left side value of this method.
+   *
+   * @param inputCounts the EnumCounters instance.
+   * @param quotaToSet the quota type to be set.
+   * @param otherQuota the other quota type besides quotaToSet.
+   * @param val the value to be set.
+   * @return the modified inputCounts.
+   */
+  private static EnumCounters<Quota> setQuotaCounter(
+      EnumCounters<Quota> inputCounts, Quota quotaToSet, Quota otherQuota,
+      long val) {
+    if (val == HdfsConstants.QUOTA_RESET
+        && inputCounts.get(otherQuota) == HdfsConstants.QUOTA_RESET) {
+      return QUOTA_RESET;
+    } else if (val == 0 && inputCounts.get(otherQuota) == 0) {
+      return QUOTA_DEFAULT;
+    } else {
+      return modify(inputCounts, ec -> ec.set(quotaToSet, val));
+    }
+  }
+
   @Override
   public String toString() {
     return "name space=" + getNameSpace() +

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ConstEnumCounters.java

@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.util;
+
+/**
+ * Const Counters for an enum type.
+ *
+ * It's the const version of EnumCounters. Any modification ends with a
+ * ConstEnumException.
+ *
+ * @see org.apache.hadoop.hdfs.util.EnumCounters
+ */
+public class ConstEnumCounters<E extends Enum<E>> extends EnumCounters<E> {
+
+  /**
+   * An exception class for modification on ConstEnumCounters.
+   */
+  public static final class ConstEnumException extends RuntimeException {
+    private ConstEnumException(String msg) {
+      super(msg);
+    }
+  }
+
+  /**
+   * Throwing this exception if any modification occurs.
+   */
+  private static final ConstEnumException CONST_ENUM_EXCEPTION =
+      new ConstEnumException("modification on const.");
+
+  public ConstEnumCounters(Class<E> enumClass, long defaultVal) {
+    super(enumClass);
+    forceReset(defaultVal);
+  }
+
+  @Override
+  public final void negation() {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void set(final E e, final long value) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void set(final EnumCounters<E> that) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void reset() {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void add(final E e, final long value) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void add(final EnumCounters<E> that) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void subtract(final E e, final long value) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void subtract(final EnumCounters<E> that) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  @Override
+  public final void reset(long val) {
+    throw CONST_ENUM_EXCEPTION;
+  }
+
+  private void forceReset(long val) {
+    super.reset(val);
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java

@@ -70,55 +70,55 @@ public class EnumCounters<E extends Enum<E>> {
   }
 
   /** Negate all counters. */
-  public final void negation() {
+  public void negation() {
     for(int i = 0; i < counters.length; i++) {
       counters[i] = -counters[i];
     }
   }
   
   /** Set counter e to the given value. */
-  public final void set(final E e, final long value) {
+  public void set(final E e, final long value) {
     counters[e.ordinal()] = value;
   }
 
   /** Set this counters to that counters. */
-  public final void set(final EnumCounters<E> that) {
+  public void set(final EnumCounters<E> that) {
     for(int i = 0; i < counters.length; i++) {
       this.counters[i] = that.counters[i];
     }
   }
 
   /** Reset all counters to zero. */
-  public final void reset() {
+  public void reset() {
     reset(0L);
   }
 
   /** Add the given value to counter e. */
-  public final void add(final E e, final long value) {
+  public void add(final E e, final long value) {
     counters[e.ordinal()] += value;
   }
 
   /** Add that counters to this counters. */
-  public final void add(final EnumCounters<E> that) {
+  public void add(final EnumCounters<E> that) {
     for(int i = 0; i < counters.length; i++) {
       this.counters[i] += that.counters[i];
     }
   }
 
   /** Subtract the given value from counter e. */
-  public final void subtract(final E e, final long value) {
+  public void subtract(final E e, final long value) {
     counters[e.ordinal()] -= value;
   }
 
   /** Subtract this counters from that counters. */
-  public final void subtract(final EnumCounters<E> that) {
+  public void subtract(final EnumCounters<E> that) {
     for(int i = 0; i < counters.length; i++) {
       this.counters[i] -= that.counters[i];
     }
   }
   
   /** @return the sum of all counters. */
-  public final long sum() {
+  public long sum() {
     long sum = 0;
     for(int i = 0; i < counters.length; i++) {
       sum += counters[i];
@@ -138,6 +138,15 @@ public class EnumCounters<E extends Enum<E>> {
         && Arrays.equals(this.counters, that.counters);
   }
 
+  /**
+   * Return a deep copy of EnumCounter.
+   */
+  public EnumCounters<E> deepCopyEnumCounter() {
+    EnumCounters<E> newCounter = new EnumCounters<>(enumClass);
+    newCounter.set(this);
+    return newCounter;
+  }
+
   @Override
   public int hashCode() {
     return Arrays.hashCode(counters);
@@ -154,7 +163,7 @@ public class EnumCounters<E extends Enum<E>> {
     return b.substring(0, b.length() - 2);
   }
 
-  public final void reset(long val) {
+  public void reset(long val) {
     for(int i = 0; i < counters.length; i++) {
       this.counters[i] = val;
     }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaCounts.java

@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+/**
+ * Test QuotaCounts.
+ */
+public class TestQuotaCounts {
+  @Test
+  public void testBuildConstEnumCounters() throws Exception {
+    QuotaCounts qc =
+        new QuotaCounts.Builder().nameSpace(HdfsConstants.QUOTA_RESET)
+            .storageSpace(HdfsConstants.QUOTA_RESET).build();
+    // compare the references
+    assertSame(QuotaCounts.QUOTA_RESET, qc.nsSsCounts);
+    assertSame(QuotaCounts.STORAGE_TYPE_DEFAULT, qc.tsCounts);
+    // compare the values
+    assertEquals(HdfsConstants.QUOTA_RESET, qc.getNameSpace());
+    assertEquals(HdfsConstants.QUOTA_RESET, qc.getStorageSpace());
+    for (StorageType st : StorageType.values()) {
+      assertEquals(0, qc.getTypeSpace(st));
+    }
+  }
+
+  @Test
+  public void testAddSpace() throws Exception {
+    QuotaCounts qc = new QuotaCounts.Builder().build();
+    qc.addNameSpace(1);
+    qc.addStorageSpace(1024);
+    assertEquals(1, qc.getNameSpace());
+    assertEquals(1024, qc.getStorageSpace());
+  }
+
+  @Test
+  public void testAdd() throws Exception {
+    QuotaCounts qc1 = new QuotaCounts.Builder().build();
+    QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512)
+        .typeSpaces(5).build();
+    qc1.add(qc2);
+    assertEquals(1, qc1.getNameSpace());
+    assertEquals(512, qc1.getStorageSpace());
+    for (StorageType type : StorageType.values()) {
+      assertEquals(5, qc1.getTypeSpace(type));
+    }
+  }
+
+  @Test
+  public void testAddTypeSpaces() throws Exception {
+    QuotaCounts qc = new QuotaCounts.Builder().build();
+    for (StorageType t : StorageType.values()) {
+      qc.addTypeSpace(t, 10);
+    }
+    for (StorageType type : StorageType.values()) {
+      assertEquals(10, qc.getTypeSpace(type));
+    }
+  }
+
+  @Test
+  public void testSubtract() throws Exception {
+    QuotaCounts qc1 = new QuotaCounts.Builder().build();
+    QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512)
+        .typeSpaces(5).build();
+    qc1.subtract(qc2);
+    assertEquals(-1, qc1.getNameSpace());
+    assertEquals(-512, qc1.getStorageSpace());
+    for (StorageType type : StorageType.values()) {
+      assertEquals(-5, qc1.getTypeSpace(type));
+    }
+  }
+
+  @Test
+  public void testSetTypeSpaces() throws Exception {
+    QuotaCounts qc1 = new QuotaCounts.Builder().build();
+    QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512)
+        .typeSpaces(5).build();
+    qc1.setTypeSpaces(qc2.getTypeSpaces());
+    for (StorageType t : StorageType.values()) {
+      assertEquals(qc2.getTypeSpace(t), qc1.getTypeSpace(t));
+    }
+
+    // test ConstEnumCounters
+    qc1.setTypeSpaces(QuotaCounts.STORAGE_TYPE_RESET);
+    assertSame(QuotaCounts.STORAGE_TYPE_RESET, qc1.tsCounts);
+  }
+
+  @Test
+  public void testSetSpaces() {
+    QuotaCounts qc = new QuotaCounts.Builder().build();
+    qc.setNameSpace(10);
+    qc.setStorageSpace(1024);
+    assertEquals(10, qc.getNameSpace());
+    assertEquals(1024, qc.getStorageSpace());
+
+    // test ConstEnumCounters
+    qc.setNameSpace(HdfsConstants.QUOTA_RESET);
+    qc.setStorageSpace(HdfsConstants.QUOTA_RESET);
+    assertSame(QuotaCounts.QUOTA_RESET, qc.nsSsCounts);
+  }
+
+  @Test
+  public void testNegation() throws Exception {
+    QuotaCounts qc = new QuotaCounts.Builder()
+        .nameSpace(HdfsConstants.QUOTA_RESET)
+        .storageSpace(HdfsConstants.QUOTA_RESET)
+        .typeSpaces(HdfsConstants.QUOTA_RESET).build();
+    qc = qc.negation();
+    assertEquals(1, qc.getNameSpace());
+    assertEquals(1, qc.getStorageSpace());
+    for (StorageType t : StorageType.values()) {
+      assertEquals(1, qc.getTypeSpace(t));
+    }
+  }
+}