HADOOP-8971. PureJavaCrc32 cache hit-ratio is low for static data (Backport of HADOOP-8926). Contributed by Gopal V.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1@1401973 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -109,6 +109,9 @@ Release 1.2.0 - unreleased
     HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
     methods.  (Brandon Li via szetszwo)
 
+    HADOOP-8971. PureJavaCrc32 cache hit-ratio is low for static data
+    (Backport of HADOOP-8926) (Gopal V via suresh)
+
   BUG FIXES
 
     HADOOP-8460. Document proper setting of HADOOP_PID_DIR and

src/core/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java

@@ -306,8 +306,8 @@ public class AuthenticationFilter implements Filter {
       if (!token.getType().equals(authHandler.getType())) {
         throw new AuthenticationException("Invalid AuthenticationToken type");
       }
-      if (token.isExpired()) {
-        throw new AuthenticationException("AuthenticationToken expired");
+      if (LOG.isDebugEnabled() && token.isExpired ()) {
+        LOG.debug("Authentication Token expired");
       }
     }
     return token;
@@ -341,7 +341,7 @@ public class AuthenticationFilter implements Filter {
         LOG.warn("AuthenticationToken ignored: " + ex.getMessage());
         token = null;
       }
-      if (token == null) {
+      if (token == null || token.isExpired ())  {
         if (LOG.isDebugEnabled()) {
           LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest));
         }

src/core/org/apache/hadoop/util/PureJavaCrc32.java

@@ -53,22 +53,38 @@ public class PureJavaCrc32 implements Checksum {
   /** {@inheritDoc} */
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -77,14 +93,24 @@ public class PureJavaCrc32 implements Checksum {
 
   /** {@inheritDoc} */
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
 
   /*
    * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
    * See also TestPureJavaCrc32.Table.
    */
-  private static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+  	/* T8_0 */
     0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 
     0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 
     0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 
@@ -148,9 +174,8 @@ public class PureJavaCrc32 implements Checksum {
     0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 
     0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 
     0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 
-    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
-  };
-  private static final int[] T8_1 = new int[] {
+    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+    /* T8_1 */
     0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, 
     0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, 
     0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, 
@@ -214,9 +239,8 @@ public class PureJavaCrc32 implements Checksum {
     0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, 
     0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, 
     0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, 
-    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
-  };
-  private static final int[] T8_2 = new int[] {
+    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+    /* T8_2 */
     0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, 
     0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, 
     0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, 
@@ -280,9 +304,8 @@ public class PureJavaCrc32 implements Checksum {
     0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, 
     0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, 
     0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, 
-    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
-  };
-  private static final int[] T8_3 = new int[] {
+    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+    /* T8_3 */
     0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, 
     0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, 
     0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, 
@@ -346,9 +369,8 @@ public class PureJavaCrc32 implements Checksum {
     0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, 
     0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, 
     0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, 
-    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
-  };
-  private static final int[] T8_4 = new int[] {
+    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+    /* T8_4 */
     0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, 
     0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, 
     0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, 
@@ -412,9 +434,8 @@ public class PureJavaCrc32 implements Checksum {
     0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, 
     0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, 
     0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, 
-    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
-  };
-  private static final int[] T8_5 = new int[] {
+    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+    /* T8_5 */
     0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, 
     0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, 
     0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, 
@@ -478,9 +499,8 @@ public class PureJavaCrc32 implements Checksum {
     0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, 
     0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, 
     0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, 
-    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
-  };
-  private static final int[] T8_6 = new int[] {
+    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+    /* T8_6 */
     0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, 
     0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, 
     0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, 
@@ -544,9 +564,8 @@ public class PureJavaCrc32 implements Checksum {
     0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, 
     0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, 
     0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, 
-    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
-  };
-  private static final int[] T8_7 = new int[] {
+    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+    /* T8_7 */
     0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, 
     0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, 
     0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, 

src/test/org/apache/hadoop/util/TestPureJavaCrc32.java

@@ -143,16 +143,14 @@ public class TestPureJavaCrc32 {
       for (int j = 0; j < tables.length; j++) {
         final int[] t = tables[j];
         final StringBuilder b = new StringBuilder();
-        b.append(String.format("  static final int[] " + nameformat
-            + " = new int[] {", j));
+        b.append(String.format("    /* "+ nameformat +" */", j));
         for (int i = 0; i < t.length;) {
           b.append("\n    ");
           for(int k = 0; k < 4; k++) {
             b.append(String.format("0x%08X, ", t[i++]));
           }
         }
-        b.setCharAt(b.length() - 2, '\n');
-        s[j] = b.toString() + " };\n";
+        s[j] = b.toString();
       }
       return s;
     }
@@ -160,10 +158,23 @@ public class TestPureJavaCrc32 {
     /** {@inheritDoc} */
     public String toString() {
       final StringBuilder b = new StringBuilder();
-      for(String s : toStrings(String.format("T%d_",
-          Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
+
+      final String tableFormat = String.format("T%d_", 
+        Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
+      final String startFormat = "  private static final int "+tableFormat+"_start = %d*256;";
+
+      for (int j = 0; j < tables.length; j++) {
+        b.append(String.format(startFormat, j, j));
+        b.append("\n");
+      }
+
+      b.append("  private static final int[] T = new int[] {");
+      for(String s : toStrings(tableFormat)) {
+        b.append("\n");
         b.append(s);
       }
+      b.setCharAt(b.length() - 2, '\n');
+      b.append(" };\n");
       return b.toString();
     }