|
@@ -0,0 +1,456 @@
|
|
|
+/**
|
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one
|
|
|
+ * or more contributor license agreements. See the NOTICE file
|
|
|
+ * distributed with this work for additional information
|
|
|
+ * regarding copyright ownership. The ASF licenses this file
|
|
|
+ * to you under the Apache License, Version 2.0 (the
|
|
|
+ * "License"); you may not use this file except in compliance
|
|
|
+ * with the License. You may obtain a copy of the License at
|
|
|
+ *
|
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+ *
|
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
+ * See the License for the specific language governing permissions and
|
|
|
+ * limitations under the License.
|
|
|
+ */
|
|
|
+package org.apache.hadoop.util;
|
|
|
+
|
|
|
+import java.util.Date;
|
|
|
+import java.util.Iterator;
|
|
|
+import java.util.Random;
|
|
|
+
|
|
|
+import org.junit.Assert;
|
|
|
+import org.junit.Test;
|
|
|
+
|
|
|
+/** Testing {@link LightWeightCache} */
|
|
|
+public class TestLightWeightCache {
|
|
|
+ private static final long starttime = Time.now();
|
|
|
+ private static final Random ran = new Random(starttime);
|
|
|
+ static {
|
|
|
+ println("Start time = " + new Date(starttime) + ", seed=" + starttime);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void print(Object s) {
|
|
|
+ System.out.print(s);
|
|
|
+ System.out.flush();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void println(Object s) {
|
|
|
+ System.out.println(s);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void testLightWeightCache() {
|
|
|
+ // test randomized creation expiration with zero access expiration
|
|
|
+ {
|
|
|
+ final long creationExpiration = ran.nextInt(1024) + 1;
|
|
|
+ check(1, creationExpiration, 0L, 1 << 10, 65537);
|
|
|
+ check(17, creationExpiration, 0L, 1 << 16, 17);
|
|
|
+ check(255, creationExpiration, 0L, 1 << 16, 65537);
|
|
|
+ }
|
|
|
+
|
|
|
+ // test randomized creation/access expiration periods
|
|
|
+ for(int i = 0; i < 3; i++) {
|
|
|
+ final long creationExpiration = ran.nextInt(1024) + 1;
|
|
|
+ final long accessExpiration = ran.nextInt(1024) + 1;
|
|
|
+
|
|
|
+ check(1, creationExpiration, accessExpiration, 1 << 10, 65537);
|
|
|
+ check(17, creationExpiration, accessExpiration, 1 << 16, 17);
|
|
|
+ check(255, creationExpiration, accessExpiration, 1 << 16, 65537);
|
|
|
+ }
|
|
|
+
|
|
|
+ // test size limit
|
|
|
+ final int dataSize = 1 << 16;
|
|
|
+ for(int i = 0; i < 10; i++) {
|
|
|
+ final int modulus = ran.nextInt(1024) + 1;
|
|
|
+ final int sizeLimit = ran.nextInt(modulus);
|
|
|
+ checkSizeLimit(sizeLimit, dataSize, modulus);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void checkSizeLimit(final int sizeLimit, final int datasize,
|
|
|
+ final int modulus) {
|
|
|
+ final LightWeightCacheTestCase test = new LightWeightCacheTestCase(
|
|
|
+ sizeLimit, sizeLimit, 1L << 32, 1L << 32, datasize, modulus);
|
|
|
+
|
|
|
+ // keep putting entries and check size limit
|
|
|
+ print(" check size ................. ");
|
|
|
+ for(int i = 0; i < test.data.size(); i++) {
|
|
|
+ test.cache.put(test.data.get(i));
|
|
|
+ Assert.assertTrue(test.cache.size() <= sizeLimit);
|
|
|
+ }
|
|
|
+ println("DONE " + test.stat());
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Test various createionExpirationPeriod and accessExpirationPeriod.
|
|
|
+ * It runs ~2 minutes. If you are changing the implementation,
|
|
|
+ * please un-comment the following line in order to run the test.
|
|
|
+ */
|
|
|
+// @Test
|
|
|
+ public void testExpirationPeriods() {
|
|
|
+ for(int k = -4; k < 10; k += 4) {
|
|
|
+ final long accessExpirationPeriod = k < 0? 0L: (1L << k);
|
|
|
+ for(int j = 0; j < 10; j += 4) {
|
|
|
+ final long creationExpirationPeriod = 1L << j;
|
|
|
+ runTests(1, creationExpirationPeriod, accessExpirationPeriod);
|
|
|
+ for(int i = 1; i < Integer.SIZE - 1; i += 8) {
|
|
|
+ runTests((1 << i) + 1, creationExpirationPeriod, accessExpirationPeriod);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** Run tests with various table lengths. */
|
|
|
+ private static void runTests(final int modulus,
|
|
|
+ final long creationExpirationPeriod,
|
|
|
+ final long accessExpirationPeriod) {
|
|
|
+ println("\n\n\n*** runTest: modulus=" + modulus
|
|
|
+ + ", creationExpirationPeriod=" + creationExpirationPeriod
|
|
|
+ + ", accessExpirationPeriod=" + accessExpirationPeriod);
|
|
|
+ for(int i = 0; i <= 16; i += 4) {
|
|
|
+ final int tablelength = (1 << i);
|
|
|
+
|
|
|
+ final int upper = i + 2;
|
|
|
+ final int steps = Math.max(1, upper/3);
|
|
|
+
|
|
|
+ for(int j = upper; j > 0; j -= steps) {
|
|
|
+ final int datasize = 1 << j;
|
|
|
+ check(tablelength, creationExpirationPeriod, accessExpirationPeriod,
|
|
|
+ datasize, modulus);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void check(int tablelength, long creationExpirationPeriod,
|
|
|
+ long accessExpirationPeriod, int datasize, int modulus) {
|
|
|
+ check(new LightWeightCacheTestCase(tablelength, -1,
|
|
|
+ creationExpirationPeriod, accessExpirationPeriod, datasize, modulus));
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * check the following operations
|
|
|
+ * (1) put
|
|
|
+ * (2) remove & put
|
|
|
+ * (3) remove
|
|
|
+ * (4) remove & put again
|
|
|
+ */
|
|
|
+ private static void check(final LightWeightCacheTestCase test) {
|
|
|
+ //check put
|
|
|
+ print(" check put .................. ");
|
|
|
+ for(int i = 0; i < test.data.size()/2; i++) {
|
|
|
+ test.put(test.data.get(i));
|
|
|
+ }
|
|
|
+ for(int i = 0; i < test.data.size(); i++) {
|
|
|
+ test.put(test.data.get(i));
|
|
|
+ }
|
|
|
+ println("DONE " + test.stat());
|
|
|
+
|
|
|
+ //check remove and put
|
|
|
+ print(" check remove & put ......... ");
|
|
|
+ for(int j = 0; j < 10; j++) {
|
|
|
+ for(int i = 0; i < test.data.size()/2; i++) {
|
|
|
+ final int r = ran.nextInt(test.data.size());
|
|
|
+ test.remove(test.data.get(r));
|
|
|
+ }
|
|
|
+ for(int i = 0; i < test.data.size()/2; i++) {
|
|
|
+ final int r = ran.nextInt(test.data.size());
|
|
|
+ test.put(test.data.get(r));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ println("DONE " + test.stat());
|
|
|
+
|
|
|
+ //check remove
|
|
|
+ print(" check remove ............... ");
|
|
|
+ for(int i = 0; i < test.data.size(); i++) {
|
|
|
+ test.remove(test.data.get(i));
|
|
|
+ }
|
|
|
+ Assert.assertEquals(0, test.cache.size());
|
|
|
+ println("DONE " + test.stat());
|
|
|
+
|
|
|
+ //check remove and put again
|
|
|
+ print(" check remove & put again ... ");
|
|
|
+ for(int j = 0; j < 10; j++) {
|
|
|
+ for(int i = 0; i < test.data.size()/2; i++) {
|
|
|
+ final int r = ran.nextInt(test.data.size());
|
|
|
+ test.remove(test.data.get(r));
|
|
|
+ }
|
|
|
+ for(int i = 0; i < test.data.size()/2; i++) {
|
|
|
+ final int r = ran.nextInt(test.data.size());
|
|
|
+ test.put(test.data.get(r));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ println("DONE " + test.stat());
|
|
|
+
|
|
|
+ final long s = (Time.now() - starttime)/1000L;
|
|
|
+ println("total time elapsed=" + s + "s\n");
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * The test case contains two data structures, a cache and a hashMap.
|
|
|
+ * The hashMap is used to verify the correctness of the cache. Note that
|
|
|
+ * no automatic eviction is performed in the hashMap. Thus, we have
|
|
|
+ * (1) If an entry exists in cache, it MUST exist in the hashMap.
|
|
|
+ * (2) If an entry does not exist in the cache, it may or may not exist in the
|
|
|
+ * hashMap. If it exists, it must be expired.
|
|
|
+ */
|
|
|
+ private static class LightWeightCacheTestCase implements GSet<IntEntry, IntEntry> {
|
|
|
+ /** hashMap will not evict entries automatically. */
|
|
|
+ final GSet<IntEntry, IntEntry> hashMap
|
|
|
+ = new GSetByHashMap<IntEntry, IntEntry>(1024, 0.75f);
|
|
|
+
|
|
|
+ final LightWeightCache<IntEntry, IntEntry> cache;
|
|
|
+ final IntData data;
|
|
|
+
|
|
|
+ final String info;
|
|
|
+ final long starttime = Time.now();
|
|
|
+ /** Determine the probability in {@link #check()}. */
|
|
|
+ final int denominator;
|
|
|
+ int iterate_count = 0;
|
|
|
+ int contain_count = 0;
|
|
|
+
|
|
|
+ private long currentTestTime = ran.nextInt();
|
|
|
+
|
|
|
+ LightWeightCacheTestCase(int tablelength, int sizeLimit,
|
|
|
+ long creationExpirationPeriod, long accessExpirationPeriod,
|
|
|
+ int datasize, int modulus) {
|
|
|
+ denominator = Math.min((datasize >> 7) + 1, 1 << 16);
|
|
|
+ info = getClass().getSimpleName() + "(" + new Date(starttime)
|
|
|
+ + "): tablelength=" + tablelength
|
|
|
+ + ", creationExpirationPeriod=" + creationExpirationPeriod
|
|
|
+ + ", accessExpirationPeriod=" + accessExpirationPeriod
|
|
|
+ + ", datasize=" + datasize
|
|
|
+ + ", modulus=" + modulus
|
|
|
+ + ", denominator=" + denominator;
|
|
|
+ println(info);
|
|
|
+
|
|
|
+ data = new IntData(datasize, modulus);
|
|
|
+ cache = new LightWeightCache<IntEntry, IntEntry>(tablelength, sizeLimit,
|
|
|
+ creationExpirationPeriod, 0, new LightWeightCache.Clock() {
|
|
|
+ @Override
|
|
|
+ long currentTime() {
|
|
|
+ return currentTestTime;
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ Assert.assertEquals(0, cache.size());
|
|
|
+ }
|
|
|
+
|
|
|
+ private boolean containsTest(IntEntry key) {
|
|
|
+ final boolean c = cache.contains(key);
|
|
|
+ if (c) {
|
|
|
+ Assert.assertTrue(hashMap.contains(key));
|
|
|
+ } else {
|
|
|
+ final IntEntry h = hashMap.remove(key);
|
|
|
+ if (h != null) {
|
|
|
+ Assert.assertTrue(cache.isExpired(h, currentTestTime));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return c;
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public boolean contains(IntEntry key) {
|
|
|
+ final boolean e = containsTest(key);
|
|
|
+ check();
|
|
|
+ return e;
|
|
|
+ }
|
|
|
+
|
|
|
+ private IntEntry getTest(IntEntry key) {
|
|
|
+ final IntEntry c = cache.get(key);
|
|
|
+ if (c != null) {
|
|
|
+ Assert.assertEquals(hashMap.get(key).id, c.id);
|
|
|
+ } else {
|
|
|
+ final IntEntry h = hashMap.remove(key);
|
|
|
+ if (h != null) {
|
|
|
+ Assert.assertTrue(cache.isExpired(h, currentTestTime));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return c;
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public IntEntry get(IntEntry key) {
|
|
|
+ final IntEntry e = getTest(key);
|
|
|
+ check();
|
|
|
+ return e;
|
|
|
+ }
|
|
|
+
|
|
|
+ private IntEntry putTest(IntEntry entry) {
|
|
|
+ final IntEntry c = cache.put(entry);
|
|
|
+ if (c != null) {
|
|
|
+ Assert.assertEquals(hashMap.put(entry).id, c.id);
|
|
|
+ } else {
|
|
|
+ final IntEntry h = hashMap.put(entry);
|
|
|
+ if (h != null && h != entry) {
|
|
|
+ // if h == entry, its expiration time is already updated
|
|
|
+ Assert.assertTrue(cache.isExpired(h, currentTestTime));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return c;
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public IntEntry put(IntEntry entry) {
|
|
|
+ final IntEntry e = putTest(entry);
|
|
|
+ check();
|
|
|
+ return e;
|
|
|
+ }
|
|
|
+
|
|
|
+ private IntEntry removeTest(IntEntry key) {
|
|
|
+ final IntEntry c = cache.remove(key);
|
|
|
+ if (c != null) {
|
|
|
+ Assert.assertEquals(c.id, hashMap.remove(key).id);
|
|
|
+ } else {
|
|
|
+ final IntEntry h = hashMap.remove(key);
|
|
|
+ if (h != null) {
|
|
|
+ Assert.assertTrue(cache.isExpired(h, currentTestTime));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return c;
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public IntEntry remove(IntEntry key) {
|
|
|
+ final IntEntry e = removeTest(key);
|
|
|
+ check();
|
|
|
+ return e;
|
|
|
+ }
|
|
|
+
|
|
|
+ private int sizeTest() {
|
|
|
+ final int c = cache.size();
|
|
|
+ Assert.assertTrue(hashMap.size() >= c);
|
|
|
+ return c;
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public int size() {
|
|
|
+ final int s = sizeTest();
|
|
|
+ check();
|
|
|
+ return s;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public Iterator<IntEntry> iterator() {
|
|
|
+ throw new UnsupportedOperationException();
|
|
|
+ }
|
|
|
+
|
|
|
+ boolean tossCoin() {
|
|
|
+ return ran.nextInt(denominator) == 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ void check() {
|
|
|
+ currentTestTime += ran.nextInt() & 0x3;
|
|
|
+
|
|
|
+ //test size
|
|
|
+ sizeTest();
|
|
|
+
|
|
|
+ if (tossCoin()) {
|
|
|
+ //test get(..), check content and test iterator
|
|
|
+ iterate_count++;
|
|
|
+ for(IntEntry i : cache) {
|
|
|
+ getTest(i);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (tossCoin()) {
|
|
|
+ //test contains(..)
|
|
|
+ contain_count++;
|
|
|
+ final int count = Math.min(data.size(), 1000);
|
|
|
+ if (count == data.size()) {
|
|
|
+ for(IntEntry i : data.integers) {
|
|
|
+ containsTest(i);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ for(int j = 0; j < count; j++) {
|
|
|
+ containsTest(data.get(ran.nextInt(data.size())));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ String stat() {
|
|
|
+ final long t = Time.now() - starttime;
|
|
|
+ return String.format(" iterate=%5d, contain=%5d, time elapsed=%5d.%03ds",
|
|
|
+ iterate_count, contain_count, t/1000, t%1000);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void clear() {
|
|
|
+ hashMap.clear();
|
|
|
+ cache.clear();
|
|
|
+ Assert.assertEquals(0, size());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class IntData {
|
|
|
+ final IntEntry[] integers;
|
|
|
+
|
|
|
+ IntData(int size, int modulus) {
|
|
|
+ integers = new IntEntry[size];
|
|
|
+ for(int i = 0; i < integers.length; i++) {
|
|
|
+ integers[i] = new IntEntry(i, ran.nextInt(modulus));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ IntEntry get(int i) {
|
|
|
+ return integers[i];
|
|
|
+ }
|
|
|
+
|
|
|
+ int size() {
|
|
|
+ return integers.length;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** Entries of {@link LightWeightCache} in this test */
|
|
|
+ private static class IntEntry implements LightWeightCache.Entry,
|
|
|
+ Comparable<IntEntry> {
|
|
|
+ private LightWeightGSet.LinkedElement next;
|
|
|
+ final int id;
|
|
|
+ final int value;
|
|
|
+ private long expirationTime = 0;
|
|
|
+
|
|
|
+ IntEntry(int id, int value) {
|
|
|
+ this.id = id;
|
|
|
+ this.value = value;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean equals(Object obj) {
|
|
|
+ return obj != null && obj instanceof IntEntry
|
|
|
+ && value == ((IntEntry)obj).value;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public int hashCode() {
|
|
|
+ return value;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public int compareTo(IntEntry that) {
|
|
|
+ return value - that.value;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public String toString() {
|
|
|
+ return id + "#" + value + ",expirationTime=" + expirationTime;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public LightWeightGSet.LinkedElement getNext() {
|
|
|
+ return next;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void setNext(LightWeightGSet.LinkedElement e) {
|
|
|
+ next = e;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void setExpirationTime(long timeNano) {
|
|
|
+ this.expirationTime = timeNano;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public long getExpirationTime() {
|
|
|
+ return expirationTime;
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|