1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.nio.ByteBuffer;
23 import java.util.Random;
24
25 import org.apache.hadoop.hbase.io.HeapSize;
26 import org.apache.hadoop.hbase.util.ClassSize;
27
28 import junit.framework.TestCase;
29
30
31
32
33
34
35
36
37 public class TestLruBlockCache extends TestCase {
38
39 public void testBackgroundEvictionThread() throws Exception {
40
41 long maxSize = 100000;
42 long blockSize = calculateBlockSizeDefault(maxSize, 9);
43
44 LruBlockCache cache = new LruBlockCache(maxSize,blockSize);
45
46 CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
47
48
49 for (CachedItem block : blocks) {
50 cache.cacheBlock(block.blockName, block);
51 }
52
53
54 int n = 0;
55 while(cache.getEvictionCount() == 0) {
56 Thread.sleep(200);
57 assertTrue(n++ < 10);
58 }
59 System.out.println("Background Evictions run: " + cache.getEvictionCount());
60
61
62 assertEquals(cache.getEvictionCount(), 1);
63 }
64
65 public void testCacheSimple() throws Exception {
66
67 long maxSize = 1000000;
68 long blockSize = calculateBlockSizeDefault(maxSize, 101);
69
70 LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
71
72 CachedItem [] blocks = generateRandomBlocks(100, blockSize);
73
74 long expectedCacheSize = cache.heapSize();
75
76
77 for (CachedItem block : blocks) {
78 assertTrue(cache.getBlock(block.blockName, true) == null);
79 }
80
81
82 for (CachedItem block : blocks) {
83 cache.cacheBlock(block.blockName, block);
84 expectedCacheSize += block.cacheBlockHeapSize();
85 }
86
87
88 assertEquals(expectedCacheSize, cache.heapSize());
89
90
91 for (CachedItem block : blocks) {
92 HeapSize buf = cache.getBlock(block.blockName, true);
93 assertTrue(buf != null);
94 assertEquals(buf.heapSize(), block.heapSize());
95 }
96
97
98 for (CachedItem block : blocks) {
99 try {
100 cache.cacheBlock(block.blockName, block);
101 assertTrue("Cache should not allow re-caching a block", false);
102 } catch(RuntimeException re) {
103
104 }
105 }
106
107
108 assertEquals(expectedCacheSize, cache.heapSize());
109
110
111 for (CachedItem block : blocks) {
112 HeapSize buf = cache.getBlock(block.blockName, true);
113 assertTrue(buf != null);
114 assertEquals(buf.heapSize(), block.heapSize());
115 }
116
117
118 assertEquals(0, cache.getEvictionCount());
119 Thread t = new LruBlockCache.StatisticsThread(cache);
120 t.start();
121 t.join();
122 }
123
124 public void testCacheEvictionSimple() throws Exception {
125
126 long maxSize = 100000;
127 long blockSize = calculateBlockSizeDefault(maxSize, 10);
128
129 LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
130
131 CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
132
133 long expectedCacheSize = cache.heapSize();
134
135
136 for (CachedItem block : blocks) {
137 cache.cacheBlock(block.blockName, block);
138 expectedCacheSize += block.cacheBlockHeapSize();
139 }
140
141
142 assertEquals(1, cache.getEvictionCount());
143
144
145 assertTrue(expectedCacheSize >
146 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
147
148
149 assertTrue(cache.heapSize() < maxSize);
150
151
152 assertTrue(cache.heapSize() <
153 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
154
155
156 assertTrue(cache.getBlock(blocks[0].blockName, true) == null);
157 assertTrue(cache.getBlock(blocks[1].blockName, true) == null);
158 for(int i=2;i<blocks.length;i++) {
159 assertEquals(cache.getBlock(blocks[i].blockName, true),
160 blocks[i]);
161 }
162 }
163
164 public void testCacheEvictionTwoPriorities() throws Exception {
165
166 long maxSize = 100000;
167 long blockSize = calculateBlockSizeDefault(maxSize, 10);
168
169 LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
170
171 CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single");
172 CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi");
173
174 long expectedCacheSize = cache.heapSize();
175
176
177 for (CachedItem block : multiBlocks) {
178 cache.cacheBlock(block.blockName, block);
179 expectedCacheSize += block.cacheBlockHeapSize();
180 assertEquals(cache.getBlock(block.blockName, true), block);
181 }
182
183
184 for (CachedItem block : singleBlocks) {
185 cache.cacheBlock(block.blockName, block);
186 expectedCacheSize += block.heapSize();
187 }
188
189
190 assertEquals(cache.getEvictionCount(), 1);
191
192
193 assertEquals(cache.getEvictedCount(), 2);
194
195
196 assertTrue(expectedCacheSize >
197 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
198
199
200 assertTrue(cache.heapSize() <= maxSize);
201
202
203 assertTrue(cache.heapSize() <=
204 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
205
206
207
208
209
210 assertTrue(cache.getBlock(singleBlocks[0].blockName, true) == null);
211 assertTrue(cache.getBlock(multiBlocks[0].blockName, true) == null);
212
213
214 for(int i=1;i<4;i++) {
215 assertEquals(cache.getBlock(singleBlocks[i].blockName, true),
216 singleBlocks[i]);
217 assertEquals(cache.getBlock(multiBlocks[i].blockName, true),
218 multiBlocks[i]);
219 }
220 }
221
222 public void testCacheEvictionThreePriorities() throws Exception {
223
224 long maxSize = 100000;
225 long blockSize = calculateBlockSize(maxSize, 10);
226
227 LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
228 (int)Math.ceil(1.2*maxSize/blockSize),
229 LruBlockCache.DEFAULT_LOAD_FACTOR,
230 LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
231 0.98f,
232 0.99f,
233 0.33f,
234 0.33f,
235 0.34f);
236
237
238 CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single");
239 CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
240 CachedItem [] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
241
242 long expectedCacheSize = cache.heapSize();
243
244
245 for(int i=0;i<3;i++) {
246
247
248 cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
249 expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
250
251
252 cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i]);
253 expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
254 cache.getBlock(multiBlocks[i].blockName, true);
255
256
257 cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i], true);
258 expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize();
259
260 }
261
262
263 assertEquals(0, cache.getEvictionCount());
264
265
266 assertEquals(expectedCacheSize, cache.heapSize());
267
268
269 cache.cacheBlock(singleBlocks[3].blockName, singleBlocks[3]);
270
271
272 assertEquals(1, cache.getEvictionCount());
273 assertEquals(1, cache.getEvictedCount());
274
275
276 assertEquals(null, cache.getBlock(singleBlocks[0].blockName, true));
277
278
279 cache.getBlock(singleBlocks[1].blockName, true);
280
281
282 cache.cacheBlock(singleBlocks[4].blockName, singleBlocks[4]);
283
284
285 assertEquals(2, cache.getEvictionCount());
286 assertEquals(2, cache.getEvictedCount());
287
288
289 assertEquals(null, cache.getBlock(multiBlocks[0].blockName, true));
290
291
292 cache.cacheBlock(memoryBlocks[3].blockName, memoryBlocks[3], true);
293
294
295 assertEquals(3, cache.getEvictionCount());
296 assertEquals(3, cache.getEvictedCount());
297
298
299 assertEquals(null, cache.getBlock(memoryBlocks[0].blockName, true));
300
301
302 CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
303 cache.cacheBlock(bigBlocks[0].blockName, bigBlocks[0]);
304
305
306 assertEquals(4, cache.getEvictionCount());
307 assertEquals(6, cache.getEvictedCount());
308
309
310 assertEquals(null, cache.getBlock(singleBlocks[2].blockName, true));
311 assertEquals(null, cache.getBlock(singleBlocks[3].blockName, true));
312 assertEquals(null, cache.getBlock(singleBlocks[4].blockName, true));
313
314
315 cache.getBlock(bigBlocks[0].blockName, true);
316
317
318 cache.cacheBlock(bigBlocks[1].blockName, bigBlocks[1]);
319
320
321 assertEquals(5, cache.getEvictionCount());
322 assertEquals(9, cache.getEvictedCount());
323
324
325 assertEquals(null, cache.getBlock(singleBlocks[1].blockName, true));
326 assertEquals(null, cache.getBlock(multiBlocks[1].blockName, true));
327 assertEquals(null, cache.getBlock(multiBlocks[2].blockName, true));
328
329
330 cache.cacheBlock(bigBlocks[2].blockName, bigBlocks[2], true);
331
332
333 assertEquals(6, cache.getEvictionCount());
334 assertEquals(12, cache.getEvictedCount());
335
336
337 assertEquals(null, cache.getBlock(memoryBlocks[1].blockName, true));
338 assertEquals(null, cache.getBlock(memoryBlocks[2].blockName, true));
339 assertEquals(null, cache.getBlock(memoryBlocks[3].blockName, true));
340
341
342 }
343
344
345 public void testScanResistance() throws Exception {
346
347 long maxSize = 100000;
348 long blockSize = calculateBlockSize(maxSize, 10);
349
350 LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
351 (int)Math.ceil(1.2*maxSize/blockSize),
352 LruBlockCache.DEFAULT_LOAD_FACTOR,
353 LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
354 0.66f,
355 0.99f,
356 0.33f,
357 0.33f,
358 0.34f);
359
360 CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
361 CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
362
363
364 for (CachedItem block : multiBlocks) {
365 cache.cacheBlock(block.blockName, block);
366 cache.getBlock(block.blockName, true);
367 }
368
369
370 for(int i=0;i<5;i++) {
371 cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
372 }
373
374
375 assertEquals(1, cache.getEvictionCount());
376
377
378 assertEquals(4, cache.getEvictedCount());
379
380
381 assertEquals(null, cache.getBlock(singleBlocks[0].blockName, true));
382 assertEquals(null, cache.getBlock(singleBlocks[1].blockName, true));
383 assertEquals(null, cache.getBlock(multiBlocks[0].blockName, true));
384 assertEquals(null, cache.getBlock(multiBlocks[1].blockName, true));
385
386
387
388
389
390
391
392
393 for(int i=5;i<18;i++) {
394 cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
395 }
396
397
398 assertEquals(4, cache.getEvictionCount());
399 assertEquals(16, cache.getEvictedCount());
400
401
402 assertEquals(7, cache.size());
403
404 }
405
406
407 public void testResizeBlockCache() throws Exception {
408
409 long maxSize = 300000;
410 long blockSize = calculateBlockSize(maxSize, 31);
411
412 LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
413 (int)Math.ceil(1.2*maxSize/blockSize),
414 LruBlockCache.DEFAULT_LOAD_FACTOR,
415 LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
416 0.98f,
417 0.99f,
418 0.33f,
419 0.33f,
420 0.34f);
421
422 CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single");
423 CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
424 CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
425
426
427 for(int i=0;i<10;i++) {
428
429
430 cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i]);
431
432
433 cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i]);
434 cache.getBlock(multiBlocks[i].blockName, true);
435
436
437 cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i], true);
438 }
439
440
441 assertEquals(0, cache.getEvictionCount());
442
443
444 cache.setMaxSize((long)(maxSize * 0.5f));
445
446
447 assertEquals(1, cache.getEvictionCount());
448
449
450 assertEquals(15, cache.getEvictedCount());
451
452
453 for(int i=0;i<5;i++) {
454 assertEquals(null, cache.getBlock(singleBlocks[i].blockName, true));
455 assertEquals(null, cache.getBlock(multiBlocks[i].blockName, true));
456 assertEquals(null, cache.getBlock(memoryBlocks[i].blockName, true));
457 }
458
459
460 for(int i=5;i<10;i++) {
461 assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].blockName, true));
462 assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].blockName, true));
463 assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].blockName, true));
464 }
465 }
466
467 private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
468 CachedItem [] blocks = new CachedItem[numBlocks];
469 for(int i=0;i<numBlocks;i++) {
470 blocks[i] = new CachedItem(pfx + i, size);
471 }
472 return blocks;
473 }
474
475 private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
476 return generateFixedBlocks(numBlocks, (int)size, pfx);
477 }
478
479 private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
480 CachedItem [] blocks = new CachedItem[numBlocks];
481 Random r = new Random();
482 for(int i=0;i<numBlocks;i++) {
483 blocks[i] = new CachedItem("block" + i, r.nextInt((int)maxSize)+1);
484 }
485 return blocks;
486 }
487
488 private long calculateBlockSize(long maxSize, int numBlocks) {
489 long roughBlockSize = maxSize / numBlocks;
490 int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
491 long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
492 ClassSize.CONCURRENT_HASHMAP +
493 (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
494 (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
495 long negateBlockSize = (long)(totalOverhead/numEntries);
496 negateBlockSize += CachedBlock.PER_BLOCK_OVERHEAD;
497 return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*0.99f));
498 }
499
500 private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
501 long roughBlockSize = maxSize / numBlocks;
502 int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
503 long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
504 ClassSize.CONCURRENT_HASHMAP +
505 (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
506 (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
507 long negateBlockSize = totalOverhead / numEntries;
508 negateBlockSize += CachedBlock.PER_BLOCK_OVERHEAD;
509 return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*
510 LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
511 }
512
513 private static class CachedItem implements Cacheable {
514 String blockName;
515 int size;
516
517 CachedItem(String blockName, int size) {
518 this.blockName = blockName;
519 this.size = size;
520 }
521
522
523 @Override
524 public long heapSize() {
525 return ClassSize.align(size);
526 }
527
528
529 public long cacheBlockHeapSize() {
530 return CachedBlock.PER_BLOCK_OVERHEAD
531 + ClassSize.align(blockName.length())
532 + ClassSize.align(size);
533 }
534
535 @Override
536 public int getSerializedLength() {
537 return 0;
538 }
539
540 @Override
541 public CacheableDeserializer<Cacheable> getDeserializer() {
542 return null;
543 }
544
545 @Override
546 public void serialize(ByteBuffer destination) {
547 }
548
549 }
550 }