1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import static org.junit.Assert.assertArrayEquals;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.nio.ByteBuffer;
30  import java.util.Arrays;
31  import java.util.HashSet;
32  import java.util.Random;
33  import java.util.concurrent.ConcurrentLinkedQueue;
34  import java.util.concurrent.atomic.AtomicInteger;
35  
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.MultithreadedTestUtil;
38  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
39  import org.apache.hadoop.hbase.io.HeapSize;
40  
41  public class CacheTestUtils {
42  
43    /*Just checks if heapsize grows when something is cached, and gets smaller when the same object is evicted*/
44  
45    public static void testHeapSizeChanges(final BlockCache toBeTested, final int blockSize){
46      HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1);
47      long heapSize = ((HeapSize) toBeTested).heapSize();
48      toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block);
49  
50      /*When we cache something HeapSize should always increase */
51      assertTrue(heapSize < ((HeapSize) toBeTested).heapSize());
52  
53      toBeTested.evictBlock(blocks[0].blockName);
54  
55      /*Post eviction, heapsize should be the same */
56      assertEquals(heapSize, ((HeapSize) toBeTested).heapSize());
57    }
58    public static void testCacheMultiThreaded(final BlockCache toBeTested,
59        final int blockSize, final int numThreads, final int numQueries,
60        final double passingScore) throws Exception {
61  
62      Configuration conf = new Configuration();
63      MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(
64          conf);
65  
66      final AtomicInteger totalQueries = new AtomicInteger();
67      final ConcurrentLinkedQueue<HFileBlockPair> blocksToTest = new ConcurrentLinkedQueue<HFileBlockPair>();
68      final AtomicInteger hits = new AtomicInteger();
69      final AtomicInteger miss = new AtomicInteger();
70  
71      HFileBlockPair[] blocks = generateHFileBlocks(numQueries, blockSize);
72      blocksToTest.addAll(Arrays.asList(blocks));
73  
74      for (int i = 0; i < numThreads; i++) {
75        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
76          @Override
77          public void doAnAction() throws Exception {
78            if (!blocksToTest.isEmpty()) {
79              HFileBlockPair ourBlock = blocksToTest.poll();
80              // if we run out of blocks to test, then we should stop the tests.
81              if (ourBlock == null) {
82                ctx.setStopFlag(true);
83                return;
84              }
85              toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block);
86              Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName,
87                  false);
88              if (retrievedBlock != null) {
89                assertEquals(ourBlock.block, retrievedBlock);
90                toBeTested.evictBlock(ourBlock.blockName);
91                hits.incrementAndGet();
92                assertNull(toBeTested.getBlock(ourBlock.blockName, false));
93              } else {
94                miss.incrementAndGet();
95              }
96              totalQueries.incrementAndGet();
97            }
98          }
99        };
100       t.setDaemon(true);
101       ctx.addThread(t);
102     }
103     ctx.startThreads();
104     while (!blocksToTest.isEmpty() && ctx.shouldRun()) {
105       Thread.sleep(10);
106     }
107     ctx.stop();
108     if (hits.get() / ((double) hits.get() + (double) miss.get()) < passingScore) {
109       fail("Too many nulls returned. Hits: " + hits.get() + " Misses: "
110           + miss.get());
111     }
112   }
113 
114   public static void testCacheSimple(BlockCache toBeTested, int blockSize,
115       int numBlocks) throws Exception {
116 
117     HFileBlockPair[] blocks = generateHFileBlocks(numBlocks, blockSize);
118     // Confirm empty
119     for (HFileBlockPair block : blocks) {
120       assertNull(toBeTested.getBlock(block.blockName, true));
121     }
122 
123     // Add blocks
124     for (HFileBlockPair block : blocks) {
125       toBeTested.cacheBlock(block.blockName, block.block);
126     }
127 
128     // Check if all blocks are properly cached and contain the right
129     // information, or the blocks are null.
130     // MapMaker makes no guarantees when it will evict, so neither can we.
131 
132     for (HFileBlockPair block : blocks) {
133       HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true);
134       if (buf != null) {
135         assertEquals(block.block, buf);
136       }
137 
138     }
139 
140     // Re-add some duplicate blocks. Hope nothing breaks.
141 
142     for (HFileBlockPair block : blocks) {
143       try {
144         if (toBeTested.getBlock(block.blockName, true) != null) {
145           toBeTested.cacheBlock(block.blockName, block.block);
146           fail("Cache should not allow re-caching a block");
147         }
148       } catch (RuntimeException re) {
149         // expected
150       }
151     }
152 
153   }
154 
155   public static void hammerSingleKey(final BlockCache toBeTested,
156       int BlockSize, int numThreads, int numQueries) throws Exception {
157     final String key = "key";
158     final byte[] buf = new byte[5 * 1024];
159     Arrays.fill(buf, (byte) 5);
160 
161     final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
162     Configuration conf = new Configuration();
163     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(
164         conf);
165 
166     final AtomicInteger totalQueries = new AtomicInteger();
167     toBeTested.cacheBlock(key, bac);
168 
169     for (int i = 0; i < numThreads; i++) {
170       TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
171         @Override
172         public void doAnAction() throws Exception {
173           ByteArrayCacheable returned = (ByteArrayCacheable) toBeTested
174               .getBlock(key, false);
175           assertArrayEquals(buf, returned.buf);
176           totalQueries.incrementAndGet();
177         }
178       };
179 
180       t.setDaemon(true);
181       ctx.addThread(t);
182     }
183 
184     ctx.startThreads();
185     while (totalQueries.get() < numQueries && ctx.shouldRun()) {
186       Thread.sleep(10);
187     }
188     ctx.stop();
189   }
190 
191   public static void hammerEviction(final BlockCache toBeTested, int BlockSize,
192       int numThreads, int numQueries) throws Exception {
193 
194     Configuration conf = new Configuration();
195     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(
196         conf);
197 
198     final AtomicInteger totalQueries = new AtomicInteger();
199 
200     for (int i = 0; i < numThreads; i++) {
201       final int finalI = i;
202 
203       final byte[] buf = new byte[5 * 1024];
204       TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
205         @Override
206         public void doAnAction() throws Exception {
207           for (int j = 0; j < 100; j++) {
208             String key = "key_" + finalI + "_" + j;
209             Arrays.fill(buf, (byte) (finalI * j));
210             final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
211 
212             ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested
213                 .getBlock(key, true);
214             if (gotBack != null) {
215               assertArrayEquals(gotBack.buf, bac.buf);
216             } else {
217               toBeTested.cacheBlock(key, bac);
218             }
219           }
220           totalQueries.incrementAndGet();
221         }
222       };
223 
224       t.setDaemon(true);
225       ctx.addThread(t);
226     }
227 
228     ctx.startThreads();
229     while (totalQueries.get() < numQueries && ctx.shouldRun()) {
230       Thread.sleep(10);
231     }
232     ctx.stop();
233 
234     assertTrue(toBeTested.getStats().getEvictedCount() > 0);
235   }
236 
237   private static class ByteArrayCacheable implements Cacheable {
238 
239     final byte[] buf;
240 
241     public ByteArrayCacheable(byte[] buf) {
242       this.buf = buf;
243     }
244 
245     @Override
246     public long heapSize() {
247       return 4 + buf.length;
248     }
249 
250     @Override
251     public int getSerializedLength() {
252       return 4 + buf.length;
253     }
254 
255     @Override
256     public void serialize(ByteBuffer destination) {
257       destination.putInt(buf.length);
258       Thread.yield();
259       destination.put(buf);
260       destination.rewind();
261     }
262 
263     @Override
264     public CacheableDeserializer<Cacheable> getDeserializer() {
265       return new CacheableDeserializer<Cacheable>() {
266 
267         @Override
268         public Cacheable deserialize(ByteBuffer b) throws IOException {
269           int len = b.getInt();
270           Thread.yield();
271           byte buf[] = new byte[len];
272           b.get(buf);
273           return new ByteArrayCacheable(buf);
274         }
275       };
276     }
277 
278   }
279 
280   private static HFileBlockPair[] generateHFileBlocks(int blockSize,
281       int numBlocks) {
282     HFileBlockPair[] returnedBlocks = new HFileBlockPair[numBlocks];
283     Random rand = new Random();
284     HashSet<String> usedStrings = new HashSet<String>();
285     for (int i = 0; i < numBlocks; i++) {
286 
287       // The buffer serialized size needs to match the size of BlockSize. So we
288       // declare our data size to be smaller than it by the serialization space
289       // required.
290 
291       ByteBuffer cachedBuffer = ByteBuffer.allocate(blockSize
292           - HFileBlock.EXTRA_SERIALIZATION_SPACE);
293       rand.nextBytes(cachedBuffer.array());
294       cachedBuffer.rewind();
295       int onDiskSizeWithoutHeader = blockSize
296           - HFileBlock.EXTRA_SERIALIZATION_SPACE;
297       int uncompressedSizeWithoutHeader = blockSize
298           - HFileBlock.EXTRA_SERIALIZATION_SPACE;
299       long prevBlockOffset = rand.nextLong();
300       BlockType.DATA.write(cachedBuffer);
301       cachedBuffer.putInt(onDiskSizeWithoutHeader);
302       cachedBuffer.putInt(uncompressedSizeWithoutHeader);
303       cachedBuffer.putLong(prevBlockOffset);
304       cachedBuffer.rewind();
305 
306       HFileBlock generated = new HFileBlock(BlockType.DATA,
307           onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
308           prevBlockOffset, cachedBuffer, false, blockSize);
309 
310       String strKey;
311       /* No conflicting keys */
312       for (strKey = new Long(rand.nextLong()).toString(); !usedStrings
313           .add(strKey); strKey = new Long(rand.nextLong()).toString())
314         ;
315 
316       returnedBlocks[i] = new HFileBlockPair();
317       returnedBlocks[i].blockName = strKey;
318       returnedBlocks[i].block = generated;
319     }
320     return returnedBlocks;
321   }
322 
323   private static class HFileBlockPair {
324     String blockName;
325     HFileBlock block;
326   }
327 }