1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.nio.ByteBuffer;
30 import java.util.Arrays;
31 import java.util.HashSet;
32 import java.util.Random;
33 import java.util.concurrent.ConcurrentLinkedQueue;
34 import java.util.concurrent.atomic.AtomicInteger;
35
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.MultithreadedTestUtil;
38 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
39 import org.apache.hadoop.hbase.io.HeapSize;
40
41 public class CacheTestUtils {
42
43
44
45 public static void testHeapSizeChanges(final BlockCache toBeTested, final int blockSize){
46 HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1);
47 long heapSize = ((HeapSize) toBeTested).heapSize();
48 toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block);
49
50
51 assertTrue(heapSize < ((HeapSize) toBeTested).heapSize());
52
53 toBeTested.evictBlock(blocks[0].blockName);
54
55
56 assertEquals(heapSize, ((HeapSize) toBeTested).heapSize());
57 }
58 public static void testCacheMultiThreaded(final BlockCache toBeTested,
59 final int blockSize, final int numThreads, final int numQueries,
60 final double passingScore) throws Exception {
61
62 Configuration conf = new Configuration();
63 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(
64 conf);
65
66 final AtomicInteger totalQueries = new AtomicInteger();
67 final ConcurrentLinkedQueue<HFileBlockPair> blocksToTest = new ConcurrentLinkedQueue<HFileBlockPair>();
68 final AtomicInteger hits = new AtomicInteger();
69 final AtomicInteger miss = new AtomicInteger();
70
71 HFileBlockPair[] blocks = generateHFileBlocks(numQueries, blockSize);
72 blocksToTest.addAll(Arrays.asList(blocks));
73
74 for (int i = 0; i < numThreads; i++) {
75 TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
76 @Override
77 public void doAnAction() throws Exception {
78 if (!blocksToTest.isEmpty()) {
79 HFileBlockPair ourBlock = blocksToTest.poll();
80
81 if (ourBlock == null) {
82 ctx.setStopFlag(true);
83 return;
84 }
85 toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block);
86 Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName,
87 false);
88 if (retrievedBlock != null) {
89 assertEquals(ourBlock.block, retrievedBlock);
90 toBeTested.evictBlock(ourBlock.blockName);
91 hits.incrementAndGet();
92 assertNull(toBeTested.getBlock(ourBlock.blockName, false));
93 } else {
94 miss.incrementAndGet();
95 }
96 totalQueries.incrementAndGet();
97 }
98 }
99 };
100 t.setDaemon(true);
101 ctx.addThread(t);
102 }
103 ctx.startThreads();
104 while (!blocksToTest.isEmpty() && ctx.shouldRun()) {
105 Thread.sleep(10);
106 }
107 ctx.stop();
108 if (hits.get() / ((double) hits.get() + (double) miss.get()) < passingScore) {
109 fail("Too many nulls returned. Hits: " + hits.get() + " Misses: "
110 + miss.get());
111 }
112 }
113
114 public static void testCacheSimple(BlockCache toBeTested, int blockSize,
115 int numBlocks) throws Exception {
116
117 HFileBlockPair[] blocks = generateHFileBlocks(numBlocks, blockSize);
118
119 for (HFileBlockPair block : blocks) {
120 assertNull(toBeTested.getBlock(block.blockName, true));
121 }
122
123
124 for (HFileBlockPair block : blocks) {
125 toBeTested.cacheBlock(block.blockName, block.block);
126 }
127
128
129
130
131
132 for (HFileBlockPair block : blocks) {
133 HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true);
134 if (buf != null) {
135 assertEquals(block.block, buf);
136 }
137
138 }
139
140
141
142 for (HFileBlockPair block : blocks) {
143 try {
144 if (toBeTested.getBlock(block.blockName, true) != null) {
145 toBeTested.cacheBlock(block.blockName, block.block);
146 fail("Cache should not allow re-caching a block");
147 }
148 } catch (RuntimeException re) {
149
150 }
151 }
152
153 }
154
155 public static void hammerSingleKey(final BlockCache toBeTested,
156 int BlockSize, int numThreads, int numQueries) throws Exception {
157 final String key = "key";
158 final byte[] buf = new byte[5 * 1024];
159 Arrays.fill(buf, (byte) 5);
160
161 final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
162 Configuration conf = new Configuration();
163 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(
164 conf);
165
166 final AtomicInteger totalQueries = new AtomicInteger();
167 toBeTested.cacheBlock(key, bac);
168
169 for (int i = 0; i < numThreads; i++) {
170 TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
171 @Override
172 public void doAnAction() throws Exception {
173 ByteArrayCacheable returned = (ByteArrayCacheable) toBeTested
174 .getBlock(key, false);
175 assertArrayEquals(buf, returned.buf);
176 totalQueries.incrementAndGet();
177 }
178 };
179
180 t.setDaemon(true);
181 ctx.addThread(t);
182 }
183
184 ctx.startThreads();
185 while (totalQueries.get() < numQueries && ctx.shouldRun()) {
186 Thread.sleep(10);
187 }
188 ctx.stop();
189 }
190
191 public static void hammerEviction(final BlockCache toBeTested, int BlockSize,
192 int numThreads, int numQueries) throws Exception {
193
194 Configuration conf = new Configuration();
195 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(
196 conf);
197
198 final AtomicInteger totalQueries = new AtomicInteger();
199
200 for (int i = 0; i < numThreads; i++) {
201 final int finalI = i;
202
203 final byte[] buf = new byte[5 * 1024];
204 TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
205 @Override
206 public void doAnAction() throws Exception {
207 for (int j = 0; j < 100; j++) {
208 String key = "key_" + finalI + "_" + j;
209 Arrays.fill(buf, (byte) (finalI * j));
210 final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
211
212 ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested
213 .getBlock(key, true);
214 if (gotBack != null) {
215 assertArrayEquals(gotBack.buf, bac.buf);
216 } else {
217 toBeTested.cacheBlock(key, bac);
218 }
219 }
220 totalQueries.incrementAndGet();
221 }
222 };
223
224 t.setDaemon(true);
225 ctx.addThread(t);
226 }
227
228 ctx.startThreads();
229 while (totalQueries.get() < numQueries && ctx.shouldRun()) {
230 Thread.sleep(10);
231 }
232 ctx.stop();
233
234 assertTrue(toBeTested.getStats().getEvictedCount() > 0);
235 }
236
237 private static class ByteArrayCacheable implements Cacheable {
238
239 final byte[] buf;
240
241 public ByteArrayCacheable(byte[] buf) {
242 this.buf = buf;
243 }
244
245 @Override
246 public long heapSize() {
247 return 4 + buf.length;
248 }
249
250 @Override
251 public int getSerializedLength() {
252 return 4 + buf.length;
253 }
254
255 @Override
256 public void serialize(ByteBuffer destination) {
257 destination.putInt(buf.length);
258 Thread.yield();
259 destination.put(buf);
260 destination.rewind();
261 }
262
263 @Override
264 public CacheableDeserializer<Cacheable> getDeserializer() {
265 return new CacheableDeserializer<Cacheable>() {
266
267 @Override
268 public Cacheable deserialize(ByteBuffer b) throws IOException {
269 int len = b.getInt();
270 Thread.yield();
271 byte buf[] = new byte[len];
272 b.get(buf);
273 return new ByteArrayCacheable(buf);
274 }
275 };
276 }
277
278 }
279
280 private static HFileBlockPair[] generateHFileBlocks(int blockSize,
281 int numBlocks) {
282 HFileBlockPair[] returnedBlocks = new HFileBlockPair[numBlocks];
283 Random rand = new Random();
284 HashSet<String> usedStrings = new HashSet<String>();
285 for (int i = 0; i < numBlocks; i++) {
286
287
288
289
290
291 ByteBuffer cachedBuffer = ByteBuffer.allocate(blockSize
292 - HFileBlock.EXTRA_SERIALIZATION_SPACE);
293 rand.nextBytes(cachedBuffer.array());
294 cachedBuffer.rewind();
295 int onDiskSizeWithoutHeader = blockSize
296 - HFileBlock.EXTRA_SERIALIZATION_SPACE;
297 int uncompressedSizeWithoutHeader = blockSize
298 - HFileBlock.EXTRA_SERIALIZATION_SPACE;
299 long prevBlockOffset = rand.nextLong();
300 BlockType.DATA.write(cachedBuffer);
301 cachedBuffer.putInt(onDiskSizeWithoutHeader);
302 cachedBuffer.putInt(uncompressedSizeWithoutHeader);
303 cachedBuffer.putLong(prevBlockOffset);
304 cachedBuffer.rewind();
305
306 HFileBlock generated = new HFileBlock(BlockType.DATA,
307 onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
308 prevBlockOffset, cachedBuffer, false, blockSize);
309
310 String strKey;
311
312 for (strKey = new Long(rand.nextLong()).toString(); !usedStrings
313 .add(strKey); strKey = new Long(rand.nextLong()).toString())
314 ;
315
316 returnedBlocks[i] = new HFileBlockPair();
317 returnedBlocks[i].blockName = strKey;
318 returnedBlocks[i].block = generated;
319 }
320 return returnedBlocks;
321 }
322
323 private static class HFileBlockPair {
324 String blockName;
325 HFileBlock block;
326 }
327 }