1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseConfiguration;
32 import org.apache.hadoop.hbase.HBaseTestCase;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.client.Delete;
39 import org.apache.hadoop.hbase.client.Get;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.io.hfile.BlockCache;
43 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
46 import org.junit.Test;
47
48 public class TestBlocksRead extends HBaseTestCase {
49 static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
50
51 private static BlockCache blockCache;
52
53 private HBaseConfiguration getConf() {
54 HBaseConfiguration conf = new HBaseConfiguration();
55
56
57 conf.setInt("hbase.hstore.compactionThreshold", 10000);
58 return conf;
59 }
60
61 HRegion region = null;
62 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63 private final String DIR = TEST_UTIL.getDataTestDir("TestBlocksRead").toString();
64
65
66
67
68 @SuppressWarnings("deprecation")
69 @Override
70 protected void setUp() throws Exception {
71 super.setUp();
72 }
73
74 @SuppressWarnings("deprecation")
75 @Override
76 protected void tearDown() throws Exception {
77 super.tearDown();
78 EnvironmentEdgeManagerTestHelper.reset();
79 }
80
81 private void initHRegion (byte [] tableName, String callingMethod,
82 HBaseConfiguration conf, byte [] ... families)
83 throws IOException {
84 HTableDescriptor htd = new HTableDescriptor(tableName);
85 for(byte [] family : families) {
86 HColumnDescriptor familyDesc = new HColumnDescriptor(
87 family,
88 HColumnDescriptor.DEFAULT_VERSIONS,
89 HColumnDescriptor.DEFAULT_COMPRESSION,
90 HColumnDescriptor.DEFAULT_IN_MEMORY,
91 HColumnDescriptor.DEFAULT_BLOCKCACHE,
92 1,
93 HColumnDescriptor.DEFAULT_TTL,
94 HColumnDescriptor.DEFAULT_BLOOMFILTER,
95 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE);
96 htd.addFamily(familyDesc);
97 }
98 HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
99 Path path = new Path(DIR + callingMethod);
100 region = HRegion.createHRegion(info, path, conf, htd);
101 blockCache = new CacheConfig(conf).getBlockCache();
102 }
103
104 private void putData(byte[] cf, String row, String col, long version)
105 throws IOException {
106 putData(cf, row, col, version, version);
107 }
108
109
110 private static byte[] genValue(String row, String col, long version) {
111 return Bytes.toBytes("Value:" + row + "#" + col + "#" + version);
112 }
113
114 private void putData(byte[] cf, String row, String col,
115 long versionStart, long versionEnd)
116 throws IOException {
117 byte columnBytes[] = Bytes.toBytes(col);
118 Put put = new Put(Bytes.toBytes(row));
119 put.setWriteToWAL(false);
120
121 for (long version = versionStart; version <= versionEnd; version++) {
122 put.add(cf, columnBytes, version, genValue(row, col, version));
123 }
124 region.put(put);
125 }
126
127 private KeyValue[] getData(byte[] cf, String row, List<String> columns,
128 int expBlocks)
129 throws IOException {
130
131 long blocksStart = getBlkAccessCount(cf);
132 Get get = new Get(Bytes.toBytes(row));
133
134 for (String column : columns) {
135 get.addColumn(cf, Bytes.toBytes(column));
136 }
137
138 KeyValue[] kvs = region.get(get, null).raw();
139 long blocksEnd = getBlkAccessCount(cf);
140 if (expBlocks != -1) {
141 assertEquals("Blocks Read Check", expBlocks, blocksEnd - blocksStart);
142 }
143 System.out.println("Blocks Read = " + (blocksEnd - blocksStart) +
144 "Expected = " + expBlocks);
145 return kvs;
146 }
147
148 private KeyValue[] getData(byte[] cf, String row, String column,
149 int expBlocks)
150 throws IOException {
151 return getData(cf, row, Arrays.asList(column), expBlocks);
152 }
153
154 private void deleteFamily(byte[] cf, String row, long version)
155 throws IOException {
156 Delete del = new Delete(Bytes.toBytes(row));
157 del.deleteFamily(cf, version);
158 region.delete(del, null, true);
159 }
160
161 private void deleteFamily(byte[] cf, String row, String column, long version)
162 throws IOException {
163 Delete del = new Delete(Bytes.toBytes(row));
164 del.deleteColumns(cf, Bytes.toBytes(column), version);
165 region.delete(del, null, true);
166 }
167
168 private static void verifyData(KeyValue kv, String expectedRow,
169 String expectedCol, long expectedVersion) {
170 assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow()));
171 assertEquals("ColumnCheck", expectedCol, Bytes.toString(kv.getQualifier()));
172 assertEquals("TSCheck", expectedVersion, kv.getTimestamp());
173 assertEquals("ValueCheck",
174 Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)),
175 Bytes.toString(kv.getValue()));
176 }
177
178 private static long getBlkAccessCount(byte[] cf) {
179 return blockCache.getStats().getRequestCount();
180 }
181
182 private static long getBlkCount() {
183 return blockCache.getBlockCount();
184 }
185
186
187
188
189
190 @Test
191 public void testBlocksRead() throws Exception {
192 byte [] TABLE = Bytes.toBytes("testBlocksRead");
193 byte [] FAMILY = Bytes.toBytes("cf1");
194 byte [][] FAMILIES = new byte[][] { FAMILY };
195 KeyValue kvs[];
196
197 HBaseConfiguration conf = getConf();
198 initHRegion(TABLE, getName(), conf, FAMILIES);
199
200 putData(FAMILY, "row", "col1", 1);
201 putData(FAMILY, "row", "col2", 2);
202 putData(FAMILY, "row", "col3", 3);
203 putData(FAMILY, "row", "col4", 4);
204 putData(FAMILY, "row", "col5", 5);
205 putData(FAMILY, "row", "col6", 6);
206 putData(FAMILY, "row", "col7", 7);
207 region.flushcache();
208
209
210 kvs = getData(FAMILY, "row", "col1", 1);
211 assertEquals(1, kvs.length);
212 verifyData(kvs[0], "row", "col1", 1);
213
214
215 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
216 assertEquals(2, kvs.length);
217 verifyData(kvs[0], "row", "col1", 1);
218 verifyData(kvs[1], "row", "col2", 2);
219
220
221 kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3);
222 assertEquals(2, kvs.length);
223 verifyData(kvs[0], "row", "col2", 2);
224 verifyData(kvs[1], "row", "col3", 3);
225
226
227 kvs = getData(FAMILY, "row", Arrays.asList("col5"), 3);
228 assertEquals(1, kvs.length);
229 verifyData(kvs[0], "row", "col5", 5);
230 }
231
232
233
234
235
236 @Test
237 public void testLazySeekBlocksRead() throws Exception {
238 byte [] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
239 byte [] FAMILY = Bytes.toBytes("cf1");
240 byte [][] FAMILIES = new byte[][] { FAMILY };
241 KeyValue kvs[];
242
243 HBaseConfiguration conf = getConf();
244 initHRegion(TABLE, getName(), conf, FAMILIES);
245
246
247 putData(FAMILY, "row", "col1", 1);
248 putData(FAMILY, "row", "col2", 2);
249 region.flushcache();
250
251
252 putData(FAMILY, "row", "col1", 3);
253 putData(FAMILY, "row", "col2", 4);
254 region.flushcache();
255
256
257 kvs = getData(FAMILY, "row", Arrays.asList("col1"), 2);
258 assertEquals(1, kvs.length);
259 verifyData(kvs[0], "row", "col1", 3);
260
261
262 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 4);
263 assertEquals(2, kvs.length);
264 verifyData(kvs[0], "row", "col1", 3);
265 verifyData(kvs[1], "row", "col2", 4);
266
267
268 putData(FAMILY, "row", "col3", 5);
269 region.flushcache();
270
271
272 kvs = getData(FAMILY, "row", "col3", 5);
273 assertEquals(1, kvs.length);
274 verifyData(kvs[0], "row", "col3", 5);
275
276
277
278 kvs = getData(FAMILY, "row", Arrays.asList("col1"), 3);
279 assertEquals(1, kvs.length);
280 verifyData(kvs[0], "row", "col1", 3);
281
282
283 deleteFamily(FAMILY, "row", 6);
284 region.flushcache();
285
286
287 kvs = getData(FAMILY, "row", "col1", 6);
288 assertEquals(0, kvs.length);
289 kvs = getData(FAMILY, "row", "col2", 6);
290 assertEquals(0, kvs.length);
291 kvs = getData(FAMILY, "row", "col3", 6);
292 assertEquals(0, kvs.length);
293 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 6);
294 assertEquals(0, kvs.length);
295
296
297 deleteFamily(FAMILY, "row", 10);
298 region.flushcache();
299
300
301
302 putData(FAMILY, "row", "col1", 7);
303 putData(FAMILY, "row", "col2", 8);
304 putData(FAMILY, "row", "col3", 9);
305 region.flushcache();
306
307
308 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 10);
309 assertEquals(0, kvs.length);
310
311
312 putData(FAMILY, "row", "col1", 11);
313 putData(FAMILY, "row", "col2", 12);
314 putData(FAMILY, "row", "col3", 13);
315 region.flushcache();
316
317
318 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 13);
319 assertEquals(3, kvs.length);
320 verifyData(kvs[0], "row", "col1", 11);
321 verifyData(kvs[1], "row", "col2", 12);
322 verifyData(kvs[2], "row", "col3", 13);
323 }
324
325
326
327
328
329 @Test
330 public void testBlocksStoredWhenCachingDisabled() throws Exception {
331 byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled");
332 byte [] FAMILY = Bytes.toBytes("cf1");
333 byte [][] FAMILIES = new byte[][] { FAMILY };
334
335 HBaseConfiguration conf = getConf();
336 initHRegion(TABLE, getName(), conf, FAMILIES);
337
338 putData(FAMILY, "row", "col1", 1);
339 putData(FAMILY, "row", "col2", 2);
340 region.flushcache();
341
342
343
344 long blocksStart = getBlkCount();
345 Scan scan = new Scan();
346 scan.setCacheBlocks(false);
347 RegionScanner rs = region.getScanner(scan);
348 List<KeyValue> result = new ArrayList<KeyValue>(2);
349 rs.next(result);
350 assertEquals(2, result.size());
351 rs.close();
352 long blocksEnd = getBlkCount();
353
354 assertEquals(blocksStart, blocksEnd);
355
356
357
358 blocksStart = blocksEnd;
359 scan.setCacheBlocks(true);
360 rs = region.getScanner(scan);
361 result = new ArrayList<KeyValue>(2);
362 rs.next(result);
363 assertEquals(2, result.size());
364 rs.close();
365 blocksEnd = getBlkCount();
366
367 assertEquals(2, blocksEnd - blocksStart);
368 }
369 }