1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataOutputStream;
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.HBaseTestCase;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.UnknownScannerException;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Result;
40 import org.apache.hadoop.hbase.client.Scan;
41 import org.apache.hadoop.hbase.filter.Filter;
42 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
43 import org.apache.hadoop.hbase.filter.PrefixFilter;
44 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
45 import org.apache.hadoop.hbase.io.hfile.Compression;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.apache.hadoop.hdfs.MiniDFSCluster;
49
50
51
52
53 public class TestScanner extends HBaseTestCase {
54 private final Log LOG = LogFactory.getLog(this.getClass());
55
56 private static final byte [] FIRST_ROW = HConstants.EMPTY_START_ROW;
57 private static final byte [][] COLS = { HConstants.CATALOG_FAMILY };
58 private static final byte [][] EXPLICIT_COLS = {
59 HConstants.REGIONINFO_QUALIFIER, HConstants.SERVER_QUALIFIER,
60
61
62 };
63
64 static final HTableDescriptor TESTTABLEDESC =
65 new HTableDescriptor("testscanner");
66 static {
67 TESTTABLEDESC.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY,
68 10,
69 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
70 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
71 HConstants.REPLICATION_SCOPE_LOCAL));
72 }
73
74 public static final HRegionInfo REGION_INFO =
75 new HRegionInfo(TESTTABLEDESC.getName(), HConstants.EMPTY_BYTE_ARRAY,
76 HConstants.EMPTY_BYTE_ARRAY);
77
78 private static final byte [] ROW_KEY = REGION_INFO.getRegionName();
79
80 private static final long START_CODE = Long.MAX_VALUE;
81
82 private HRegion r;
83 private HRegionIncommon region;
84
85
86
87
88
89 public void testStopRow() throws Exception {
90 byte [] startrow = Bytes.toBytes("bbb");
91 byte [] stoprow = Bytes.toBytes("ccc");
92 try {
93 this.r = createNewHRegion(TESTTABLEDESC, null, null);
94 addContent(this.r, HConstants.CATALOG_FAMILY);
95 List<KeyValue> results = new ArrayList<KeyValue>();
96
97 Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
98 scan.addFamily(HConstants.CATALOG_FAMILY);
99
100 InternalScanner s = r.getScanner(scan);
101 int count = 0;
102 while (s.next(results)) {
103 count++;
104 }
105 s.close();
106 assertEquals(0, count);
107
108 scan = new Scan(startrow, stoprow);
109 scan.addFamily(HConstants.CATALOG_FAMILY);
110
111 s = r.getScanner(scan);
112 count = 0;
113 KeyValue kv = null;
114 results = new ArrayList<KeyValue>();
115 for (boolean first = true; s.next(results);) {
116 kv = results.get(0);
117 if (first) {
118 assertTrue(Bytes.BYTES_COMPARATOR.compare(startrow, kv.getRow()) == 0);
119 first = false;
120 }
121 count++;
122 }
123 assertTrue(Bytes.BYTES_COMPARATOR.compare(stoprow, kv.getRow()) > 0);
124
125 assertTrue(count > 10);
126 s.close();
127 } finally {
128 this.r.close();
129 this.r.getLog().closeAndDelete();
130 }
131 }
132
133 void rowPrefixFilter(Scan scan) throws IOException {
134 List<KeyValue> results = new ArrayList<KeyValue>();
135 scan.addFamily(HConstants.CATALOG_FAMILY);
136 InternalScanner s = r.getScanner(scan);
137 boolean hasMore = true;
138 while (hasMore) {
139 hasMore = s.next(results);
140 for (KeyValue kv : results) {
141 assertEquals((byte)'a', kv.getRow()[0]);
142 assertEquals((byte)'b', kv.getRow()[1]);
143 }
144 results.clear();
145 }
146 s.close();
147 }
148
149 void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException {
150 List<KeyValue> results = new ArrayList<KeyValue>();
151 scan.addFamily(HConstants.CATALOG_FAMILY);
152 InternalScanner s = r.getScanner(scan);
153 boolean hasMore = true;
154 while (hasMore) {
155 hasMore = s.next(results);
156 for (KeyValue kv : results) {
157 assertTrue(Bytes.compareTo(kv.getRow(), stopRow) <= 0);
158 }
159 results.clear();
160 }
161 s.close();
162 }
163
164 public void testFilters() throws IOException {
165 try {
166 this.r = createNewHRegion(TESTTABLEDESC, null, null);
167 addContent(this.r, HConstants.CATALOG_FAMILY);
168 byte [] prefix = Bytes.toBytes("ab");
169 Filter newFilter = new PrefixFilter(prefix);
170 Scan scan = new Scan();
171 scan.setFilter(newFilter);
172 rowPrefixFilter(scan);
173
174 byte[] stopRow = Bytes.toBytes("bbc");
175 newFilter = new WhileMatchFilter(new InclusiveStopFilter(stopRow));
176 scan = new Scan();
177 scan.setFilter(newFilter);
178 rowInclusiveStopFilter(scan, stopRow);
179
180 } finally {
181 this.r.close();
182 this.r.getLog().closeAndDelete();
183 }
184 }
185
186
187
188
189
190
191 public void testRaceBetweenClientAndTimeout() throws Exception {
192 try {
193 this.r = createNewHRegion(TESTTABLEDESC, null, null);
194 addContent(this.r, HConstants.CATALOG_FAMILY);
195 Scan scan = new Scan();
196 InternalScanner s = r.getScanner(scan);
197 List<KeyValue> results = new ArrayList<KeyValue>();
198 try {
199 s.next(results);
200 s.close();
201 s.next(results);
202 fail("We don't want anything more, we should be failing");
203 } catch (UnknownScannerException ex) {
204
205 return;
206 }
207 } finally {
208 this.r.close();
209 this.r.getLog().closeAndDelete();
210 }
211 }
212
213
214
215
216 public void testScanner() throws IOException {
217 try {
218 r = createNewHRegion(TESTTABLEDESC, null, null);
219 region = new HRegionIncommon(r);
220
221
222
223 Put put = new Put(ROW_KEY, System.currentTimeMillis(), null);
224
225 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
226 DataOutputStream s = new DataOutputStream(byteStream);
227 REGION_INFO.write(s);
228 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
229 byteStream.toByteArray());
230 region.put(put);
231
232
233
234
235 scan(false, null);
236 getRegionInfo();
237
238
239
240 r.close();
241 r = openClosedRegion(r);
242 region = new HRegionIncommon(r);
243
244
245
246 scan(false, null);
247 getRegionInfo();
248
249
250
251 String address = "www.example.com:1234";
252
253 put = new Put(ROW_KEY, System.currentTimeMillis(), null);
254 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
255 Bytes.toBytes(address));
256
257
258
259 region.put(put);
260
261
262
263
264 scan(true, address.toString());
265 getRegionInfo();
266
267
268
269 region.flushcache();
270
271
272
273 scan(true, address.toString());
274 getRegionInfo();
275
276
277
278 r.close();
279 r = openClosedRegion(r);
280 region = new HRegionIncommon(r);
281
282
283
284 scan(true, address.toString());
285 getRegionInfo();
286
287
288
289 address = "bar.foo.com:4321";
290
291 put = new Put(ROW_KEY, System.currentTimeMillis(), null);
292
293 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
294 Bytes.toBytes(address));
295 region.put(put);
296
297
298
299 scan(true, address.toString());
300 getRegionInfo();
301
302
303
304 region.flushcache();
305
306
307
308 scan(true, address.toString());
309 getRegionInfo();
310
311
312
313 r.close();
314 r = openClosedRegion(r);
315 region = new HRegionIncommon(r);
316
317
318
319 scan(true, address.toString());
320 getRegionInfo();
321
322 } finally {
323
324 r.close();
325 r.getLog().closeAndDelete();
326 }
327 }
328
329
330 private void validateRegionInfo(byte [] regionBytes) throws IOException {
331 HRegionInfo info =
332 (HRegionInfo) Writables.getWritable(regionBytes, new HRegionInfo());
333
334 assertEquals(REGION_INFO.getRegionId(), info.getRegionId());
335 assertEquals(0, info.getStartKey().length);
336 assertEquals(0, info.getEndKey().length);
337 assertEquals(0, Bytes.compareTo(info.getRegionName(), REGION_INFO.getRegionName()));
338
339 }
340
341
342 private void scan(boolean validateStartcode, String serverName)
343 throws IOException {
344 InternalScanner scanner = null;
345 Scan scan = null;
346 List<KeyValue> results = new ArrayList<KeyValue>();
347 byte [][][] scanColumns = {
348 COLS,
349 EXPLICIT_COLS
350 };
351
352 for(int i = 0; i < scanColumns.length; i++) {
353 try {
354 scan = new Scan(FIRST_ROW);
355 for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) {
356 scan.addColumn(COLS[0], EXPLICIT_COLS[ii]);
357 }
358 scanner = r.getScanner(scan);
359 while (scanner.next(results)) {
360 assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
361 HConstants.REGIONINFO_QUALIFIER));
362 byte [] val = getColumn(results, HConstants.CATALOG_FAMILY,
363 HConstants.REGIONINFO_QUALIFIER).getValue();
364 validateRegionInfo(val);
365 if(validateStartcode) {
366
367
368
369
370 assertNotNull(val);
371 assertFalse(val.length == 0);
372 long startCode = Bytes.toLong(val);
373 assertEquals(START_CODE, startCode);
374 }
375
376 if(serverName != null) {
377 assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
378 HConstants.SERVER_QUALIFIER));
379 val = getColumn(results, HConstants.CATALOG_FAMILY,
380 HConstants.SERVER_QUALIFIER).getValue();
381 assertNotNull(val);
382 assertFalse(val.length == 0);
383 String server = Bytes.toString(val);
384 assertEquals(0, server.compareTo(serverName));
385 }
386 }
387 } finally {
388 InternalScanner s = scanner;
389 scanner = null;
390 if(s != null) {
391 s.close();
392 }
393 }
394 }
395 }
396
397 private boolean hasColumn(final List<KeyValue> kvs, final byte [] family,
398 final byte [] qualifier) {
399 for (KeyValue kv: kvs) {
400 if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
401 return true;
402 }
403 }
404 return false;
405 }
406
407 private KeyValue getColumn(final List<KeyValue> kvs, final byte [] family,
408 final byte [] qualifier) {
409 for (KeyValue kv: kvs) {
410 if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
411 return kv;
412 }
413 }
414 return null;
415 }
416
417
418
419 private void getRegionInfo() throws IOException {
420 Get get = new Get(ROW_KEY);
421 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
422 Result result = region.get(get, null);
423 byte [] bytes = result.value();
424 validateRegionInfo(bytes);
425 }
426
427
428
429
430
431
432
433 public void testScanAndSyncFlush() throws Exception {
434 this.r = createNewHRegion(TESTTABLEDESC, null, null);
435 HRegionIncommon hri = new HRegionIncommon(r);
436 try {
437 LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
438 Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
439 int count = count(hri, -1, false);
440 assertEquals(count, count(hri, 100, false));
441 } catch (Exception e) {
442 LOG.error("Failed", e);
443 throw e;
444 } finally {
445 this.r.close();
446 this.r.getLog().closeAndDelete();
447 }
448 }
449
450
451
452
453
454
455
456 public void testScanAndRealConcurrentFlush() throws Exception {
457 this.r = createNewHRegion(TESTTABLEDESC, null, null);
458 HRegionIncommon hri = new HRegionIncommon(r);
459 try {
460 LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
461 Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
462 int count = count(hri, -1, false);
463 assertEquals(count, count(hri, 100, true));
464 } catch (Exception e) {
465 LOG.error("Failed", e);
466 throw e;
467 } finally {
468 this.r.close();
469 this.r.getLog().closeAndDelete();
470 }
471 }
472
473
474
475
476
477
478
479
480
481 private int count(final HRegionIncommon hri, final int flushIndex,
482 boolean concurrent)
483 throws IOException {
484 LOG.info("Taking out counting scan");
485 ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
486 HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
487 List<KeyValue> values = new ArrayList<KeyValue>();
488 int count = 0;
489 boolean justFlushed = false;
490 while (s.next(values)) {
491 if (justFlushed) {
492 LOG.info("after next() just after next flush");
493 justFlushed=false;
494 }
495 count++;
496 if (flushIndex == count) {
497 LOG.info("Starting flush at flush index " + flushIndex);
498 Thread t = new Thread() {
499 public void run() {
500 try {
501 hri.flushcache();
502 LOG.info("Finishing flush");
503 } catch (IOException e) {
504 LOG.info("Failed flush cache");
505 }
506 }
507 };
508 if (concurrent) {
509 t.start();
510 } else {
511 t.run();
512 }
513 LOG.info("Continuing on after kicking off background flush");
514 justFlushed = true;
515 }
516 }
517 s.close();
518 LOG.info("Found " + count + " items");
519 return count;
520 }
521 }