1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.UnsupportedEncodingException;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.NavigableMap;
28
29 import junit.framework.AssertionFailedError;
30 import junit.framework.TestCase;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.HTable;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.ResultScanner;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.InternalScanner;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48
49
50
51
52
53
54 public abstract class HBaseTestCase extends TestCase {
55 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56
57
58
59
60 private static final String TEST_DIRECTORY_KEY = "test.build.data";
61
62
63
64
65
66
67 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
68 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
69 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
70
71 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
72
73 private boolean localfs = false;
74 protected Path testDir = null;
75 protected FileSystem fs = null;
76 protected HRegion root = null;
77 protected HRegion meta = null;
78 protected static final char FIRST_CHAR = 'a';
79 protected static final char LAST_CHAR = 'z';
80 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
81 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
82 protected String START_KEY;
83 protected static final int MAXVERSIONS = 3;
84
85 static {
86 initialize();
87 }
88
89 public volatile Configuration conf;
90
91
92 public HBaseTestCase() {
93 super();
94 init();
95 }
96
97
98
99
100 public HBaseTestCase(String name) {
101 super(name);
102 init();
103 }
104
105 private void init() {
106 conf = HBaseConfiguration.create();
107 try {
108 START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
109 } catch (UnsupportedEncodingException e) {
110 LOG.fatal("error during initialization", e);
111 fail();
112 }
113 }
114
115
116
117
118
119 @Override
120 protected void setUp() throws Exception {
121 super.setUp();
122 localfs =
123 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
124
125 if (fs == null) {
126 this.fs = FileSystem.get(conf);
127 }
128 try {
129 if (localfs) {
130 this.testDir = getUnitTestdir(getName());
131 if (fs.exists(testDir)) {
132 fs.delete(testDir, true);
133 }
134 } else {
135 this.testDir =
136 this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
137 }
138 } catch (Exception e) {
139 LOG.fatal("error during setup", e);
140 throw e;
141 }
142 }
143
144 @Override
145 protected void tearDown() throws Exception {
146 try {
147 if (localfs) {
148 if (this.fs.exists(testDir)) {
149 this.fs.delete(testDir, true);
150 }
151 }
152 } catch (Exception e) {
153 LOG.fatal("error during tear down", e);
154 }
155 super.tearDown();
156 }
157
158
159
160
161
162
163 protected Path getUnitTestdir(String testName) {
164 return new Path(
165 System.getProperty(
166 HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
167 HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY
168 ),
169 testName
170 );
171 }
172
173 protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
174 byte [] endKey)
175 throws IOException {
176 FileSystem filesystem = FileSystem.get(conf);
177 HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
178 return HRegion.createHRegion(hri, testDir, conf, desc);
179 }
180
181 protected HRegion openClosedRegion(final HRegion closedRegion)
182 throws IOException {
183 HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
184 closedRegion.getFilesystem(), closedRegion.getConf(),
185 closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
186 r.initialize();
187 return r;
188 }
189
190
191
192
193
194
195
196 protected HTableDescriptor createTableDescriptor(final String name) {
197 return createTableDescriptor(name, MAXVERSIONS);
198 }
199
200
201
202
203
204
205
206
207 protected HTableDescriptor createTableDescriptor(final String name,
208 final int versions) {
209 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
210 versions, HConstants.FOREVER);
211 }
212
213
214
215
216
217
218
219
220 protected HTableDescriptor createTableDescriptor(final String name,
221 final int minVersions, final int versions, final int ttl) {
222 HTableDescriptor htd = new HTableDescriptor(name);
223 htd.addFamily(new HColumnDescriptor(fam1, minVersions, versions,
224 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
225 HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
226 HColumnDescriptor.DEFAULT_BLOOMFILTER,
227 HConstants.REPLICATION_SCOPE_LOCAL));
228 htd.addFamily(new HColumnDescriptor(fam2, minVersions, versions,
229 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
230 HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
231 HColumnDescriptor.DEFAULT_BLOOMFILTER,
232 HConstants.REPLICATION_SCOPE_LOCAL));
233 htd.addFamily(new HColumnDescriptor(fam3, minVersions, versions,
234 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
235 HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
236 HColumnDescriptor.DEFAULT_BLOOMFILTER,
237 HConstants.REPLICATION_SCOPE_LOCAL));
238 return htd;
239 }
240
241
242
243
244
245
246
247
248
249
250 protected static long addContent(final HRegion r, final byte [] columnFamily)
251 throws IOException {
252 byte [] startKey = r.getRegionInfo().getStartKey();
253 byte [] endKey = r.getRegionInfo().getEndKey();
254 byte [] startKeyBytes = startKey;
255 if (startKeyBytes == null || startKeyBytes.length == 0) {
256 startKeyBytes = START_KEY_BYTES;
257 }
258 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
259 startKeyBytes, endKey, -1);
260 }
261
262
263
264
265
266
267
268
269
270
271 protected static long addContent(final Incommon updater,
272 final String columnFamily) throws IOException {
273 return addContent(updater, columnFamily, START_KEY_BYTES, null);
274 }
275
276 protected static long addContent(final Incommon updater, final String family,
277 final String column) throws IOException {
278 return addContent(updater, family, column, START_KEY_BYTES, null);
279 }
280
281
282
283
284
285
286
287
288
289
290
291
292 protected static long addContent(final Incommon updater, final String columnFamily,
293 final byte [] startKeyBytes, final byte [] endKey)
294 throws IOException {
295 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
296 }
297
298 protected static long addContent(final Incommon updater, final String family,
299 final String column, final byte [] startKeyBytes,
300 final byte [] endKey) throws IOException {
301 return addContent(updater, family, column, startKeyBytes, endKey, -1);
302 }
303
304
305
306
307
308
309
310
311
312
313
314
315
316 protected static long addContent(final Incommon updater,
317 final String columnFamily,
318 final String column,
319 final byte [] startKeyBytes, final byte [] endKey, final long ts)
320 throws IOException {
321 long count = 0;
322
323
324
325
326 char secondCharStart = (char)startKeyBytes[1];
327 char thirdCharStart = (char)startKeyBytes[2];
328 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
329 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
330 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
331 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
332 if (endKey != null && endKey.length > 0
333 && Bytes.compareTo(endKey, t) <= 0) {
334 break EXIT;
335 }
336 try {
337 Put put;
338 if(ts != -1) {
339 put = new Put(t, ts, null);
340 } else {
341 put = new Put(t);
342 }
343 try {
344 StringBuilder sb = new StringBuilder();
345 if (column != null && column.contains(":")) {
346 sb.append(column);
347 } else {
348 if (columnFamily != null) {
349 sb.append(columnFamily);
350 if (!columnFamily.endsWith(":")) {
351 sb.append(":");
352 }
353 if (column != null) {
354 sb.append(column);
355 }
356 }
357 }
358 byte[][] split =
359 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
360 if(split.length == 1) {
361 put.add(split[0], new byte[0], t);
362 } else {
363 put.add(split[0], split[1], t);
364 }
365 updater.put(put);
366 count++;
367 } catch (RuntimeException ex) {
368 ex.printStackTrace();
369 throw ex;
370 } catch (IOException ex) {
371 ex.printStackTrace();
372 throw ex;
373 }
374 } catch (RuntimeException ex) {
375 ex.printStackTrace();
376 throw ex;
377 } catch (IOException ex) {
378 ex.printStackTrace();
379 throw ex;
380 }
381 }
382
383 thirdCharStart = FIRST_CHAR;
384 }
385 secondCharStart = FIRST_CHAR;
386 }
387 return count;
388 }
389
390
391
392
393 public static interface FlushCache {
394
395
396
397 public void flushcache() throws IOException;
398 }
399
400
401
402
403
404
405
406 public static interface Incommon {
407
408
409
410
411
412
413
414 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
415 throws IOException;
416
417
418
419
420
421 public void put(Put put) throws IOException;
422
423 public Result get(Get get) throws IOException;
424
425
426
427
428
429
430
431
432
433 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
434 byte [] firstRow, long ts)
435 throws IOException;
436 }
437
438
439
440
441 public static class HRegionIncommon implements Incommon, FlushCache {
442 final HRegion region;
443
444
445
446
447 public HRegionIncommon(final HRegion HRegion) {
448 this.region = HRegion;
449 }
450
451 public void put(Put put) throws IOException {
452 region.put(put);
453 }
454
455 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
456 throws IOException {
457 this.region.delete(delete, lockid, writeToWAL);
458 }
459
460 public Result get(Get get) throws IOException {
461 return region.get(get, null);
462 }
463
464 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
465 byte [] firstRow, long ts)
466 throws IOException {
467 Scan scan = new Scan(firstRow);
468 if(qualifiers == null || qualifiers.length == 0) {
469 scan.addFamily(family);
470 } else {
471 for(int i=0; i<qualifiers.length; i++){
472 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
473 }
474 }
475 scan.setTimeRange(0, ts);
476 return new
477 InternalScannerIncommon(region.getScanner(scan));
478 }
479
480 public Result get(Get get, Integer lockid) throws IOException{
481 return this.region.get(get, lockid);
482 }
483
484
485 public void flushcache() throws IOException {
486 this.region.flushcache();
487 }
488 }
489
490
491
492
493 public static class HTableIncommon implements Incommon {
494 final HTable table;
495
496
497
498
499 public HTableIncommon(final HTable table) {
500 super();
501 this.table = table;
502 }
503
504 public void put(Put put) throws IOException {
505 table.put(put);
506 }
507
508
509 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
510 throws IOException {
511 this.table.delete(delete);
512 }
513
514 public Result get(Get get) throws IOException {
515 return table.get(get);
516 }
517
518 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
519 byte [] firstRow, long ts)
520 throws IOException {
521 Scan scan = new Scan(firstRow);
522 if(qualifiers == null || qualifiers.length == 0) {
523 scan.addFamily(family);
524 } else {
525 for(int i=0; i<qualifiers.length; i++){
526 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
527 }
528 }
529 scan.setTimeRange(0, ts);
530 return new
531 ClientScannerIncommon(table.getScanner(scan));
532 }
533 }
534
535 public interface ScannerIncommon
536 extends Iterable<Result> {
537 public boolean next(List<KeyValue> values)
538 throws IOException;
539
540 public void close() throws IOException;
541 }
542
543 public static class ClientScannerIncommon implements ScannerIncommon {
544 ResultScanner scanner;
545 public ClientScannerIncommon(ResultScanner scanner) {
546 this.scanner = scanner;
547 }
548
549 public boolean next(List<KeyValue> values)
550 throws IOException {
551 Result results = scanner.next();
552 if (results == null) {
553 return false;
554 }
555 values.clear();
556 values.addAll(results.list());
557 return true;
558 }
559
560 public void close() throws IOException {
561 scanner.close();
562 }
563
564 @SuppressWarnings("unchecked")
565 public Iterator iterator() {
566 return scanner.iterator();
567 }
568 }
569
570 public static class InternalScannerIncommon implements ScannerIncommon {
571 InternalScanner scanner;
572
573 public InternalScannerIncommon(InternalScanner scanner) {
574 this.scanner = scanner;
575 }
576
577 public boolean next(List<KeyValue> results)
578 throws IOException {
579 return scanner.next(results);
580 }
581
582 public void close() throws IOException {
583 scanner.close();
584 }
585
586 public Iterator<Result> iterator() {
587 throw new UnsupportedOperationException();
588 }
589 }
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611 protected void assertResultEquals(final HRegion region, final byte [] row,
612 final byte [] family, final byte [] qualifier, final long timestamp,
613 final byte [] value)
614 throws IOException {
615 Get get = new Get(row);
616 get.setTimeStamp(timestamp);
617 Result res = region.get(get, null);
618 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
619 res.getMap();
620 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
621
622 if (value == null) {
623 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
624 " at timestamp " + timestamp, null, res_value);
625 } else {
626 if (res_value == null) {
627 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
628 " at timestamp " + timestamp + "\" was expected to be \"" +
629 Bytes.toStringBinary(value) + " but was null");
630 }
631 if (res_value != null) {
632 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
633 " at timestamp " +
634 timestamp, value, new String(res_value));
635 }
636 }
637 }
638
639
640
641
642
643
644
645
646
647 public static void initialize() {
648 if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
649 System.setProperty(TEST_DIRECTORY_KEY, new File(
650 "build/hbase/test").getAbsolutePath());
651 }
652 }
653
654
655
656
657
658
659 public static void shutdownDfs(MiniDFSCluster cluster) {
660 if (cluster != null) {
661 LOG.info("Shutting down Mini DFS ");
662 try {
663 cluster.shutdown();
664 } catch (Exception e) {
665
666
667
668 }
669 try {
670 FileSystem fs = cluster.getFileSystem();
671 if (fs != null) {
672 LOG.info("Shutting down FileSystem");
673 fs.close();
674 }
675 FileSystem.closeAll();
676 } catch (IOException e) {
677 LOG.error("error closing file system", e);
678 }
679 }
680 }
681
682 protected void createRootAndMetaRegions() throws IOException {
683 root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir,
684 conf, HTableDescriptor.ROOT_TABLEDESC);
685 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
686 conf, HTableDescriptor.META_TABLEDESC);
687 HRegion.addRegionToMETA(root, meta);
688 }
689
690 protected void closeRootAndMeta() throws IOException {
691 if (meta != null) {
692 meta.close();
693 meta.getLog().closeAndDelete();
694 }
695 if (root != null) {
696 root.close();
697 root.getLog().closeAndDelete();
698 }
699 }
700
701 public static void assertByteEquals(byte[] expected,
702 byte[] actual) {
703 if (Bytes.compareTo(expected, actual) != 0) {
704 throw new AssertionFailedError("expected:<" +
705 Bytes.toString(expected) + "> but was:<" +
706 Bytes.toString(actual) + ">");
707 }
708 }
709
710 public static void assertEquals(byte[] expected,
711 byte[] actual) {
712 if (Bytes.compareTo(expected, actual) != 0) {
713 throw new AssertionFailedError("expected:<" +
714 Bytes.toStringBinary(expected) + "> but was:<" +
715 Bytes.toStringBinary(actual) + ">");
716 }
717 }
718
719 }