1   /**
2    * Copyright 2007 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.io.UnsupportedEncodingException;
25  import java.util.Iterator;
26  import java.util.List;
27  import java.util.NavigableMap;
28  
29  import junit.framework.AssertionFailedError;
30  import junit.framework.TestCase;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.client.Delete;
38  import org.apache.hadoop.hbase.client.Get;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.ResultScanner;
43  import org.apache.hadoop.hbase.client.Scan;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.InternalScanner;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hdfs.MiniDFSCluster;
48  
49  /**
50   * Abstract HBase test class.  Initializes a few things that can come in handly
51   * like an HBaseConfiguration and filesystem.
52   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
53   */
54  public abstract class HBaseTestCase extends TestCase {
55    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56  
57    /** configuration parameter name for test directory
58     * @deprecated see HBaseTestingUtility#TEST_DIRECTORY_KEY
59     **/
60    private static final String TEST_DIRECTORY_KEY = "test.build.data";
61  
62  /*
63    protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
64    protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
65    protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
66  */
67    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
68    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
69    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
70  
71    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
72  
73    private boolean localfs = false;
74    protected Path testDir = null;
75    protected FileSystem fs = null;
76    protected HRegion root = null;
77    protected HRegion meta = null;
78    protected static final char FIRST_CHAR = 'a';
79    protected static final char LAST_CHAR = 'z';
80    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
81    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
82    protected String START_KEY;
83    protected static final int MAXVERSIONS = 3;
84  
85    static {
86      initialize();
87    }
88  
89    public volatile Configuration conf;
90  
91    /** constructor */
92    public HBaseTestCase() {
93      super();
94      init();
95    }
96  
97    /**
98     * @param name
99     */
100   public HBaseTestCase(String name) {
101     super(name);
102     init();
103   }
104 
105   private void init() {
106     conf = HBaseConfiguration.create();
107     try {
108       START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
109     } catch (UnsupportedEncodingException e) {
110       LOG.fatal("error during initialization", e);
111       fail();
112     }
113   }
114 
115   /**
116    * Note that this method must be called after the mini hdfs cluster has
117    * started or we end up with a local file system.
118    */
119   @Override
120   protected void setUp() throws Exception {
121     super.setUp();
122     localfs =
123       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
124 
125     if (fs == null) {
126       this.fs = FileSystem.get(conf);
127     }
128     try {
129       if (localfs) {
130         this.testDir = getUnitTestdir(getName());
131         if (fs.exists(testDir)) {
132           fs.delete(testDir, true);
133         }
134       } else {
135         this.testDir =
136           this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
137       }
138     } catch (Exception e) {
139       LOG.fatal("error during setup", e);
140       throw e;
141     }
142   }
143 
144   @Override
145   protected void tearDown() throws Exception {
146     try {
147       if (localfs) {
148         if (this.fs.exists(testDir)) {
149           this.fs.delete(testDir, true);
150         }
151       }
152     } catch (Exception e) {
153       LOG.fatal("error during tear down", e);
154     }
155     super.tearDown();
156   }
157 
158   /**
159    * @see HBaseTestingUtility#getBaseTestDir
160    * @param testName
161    * @return directory to use for this test
162    */
163     protected Path getUnitTestdir(String testName) {
164       return new Path(
165           System.getProperty(
166             HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
167             HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY
168             ),
169         testName
170       );
171     }
172 
173   protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
174       byte [] endKey)
175   throws IOException {
176     FileSystem filesystem = FileSystem.get(conf);
177     HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
178     return HRegion.createHRegion(hri, testDir, conf, desc);
179   }
180 
181   protected HRegion openClosedRegion(final HRegion closedRegion)
182   throws IOException {
183     HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
184         closedRegion.getFilesystem(), closedRegion.getConf(),
185         closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
186     r.initialize();
187     return r;
188   }
189 
190   /**
191    * Create a table of name <code>name</code> with {@link COLUMNS} for
192    * families.
193    * @param name Name to give table.
194    * @return Column descriptor.
195    */
196   protected HTableDescriptor createTableDescriptor(final String name) {
197     return createTableDescriptor(name, MAXVERSIONS);
198   }
199 
200   /**
201    * Create a table of name <code>name</code> with {@link COLUMNS} for
202    * families.
203    * @param name Name to give table.
204    * @param versions How many versions to allow per column.
205    * @return Column descriptor.
206    */
207   protected HTableDescriptor createTableDescriptor(final String name,
208       final int versions) {
209     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
210         versions, HConstants.FOREVER);
211   }
212 
213   /**
214    * Create a table of name <code>name</code> with {@link COLUMNS} for
215    * families.
216    * @param name Name to give table.
217    * @param versions How many versions to allow per column.
218    * @return Column descriptor.
219    */
220   protected HTableDescriptor createTableDescriptor(final String name,
221       final int minVersions, final int versions, final int ttl) {
222     HTableDescriptor htd = new HTableDescriptor(name);
223     htd.addFamily(new HColumnDescriptor(fam1, minVersions, versions,
224       HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
225       HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
226       HColumnDescriptor.DEFAULT_BLOOMFILTER,
227       HConstants.REPLICATION_SCOPE_LOCAL));
228     htd.addFamily(new HColumnDescriptor(fam2, minVersions, versions,
229         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
230         HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
231         HColumnDescriptor.DEFAULT_BLOOMFILTER,
232         HConstants.REPLICATION_SCOPE_LOCAL));
233     htd.addFamily(new HColumnDescriptor(fam3, minVersions, versions,
234         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
235         HColumnDescriptor.DEFAULT_BLOCKSIZE,  ttl,
236         HColumnDescriptor.DEFAULT_BLOOMFILTER,
237         HConstants.REPLICATION_SCOPE_LOCAL));
238     return htd;
239   }
240 
241   /**
242    * Add content to region <code>r</code> on the passed column
243    * <code>column</code>.
244    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
245    * @param r
246    * @param columnFamily
247    * @throws IOException
248    * @return count of what we added.
249    */
250   protected static long addContent(final HRegion r, final byte [] columnFamily)
251   throws IOException {
252     byte [] startKey = r.getRegionInfo().getStartKey();
253     byte [] endKey = r.getRegionInfo().getEndKey();
254     byte [] startKeyBytes = startKey;
255     if (startKeyBytes == null || startKeyBytes.length == 0) {
256       startKeyBytes = START_KEY_BYTES;
257     }
258     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
259       startKeyBytes, endKey, -1);
260   }
261 
262   /**
263    * Add content to region <code>r</code> on the passed column
264    * <code>column</code>.
265    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
266    * @param updater  An instance of {@link Incommon}.
267    * @param columnFamily
268    * @throws IOException
269    * @return count of what we added.
270    */
271   protected static long addContent(final Incommon updater,
272                                    final String columnFamily) throws IOException {
273     return addContent(updater, columnFamily, START_KEY_BYTES, null);
274   }
275 
276   protected static long addContent(final Incommon updater, final String family,
277                                    final String column) throws IOException {
278     return addContent(updater, family, column, START_KEY_BYTES, null);
279   }
280 
281   /**
282    * Add content to region <code>r</code> on the passed column
283    * <code>column</code>.
284    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
285    * @param updater  An instance of {@link Incommon}.
286    * @param columnFamily
287    * @param startKeyBytes Where to start the rows inserted
288    * @param endKey Where to stop inserting rows.
289    * @return count of what we added.
290    * @throws IOException
291    */
292   protected static long addContent(final Incommon updater, final String columnFamily,
293       final byte [] startKeyBytes, final byte [] endKey)
294   throws IOException {
295     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
296   }
297 
298   protected static long addContent(final Incommon updater, final String family,
299                                    final String column, final byte [] startKeyBytes,
300                                    final byte [] endKey) throws IOException {
301     return addContent(updater, family, column, startKeyBytes, endKey, -1);
302   }
303 
304   /**
305    * Add content to region <code>r</code> on the passed column
306    * <code>column</code>.
307    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
308    * @param updater  An instance of {@link Incommon}.
309    * @param column
310    * @param startKeyBytes Where to start the rows inserted
311    * @param endKey Where to stop inserting rows.
312    * @param ts Timestamp to write the content with.
313    * @return count of what we added.
314    * @throws IOException
315    */
316   protected static long addContent(final Incommon updater,
317                                    final String columnFamily,
318                                    final String column,
319       final byte [] startKeyBytes, final byte [] endKey, final long ts)
320   throws IOException {
321     long count = 0;
322     // Add rows of three characters.  The first character starts with the
323     // 'a' character and runs up to 'z'.  Per first character, we run the
324     // second character over same range.  And same for the third so rows
325     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
326     char secondCharStart = (char)startKeyBytes[1];
327     char thirdCharStart = (char)startKeyBytes[2];
328     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
329       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
330         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
331           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
332           if (endKey != null && endKey.length > 0
333               && Bytes.compareTo(endKey, t) <= 0) {
334             break EXIT;
335           }
336           try {
337             Put put;
338             if(ts != -1) {
339               put = new Put(t, ts, null);
340             } else {
341               put = new Put(t);
342             }
343             try {
344               StringBuilder sb = new StringBuilder();
345               if (column != null && column.contains(":")) {
346                 sb.append(column);
347               } else {
348                 if (columnFamily != null) {
349                   sb.append(columnFamily);
350                   if (!columnFamily.endsWith(":")) {
351                     sb.append(":");
352                   }
353                   if (column != null) {
354                     sb.append(column);
355                   }
356                 }
357               }
358               byte[][] split =
359                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
360               if(split.length == 1) {
361                 put.add(split[0], new byte[0], t);
362               } else {
363                 put.add(split[0], split[1], t);
364               }
365               updater.put(put);
366               count++;
367             } catch (RuntimeException ex) {
368               ex.printStackTrace();
369               throw ex;
370             } catch (IOException ex) {
371               ex.printStackTrace();
372               throw ex;
373             }
374           } catch (RuntimeException ex) {
375             ex.printStackTrace();
376             throw ex;
377           } catch (IOException ex) {
378             ex.printStackTrace();
379             throw ex;
380           }
381         }
382         // Set start character back to FIRST_CHAR after we've done first loop.
383         thirdCharStart = FIRST_CHAR;
384       }
385       secondCharStart = FIRST_CHAR;
386     }
387     return count;
388   }
389 
390   /**
391    * Implementors can flushcache.
392    */
393   public static interface FlushCache {
394     /**
395      * @throws IOException
396      */
397     public void flushcache() throws IOException;
398   }
399 
400   /**
401    * Interface used by tests so can do common operations against an HTable
402    * or an HRegion.
403    *
404    * TOOD: Come up w/ a better name for this interface.
405    */
406   public static interface Incommon {
407     /**
408      *
409      * @param delete
410      * @param lockid
411      * @param writeToWAL
412      * @throws IOException
413      */
414     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
415     throws IOException;
416 
417     /**
418      * @param put
419      * @throws IOException
420      */
421     public void put(Put put) throws IOException;
422 
423     public Result get(Get get) throws IOException;
424 
425     /**
426      * @param family
427      * @param qualifiers
428      * @param firstRow
429      * @param ts
430      * @return scanner for specified columns, first row and timestamp
431      * @throws IOException
432      */
433     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
434         byte [] firstRow, long ts)
435     throws IOException;
436   }
437 
438   /**
439    * A class that makes a {@link Incommon} out of a {@link HRegion}
440    */
441   public static class HRegionIncommon implements Incommon, FlushCache {
442     final HRegion region;
443 
444     /**
445      * @param HRegion
446      */
447     public HRegionIncommon(final HRegion HRegion) {
448       this.region = HRegion;
449     }
450 
451     public void put(Put put) throws IOException {
452       region.put(put);
453     }
454 
455     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
456     throws IOException {
457       this.region.delete(delete, lockid, writeToWAL);
458     }
459 
460     public Result get(Get get) throws IOException {
461       return region.get(get, null);
462     }
463 
464     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
465         byte [] firstRow, long ts)
466       throws IOException {
467         Scan scan = new Scan(firstRow);
468         if(qualifiers == null || qualifiers.length == 0) {
469           scan.addFamily(family);
470         } else {
471           for(int i=0; i<qualifiers.length; i++){
472             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
473           }
474         }
475         scan.setTimeRange(0, ts);
476         return new
477           InternalScannerIncommon(region.getScanner(scan));
478       }
479 
480     public Result get(Get get, Integer lockid) throws IOException{
481       return this.region.get(get, lockid);
482     }
483 
484 
485     public void flushcache() throws IOException {
486       this.region.flushcache();
487     }
488   }
489 
490   /**
491    * A class that makes a {@link Incommon} out of a {@link HTable}
492    */
493   public static class HTableIncommon implements Incommon {
494     final HTable table;
495 
496     /**
497      * @param table
498      */
499     public HTableIncommon(final HTable table) {
500       super();
501       this.table = table;
502     }
503 
504     public void put(Put put) throws IOException {
505       table.put(put);
506     }
507 
508 
509     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
510     throws IOException {
511       this.table.delete(delete);
512     }
513 
514     public Result get(Get get) throws IOException {
515       return table.get(get);
516     }
517 
518     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
519         byte [] firstRow, long ts)
520       throws IOException {
521       Scan scan = new Scan(firstRow);
522       if(qualifiers == null || qualifiers.length == 0) {
523         scan.addFamily(family);
524       } else {
525         for(int i=0; i<qualifiers.length; i++){
526           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
527         }
528       }
529       scan.setTimeRange(0, ts);
530       return new
531         ClientScannerIncommon(table.getScanner(scan));
532     }
533   }
534 
535   public interface ScannerIncommon
536   extends Iterable<Result> {
537     public boolean next(List<KeyValue> values)
538     throws IOException;
539 
540     public void close() throws IOException;
541   }
542 
543   public static class ClientScannerIncommon implements ScannerIncommon {
544     ResultScanner scanner;
545     public ClientScannerIncommon(ResultScanner scanner) {
546       this.scanner = scanner;
547     }
548 
549     public boolean next(List<KeyValue> values)
550     throws IOException {
551       Result results = scanner.next();
552       if (results == null) {
553         return false;
554       }
555       values.clear();
556       values.addAll(results.list());
557       return true;
558     }
559 
560     public void close() throws IOException {
561       scanner.close();
562     }
563 
564     @SuppressWarnings("unchecked")
565     public Iterator iterator() {
566       return scanner.iterator();
567     }
568   }
569 
570   public static class InternalScannerIncommon implements ScannerIncommon {
571     InternalScanner scanner;
572 
573     public InternalScannerIncommon(InternalScanner scanner) {
574       this.scanner = scanner;
575     }
576 
577     public boolean next(List<KeyValue> results)
578     throws IOException {
579       return scanner.next(results);
580     }
581 
582     public void close() throws IOException {
583       scanner.close();
584     }
585 
586     public Iterator<Result> iterator() {
587       throw new UnsupportedOperationException();
588     }
589   }
590 
591 //  protected void assertCellEquals(final HRegion region, final byte [] row,
592 //    final byte [] column, final long timestamp, final String value)
593 //  throws IOException {
594 //    Map<byte [], Cell> result = region.getFull(row, null, timestamp, 1, null);
595 //    Cell cell_value = result.get(column);
596 //    if (value == null) {
597 //      assertEquals(Bytes.toString(column) + " at timestamp " + timestamp, null,
598 //        cell_value);
599 //    } else {
600 //      if (cell_value == null) {
601 //        fail(Bytes.toString(column) + " at timestamp " + timestamp +
602 //          "\" was expected to be \"" + value + " but was null");
603 //      }
604 //      if (cell_value != null) {
605 //        assertEquals(Bytes.toString(column) + " at timestamp "
606 //            + timestamp, value, new String(cell_value.getValue()));
607 //      }
608 //    }
609 //  }
610 
611   protected void assertResultEquals(final HRegion region, final byte [] row,
612       final byte [] family, final byte [] qualifier, final long timestamp,
613       final byte [] value)
614     throws IOException {
615       Get get = new Get(row);
616       get.setTimeStamp(timestamp);
617       Result res = region.get(get, null);
618       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
619         res.getMap();
620       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
621 
622       if (value == null) {
623         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
624             " at timestamp " + timestamp, null, res_value);
625       } else {
626         if (res_value == null) {
627           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
628               " at timestamp " + timestamp + "\" was expected to be \"" +
629               Bytes.toStringBinary(value) + " but was null");
630         }
631         if (res_value != null) {
632           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
633               " at timestamp " +
634               timestamp, value, new String(res_value));
635         }
636       }
637     }
638 
639   /**
640    * Initializes parameters used in the test environment:
641    *
642    * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
643    * Sets the boolean debugging if "DEBUGGING" is set in the environment.
644    * If debugging is enabled, reconfigures logging so that the root log level is
645    * set to WARN and the logging level for the package is set to DEBUG.
646    */
647   public static void initialize() {
648     if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
649       System.setProperty(TEST_DIRECTORY_KEY, new File(
650           "build/hbase/test").getAbsolutePath());
651     }
652   }
653 
654   /**
655    * Common method to close down a MiniDFSCluster and the associated file system
656    *
657    * @param cluster
658    */
659   public static void shutdownDfs(MiniDFSCluster cluster) {
660     if (cluster != null) {
661       LOG.info("Shutting down Mini DFS ");
662       try {
663         cluster.shutdown();
664       } catch (Exception e) {
665         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
666         // here because of an InterruptedException. Don't let exceptions in
667         // here be cause of test failure.
668       }
669       try {
670         FileSystem fs = cluster.getFileSystem();
671         if (fs != null) {
672           LOG.info("Shutting down FileSystem");
673           fs.close();
674         }
675         FileSystem.closeAll();
676       } catch (IOException e) {
677         LOG.error("error closing file system", e);
678       }
679     }
680   }
681 
682   protected void createRootAndMetaRegions() throws IOException {
683     root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir,
684         conf, HTableDescriptor.ROOT_TABLEDESC);
685     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
686         conf, HTableDescriptor.META_TABLEDESC);
687     HRegion.addRegionToMETA(root, meta);
688   }
689 
690   protected void closeRootAndMeta() throws IOException {
691     if (meta != null) {
692       meta.close();
693       meta.getLog().closeAndDelete();
694     }
695     if (root != null) {
696       root.close();
697       root.getLog().closeAndDelete();
698     }
699   }
700 
701   public static void assertByteEquals(byte[] expected,
702                                byte[] actual) {
703     if (Bytes.compareTo(expected, actual) != 0) {
704       throw new AssertionFailedError("expected:<" +
705       Bytes.toString(expected) + "> but was:<" +
706       Bytes.toString(actual) + ">");
707     }
708   }
709 
710   public static void assertEquals(byte[] expected,
711                                byte[] actual) {
712     if (Bytes.compareTo(expected, actual) != 0) {
713       throw new AssertionFailedError("expected:<" +
714       Bytes.toStringBinary(expected) + "> but was:<" +
715       Bytes.toStringBinary(actual) + ">");
716     }
717   }
718 
719 }