1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
23  import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
24  import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
25  import static org.junit.Assert.assertEquals;
26  
27  import java.io.IOException;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HServerAddress;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.ServerName;
44  import org.apache.hadoop.hbase.client.Delete;
45  import org.apache.hadoop.hbase.client.HBaseAdmin;
46  import org.apache.hadoop.hbase.client.HTable;
47  import org.apache.hadoop.hbase.client.Put;
48  import org.apache.hadoop.hbase.client.Result;
49  import org.apache.hadoop.hbase.client.ResultScanner;
50  import org.apache.hadoop.hbase.client.Scan;
51  import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
52  import org.apache.zookeeper.KeeperException;
53  import org.junit.AfterClass;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  
57  /**
58   * This tests HBaseFsck's ability to detect reasons for inconsistent tables.
59   */
60  public class TestHBaseFsck {
61    final Log LOG = LogFactory.getLog(getClass());
62    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63    private final static Configuration conf = TEST_UTIL.getConfiguration();
64    private final static byte[] FAM = Bytes.toBytes("fam");
65  
66    // for the instance, reset every test run
67    private HTable tbl;
68    private final static byte[][] splits= new byte[][] { Bytes.toBytes("A"), 
69      Bytes.toBytes("B"), Bytes.toBytes("C") };
70    
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      TEST_UTIL.getConfiguration().setBoolean("hbase.master.distributed.log.splitting", false);
74      TEST_UTIL.startMiniCluster(3);
75    }
76  
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      TEST_UTIL.shutdownMiniCluster();
80    }
81  
82    @Test
83    public void testHBaseFsck() throws Exception {
84      assertNoErrors(doFsck(conf, false));
85      String table = "tableBadMetaAssign"; 
86      TEST_UTIL.createTable(Bytes.toBytes(table), FAM);
87  
88      // We created 1 table, should be fine
89      assertNoErrors(doFsck(conf, false));
90  
91      // Now let's mess it up and change the assignment in .META. to
92      // point to a different region server
93      HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
94      ResultScanner scanner = meta.getScanner(new Scan());
95  
96      resforloop:
97      for (Result res : scanner) {
98        long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
99            HConstants.STARTCODE_QUALIFIER));
100 
101       for (JVMClusterUtil.RegionServerThread rs :
102           TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
103 
104         ServerName sn = rs.getRegionServer().getServerName();
105 
106         // When we find a diff RS, change the assignment and break
107         if (startCode != sn.getStartcode()) {
108           Put put = new Put(res.getRow());
109           put.setWriteToWAL(false);
110           put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
111             Bytes.toBytes(sn.getHostAndPort()));
112           put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
113             Bytes.toBytes(sn.getStartcode()));
114           meta.put(put);
115           break resforloop;
116         }
117       }
118     }
119 
120     // Try to fix the data
121     assertErrors(doFsck(conf, true), new ERROR_CODE[]{
122         ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
123 
124     // fixing assignements require opening regions is not synchronous.  To make
125     // the test pass consistentyl so for now we bake in some sleep to let it
126     // finish.  1s seems sufficient.
127     Thread.sleep(1000);
128 
129     // Should be fixed now
130     assertNoErrors(doFsck(conf, false));
131 
132     // comment needed - what is the purpose of this line
133     new HTable(conf, Bytes.toBytes(table)).getScanner(new Scan());
134   }
135 
136   private HRegionInfo createRegion(Configuration conf, final HTableDescriptor
137       htd, byte[] startKey, byte[] endKey)
138       throws IOException {
139     HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
140     HRegionInfo hri = new HRegionInfo(htd.getName(), startKey, endKey);
141     Put put = new Put(hri.getRegionName());
142     put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
143         Writables.getBytes(hri));
144     meta.put(put);
145     return hri;
146   }
147 
148   public void dumpMeta(HTableDescriptor htd) throws IOException {
149     List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getName());
150     for (byte[] row : metaRows) {
151       LOG.info(Bytes.toString(row));
152     }
153   }
154 
155   private void deleteRegion(Configuration conf, final HTableDescriptor htd, 
156       byte[] startKey, byte[] endKey) throws IOException {
157 
158     LOG.info("Before delete:");
159     dumpMeta(htd);
160 
161     Map<HRegionInfo, HServerAddress> hris = tbl.getRegionsInfo();
162     for (Entry<HRegionInfo, HServerAddress> e: hris.entrySet()) {
163       HRegionInfo hri = e.getKey();
164       HServerAddress hsa = e.getValue();
165       if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 
166           && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
167 
168         LOG.info("RegionName: " +hri.getRegionNameAsString());
169         byte[] deleteRow = hri.getRegionName();
170         TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
171 
172         LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
173         Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
174         FileSystem fs = rootDir.getFileSystem(conf);
175         Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
176         fs.delete(p, true);
177 
178         HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
179         Delete delete = new Delete(deleteRow);
180         meta.delete(delete);
181       }
182       LOG.info(hri.toString() + hsa.toString());
183     }
184 
185     TEST_UTIL.getMetaTableRows(htd.getName());
186     LOG.info("After delete:");
187     dumpMeta(htd);
188 
189   }
190 
191   /**
192    * Setup a clean table before we start mucking with it.
193    * 
194    * @throws IOException
195    * @throws InterruptedException
196    * @throws KeeperException
197    */
198   HTable setupTable(String tablename) throws Exception {
199     HTableDescriptor desc = new HTableDescriptor(tablename);
200     HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
201     desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
202     TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
203     tbl = new HTable(TEST_UTIL.getConfiguration(), tablename);
204     return tbl;
205   }
206 
207 
208   /**
209    * delete table in preparation for next test
210    * 
211    * @param tablename
212    * @throws IOException
213    */
214   void deleteTable(String tablename) throws IOException {
215     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
216     byte[] tbytes = Bytes.toBytes(tablename);
217     admin.disableTable(tbytes);
218     admin.deleteTable(tbytes);
219   }
220 
221 
222   
223   /**
224    * This creates a clean table and confirms that the table is clean.
225    */
226   @Test
227   public void testHBaseFsckClean() throws Exception {
228     assertNoErrors(doFsck(conf, false));
229     String table = "tableClean";
230     try {
231       HBaseFsck hbck = doFsck(conf, false);
232       assertNoErrors(hbck);
233 
234       setupTable(table);
235       
236       // We created 1 table, should be fine
237       hbck = doFsck(conf, false);
238       assertNoErrors(hbck);
239       assertEquals(0, hbck.getOverlapGroups(table).size());
240     } finally {
241       deleteTable(table);
242     }
243   }
244 
245   /**
246    * This creates a bad table with regions that have a duplicate start key
247    */
248   @Test
249   public void testDupeStartKey() throws Exception {
250     String table = "tableDupeStartKey";
251     try {
252       setupTable(table);
253       assertNoErrors(doFsck(conf, false));
254 
255       // Now let's mess it up, by adding a region with a duplicate startkey
256       HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
257           Bytes.toBytes("A"), Bytes.toBytes("A2"));
258       TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
259       TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
260           .waitForAssignment(hriDupe);
261 
262       HBaseFsck hbck = doFsck(conf, false);
263       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
264             ERROR_CODE.DUPE_STARTKEYS});
265       assertEquals(2, hbck.getOverlapGroups(table).size());
266     } finally {
267       deleteTable(table);
268     }
269   }
270   
271   /**
272    * This creates a bad table with regions that has startkey == endkey
273    */
274   @Test
275   public void testDegenerateRegions() throws Exception {
276     String table = "tableDegenerateRegions";
277     try {
278       setupTable(table);
279       assertNoErrors(doFsck(conf,false));
280 
281       // Now let's mess it up, by adding a region with a duplicate startkey
282       HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
283           Bytes.toBytes("B"), Bytes.toBytes("B"));
284       TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
285       TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
286           .waitForAssignment(hriDupe);
287 
288       HBaseFsck hbck = doFsck(conf,false);
289       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION,
290           ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS});
291       assertEquals(2, hbck.getOverlapGroups(table).size());
292     } finally {
293       deleteTable(table);
294     }
295   }
296 
297   /**
298    * This creates a bad table where a start key contained in another region.
299    */
300   @Test
301   public void testCoveredStartKey() throws Exception {
302     String table = "tableCoveredStartKey";
303     try {
304       setupTable(table);
305 
306       // Mess it up by creating an overlap in the metadata
307       HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
308           Bytes.toBytes("A2"), Bytes.toBytes("B2"));
309       TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
310       TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
311           .waitForAssignment(hriOverlap);
312 
313       HBaseFsck hbck = doFsck(conf, false);
314       assertErrors(hbck, new ERROR_CODE[] {
315           ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
316           ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
317       assertEquals(3, hbck.getOverlapGroups(table).size());
318     } finally {
319       deleteTable(table);
320     }
321   }
322 
323   /**
324    * This creates a bad table with a hole in meta.
325    */
326   @Test
327   public void testMetaHole() throws Exception {
328     String table = "tableMetaHole";
329     try {
330       setupTable(table);
331 
332       // Mess it up by leaving a hole in the meta data
333       HRegionInfo hriHole = createRegion(conf, tbl.getTableDescriptor(),
334           Bytes.toBytes("D"), Bytes.toBytes(""));
335       TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriHole);
336       TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
337           .waitForAssignment(hriHole);
338 
339       TEST_UTIL.getHBaseAdmin().disableTable(table);
340       deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""));
341       TEST_UTIL.getHBaseAdmin().enableTable(table);
342 
343       HBaseFsck hbck = doFsck(conf, false);
344       assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN });
345       // holes are separate from overlap groups
346       assertEquals(0, hbck.getOverlapGroups(table).size());
347     } finally {
348       deleteTable(table);
349     }
350   }
351 }