1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
24 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
25 import static org.junit.Assert.assertEquals;
26
27 import java.io.IOException;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Map.Entry;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HServerAddress;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.ServerName;
44 import org.apache.hadoop.hbase.client.Delete;
45 import org.apache.hadoop.hbase.client.HBaseAdmin;
46 import org.apache.hadoop.hbase.client.HTable;
47 import org.apache.hadoop.hbase.client.Put;
48 import org.apache.hadoop.hbase.client.Result;
49 import org.apache.hadoop.hbase.client.ResultScanner;
50 import org.apache.hadoop.hbase.client.Scan;
51 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
52 import org.apache.zookeeper.KeeperException;
53 import org.junit.AfterClass;
54 import org.junit.BeforeClass;
55 import org.junit.Test;
56
57
58
59
60 public class TestHBaseFsck {
61 final Log LOG = LogFactory.getLog(getClass());
62 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63 private final static Configuration conf = TEST_UTIL.getConfiguration();
64 private final static byte[] FAM = Bytes.toBytes("fam");
65
66
67 private HTable tbl;
68 private final static byte[][] splits= new byte[][] { Bytes.toBytes("A"),
69 Bytes.toBytes("B"), Bytes.toBytes("C") };
70
71 @BeforeClass
72 public static void setUpBeforeClass() throws Exception {
73 TEST_UTIL.getConfiguration().setBoolean("hbase.master.distributed.log.splitting", false);
74 TEST_UTIL.startMiniCluster(3);
75 }
76
77 @AfterClass
78 public static void tearDownAfterClass() throws Exception {
79 TEST_UTIL.shutdownMiniCluster();
80 }
81
82 @Test
83 public void testHBaseFsck() throws Exception {
84 assertNoErrors(doFsck(conf, false));
85 String table = "tableBadMetaAssign";
86 TEST_UTIL.createTable(Bytes.toBytes(table), FAM);
87
88
89 assertNoErrors(doFsck(conf, false));
90
91
92
93 HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
94 ResultScanner scanner = meta.getScanner(new Scan());
95
96 resforloop:
97 for (Result res : scanner) {
98 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
99 HConstants.STARTCODE_QUALIFIER));
100
101 for (JVMClusterUtil.RegionServerThread rs :
102 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
103
104 ServerName sn = rs.getRegionServer().getServerName();
105
106
107 if (startCode != sn.getStartcode()) {
108 Put put = new Put(res.getRow());
109 put.setWriteToWAL(false);
110 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
111 Bytes.toBytes(sn.getHostAndPort()));
112 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
113 Bytes.toBytes(sn.getStartcode()));
114 meta.put(put);
115 break resforloop;
116 }
117 }
118 }
119
120
121 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
122 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
123
124
125
126
127 Thread.sleep(1000);
128
129
130 assertNoErrors(doFsck(conf, false));
131
132
133 new HTable(conf, Bytes.toBytes(table)).getScanner(new Scan());
134 }
135
136 private HRegionInfo createRegion(Configuration conf, final HTableDescriptor
137 htd, byte[] startKey, byte[] endKey)
138 throws IOException {
139 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
140 HRegionInfo hri = new HRegionInfo(htd.getName(), startKey, endKey);
141 Put put = new Put(hri.getRegionName());
142 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
143 Writables.getBytes(hri));
144 meta.put(put);
145 return hri;
146 }
147
148 public void dumpMeta(HTableDescriptor htd) throws IOException {
149 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getName());
150 for (byte[] row : metaRows) {
151 LOG.info(Bytes.toString(row));
152 }
153 }
154
155 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
156 byte[] startKey, byte[] endKey) throws IOException {
157
158 LOG.info("Before delete:");
159 dumpMeta(htd);
160
161 Map<HRegionInfo, HServerAddress> hris = tbl.getRegionsInfo();
162 for (Entry<HRegionInfo, HServerAddress> e: hris.entrySet()) {
163 HRegionInfo hri = e.getKey();
164 HServerAddress hsa = e.getValue();
165 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
166 && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
167
168 LOG.info("RegionName: " +hri.getRegionNameAsString());
169 byte[] deleteRow = hri.getRegionName();
170 TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
171
172 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
173 Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
174 FileSystem fs = rootDir.getFileSystem(conf);
175 Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
176 fs.delete(p, true);
177
178 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
179 Delete delete = new Delete(deleteRow);
180 meta.delete(delete);
181 }
182 LOG.info(hri.toString() + hsa.toString());
183 }
184
185 TEST_UTIL.getMetaTableRows(htd.getName());
186 LOG.info("After delete:");
187 dumpMeta(htd);
188
189 }
190
191
192
193
194
195
196
197
198 HTable setupTable(String tablename) throws Exception {
199 HTableDescriptor desc = new HTableDescriptor(tablename);
200 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
201 desc.addFamily(hcd);
202 TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
203 tbl = new HTable(TEST_UTIL.getConfiguration(), tablename);
204 return tbl;
205 }
206
207
208
209
210
211
212
213
214 void deleteTable(String tablename) throws IOException {
215 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
216 byte[] tbytes = Bytes.toBytes(tablename);
217 admin.disableTable(tbytes);
218 admin.deleteTable(tbytes);
219 }
220
221
222
223
224
225
226 @Test
227 public void testHBaseFsckClean() throws Exception {
228 assertNoErrors(doFsck(conf, false));
229 String table = "tableClean";
230 try {
231 HBaseFsck hbck = doFsck(conf, false);
232 assertNoErrors(hbck);
233
234 setupTable(table);
235
236
237 hbck = doFsck(conf, false);
238 assertNoErrors(hbck);
239 assertEquals(0, hbck.getOverlapGroups(table).size());
240 } finally {
241 deleteTable(table);
242 }
243 }
244
245
246
247
248 @Test
249 public void testDupeStartKey() throws Exception {
250 String table = "tableDupeStartKey";
251 try {
252 setupTable(table);
253 assertNoErrors(doFsck(conf, false));
254
255
256 HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
257 Bytes.toBytes("A"), Bytes.toBytes("A2"));
258 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
259 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
260 .waitForAssignment(hriDupe);
261
262 HBaseFsck hbck = doFsck(conf, false);
263 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
264 ERROR_CODE.DUPE_STARTKEYS});
265 assertEquals(2, hbck.getOverlapGroups(table).size());
266 } finally {
267 deleteTable(table);
268 }
269 }
270
271
272
273
274 @Test
275 public void testDegenerateRegions() throws Exception {
276 String table = "tableDegenerateRegions";
277 try {
278 setupTable(table);
279 assertNoErrors(doFsck(conf,false));
280
281
282 HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
283 Bytes.toBytes("B"), Bytes.toBytes("B"));
284 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
285 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
286 .waitForAssignment(hriDupe);
287
288 HBaseFsck hbck = doFsck(conf,false);
289 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION,
290 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS});
291 assertEquals(2, hbck.getOverlapGroups(table).size());
292 } finally {
293 deleteTable(table);
294 }
295 }
296
297
298
299
300 @Test
301 public void testCoveredStartKey() throws Exception {
302 String table = "tableCoveredStartKey";
303 try {
304 setupTable(table);
305
306
307 HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
308 Bytes.toBytes("A2"), Bytes.toBytes("B2"));
309 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
310 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
311 .waitForAssignment(hriOverlap);
312
313 HBaseFsck hbck = doFsck(conf, false);
314 assertErrors(hbck, new ERROR_CODE[] {
315 ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
316 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
317 assertEquals(3, hbck.getOverlapGroups(table).size());
318 } finally {
319 deleteTable(table);
320 }
321 }
322
323
324
325
326 @Test
327 public void testMetaHole() throws Exception {
328 String table = "tableMetaHole";
329 try {
330 setupTable(table);
331
332
333 HRegionInfo hriHole = createRegion(conf, tbl.getTableDescriptor(),
334 Bytes.toBytes("D"), Bytes.toBytes(""));
335 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriHole);
336 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
337 .waitForAssignment(hriHole);
338
339 TEST_UTIL.getHBaseAdmin().disableTable(table);
340 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""));
341 TEST_UTIL.getHBaseAdmin().enableTable(table);
342
343 HBaseFsck hbck = doFsck(conf, false);
344 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN });
345
346 assertEquals(0, hbck.getOverlapGroups(table).size());
347 } finally {
348 deleteTable(table);
349 }
350 }
351 }