1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util.hbck;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Map.Entry;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HColumnDescriptor;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HServerAddress;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.client.Delete;
41 import org.apache.hadoop.hbase.client.HBaseAdmin;
42 import org.apache.hadoop.hbase.client.HConnectionManager;
43 import org.apache.hadoop.hbase.client.HTable;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.client.ResultScanner;
47 import org.apache.hadoop.hbase.client.Scan;
48 import org.apache.hadoop.hbase.regionserver.HRegion;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.Writables;
51 import org.apache.zookeeper.KeeperException;
52 import org.junit.After;
53 import org.junit.Before;
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68 public class OfflineMetaRebuildTestCore {
69 protected final static Log LOG = LogFactory
70 .getLog(OfflineMetaRebuildTestCore.class);
71 protected HBaseTestingUtility TEST_UTIL;
72 protected Configuration conf;
73 private final static byte[] FAM = Bytes.toBytes("fam");
74
75
76 protected HTable htbl;
77 protected final static byte[][] splits = new byte[][] { Bytes.toBytes("A"),
78 Bytes.toBytes("B"), Bytes.toBytes("C") };
79
80 private final static String TABLE_BASE = "tableMetaRebuild";
81 private static int tableIdx = 0;
82 protected String table = "tableMetaRebuild";
83
84 @Before
85 public void setUpBefore() throws Exception {
86 TEST_UTIL = new HBaseTestingUtility();
87 TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
88 TEST_UTIL.startMiniCluster(3);
89 conf = TEST_UTIL.getConfiguration();
90 assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);
91
92
93 table = TABLE_BASE + "-" + tableIdx;
94 tableIdx++;
95 htbl = setupTable(table);
96 populateTable(htbl);
97 assertEquals(4, scanMeta());
98 LOG.info("Table " + table + " has " + tableRowCount(conf, table)
99 + " entries.");
100 assertEquals(16, tableRowCount(conf, table));
101 TEST_UTIL.getHBaseAdmin().disableTable(table);
102 assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
103 }
104
105 @After
106 public void tearDownAfter() throws Exception {
107 TEST_UTIL.shutdownMiniCluster();
108 HConnectionManager.deleteConnection(conf, true);
109 }
110
111
112
113
114
115
116
117
118 private HTable setupTable(String tablename) throws Exception {
119 HTableDescriptor desc = new HTableDescriptor(tablename);
120 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
121 desc.addFamily(hcd);
122 TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
123 return new HTable(TEST_UTIL.getConfiguration(), tablename);
124 }
125
126 private void dumpMeta(HTableDescriptor htd) throws IOException {
127 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getName());
128 for (byte[] row : metaRows) {
129 LOG.info(Bytes.toString(row));
130 }
131 }
132
133 private void populateTable(HTable tbl) throws IOException {
134 byte[] values = { 'A', 'B', 'C', 'D' };
135 for (int i = 0; i < values.length; i++) {
136 for (int j = 0; j < values.length; j++) {
137 Put put = new Put(new byte[] { values[i], values[j] });
138 put.add(Bytes.toBytes("fam"), new byte[] {}, new byte[] { values[i],
139 values[j] });
140 tbl.put(put);
141 }
142 }
143 tbl.flushCommits();
144 }
145
146
147
148
149
150
151
152 void deleteTable(HBaseAdmin admin, String tablename) throws IOException {
153 try {
154 byte[] tbytes = Bytes.toBytes(tablename);
155 admin.disableTable(tbytes);
156 admin.deleteTable(tbytes);
157 } catch (Exception e) {
158
159 }
160 }
161
162 protected void deleteRegion(Configuration conf, final HTable tbl,
163 byte[] startKey, byte[] endKey) throws IOException {
164
165 LOG.info("Before delete:");
166 HTableDescriptor htd = tbl.getTableDescriptor();
167 dumpMeta(htd);
168
169 Map<HRegionInfo, HServerAddress> hris = tbl.getRegionsInfo();
170 for (Entry<HRegionInfo, HServerAddress> e : hris.entrySet()) {
171 HRegionInfo hri = e.getKey();
172 HServerAddress hsa = e.getValue();
173 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
174 && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
175
176 LOG.info("RegionName: " + hri.getRegionNameAsString());
177 byte[] deleteRow = hri.getRegionName();
178 TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
179
180 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
181 Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
182 FileSystem fs = rootDir.getFileSystem(conf);
183 Path p = new Path(rootDir + "/" + htd.getNameAsString(),
184 hri.getEncodedName());
185 fs.delete(p, true);
186
187 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
188 Delete delete = new Delete(deleteRow);
189 meta.delete(delete);
190 }
191 LOG.info(hri.toString() + hsa.toString());
192 }
193
194 TEST_UTIL.getMetaTableRows(htd.getName());
195 LOG.info("After delete:");
196 dumpMeta(htd);
197 }
198
199 protected HRegionInfo createRegion(Configuration conf, final HTable htbl,
200 byte[] startKey, byte[] endKey) throws IOException {
201 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
202 HTableDescriptor htd = htbl.getTableDescriptor();
203 HRegionInfo hri = new HRegionInfo(htbl.getTableName(), startKey, endKey);
204
205 LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
206 Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
207 FileSystem fs = rootDir.getFileSystem(conf);
208 Path p = new Path(rootDir + "/" + htd.getNameAsString(),
209 hri.getEncodedName());
210 fs.mkdirs(p);
211 Path riPath = new Path(p, HRegion.REGIONINFO_FILE);
212 FSDataOutputStream out = fs.create(riPath);
213 hri.write(out);
214 out.close();
215
216
217 Put put = new Put(hri.getRegionName());
218 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
219 Writables.getBytes(hri));
220 meta.put(put);
221 meta.flushCommits();
222 return hri;
223 }
224
225 protected void wipeOutMeta() throws IOException {
226
227 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
228 Scan s = new Scan();
229 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
230 ResultScanner scanner = meta.getScanner(s);
231 List<Delete> dels = new ArrayList<Delete>();
232 for (Result r : scanner) {
233 Delete d = new Delete(r.getRow());
234 dels.add(d);
235 admin.unassign(r.getRow(), true);
236 }
237 meta.delete(dels);
238 meta.flushCommits();
239 }
240
241
242
243
244
245
246
247 protected int tableRowCount(Configuration conf, String table)
248 throws IOException {
249 HTable t = new HTable(conf, table);
250 Scan st = new Scan();
251
252 ResultScanner rst = t.getScanner(st);
253 int count = 0;
254 for (@SuppressWarnings("unused")
255 Result rt : rst) {
256 count++;
257 }
258 return count;
259 }
260
261
262
263
264
265
266 protected int scanMeta() throws IOException {
267 int count = 0;
268 HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
269 ResultScanner scanner = meta.getScanner(new Scan());
270 LOG.info("Table: " + Bytes.toString(meta.getTableName()));
271 for (Result res : scanner) {
272 LOG.info(Bytes.toString(res.getRow()));
273 count++;
274 }
275 return count;
276 }
277 }