1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.coprocessor;
22
23 import java.io.IOException;
24 import java.lang.reflect.Method;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.List;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.Coprocessor;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.MiniHBaseCluster;
39 import org.apache.hadoop.hbase.client.*;
40 import org.apache.hadoop.hbase.regionserver.HRegion;
41 import org.apache.hadoop.hbase.regionserver.InternalScanner;
42 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
43 import org.apache.hadoop.hbase.regionserver.Store;
44 import org.apache.hadoop.hbase.regionserver.StoreFile;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
47 import org.apache.hadoop.hbase.util.JVMClusterUtil;
48
49 import org.junit.AfterClass;
50 import org.junit.BeforeClass;
51 import org.junit.Test;
52
53 import static org.junit.Assert.*;
54
55 public class TestRegionObserverInterface {
56 static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
57 static final String DIR = "test/build/data/TestRegionObserver/";
58
59 public static final byte[] TEST_TABLE = Bytes.toBytes("TestTable");
60 public final static byte[] A = Bytes.toBytes("a");
61 public final static byte[] B = Bytes.toBytes("b");
62 public final static byte[] C = Bytes.toBytes("c");
63 public final static byte[] ROW = Bytes.toBytes("testrow");
64
65 private static HBaseTestingUtility util = new HBaseTestingUtility();
66 private static MiniHBaseCluster cluster = null;
67
68 @BeforeClass
69 public static void setupBeforeClass() throws Exception {
70
71 Configuration conf = util.getConfiguration();
72 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
73 "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver");
74
75 util.startMiniCluster();
76 cluster = util.getMiniHBaseCluster();
77 }
78
79 @AfterClass
80 public static void tearDownAfterClass() throws Exception {
81 util.shutdownMiniCluster();
82 }
83
84 @Test
85 public void testRegionObserver() throws IOException {
86 byte[] tableName = TEST_TABLE;
87
88
89 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
90 verifyMethodResult(SimpleRegionObserver.class,
91 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
92 "hadDelete"},
93 TEST_TABLE,
94 new Boolean[] {false, false, false, false, false});
95
96 Put put = new Put(ROW);
97 put.add(A, A, A);
98 put.add(B, B, B);
99 put.add(C, C, C);
100 table.put(put);
101
102 verifyMethodResult(SimpleRegionObserver.class,
103 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
104 "hadDelete"},
105 TEST_TABLE,
106 new Boolean[] {false, false, true, true, false}
107 );
108
109 Get get = new Get(ROW);
110 get.addColumn(A, A);
111 get.addColumn(B, B);
112 get.addColumn(C, C);
113 table.get(get);
114
115 verifyMethodResult(SimpleRegionObserver.class,
116 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
117 "hadDelete"},
118 TEST_TABLE,
119 new Boolean[] {true, true, true, true, false}
120 );
121
122 Delete delete = new Delete(ROW);
123 delete.deleteColumn(A, A);
124 delete.deleteColumn(B, B);
125 delete.deleteColumn(C, C);
126 table.delete(delete);
127
128 verifyMethodResult(SimpleRegionObserver.class,
129 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
130 "hadDelete"},
131 TEST_TABLE,
132 new Boolean[] {true, true, true, true, true}
133 );
134 util.deleteTable(tableName);
135 }
136
137 @Test
138 public void testIncrementHook() throws IOException {
139 byte[] tableName = TEST_TABLE;
140
141 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
142 Increment inc = new Increment(Bytes.toBytes(0));
143 inc.addColumn(A, A, 1);
144
145 verifyMethodResult(SimpleRegionObserver.class,
146 new String[] {"hadPreIncrement", "hadPostIncrement"},
147 tableName,
148 new Boolean[] {false, false}
149 );
150
151 table.increment(inc);
152
153 verifyMethodResult(SimpleRegionObserver.class,
154 new String[] {"hadPreIncrement", "hadPostIncrement"},
155 tableName,
156 new Boolean[] {true, true}
157 );
158 util.deleteTable(tableName);
159 }
160
161 @Test
162
163 public void testHBase3583() throws IOException {
164 byte[] tableName = Bytes.toBytes("testHBase3583");
165 util.createTable(tableName, new byte[][] {A, B, C});
166
167 verifyMethodResult(SimpleRegionObserver.class,
168 new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
169 "wasScannerCloseCalled"},
170 tableName,
171 new Boolean[] {false, false, false, false}
172 );
173
174 HTable table = new HTable(util.getConfiguration(), tableName);
175 Put put = new Put(ROW);
176 put.add(A, A, A);
177 table.put(put);
178
179 Get get = new Get(ROW);
180 get.addColumn(A, A);
181 table.get(get);
182
183
184
185 verifyMethodResult(SimpleRegionObserver.class,
186 new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
187 "wasScannerCloseCalled"},
188 tableName,
189 new Boolean[] {true, true, false, false}
190 );
191
192 Scan s = new Scan();
193 ResultScanner scanner = table.getScanner(s);
194 try {
195 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
196 }
197 } finally {
198 scanner.close();
199 }
200
201
202 verifyMethodResult(SimpleRegionObserver.class,
203 new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
204 tableName,
205 new Boolean[] {true, true}
206 );
207 util.deleteTable(tableName);
208 }
209
210 @Test
211
212 public void testHBase3758() throws IOException {
213 byte[] tableName = Bytes.toBytes("testHBase3758");
214 util.createTable(tableName, new byte[][] {A, B, C});
215
216 verifyMethodResult(SimpleRegionObserver.class,
217 new String[] {"hadDeleted", "wasScannerOpenCalled"},
218 tableName,
219 new Boolean[] {false, false}
220 );
221
222 HTable table = new HTable(util.getConfiguration(), tableName);
223 Put put = new Put(ROW);
224 put.add(A, A, A);
225 table.put(put);
226
227 Delete delete = new Delete(ROW);
228 table.delete(delete);
229
230 verifyMethodResult(SimpleRegionObserver.class,
231 new String[] {"hadDeleted", "wasScannerOpenCalled"},
232 tableName,
233 new Boolean[] {true, false}
234 );
235
236 Scan s = new Scan();
237 ResultScanner scanner = table.getScanner(s);
238 try {
239 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
240 }
241 } finally {
242 scanner.close();
243 }
244
245
246 verifyMethodResult(SimpleRegionObserver.class,
247 new String[] {"wasScannerOpenCalled"},
248 tableName,
249 new Boolean[] {true}
250 );
251 util.deleteTable(tableName);
252 }
253
254
255 public static class EvenOnlyCompactor extends BaseRegionObserver {
256 long lastCompaction;
257 long lastFlush;
258
259 @Override
260 public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
261 Store store, final InternalScanner scanner) {
262 return new InternalScanner() {
263 @Override
264 public boolean next(List<KeyValue> results) throws IOException {
265 return next(results, -1);
266 }
267
268 @Override
269 public boolean next(List<KeyValue> results, int limit) throws IOException {
270 List<KeyValue> internalResults = new ArrayList<KeyValue>();
271 boolean hasMore;
272 do {
273 hasMore = scanner.next(internalResults, limit);
274 if (!internalResults.isEmpty()) {
275 long row = Bytes.toLong(internalResults.get(0).getRow());
276 if (row % 2 == 0) {
277
278 break;
279 }
280
281 internalResults.clear();
282 }
283 } while (hasMore);
284
285 if (!internalResults.isEmpty()) {
286 results.addAll(internalResults);
287 }
288 return hasMore;
289 }
290
291 @Override
292 public void close() throws IOException {
293 scanner.close();
294 }
295 };
296 }
297
298 @Override
299 public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
300 Store store, StoreFile resultFile) {
301 lastCompaction = EnvironmentEdgeManager.currentTimeMillis();
302 }
303
304 @Override
305 public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) {
306 lastFlush = EnvironmentEdgeManager.currentTimeMillis();
307 }
308 }
309
310
311
312
313 @Test
314 public void testCompactionOverride() throws Exception {
315 byte[] compactTable = Bytes.toBytes("TestCompactionOverride");
316 HBaseAdmin admin = util.getHBaseAdmin();
317 if (admin.tableExists(compactTable)) {
318 admin.disableTable(compactTable);
319 admin.deleteTable(compactTable);
320 }
321
322 HTableDescriptor htd = new HTableDescriptor(compactTable);
323 htd.addFamily(new HColumnDescriptor(A));
324 htd.addCoprocessor(EvenOnlyCompactor.class.getName());
325 admin.createTable(htd);
326
327 HTable table = new HTable(util.getConfiguration(), compactTable);
328 for (long i=1; i<=10; i++) {
329 byte[] iBytes = Bytes.toBytes(i);
330 Put put = new Put(iBytes);
331 put.setWriteToWAL(false);
332 put.add(A, A, iBytes);
333 table.put(put);
334 }
335
336 HRegion firstRegion = cluster.getRegions(compactTable).get(0);
337 Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor(
338 EvenOnlyCompactor.class.getName());
339 assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
340 EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp;
341
342
343 long ts = System.currentTimeMillis();
344 admin.flush(compactTable);
345
346 for (int i=0; i<10; i++) {
347 if (compactor.lastFlush >= ts) {
348 break;
349 }
350 Thread.sleep(1000);
351 }
352 assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
353 LOG.debug("Flush complete");
354
355 ts = compactor.lastFlush;
356 admin.majorCompact(compactTable);
357
358 for (int i=0; i<30; i++) {
359 if (compactor.lastCompaction >= ts) {
360 break;
361 }
362 Thread.sleep(1000);
363 }
364 LOG.debug("Last compaction was at "+compactor.lastCompaction);
365 assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
366
367
368 ResultScanner scanner = table.getScanner(new Scan());
369 try {
370 for (long i=2; i<=10; i+=2) {
371 Result r = scanner.next();
372 assertNotNull(r);
373 assertFalse(r.isEmpty());
374 byte[] iBytes = Bytes.toBytes(i);
375 assertArrayEquals("Row should be "+i, r.getRow(), iBytes);
376 assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes);
377 }
378 } finally {
379 scanner.close();
380 }
381 }
382
383
384 private void verifyMethodResult(Class c, String methodName[], byte[] tableName,
385 Object value[]) throws IOException {
386 try {
387 for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
388 for (HRegionInfo r : t.getRegionServer().getOnlineRegions()) {
389 if (!Arrays.equals(r.getTableName(), tableName)) {
390 continue;
391 }
392 RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()).
393 getCoprocessorHost();
394
395 Coprocessor cp = cph.findCoprocessor(c.getName());
396 assertNotNull(cp);
397 for (int i = 0; i < methodName.length; ++i) {
398 Method m = c.getMethod(methodName[i]);
399 Object o = m.invoke(cp);
400 assertTrue("Result of " + c.getName() + "." + methodName[i]
401 + " is expected to be " + value[i].toString()
402 + ", while we get " + o.toString(), o.equals(value[i]));
403 }
404 }
405 }
406 } catch (Exception e) {
407 throw new IOException(e.toString());
408 }
409 }
410
411 private static byte [][] makeN(byte [] base, int n) {
412 byte [][] ret = new byte[n][];
413 for(int i=0;i<n;i++) {
414 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
415 }
416 return ret;
417 }
418 }
419