1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import java.io.IOException;
24  import java.lang.reflect.Method;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.Coprocessor;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.MiniHBaseCluster;
39  import org.apache.hadoop.hbase.client.*;
40  import org.apache.hadoop.hbase.regionserver.HRegion;
41  import org.apache.hadoop.hbase.regionserver.InternalScanner;
42  import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
43  import org.apache.hadoop.hbase.regionserver.Store;
44  import org.apache.hadoop.hbase.regionserver.StoreFile;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
47  import org.apache.hadoop.hbase.util.JVMClusterUtil;
48  
49  import org.junit.AfterClass;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  
53  import static org.junit.Assert.*;
54  
55  public class TestRegionObserverInterface {
56    static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
57    static final String DIR = "test/build/data/TestRegionObserver/";
58  
59    public static final byte[] TEST_TABLE = Bytes.toBytes("TestTable");
60    public final static byte[] A = Bytes.toBytes("a");
61    public final static byte[] B = Bytes.toBytes("b");
62    public final static byte[] C = Bytes.toBytes("c");
63    public final static byte[] ROW = Bytes.toBytes("testrow");
64  
65    private static HBaseTestingUtility util = new HBaseTestingUtility();
66    private static MiniHBaseCluster cluster = null;
67  
68    @BeforeClass
69    public static void setupBeforeClass() throws Exception {
70      // set configure to indicate which cp should be loaded
71      Configuration conf = util.getConfiguration();
72      conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
73          "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver");
74  
75      util.startMiniCluster();
76      cluster = util.getMiniHBaseCluster();
77    }
78  
79    @AfterClass
80    public static void tearDownAfterClass() throws Exception {
81      util.shutdownMiniCluster();
82    }
83  
84    @Test
85    public void testRegionObserver() throws IOException {
86      byte[] tableName = TEST_TABLE;
87      // recreate table every time in order to reset the status of the
88      // coproccessor.
89      HTable table = util.createTable(tableName, new byte[][] {A, B, C});
90      verifyMethodResult(SimpleRegionObserver.class,
91          new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
92              "hadDelete"},
93          TEST_TABLE,
94          new Boolean[] {false, false, false, false, false});
95  
96      Put put = new Put(ROW);
97      put.add(A, A, A);
98      put.add(B, B, B);
99      put.add(C, C, C);
100     table.put(put);
101 
102     verifyMethodResult(SimpleRegionObserver.class,
103         new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
104             "hadDelete"},
105         TEST_TABLE,
106         new Boolean[] {false, false, true, true, false}
107     );
108 
109     Get get = new Get(ROW);
110     get.addColumn(A, A);
111     get.addColumn(B, B);
112     get.addColumn(C, C);
113     table.get(get);
114 
115     verifyMethodResult(SimpleRegionObserver.class,
116         new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
117             "hadDelete"},
118         TEST_TABLE,
119         new Boolean[] {true, true, true, true, false}
120     );
121 
122     Delete delete = new Delete(ROW);
123     delete.deleteColumn(A, A);
124     delete.deleteColumn(B, B);
125     delete.deleteColumn(C, C);
126     table.delete(delete);
127 
128     verifyMethodResult(SimpleRegionObserver.class,
129         new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
130             "hadDelete"},
131         TEST_TABLE,
132         new Boolean[] {true, true, true, true, true}
133     );
134     util.deleteTable(tableName);
135   }
136 
137   @Test
138   public void testIncrementHook() throws IOException {
139     byte[] tableName = TEST_TABLE;
140 
141     HTable table = util.createTable(tableName, new byte[][] {A, B, C});
142     Increment inc = new Increment(Bytes.toBytes(0));
143     inc.addColumn(A, A, 1);
144 
145     verifyMethodResult(SimpleRegionObserver.class,
146         new String[] {"hadPreIncrement", "hadPostIncrement"},
147         tableName,
148         new Boolean[] {false, false}
149     );
150 
151     table.increment(inc);
152 
153     verifyMethodResult(SimpleRegionObserver.class,
154         new String[] {"hadPreIncrement", "hadPostIncrement"},
155         tableName,
156         new Boolean[] {true, true}
157     );
158     util.deleteTable(tableName);
159   }
160 
161   @Test
162   // HBase-3583
163   public void testHBase3583() throws IOException {
164     byte[] tableName = Bytes.toBytes("testHBase3583");
165     util.createTable(tableName, new byte[][] {A, B, C});
166 
167     verifyMethodResult(SimpleRegionObserver.class,
168         new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
169             "wasScannerCloseCalled"},
170         tableName,
171         new Boolean[] {false, false, false, false}
172     );
173 
174     HTable table = new HTable(util.getConfiguration(), tableName);
175     Put put = new Put(ROW);
176     put.add(A, A, A);
177     table.put(put);
178 
179     Get get = new Get(ROW);
180     get.addColumn(A, A);
181     table.get(get);
182 
183     // verify that scannerNext and scannerClose upcalls won't be invoked
184     // when we perform get().
185     verifyMethodResult(SimpleRegionObserver.class,
186         new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
187             "wasScannerCloseCalled"},
188         tableName,
189         new Boolean[] {true, true, false, false}
190     );
191 
192     Scan s = new Scan();
193     ResultScanner scanner = table.getScanner(s);
194     try {
195       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
196       }
197     } finally {
198       scanner.close();
199     }
200 
201     // now scanner hooks should be invoked.
202     verifyMethodResult(SimpleRegionObserver.class,
203         new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
204         tableName,
205         new Boolean[] {true, true}
206     );
207     util.deleteTable(tableName);
208   }
209 
210   @Test
211   // HBase-3758
212   public void testHBase3758() throws IOException {
213     byte[] tableName = Bytes.toBytes("testHBase3758");
214     util.createTable(tableName, new byte[][] {A, B, C});
215 
216     verifyMethodResult(SimpleRegionObserver.class,
217         new String[] {"hadDeleted", "wasScannerOpenCalled"},
218         tableName,
219         new Boolean[] {false, false}
220     );
221 
222     HTable table = new HTable(util.getConfiguration(), tableName);
223     Put put = new Put(ROW);
224     put.add(A, A, A);
225     table.put(put);
226 
227     Delete delete = new Delete(ROW);
228     table.delete(delete);
229 
230     verifyMethodResult(SimpleRegionObserver.class,
231         new String[] {"hadDeleted", "wasScannerOpenCalled"},
232         tableName,
233         new Boolean[] {true, false}
234     );
235 
236     Scan s = new Scan();
237     ResultScanner scanner = table.getScanner(s);
238     try {
239       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
240       }
241     } finally {
242       scanner.close();
243     }
244 
245     // now scanner hooks should be invoked.
246     verifyMethodResult(SimpleRegionObserver.class,
247         new String[] {"wasScannerOpenCalled"},
248         tableName,
249         new Boolean[] {true}
250     );
251     util.deleteTable(tableName);
252   }
253 
254   /* Overrides compaction to only output rows with keys that are even numbers */
255   public static class EvenOnlyCompactor extends BaseRegionObserver {
256     long lastCompaction;
257     long lastFlush;
258 
259     @Override
260     public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
261         Store store, final InternalScanner scanner) {
262       return new InternalScanner() {
263         @Override
264         public boolean next(List<KeyValue> results) throws IOException {
265           return next(results, -1);
266         }
267 
268         @Override
269         public boolean next(List<KeyValue> results, int limit) throws IOException {
270           List<KeyValue> internalResults = new ArrayList<KeyValue>();
271           boolean hasMore;
272           do {
273             hasMore = scanner.next(internalResults, limit);
274             if (!internalResults.isEmpty()) {
275               long row = Bytes.toLong(internalResults.get(0).getRow());
276               if (row % 2 == 0) {
277                 // return this row
278                 break;
279               }
280               // clear and continue
281               internalResults.clear();
282             }
283           } while (hasMore);
284 
285           if (!internalResults.isEmpty()) {
286             results.addAll(internalResults);
287           }
288           return hasMore;
289         }
290 
291         @Override
292         public void close() throws IOException {
293           scanner.close();
294         }
295       };
296     }
297 
298     @Override
299     public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
300         Store store, StoreFile resultFile) {
301       lastCompaction = EnvironmentEdgeManager.currentTimeMillis();
302     }
303 
304     @Override
305     public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) {
306       lastFlush = EnvironmentEdgeManager.currentTimeMillis();
307     }
308   }
309   /**
310    * Tests overriding compaction handling via coprocessor hooks
311    * @throws Exception
312    */
313   @Test
314   public void testCompactionOverride() throws Exception {
315     byte[] compactTable = Bytes.toBytes("TestCompactionOverride");
316     HBaseAdmin admin = util.getHBaseAdmin();
317     if (admin.tableExists(compactTable)) {
318       admin.disableTable(compactTable);
319       admin.deleteTable(compactTable);
320     }
321 
322     HTableDescriptor htd = new HTableDescriptor(compactTable);
323     htd.addFamily(new HColumnDescriptor(A));
324     htd.addCoprocessor(EvenOnlyCompactor.class.getName());
325     admin.createTable(htd);
326 
327     HTable table = new HTable(util.getConfiguration(), compactTable);
328     for (long i=1; i<=10; i++) {
329       byte[] iBytes = Bytes.toBytes(i);
330       Put put = new Put(iBytes);
331       put.setWriteToWAL(false);
332       put.add(A, A, iBytes);
333       table.put(put);
334     }
335 
336     HRegion firstRegion = cluster.getRegions(compactTable).get(0);
337     Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor(
338         EvenOnlyCompactor.class.getName());
339     assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
340     EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp;
341 
342     // force a compaction
343     long ts = System.currentTimeMillis();
344     admin.flush(compactTable);
345     // wait for flush
346     for (int i=0; i<10; i++) {
347       if (compactor.lastFlush >= ts) {
348         break;
349       }
350       Thread.sleep(1000);
351     }
352     assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
353     LOG.debug("Flush complete");
354 
355     ts = compactor.lastFlush;
356     admin.majorCompact(compactTable);
357     // wait for compaction
358     for (int i=0; i<30; i++) {
359       if (compactor.lastCompaction >= ts) {
360         break;
361       }
362       Thread.sleep(1000);
363     }
364     LOG.debug("Last compaction was at "+compactor.lastCompaction);
365     assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
366 
367     // only even rows should remain
368     ResultScanner scanner = table.getScanner(new Scan());
369     try {
370       for (long i=2; i<=10; i+=2) {
371         Result r = scanner.next();
372         assertNotNull(r);
373         assertFalse(r.isEmpty());
374         byte[] iBytes = Bytes.toBytes(i);
375         assertArrayEquals("Row should be "+i, r.getRow(), iBytes);
376         assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes);
377       }
378     } finally {
379       scanner.close();
380     }
381   }
382 
383   // check each region whether the coprocessor upcalls are called or not.
384   private void verifyMethodResult(Class c, String methodName[], byte[] tableName,
385                                   Object value[]) throws IOException {
386     try {
387       for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
388         for (HRegionInfo r : t.getRegionServer().getOnlineRegions()) {
389           if (!Arrays.equals(r.getTableName(), tableName)) {
390             continue;
391           }
392           RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()).
393               getCoprocessorHost();
394 
395           Coprocessor cp = cph.findCoprocessor(c.getName());
396           assertNotNull(cp);
397           for (int i = 0; i < methodName.length; ++i) {
398             Method m = c.getMethod(methodName[i]);
399             Object o = m.invoke(cp);
400             assertTrue("Result of " + c.getName() + "." + methodName[i]
401                 + " is expected to be " + value[i].toString()
402                 + ", while we get " + o.toString(), o.equals(value[i]));
403           }
404         }
405       }
406     } catch (Exception e) {
407       throw new IOException(e.toString());
408     }
409   }
410 
411   private static byte [][] makeN(byte [] base, int n) {
412     byte [][] ret = new byte[n][];
413     for(int i=0;i<n;i++) {
414       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
415     }
416     return ret;
417   }
418 }
419