1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.Abortable;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.MiniHBaseCluster;
33  import org.apache.hadoop.hbase.client.HBaseAdmin;
34  import org.apache.hadoop.hbase.CoprocessorEnvironment;
35  import org.apache.hadoop.hbase.master.HMaster;
36  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
39  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
40  import org.junit.AfterClass;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import static org.junit.Assert.*;
44  
45  /**
46   * Tests unhandled exceptions thrown by coprocessors running on master.
47   * Expected result is that the master will abort with an informative
48   * error message describing the set of its loaded coprocessors for crash diagnosis.
49   * (HBASE-4014).
50   */
51  public class TestMasterCoprocessorExceptionWithAbort {
52  
53    public static class MasterTracker extends ZooKeeperNodeTracker {
54      public boolean masterZKNodeWasDeleted = false;
55  
56      public MasterTracker(ZooKeeperWatcher zkw, String masterNode, Abortable abortable) {
57        super(zkw, masterNode, abortable);
58      }
59  
60      @Override
61      public synchronized void nodeDeleted(String path) {
62        if (path.equals("/hbase/master")) {
63          masterZKNodeWasDeleted = true;
64        }
65      }
66    }
67  
68    public static class CreateTableThread extends Thread {
69      HBaseTestingUtility UTIL;
70      public CreateTableThread(HBaseTestingUtility UTIL) {
71        this.UTIL = UTIL;
72      }
73  
74      @Override
75      public void run() {
76        // create a table : master coprocessor will throw an exception and not
77        // catch it.
78        HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
79        htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
80        try {
81          HBaseAdmin admin = UTIL.getHBaseAdmin();
82          admin.createTable(htd);
83          fail("BuggyMasterObserver failed to throw an exception.");
84        } catch (IOException e) {
85          assertEquals("HBaseAdmin threw an interrupted IOException as expected.",
86              e.getClass().getName(), "java.io.InterruptedIOException");
87        }
88     }
89    }
90  
91    public static class BuggyMasterObserver extends BaseMasterObserver {
92      private boolean preCreateTableCalled;
93      private boolean postCreateTableCalled;
94      private boolean startCalled;
95      private boolean postStartMasterCalled;
96  
97      @Override
98      public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
99          HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
100       // cause a NullPointerException and don't catch it: this will cause the
101       // master to abort().
102       Integer i;
103       i = null;
104       i = i++;
105     }
106 
107     public boolean wasCreateTableCalled() {
108       return preCreateTableCalled && postCreateTableCalled;
109     }
110 
111     @Override
112     public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
113         throws IOException {
114       postStartMasterCalled = true;
115     }
116 
117     public boolean wasStartMasterCalled() {
118       return postStartMasterCalled;
119     }
120 
121     @Override
122     public void start(CoprocessorEnvironment env) throws IOException {
123       startCalled = true;
124     }
125 
126     public boolean wasStarted() {
127       return startCalled;
128     }
129   }
130 
131   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
132   private static byte[] TEST_TABLE = Bytes.toBytes("observed_table");
133   private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
134 
135   @BeforeClass
136   public static void setupBeforeClass() throws Exception {
137     Configuration conf = UTIL.getConfiguration();
138     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
139         BuggyMasterObserver.class.getName());
140     conf.set("hbase.coprocessor.abortonerror", "true");
141     UTIL.startMiniCluster();
142   }
143 
144   @AfterClass
145   public static void teardownAfterClass() throws Exception {
146     UTIL.shutdownMiniCluster();
147   }
148 
149   @Test(timeout=30000)
150   public void testExceptionFromCoprocessorWhenCreatingTable()
151       throws IOException {
152     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
153 
154     HMaster master = cluster.getMaster();
155     MasterCoprocessorHost host = master.getCoprocessorHost();
156     BuggyMasterObserver cp = (BuggyMasterObserver)host.findCoprocessor(
157         BuggyMasterObserver.class.getName());
158     assertFalse("No table created yet", cp.wasCreateTableCalled());
159 
160     // set a watch on the zookeeper /hbase/master node. If the master dies,
161     // the node will be deleted.
162     ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(),
163       "unittest", new Abortable() {
164       @Override
165       public void abort(String why, Throwable e) {
166         throw new RuntimeException("Fatal ZK error: " + why, e);
167       }
168       @Override
169       public boolean isAborted() {
170         return false;
171       }
172     });
173 
174     MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master",
175         new Abortable() {
176           @Override
177           public void abort(String why, Throwable e) {
178             throw new RuntimeException("Fatal ZK master tracker error, why=", e);
179           }
180           @Override
181           public boolean isAborted() {
182             return false;
183           }
184         });
185 
186     masterTracker.start();
187     zkw.registerListener(masterTracker);
188 
189     // Test (part of the) output that should have be printed by master when it aborts:
190     // (namely the part that shows the set of loaded coprocessors).
191     // In this test, there is only a single coprocessor (BuggyMasterObserver).
192     assertTrue(master.getLoadedCoprocessors().
193       equals("[" +
194           TestMasterCoprocessorExceptionWithAbort.BuggyMasterObserver.class.getName() +
195           "]"));
196 
197     CreateTableThread createTableThread = new CreateTableThread(UTIL);
198 
199     // Attempting to create a table (using createTableThread above) triggers an NPE in BuggyMasterObserver.
200     // Master will then abort and the /hbase/master zk node will be deleted.
201     createTableThread.start();
202 
203     // Wait up to 30 seconds for master's /hbase/master zk node to go away after master aborts.
204     for (int i = 0; i < 30; i++) {
205       if (masterTracker.masterZKNodeWasDeleted == true) {
206         break;
207       }
208       try {
209         Thread.sleep(1000);
210       } catch (InterruptedException e) {
211         fail("InterruptedException while waiting for master zk node to "
212             + "be deleted.");
213       }
214     }
215 
216     assertTrue("Master aborted on coprocessor exception, as expected.",
217         masterTracker.masterZKNodeWasDeleted);
218 
219     createTableThread.interrupt();
220     try {
221       createTableThread.join(1000);
222     } catch (InterruptedException e) {
223       assertTrue("Ignoring InterruptedException while waiting for " +
224           " createTableThread.join().", true);
225     }
226   }
227 
228 }