1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.Abortable;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.MiniHBaseCluster;
33  import org.apache.hadoop.hbase.client.HBaseAdmin;
34  import org.apache.hadoop.hbase.CoprocessorEnvironment;
35  import org.apache.hadoop.hbase.master.HMaster;
36  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
39  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
40  import org.junit.AfterClass;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import static org.junit.Assert.*;
44  
45  /**
46   * Tests unhandled exceptions thrown by coprocessors running on master.
47   * Expected result is that the master will remove the buggy coprocessor from
48   * its set of coprocessors and throw a org.apache.hadoop.hbase.DoNotRetryIOException
49   * back to the client.
50   * (HBASE-4014).
51   */
52  public class TestMasterCoprocessorExceptionWithRemove {
53  
54    public static class MasterTracker extends ZooKeeperNodeTracker {
55      public boolean masterZKNodeWasDeleted = false;
56  
57      public MasterTracker(ZooKeeperWatcher zkw, String masterNode, Abortable abortable) {
58        super(zkw, masterNode, abortable);
59      }
60  
61      @Override
62      public synchronized void nodeDeleted(String path) {
63        if (path.equals("/hbase/master")) {
64          masterZKNodeWasDeleted = true;
65        }
66      }
67    }
68  
69    public static class BuggyMasterObserver extends BaseMasterObserver {
70      private boolean preCreateTableCalled;
71      private boolean postCreateTableCalled;
72      private boolean startCalled;
73      private boolean postStartMasterCalled;
74  
75      @Override
76      public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
77          HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
78        // Cause a NullPointerException and don't catch it: this should cause the
79        // master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the
80        // client.
81        Integer i;
82        i = null;
83        i = i++;
84      }
85  
86      public boolean wasCreateTableCalled() {
87        return preCreateTableCalled && postCreateTableCalled;
88      }
89  
90      @Override
91      public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
92          throws IOException {
93        postStartMasterCalled = true;
94      }
95  
96      public boolean wasStartMasterCalled() {
97        return postStartMasterCalled;
98      }
99  
100     @Override
101     public void start(CoprocessorEnvironment env) throws IOException {
102       startCalled = true;
103     }
104 
105     public boolean wasStarted() {
106       return startCalled;
107     }
108   }
109 
110   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
111 
112   private static byte[] TEST_TABLE1 = Bytes.toBytes("observed_table1");
113   private static byte[] TEST_FAMILY1 = Bytes.toBytes("fam1");
114 
115   private static byte[] TEST_TABLE2 = Bytes.toBytes("table2");
116   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
117 
118   @BeforeClass
119   public static void setupBeforeClass() throws Exception {
120     Configuration conf = UTIL.getConfiguration();
121     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
122         BuggyMasterObserver.class.getName());
123     UTIL.startMiniCluster();
124   }
125 
126   @AfterClass
127   public static void teardownAfterClass() throws Exception {
128     UTIL.shutdownMiniCluster();
129   }
130 
131   @Test(timeout=30000)
132   public void testExceptionFromCoprocessorWhenCreatingTable()
133       throws IOException {
134     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
135 
136     HMaster master = cluster.getMaster();
137     MasterCoprocessorHost host = master.getCoprocessorHost();
138     BuggyMasterObserver cp = (BuggyMasterObserver)host.findCoprocessor(
139         BuggyMasterObserver.class.getName());
140     assertFalse("No table created yet", cp.wasCreateTableCalled());
141 
142     // Set a watch on the zookeeper /hbase/master node. If the master dies,
143     // the node will be deleted.
144     // Master should *NOT* die:
145     // we are testing that the default setting of hbase.coprocessor.abortonerror
146     // =false
147     // is respected.
148     ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(),
149       "unittest", new Abortable() {
150       @Override
151       public void abort(String why, Throwable e) {
152         throw new RuntimeException("Fatal ZK error: " + why, e);
153       }
154       @Override
155       public boolean isAborted() {
156         return false;
157       }
158     });
159 
160     MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master",
161         new Abortable() {
162           @Override
163           public void abort(String why, Throwable e) {
164             throw new RuntimeException("Fatal Zookeeper tracker error, why=", e);
165           }
166           @Override
167           public boolean isAborted() {
168             return false;
169           }
170         });
171 
172     masterTracker.start();
173     zkw.registerListener(masterTracker);
174 
175     // Test (part of the) output that should have be printed by master when it aborts:
176     // (namely the part that shows the set of loaded coprocessors).
177     // In this test, there is only a single coprocessor (BuggyMasterObserver).
178     String coprocessorName =
179         BuggyMasterObserver.class.getName();
180     assertTrue(master.getLoadedCoprocessors().equals("[" + coprocessorName + "]"));
181 
182     HTableDescriptor htd1 = new HTableDescriptor(TEST_TABLE1);
183     htd1.addFamily(new HColumnDescriptor(TEST_FAMILY1));
184 
185     boolean threwDNRE = false;
186     try {
187       HBaseAdmin admin = UTIL.getHBaseAdmin();
188       admin.createTable(htd1);
189     } catch (IOException e) {
190       if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) {
191         threwDNRE = true;
192       }
193     } finally {
194       assertTrue(threwDNRE);
195     }
196 
197     // wait for a few seconds to make sure that the Master hasn't aborted.
198     try {
199       Thread.sleep(3000);
200     } catch (InterruptedException e) {
201       fail("InterruptedException while sleeping.");
202     }
203 
204     assertFalse("Master survived coprocessor NPE, as expected.",
205         masterTracker.masterZKNodeWasDeleted);
206 
207     String loadedCoprocessors = master.getLoadedCoprocessors();
208     assertTrue(loadedCoprocessors.equals("[" + coprocessorName + "]"));
209 
210     // Verify that BuggyMasterObserver has been removed due to its misbehavior
211     // by creating another table: should not have a problem this time.
212     HTableDescriptor htd2 = new HTableDescriptor(TEST_TABLE2);
213     htd2.addFamily(new HColumnDescriptor(TEST_FAMILY2));
214     HBaseAdmin admin = UTIL.getHBaseAdmin();
215     try {
216       admin.createTable(htd2);
217     } catch (IOException e) {
218       fail("Failed to create table after buggy coprocessor removal: " + e);
219     }
220   }
221 }