1   /**
2    * Copyright 2008 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.IOException;
23  import java.security.PrivilegedAction;
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.hbase.client.HConnectionManager;
32  import org.apache.hadoop.hbase.master.HMaster;
33  import org.apache.hadoop.hbase.regionserver.HRegion;
34  import org.apache.hadoop.hbase.regionserver.HRegionServer;
35  import org.apache.hadoop.hbase.security.User;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hbase.util.JVMClusterUtil;
38  import org.apache.hadoop.hbase.util.Threads;
39  import org.apache.hadoop.io.MapWritable;
40  
41  /**
42   * This class creates a single process HBase cluster.
43   * each server.  The master uses the 'default' FileSystem.  The RegionServers,
44   * if we are running on DistributedFilesystem, create a FileSystem instance
45   * each and will close down their instance on the way out.
46   */
47  public class MiniHBaseCluster {
48    static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName());
49    private Configuration conf;
50    public LocalHBaseCluster hbaseCluster;
51    private static int index;
52  
53    /**
54     * Start a MiniHBaseCluster.
55     * @param conf Configuration to be used for cluster
56     * @param numRegionServers initial number of region servers to start.
57     * @throws IOException
58     */
59    public MiniHBaseCluster(Configuration conf, int numRegionServers)
60    throws IOException, InterruptedException {
61      this(conf, 1, numRegionServers);
62    }
63  
64    /**
65     * Start a MiniHBaseCluster.
66     * @param conf Configuration to be used for cluster
67     * @param numMasters initial number of masters to start.
68     * @param numRegionServers initial number of region servers to start.
69     * @throws IOException
70     */
71    public MiniHBaseCluster(Configuration conf, int numMasters,
72        int numRegionServers)
73    throws IOException, InterruptedException {
74      this.conf = conf;
75      conf.set(HConstants.MASTER_PORT, "0");
76      init(numMasters, numRegionServers);
77    }
78  
79    public Configuration getConfiguration() {
80      return this.conf;
81    }
82  
83    /**
84     * Subclass so can get at protected methods (none at moment).  Also, creates
85     * a FileSystem instance per instantiation.  Adds a shutdown own FileSystem
86     * on the way out. Shuts down own Filesystem only, not All filesystems as
87     * the FileSystem system exit hook does.
88     */
89    public static class MiniHBaseClusterRegionServer extends HRegionServer {
90      private Thread shutdownThread = null;
91      private User user = null;
92      public static boolean TEST_SKIP_CLOSE = false;
93  
94      public MiniHBaseClusterRegionServer(Configuration conf)
95          throws IOException, InterruptedException {
96        super(conf);
97        this.user = User.getCurrent();
98      }
99  
100     /*
101      * @param c
102      * @param currentfs We return this if we did not make a new one.
103      * @param uniqueName Same name used to help identify the created fs.
104      * @return A new fs instance if we are up on DistributeFileSystem.
105      * @throws IOException
106      */
107 
108     @Override
109     protected void handleReportForDutyResponse(MapWritable c) throws IOException {
110       super.handleReportForDutyResponse(c);
111       // Run this thread to shutdown our filesystem on way out.
112       this.shutdownThread = new SingleFileSystemShutdownThread(getFileSystem());
113     }
114 
115     @Override
116     public void run() {
117       try {
118         this.user.runAs(new PrivilegedAction<Object>(){
119           public Object run() {
120             runRegionServer();
121             return null;
122           }
123         });
124       } catch (Throwable t) {
125         LOG.error("Exception in run", t);
126       } finally {
127         // Run this on the way out.
128         if (this.shutdownThread != null) {
129           this.shutdownThread.start();
130           Threads.shutdown(this.shutdownThread, 30000);
131         }
132       }
133     }
134 
135     private void runRegionServer() {
136       super.run();
137     }
138 
139     @Override
140     public void kill() {
141       super.kill();
142     }
143 
144     public void abort(final String reason, final Throwable cause) {
145       this.user.runAs(new PrivilegedAction<Object>() {
146         public Object run() {
147           abortRegionServer(reason, cause);
148           return null;
149         }
150       });
151     }
152 
153     private void abortRegionServer(String reason, Throwable cause) {
154       super.abort(reason, cause);
155     }
156   }
157 
158   /**
159    * Alternate shutdown hook.
160    * Just shuts down the passed fs, not all as default filesystem hook does.
161    */
162   static class SingleFileSystemShutdownThread extends Thread {
163     private final FileSystem fs;
164     SingleFileSystemShutdownThread(final FileSystem fs) {
165       super("Shutdown of " + fs);
166       this.fs = fs;
167     }
168     @Override
169     public void run() {
170       try {
171         LOG.info("Hook closing fs=" + this.fs);
172         this.fs.close();
173       } catch (NullPointerException npe) {
174         LOG.debug("Need to fix these: " + npe.toString());
175       } catch (IOException e) {
176         LOG.warn("Running hook", e);
177       }
178     }
179   }
180 
181   private void init(final int nMasterNodes, final int nRegionNodes)
182   throws IOException, InterruptedException {
183     try {
184       // start up a LocalHBaseCluster
185       hbaseCluster = new LocalHBaseCluster(conf, nMasterNodes, 0,
186         HMaster.class, MiniHBaseCluster.MiniHBaseClusterRegionServer.class);
187 
188       // manually add the regionservers as other users
189       for (int i=0; i<nRegionNodes; i++) {
190         Configuration rsConf = HBaseConfiguration.create(conf);
191         User user = HBaseTestingUtility.getDifferentUser(rsConf,
192             ".hfs."+index++);
193         hbaseCluster.addRegionServer(rsConf, i, user);
194       }
195 
196       hbaseCluster.startup();
197     } catch (IOException e) {
198       shutdown();
199       throw e;
200     } catch (Throwable t) {
201       LOG.error("Error starting cluster", t);
202       shutdown();
203       throw new IOException("Shutting down", t);
204     }
205   }
206 
207   /**
208    * Starts a region server thread running
209    *
210    * @throws IOException
211    * @return New RegionServerThread
212    */
213   public JVMClusterUtil.RegionServerThread startRegionServer()
214       throws IOException {
215     final Configuration newConf = HBaseConfiguration.create(conf);
216     User rsUser =
217         HBaseTestingUtility.getDifferentUser(newConf, ".hfs."+index++);
218     JVMClusterUtil.RegionServerThread t =  null;
219     try {
220       t = hbaseCluster.addRegionServer(
221           newConf, hbaseCluster.getRegionServers().size(), rsUser);
222       t.start();
223       t.waitForServerOnline();
224     } catch (InterruptedException ie) {
225       throw new IOException("Interrupted adding regionserver to cluster", ie);
226     }
227     return t;
228   }
229 
230   /**
231    * Cause a region server to exit doing basic clean up only on its way out.
232    * @param serverNumber  Used as index into a list.
233    */
234   public String abortRegionServer(int serverNumber) {
235     HRegionServer server = getRegionServer(serverNumber);
236     LOG.info("Aborting " + server.toString());
237     server.abort("Aborting for tests", new Exception("Trace info"));
238     return server.toString();
239   }
240 
241   /**
242    * Shut down the specified region server cleanly
243    *
244    * @param serverNumber  Used as index into a list.
245    * @return the region server that was stopped
246    */
247   public JVMClusterUtil.RegionServerThread stopRegionServer(int serverNumber) {
248     return stopRegionServer(serverNumber, true);
249   }
250 
251   /**
252    * Shut down the specified region server cleanly
253    *
254    * @param serverNumber  Used as index into a list.
255    * @param shutdownFS True is we are to shutdown the filesystem as part of this
256    * regionserver's shutdown.  Usually we do but you do not want to do this if
257    * you are running multiple regionservers in a test and you shut down one
258    * before end of the test.
259    * @return the region server that was stopped
260    */
261   public JVMClusterUtil.RegionServerThread stopRegionServer(int serverNumber,
262       final boolean shutdownFS) {
263     JVMClusterUtil.RegionServerThread server =
264       hbaseCluster.getRegionServers().get(serverNumber);
265     LOG.info("Stopping " + server.toString());
266     server.getRegionServer().stop("Stopping rs " + serverNumber);
267     return server;
268   }
269 
270   /**
271    * Wait for the specified region server to stop. Removes this thread from list
272    * of running threads.
273    * @param serverNumber
274    * @return Name of region server that just went down.
275    */
276   public String waitOnRegionServer(final int serverNumber) {
277     return this.hbaseCluster.waitOnRegionServer(serverNumber);
278   }
279 
280 
281   /**
282    * Starts a master thread running
283    *
284    * @throws IOException
285    * @return New RegionServerThread
286    */
287   public JVMClusterUtil.MasterThread startMaster() throws IOException {
288     Configuration c = HBaseConfiguration.create(conf);
289     User user =
290         HBaseTestingUtility.getDifferentUser(c, ".hfs."+index++);
291 
292     JVMClusterUtil.MasterThread t = null;
293     try {
294       t = hbaseCluster.addMaster(c, hbaseCluster.getMasters().size(), user);
295       t.start();
296       t.waitForServerOnline();
297     } catch (InterruptedException ie) {
298       throw new IOException("Interrupted adding master to cluster", ie);
299     }
300     return t;
301   }
302 
303   /**
304    * Returns the current active master, if available.
305    * @return the active HMaster, null if none is active.
306    */
307   public HMaster getMaster() {
308     return this.hbaseCluster.getActiveMaster();
309   }
310 
311   /**
312    * Returns the master at the specified index, if available.
313    * @return the active HMaster, null if none is active.
314    */
315   public HMaster getMaster(final int serverNumber) {
316     return this.hbaseCluster.getMaster(serverNumber);
317   }
318 
319   /**
320    * Cause a master to exit without shutting down entire cluster.
321    * @param serverNumber  Used as index into a list.
322    */
323   public String abortMaster(int serverNumber) {
324     HMaster server = getMaster(serverNumber);
325     LOG.info("Aborting " + server.toString());
326     server.abort("Aborting for tests", new Exception("Trace info"));
327     return server.toString();
328   }
329 
330   /**
331    * Shut down the specified master cleanly
332    *
333    * @param serverNumber  Used as index into a list.
334    * @return the region server that was stopped
335    */
336   public JVMClusterUtil.MasterThread stopMaster(int serverNumber) {
337     return stopMaster(serverNumber, true);
338   }
339 
340   /**
341    * Shut down the specified master cleanly
342    *
343    * @param serverNumber  Used as index into a list.
344    * @param shutdownFS True is we are to shutdown the filesystem as part of this
345    * master's shutdown.  Usually we do but you do not want to do this if
346    * you are running multiple master in a test and you shut down one
347    * before end of the test.
348    * @return the master that was stopped
349    */
350   public JVMClusterUtil.MasterThread stopMaster(int serverNumber,
351       final boolean shutdownFS) {
352     JVMClusterUtil.MasterThread server =
353       hbaseCluster.getMasters().get(serverNumber);
354     LOG.info("Stopping " + server.toString());
355     server.getMaster().stop("Stopping master " + serverNumber);
356     return server;
357   }
358 
359   /**
360    * Wait for the specified master to stop. Removes this thread from list
361    * of running threads.
362    * @param serverNumber
363    * @return Name of master that just went down.
364    */
365   public String waitOnMaster(final int serverNumber) {
366     return this.hbaseCluster.waitOnMaster(serverNumber);
367   }
368 
369   /**
370    * Blocks until there is an active master and that master has completed
371    * initialization.
372    *
373    * @return true if an active master becomes available.  false if there are no
374    *         masters left.
375    * @throws InterruptedException
376    */
377   public boolean waitForActiveAndReadyMaster() throws InterruptedException {
378     List<JVMClusterUtil.MasterThread> mts;
379     while (!(mts = getMasterThreads()).isEmpty()) {
380       for (JVMClusterUtil.MasterThread mt : mts) {
381         if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) {
382           return true;
383         }
384       }
385       Thread.sleep(200);
386     }
387     return false;
388   }
389 
390   /**
391    * @return List of master threads.
392    */
393   public List<JVMClusterUtil.MasterThread> getMasterThreads() {
394     return this.hbaseCluster.getMasters();
395   }
396 
397   /**
398    * @return List of live master threads (skips the aborted and the killed)
399    */
400   public List<JVMClusterUtil.MasterThread> getLiveMasterThreads() {
401     return this.hbaseCluster.getLiveMasters();
402   }
403 
404   /**
405    * Wait for Mini HBase Cluster to shut down.
406    */
407   public void join() {
408     this.hbaseCluster.join();
409   }
410 
411   /**
412    * Shut down the mini HBase cluster
413    * @throws IOException
414    */
415   public void shutdown() throws IOException {
416     if (this.hbaseCluster != null) {
417       this.hbaseCluster.shutdown();
418     }
419     HConnectionManager.deleteAllConnections(false);
420   }
421 
422   /**
423    * Call flushCache on all regions on all participating regionservers.
424    * @throws IOException
425    */
426   public void flushcache() throws IOException {
427     for (JVMClusterUtil.RegionServerThread t:
428         this.hbaseCluster.getRegionServers()) {
429       for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
430         r.flushcache();
431       }
432     }
433   }
434 
435   /**
436    * Call flushCache on all regions of the specified table.
437    * @throws IOException
438    */
439   public void flushcache(byte [] tableName) throws IOException {
440     for (JVMClusterUtil.RegionServerThread t:
441         this.hbaseCluster.getRegionServers()) {
442       for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
443         if(Bytes.equals(r.getTableDesc().getName(), tableName)) {
444           r.flushcache();
445         }
446       }
447     }
448   }
449 
450   /**
451    * @return List of region server threads.
452    */
453   public List<JVMClusterUtil.RegionServerThread> getRegionServerThreads() {
454     return this.hbaseCluster.getRegionServers();
455   }
456 
457   /**
458    * @return List of live region server threads (skips the aborted and the killed)
459    */
460   public List<JVMClusterUtil.RegionServerThread> getLiveRegionServerThreads() {
461     return this.hbaseCluster.getLiveRegionServers();
462   }
463 
464   /**
465    * Grab a numbered region server of your choice.
466    * @param serverNumber
467    * @return region server
468    */
469   public HRegionServer getRegionServer(int serverNumber) {
470     return hbaseCluster.getRegionServer(serverNumber);
471   }
472 
473   public List<HRegion> getRegions(byte[] tableName) {
474     List<HRegion> ret = new ArrayList<HRegion>();
475     for (JVMClusterUtil.RegionServerThread rst : getRegionServerThreads()) {
476       HRegionServer hrs = rst.getRegionServer();
477       for (HRegion region : hrs.getOnlineRegionsLocalContext()) {
478         if (Bytes.equals(region.getTableDesc().getName(), tableName)) {
479           ret.add(region);
480         }
481       }
482     }
483     return ret;
484   }
485 
486   /**
487    * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()}
488    * of HRS carrying regionName. Returns -1 if none found.
489    */
490   public int getServerWithMeta() {
491     return getServerWith(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
492   }
493 
494   /**
495    * Get the location of the specified region
496    * @param regionName Name of the region in bytes
497    * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()}
498    * of HRS carrying .META.. Returns -1 if none found.
499    */
500   public int getServerWith(byte[] regionName) {
501     int index = -1;
502     int count = 0;
503     for (JVMClusterUtil.RegionServerThread rst: getRegionServerThreads()) {
504       HRegionServer hrs = rst.getRegionServer();
505       HRegion metaRegion =
506         hrs.getOnlineRegion(regionName);
507       if (metaRegion != null) {
508         index = count;
509         break;
510       }
511       count++;
512     }
513     return index;
514   }
515 
516   /**
517    * Counts the total numbers of regions being served by the currently online
518    * region servers by asking each how many regions they have.  Does not look
519    * at META at all.  Count includes catalog tables.
520    * @return number of regions being served by all region servers
521    */
522   public long countServedRegions() {
523     long count = 0;
524     for (JVMClusterUtil.RegionServerThread rst : getLiveRegionServerThreads()) {
525       count += rst.getRegionServer().getNumberOfOnlineRegions();
526     }
527     return count;
528   }
529 }