View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import java.io.EOFException;
21  import java.io.IOException;
22  import java.net.ConnectException;
23  import java.net.NoRouteToHostException;
24  import java.net.SocketException;
25  import java.net.SocketTimeoutException;
26  import java.util.concurrent.atomic.AtomicBoolean;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.Abortable;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
34  import org.apache.hadoop.hbase.ServerName;
35  import org.apache.hadoop.hbase.client.HConnection;
36  import org.apache.hadoop.hbase.client.HConnectionManager;
37  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
38  import org.apache.hadoop.hbase.ipc.HRegionInterface;
39  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.zookeeper.MetaNodeTracker;
42  import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
43  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
44  import org.apache.hadoop.ipc.RemoteException;
45  
46  /**
47   * Tracks the availability of the catalog tables <code>-ROOT-</code> and
48   * <code>.META.</code>.
49   * 
50   * This class is "read-only" in that the locations of the catalog tables cannot
51   * be explicitly set.  Instead, ZooKeeper is used to learn of the availability
52   * and location of <code>-ROOT-</code>.  <code>-ROOT-</code> is used to learn of
53   * the location of <code>.META.</code>  If not available in <code>-ROOT-</code>,
54   * ZooKeeper is used to monitor for a new location of <code>.META.</code>.
55   *
56   * <p>Call {@link #start()} to start up operation.  Call {@link #stop()}} to
57   * interrupt waits and close up shop.
58   */
59  public class CatalogTracker {
60    // TODO: This class needs a rethink.  The original intent was that it would be
61    // the one-stop-shop for root and meta locations and that it would get this
62    // info from reading and watching zk state.  The class was to be used by
63    // servers when they needed to know of root and meta movement but also by
64    // client-side (inside in HTable) so rather than figure root and meta
65    // locations on fault, the client would instead get notifications out of zk.
66    // 
67    // But this original intent is frustrated by the fact that this class has to
68    // read an hbase table, the -ROOT- table, to figure out the .META. region
69    // location which means we depend on an HConnection.  HConnection will do
70    // retrying but also, it has its own mechanism for finding root and meta
71    // locations (and for 'verifying'; it tries the location and if it fails, does
72    // new lookup, etc.).  So, at least for now, HConnection (or HTable) can't
73    // have a CT since CT needs a HConnection (Even then, do want HT to have a CT?
74    // For HT keep up a session with ZK?  Rather, shouldn't we do like asynchbase
75    // where we'd open a connection to zk, read what we need then let the
76    // connection go?).  The 'fix' is make it so both root and meta addresses
77    // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta).
78    //
79    // But even then, this class does 'verification' of the location and it does
80    // this by making a call over an HConnection (which will do its own root
81    // and meta lookups).  Isn't this verification 'useless' since when we
82    // return, whatever is dependent on the result of this call then needs to
83    // use HConnection; what we have verified may change in meantime (HConnection
84    // uses the CT primitives, the root and meta trackers finding root locations).
85    //
86    // When meta is moved to zk, this class may make more sense.  In the
87    // meantime, it does not cohere.  It should just watch meta and root and not
88    // NOT do verification -- let that be out in HConnection since its going to
89    // be done there ultimately anyways.
90    //
91    // This class has spread throughout the codebase.  It needs to be reigned in.
92    // This class should be used server-side only, even if we move meta location
93    // up into zk.  Currently its used over in the client package. Its used in
94    // MetaReader and MetaEditor classes usually just to get the Configuration
95    // its using (It does this indirectly by asking its HConnection for its
96    // Configuration and even then this is just used to get an HConnection out on
97    // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for
98    // doing CT fixup. St.Ack 09/30/2011.
99    //
100   private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
101   private final HConnection connection;
102   private final ZooKeeperWatcher zookeeper;
103   private final RootRegionTracker rootRegionTracker;
104   private final MetaNodeTracker metaNodeTracker;
105   private final AtomicBoolean metaAvailable = new AtomicBoolean(false);
106   private boolean instantiatedzkw = false;
107 
108   /*
109    * Do not clear this address once set.  Its needed when we do
110    * server shutdown processing -- we need to know who had .META. last.  If you
111    * want to know if the address is good, rely on {@link #metaAvailable} value.
112    */
113   private ServerName metaLocation;
114 
115   /*
116    * Timeout waiting on root or meta to be set.
117    */
118   private final int defaultTimeout;
119 
120   private boolean stopped = false;
121 
122   static final byte [] ROOT_REGION_NAME =
123     HRegionInfo.ROOT_REGIONINFO.getRegionName();
124   static final byte [] META_REGION_NAME =
125     HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
126 
127   /**
128    * Constructs a catalog tracker. Find current state of catalog tables.
129    * Begin active tracking by executing {@link #start()} post construction. Does
130    * not timeout.
131    *
132    * @param conf
133    *          the {@link Configuration} from which a {@link HConnection} will be
134    *          obtained; if problem, this connections
135    *          {@link HConnection#abort(String, Throwable)} will be called.
136    * @throws IOException
137    */
138   public CatalogTracker(final Configuration conf) throws IOException {
139     this(null, conf, null);
140   }
141 
142   /**
143    * Constructs the catalog tracker.  Find current state of catalog tables.
144    * Begin active tracking by executing {@link #start()} post construction.
145    * Does not timeout.
146    * @param zk If zk is null, we'll create an instance (and shut it down
147    * when {@link #stop()} is called) else we'll use what is passed.
148    * @param conf
149    * @param abortable If fatal exception we'll call abort on this.  May be null.
150    * If it is we'll use the Connection associated with the passed
151    * {@link Configuration} as our Abortable.
152    * @throws IOException 
153    */
154   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
155       final Abortable abortable)
156   throws IOException {
157     this(zk, conf, abortable,
158       conf.getInt("hbase.catalogtracker.default.timeout", 1000));
159   }
160 
161   /**
162    * Constructs the catalog tracker.  Find current state of catalog tables.
163    * Begin active tracking by executing {@link #start()} post construction.
164    * @param zk If zk is null, we'll create an instance (and shut it down
165    * when {@link #stop()} is called) else we'll use what is passed.
166    * @param conf
167    * @param abortable If fatal exception we'll call abort on this.  May be null.
168    * If it is we'll use the Connection associated with the passed
169    * {@link Configuration} as our Abortable.
170    * @param defaultTimeout Timeout to use.  Pass zero for no timeout
171    * ({@link Object#wait(long)} when passed a <code>0</code> waits for ever).
172    * @throws IOException
173    */
174   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
175       Abortable abortable, final int defaultTimeout)
176   throws IOException {
177     this(zk, conf, HConnectionManager.getConnection(conf), abortable, defaultTimeout);
178   }
179 
180   CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
181       HConnection connection, Abortable abortable, final int defaultTimeout)
182   throws IOException {
183     this.connection = connection;
184     if (abortable == null) {
185       // A connection is abortable.
186       abortable = this.connection;
187     }
188     if (zk == null) {
189       // Create our own.  Set flag so we tear it down on stop.
190       this.zookeeper =
191         new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(),
192           abortable);
193       instantiatedzkw = true;
194     } else {
195       this.zookeeper = zk;
196     }
197     this.rootRegionTracker = new RootRegionTracker(zookeeper, abortable);
198     final CatalogTracker ct = this;
199     // Override nodeDeleted so we get notified when meta node deleted
200     this.metaNodeTracker = new MetaNodeTracker(zookeeper, abortable) {
201       public void nodeDeleted(String path) {
202         if (!path.equals(node)) return;
203         ct.resetMetaLocation();
204       }
205     };
206     this.defaultTimeout = defaultTimeout;
207   }
208 
209   /**
210    * Starts the catalog tracker.
211    * Determines current availability of catalog tables and ensures all further
212    * transitions of either region are tracked.
213    * @throws IOException
214    * @throws InterruptedException 
215    */
216   public void start() throws IOException, InterruptedException {
217     LOG.debug("Starting catalog tracker " + this);
218     this.rootRegionTracker.start();
219     this.metaNodeTracker.start();
220   }
221 
222   /**
223    * Stop working.
224    * Interrupts any ongoing waits.
225    */
226   public void stop() {
227     if (!this.stopped) {
228       LOG.debug("Stopping catalog tracker " + this);
229       this.stopped = true;
230       this.rootRegionTracker.stop();
231       this.metaNodeTracker.stop();
232       try {
233         if (this.connection != null) {
234           this.connection.close();
235         }
236       } catch (IOException e) {
237         // Although the {@link Closeable} interface throws an {@link
238         // IOException}, in reality, the implementation would never do that.
239         LOG.error("Attempt to close catalog tracker's connection failed.", e);
240       }
241       if (this.instantiatedzkw) {
242         this.zookeeper.close();
243       }
244       // Call this and it will interrupt any ongoing waits on meta.
245       synchronized (this.metaAvailable) {
246         this.metaAvailable.notifyAll();
247       }
248     }
249   }
250 
251   /**
252    * Gets the current location for <code>-ROOT-</code> or null if location is
253    * not currently available.
254    * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
255    * if none available
256    * @throws InterruptedException 
257    */
258   public ServerName getRootLocation() throws InterruptedException {
259     return this.rootRegionTracker.getRootRegionLocation();
260   }
261 
262   /**
263    * @return {@link ServerName} for server hosting <code>.META.</code> or null
264    * if none available
265    */
266   public ServerName getMetaLocation() {
267     return this.metaLocation;
268   }
269 
270   /**
271    * Method used by master on startup trying to figure state of cluster.
272    * Returns the current meta location unless its null.  In this latter case,
273    * it has not yet been set so go check whats up in <code>-ROOT-</code> and
274    * return that.
275    * @return {@link ServerName} for server hosting <code>.META.</code> or if null,
276    * we'll read the location that is up in <code>-ROOT-</code> table (which
277    * could be null or just plain stale).
278    * @throws IOException
279    */
280   public ServerName getMetaLocationOrReadLocationFromRoot() throws IOException {
281     ServerName sn = getMetaLocation();
282     return sn != null? sn: MetaReader.getMetaRegionLocation(this);
283   }
284 
285   /**
286    * Waits indefinitely for availability of <code>-ROOT-</code>.  Used during
287    * cluster startup.
288    * @throws InterruptedException if interrupted while waiting
289    */
290   public void waitForRoot()
291   throws InterruptedException {
292     this.rootRegionTracker.blockUntilAvailable();
293   }
294 
295   /**
296    * Gets the current location for <code>-ROOT-</code> if available and waits
297    * for up to the specified timeout if not immediately available.  Returns null
298    * if the timeout elapses before root is available.
299    * @param timeout maximum time to wait for root availability, in milliseconds
300    * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
301    * if none available
302    * @throws InterruptedException if interrupted while waiting
303    * @throws NotAllMetaRegionsOnlineException if root not available before
304    * timeout
305    */
306   ServerName waitForRoot(final long timeout)
307   throws InterruptedException, NotAllMetaRegionsOnlineException {
308     ServerName sn = rootRegionTracker.waitRootRegionLocation(timeout);
309     if (sn == null) {
310       throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms");
311     }
312     return sn;
313   }
314 
315   /**
316    * Gets a connection to the server hosting root, as reported by ZooKeeper,
317    * waiting up to the specified timeout for availability.
318    * @param timeout How long to wait on root location
319    * @see #waitForRoot(long) for additional information
320    * @return connection to server hosting root
321    * @throws InterruptedException
322    * @throws NotAllMetaRegionsOnlineException if timed out waiting
323    * @throws IOException
324    * @deprecated Use #getRootServerConnection(long)
325    */
326   public HRegionInterface waitForRootServerConnection(long timeout)
327   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
328     return getRootServerConnection(timeout);
329   }
330 
331   /**
332    * Gets a connection to the server hosting root, as reported by ZooKeeper,
333    * waiting up to the specified timeout for availability.
334    * <p>WARNING: Does not retry.  Use an {@link HTable} instead.
335    * @param timeout How long to wait on root location
336    * @see #waitForRoot(long) for additional information
337    * @return connection to server hosting root
338    * @throws InterruptedException
339    * @throws NotAllMetaRegionsOnlineException if timed out waiting
340    * @throws IOException
341    */
342   HRegionInterface getRootServerConnection(long timeout)
343   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
344     return getCachedConnection(waitForRoot(timeout));
345   }
346 
347   /**
348    * Gets a connection to the server hosting root, as reported by ZooKeeper,
349    * waiting for the default timeout specified on instantiation.
350    * @see #waitForRoot(long) for additional information
351    * @return connection to server hosting root
352    * @throws NotAllMetaRegionsOnlineException if timed out waiting
353    * @throws IOException
354    * @deprecated Use #getRootServerConnection(long)
355    */
356   public HRegionInterface waitForRootServerConnectionDefault()
357   throws NotAllMetaRegionsOnlineException, IOException {
358     try {
359       return getRootServerConnection(this.defaultTimeout);
360     } catch (InterruptedException e) {
361       throw new NotAllMetaRegionsOnlineException("Interrupted");
362     }
363   }
364 
365   /**
366    * Gets a connection to the server currently hosting <code>.META.</code> or
367    * null if location is not currently available.
368    * <p>
369    * If a location is known, a connection to the cached location is returned.
370    * If refresh is true, the cached connection is verified first before
371    * returning.  If the connection is not valid, it is reset and rechecked.
372    * <p>
373    * If no location for meta is currently known, method checks ROOT for a new
374    * location, verifies META is currently there, and returns a cached connection
375    * to the server hosting META.
376    *
377    * @return connection to server hosting meta, null if location not available
378    * @throws IOException
379    * @throws InterruptedException
380    */
381   private HRegionInterface getMetaServerConnection()
382   throws IOException, InterruptedException {
383     synchronized (metaAvailable) {
384       if (metaAvailable.get()) {
385         HRegionInterface current = getCachedConnection(this.metaLocation);
386         // If we are to refresh, verify we have a good connection by making
387         // an invocation on it.
388         if (verifyRegionLocation(current, this.metaLocation, META_REGION_NAME)) {
389           return current;
390         }
391         resetMetaLocation();
392       }
393       // We got here because there is no meta available or because whats
394       // available is bad.
395 
396       // Now read the current .META. content from -ROOT-.  Note: This goes via
397       // an HConnection.  It has its own way of figuring root and meta locations
398       // which we have to wait on.
399       ServerName newLocation = MetaReader.getMetaRegionLocation(this);
400       if (newLocation == null) return null;
401 
402       HRegionInterface newConnection = getCachedConnection(newLocation);
403       if (verifyRegionLocation(newConnection, newLocation, META_REGION_NAME)) {
404         setMetaLocation(newLocation);
405         return newConnection;
406       } else {
407         if (LOG.isTraceEnabled()) {
408           LOG.trace("New .META. server: " + newLocation + " isn't valid." +
409             " Cached .META. server: " + this.metaLocation);
410         }
411       }
412       return null;
413     }
414   }
415 
416   /**
417    * Waits indefinitely for availability of <code>.META.</code>.  Used during
418    * cluster startup.  Does not verify meta, just that something has been
419    * set up in zk.
420    * @see #waitForMeta(long)
421    * @throws InterruptedException if interrupted while waiting
422    */
423   public void waitForMeta() throws InterruptedException {
424     while (!this.stopped) {
425       try {
426         if (waitForMeta(100) != null) break;
427       } catch (NotAllMetaRegionsOnlineException e) {
428         if (LOG.isTraceEnabled()) {
429           LOG.info(".META. still not available, sleeping and retrying." +
430           " Reason: " + e.getMessage());
431         }
432       } catch (IOException e) {
433         LOG.info("Retrying", e);
434       }
435     }
436   }
437 
438   /**
439    * Gets the current location for <code>.META.</code> if available and waits
440    * for up to the specified timeout if not immediately available.  Throws an
441    * exception if timed out waiting.  This method differs from {@link #waitForMeta()}
442    * in that it will go ahead and verify the location gotten from ZooKeeper and
443    * -ROOT- region by trying to use returned connection.
444    * @param timeout maximum time to wait for meta availability, in milliseconds
445    * @return {@link ServerName} for server hosting <code>.META.</code> or null
446    * if none available
447    * @throws InterruptedException if interrupted while waiting
448    * @throws IOException unexpected exception connecting to meta server
449    * @throws NotAllMetaRegionsOnlineException if meta not available before
450    * timeout
451    */
452   public ServerName waitForMeta(long timeout)
453   throws InterruptedException, IOException, NotAllMetaRegionsOnlineException {
454     long stop = System.currentTimeMillis() + timeout;
455     long waitTime = Math.min(50, timeout);
456     synchronized (metaAvailable) {
457       while(!stopped && (timeout == 0 || System.currentTimeMillis() < stop)) {
458         if (getMetaServerConnection() != null) {
459           return metaLocation;
460         }
461         // perhaps -ROOT- region isn't available, let us wait a bit and retry.
462         metaAvailable.wait(waitTime);
463       }
464       if (getMetaServerConnection() == null) {
465         throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)");
466       }
467       return metaLocation;
468     }
469   }
470 
471   /**
472    * Gets a connection to the server hosting meta, as reported by ZooKeeper,
473    * waiting up to the specified timeout for availability.
474    * @see #waitForMeta(long) for additional information
475    * @return connection to server hosting meta
476    * @throws InterruptedException
477    * @throws NotAllMetaRegionsOnlineException if timed out waiting
478    * @throws IOException
479    * @deprecated Does not retry; use an HTable instance instead.
480    */
481   public HRegionInterface waitForMetaServerConnection(long timeout)
482   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
483     return getCachedConnection(waitForMeta(timeout));
484   }
485 
486   /**
487    * Gets a connection to the server hosting meta, as reported by ZooKeeper,
488    * waiting up to the specified timeout for availability.
489    * Used in tests.
490    * @see #waitForMeta(long) for additional information
491    * @return connection to server hosting meta
492    * @throws NotAllMetaRegionsOnlineException if timed out or interrupted
493    * @throws IOException
494    * @deprecated Does not retry; use an HTable instance instead.
495    */
496   public HRegionInterface waitForMetaServerConnectionDefault()
497   throws NotAllMetaRegionsOnlineException, IOException {
498     try {
499       return getCachedConnection(waitForMeta(defaultTimeout));
500     } catch (InterruptedException e) {
501       throw new NotAllMetaRegionsOnlineException("Interrupted");
502     }
503   }
504 
505   /**
506    * Called when we figure current meta is off (called from zk callback).
507    */
508   public void resetMetaLocation() {
509     LOG.debug("Current cached META location, " + metaLocation +
510       ", is not valid, resetting");
511     synchronized(this.metaAvailable) {
512       this.metaAvailable.set(false);
513       this.metaAvailable.notifyAll();
514     }
515   }
516 
517   /**
518    * @param metaLocation
519    */
520   void setMetaLocation(final ServerName metaLocation) {
521     LOG.debug("Set new cached META location: " + metaLocation);
522     synchronized (this.metaAvailable) {
523       this.metaLocation = metaLocation;
524       this.metaAvailable.set(true);
525       // no synchronization because these are private and already under lock
526       this.metaAvailable.notifyAll();
527     }
528   }
529 
530   /**
531    * @param sn ServerName to get a connection against.
532    * @return The HRegionInterface we got when we connected to <code>sn</code>
533    * May have come from cache, may not be good, may have been setup by this
534    * invocation, or may be null.
535    * @throws IOException
536    */
537   private HRegionInterface getCachedConnection(ServerName sn)
538   throws IOException {
539     if (sn == null) {
540       return null;
541     }
542     HRegionInterface protocol = null;
543     try {
544       protocol = connection.getHRegionConnection(sn.getHostname(), sn.getPort());
545     } catch (RetriesExhaustedException e) {
546       if (e.getCause() != null && e.getCause() instanceof ConnectException) {
547         // Catch this; presume it means the cached connection has gone bad.
548       } else {
549         throw e;
550       }
551     } catch (SocketTimeoutException e) {
552       LOG.debug("Timed out connecting to " + sn);
553     } catch (NoRouteToHostException e) {
554       LOG.debug("Connecting to " + sn, e);
555     } catch (SocketException e) {
556       LOG.debug("Exception connecting to " + sn);
557     } catch (IOException ioe) {
558       Throwable cause = ioe.getCause();
559       if (cause != null && cause instanceof EOFException) {
560         // Catch. Other end disconnected us.
561       } else if (cause != null && cause.getMessage() != null &&
562         cause.getMessage().toLowerCase().contains("connection reset")) {
563         // Catch. Connection reset.
564       } else {
565         throw ioe;
566       }
567       
568     }
569     return protocol;
570   }
571 
572   /**
573    * Verify we can connect to <code>hostingServer</code> and that its carrying
574    * <code>regionName</code>.
575    * @param hostingServer Interface to the server hosting <code>regionName</code>
576    * @param serverName The servername that goes with the <code>metaServer</code>
577    * Interface.  Used logging.
578    * @param regionName The regionname we are interested in.
579    * @return True if we were able to verify the region located at other side of
580    * the Interface.
581    * @throws IOException
582    */
583   // TODO: We should be able to get the ServerName from the HRegionInterface
584   // rather than have to pass it in.  Its made awkward by the fact that the
585   // HRI is likely a proxy against remote server so the getServerName needs
586   // to be fixed to go to a local method or to a cache before we can do this.
587   private boolean verifyRegionLocation(HRegionInterface hostingServer,
588       final ServerName address, final byte [] regionName)
589   throws IOException {
590     if (hostingServer == null) {
591       LOG.info("Passed hostingServer is null");
592       return false;
593     }
594     Throwable t = null;
595     try {
596       // Try and get regioninfo from the hosting server.
597       return hostingServer.getRegionInfo(regionName) != null;
598     } catch (ConnectException e) {
599       t = e;
600     } catch (RemoteException e) {
601       IOException ioe = e.unwrapRemoteException();
602       t = ioe;
603     } catch (IOException e) {
604       Throwable cause = e.getCause();
605       if (cause != null && cause instanceof EOFException) {
606         t = cause;
607       } else if (cause != null && cause.getMessage() != null
608           && cause.getMessage().contains("Connection reset")) {
609         t = cause;
610       } else {
611         t = e;
612       }
613     }
614     LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) +
615       " at address=" + address + "; " + t);
616     return false;
617   }
618 
619   /**
620    * Verify <code>-ROOT-</code> is deployed and accessible.
621    * @param timeout How long to wait on zk for root address (passed through to
622    * the internal call to {@link #waitForRootServerConnection(long)}.
623    * @return True if the <code>-ROOT-</code> location is healthy.
624    * @throws IOException
625    * @throws InterruptedException 
626    */
627   public boolean verifyRootRegionLocation(final long timeout)
628   throws InterruptedException, IOException {
629     HRegionInterface connection = null;
630     try {
631       connection = waitForRootServerConnection(timeout);
632     } catch (NotAllMetaRegionsOnlineException e) {
633       // Pass
634     } catch (ServerNotRunningYetException e) {
635       // Pass -- remote server is not up so can't be carrying root
636     }
637     return (connection == null)? false:
638       verifyRegionLocation(connection,
639         this.rootRegionTracker.getRootRegionLocation(), ROOT_REGION_NAME);
640   }
641 
642   /**
643    * Verify <code>.META.</code> is deployed and accessible.
644    * @param timeout How long to wait on zk for <code>.META.</code> address
645    * (passed through to the internal call to {@link #waitForMetaServerConnection(long)}.
646    * @return True if the <code>.META.</code> location is healthy.
647    * @throws IOException Some unexpected IOE.
648    * @throws InterruptedException
649    */
650   public boolean verifyMetaRegionLocation(final long timeout)
651   throws InterruptedException, IOException {
652     HRegionInterface connection = null;
653     try {
654       connection = waitForMetaServerConnection(timeout);
655     } catch (NotAllMetaRegionsOnlineException e) {
656       // Pass
657     } catch (ServerNotRunningYetException e) {
658       // Pass -- remote server is not up so can't be carrying .META.
659     }
660     return connection != null;
661   }
662 
663   // Used by tests.
664   MetaNodeTracker getMetaNodeTracker() {
665     return this.metaNodeTracker;
666   }
667 
668   public HConnection getConnection() {
669     return this.connection;
670   }
671 }