View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.ipc;
22  
23  import java.io.BufferedInputStream;
24  import java.io.BufferedOutputStream;
25  import java.io.DataInputStream;
26  import java.io.DataOutputStream;
27  import java.io.FilterInputStream;
28  import java.io.IOException;
29  import java.io.InputStream;
30  import java.net.ConnectException;
31  import java.net.InetSocketAddress;
32  import java.net.Socket;
33  import java.net.SocketException;
34  import java.net.SocketTimeoutException;
35  import java.net.UnknownHostException;
36  import java.util.Iterator;
37  import java.util.Map.Entry;
38  import java.util.concurrent.ConcurrentSkipListMap;
39  import java.util.concurrent.atomic.AtomicBoolean;
40  import java.util.concurrent.atomic.AtomicLong;
41  
42  import javax.net.SocketFactory;
43  
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.hbase.HConstants;
48  import org.apache.hadoop.hbase.security.User;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.PoolMap;
51  import org.apache.hadoop.hbase.util.PoolMap.PoolType;
52  import org.apache.hadoop.io.DataOutputBuffer;
53  import org.apache.hadoop.io.IOUtils;
54  import org.apache.hadoop.io.Writable;
55  import org.apache.hadoop.io.WritableUtils;
56  import org.apache.hadoop.ipc.RemoteException;
57  import org.apache.hadoop.net.NetUtils;
58  import org.apache.hadoop.util.ReflectionUtils;
59  
60  /** A client for an IPC service.  IPC calls take a single {@link Writable} as a
61   * parameter, and return a {@link Writable} as their value.  A service runs on
62   * a port and is defined by a parameter class and a value class.
63   *
64   * <p>This is the org.apache.hadoop.ipc.Client renamed as HBaseClient and
65   * moved into this package so can access package-private methods.
66   *
67   * @see HBaseServer
68   */
69  public class HBaseClient {
70  
71    private static final Log LOG =
72      LogFactory.getLog("org.apache.hadoop.ipc.HBaseClient");
73    protected final PoolMap<ConnectionId, Connection> connections;
74  
75    protected final Class<? extends Writable> valueClass;   // class of call values
76    protected int counter;                            // counter for call ids
77    protected final AtomicBoolean running = new AtomicBoolean(true); // if client runs
78    final protected Configuration conf;
79    final protected int maxIdleTime; // connections will be culled if it was idle for
80                             // maxIdleTime microsecs
81    final protected int maxRetries; //the max. no. of retries for socket connections
82    final protected long failureSleep; // Time to sleep before retry on failure.
83    protected final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
84    protected final boolean tcpKeepAlive; // if T then use keepalives
85    protected int pingInterval; // how often sends ping to the server in msecs
86    protected int socketTimeout; // socket timeout
87  
88    protected final SocketFactory socketFactory;           // how to create sockets
89    private int refCount = 1;
90    protected String clusterId;
91  
92    final private static String PING_INTERVAL_NAME = "ipc.ping.interval";
93    final private static String SOCKET_TIMEOUT = "ipc.socket.timeout";
94    final static int DEFAULT_PING_INTERVAL = 60000;  // 1 min
95    final static int DEFAULT_SOCKET_TIMEOUT = 20000; // 20 seconds
96    final static int PING_CALL_ID = -1;
97  
98    /**
99     * set the ping interval value in configuration
100    *
101    * @param conf Configuration
102    * @param pingInterval the ping interval
103    */
104   public static void setPingInterval(Configuration conf, int pingInterval) {
105     conf.setInt(PING_INTERVAL_NAME, pingInterval);
106   }
107 
108   /**
109    * Get the ping interval from configuration;
110    * If not set in the configuration, return the default value.
111    *
112    * @param conf Configuration
113    * @return the ping interval
114    */
115   static int getPingInterval(Configuration conf) {
116     return conf.getInt(PING_INTERVAL_NAME, DEFAULT_PING_INTERVAL);
117   }
118 
119   /**
120    * Set the socket timeout
121    * @param conf Configuration
122    * @param socketTimeout the socket timeout
123    */
124   public static void setSocketTimeout(Configuration conf, int socketTimeout) {
125     conf.setInt(SOCKET_TIMEOUT, socketTimeout);
126   }
127 
128   /**
129    * @return the socket timeout
130    */
131   static int getSocketTimeout(Configuration conf) {
132     return conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT);
133   }
134 
135   /**
136    * Increment this client's reference count
137    *
138    */
139   synchronized void incCount() {
140     refCount++;
141   }
142 
143   /**
144    * Decrement this client's reference count
145    *
146    */
147   synchronized void decCount() {
148     refCount--;
149   }
150 
151   /**
152    * Return if this client has no reference
153    *
154    * @return true if this client has no reference; false otherwise
155    */
156   synchronized boolean isZeroReference() {
157     return refCount==0;
158   }
159 
160   /** A call waiting for a value. */
161   protected class Call {
162     final int id;                                       // call id
163     final Writable param;                               // parameter
164     Writable value;                               // value, null if error
165     IOException error;                            // exception, null if value
166     boolean done;                                 // true when call is done
167     long startTime;
168 
169     protected Call(Writable param) {
170       this.param = param;
171       this.startTime = System.currentTimeMillis();
172       synchronized (HBaseClient.this) {
173         this.id = counter++;
174       }
175     }
176 
177     /** Indicate when the call is complete and the
178      * value or error are available.  Notifies by default.  */
179     protected synchronized void callComplete() {
180       this.done = true;
181       notify();                                 // notify caller
182     }
183 
184     /** Set the exception when there is an error.
185      * Notify the caller the call is done.
186      *
187      * @param error exception thrown by the call; either local or remote
188      */
189     public synchronized void setException(IOException error) {
190       this.error = error;
191       callComplete();
192     }
193 
194     /** Set the return value when there is no error.
195      * Notify the caller the call is done.
196      *
197      * @param value return value of the call.
198      */
199     public synchronized void setValue(Writable value) {
200       this.value = value;
201       callComplete();
202     }
203 
204     public long getStartTime() {
205       return this.startTime;
206     }
207   }
208 
209   /** Thread that reads responses and notifies callers.  Each connection owns a
210    * socket connected to a remote address.  Calls are multiplexed through this
211    * socket: responses may be delivered out of order. */
212   protected class Connection extends Thread {
213     private ConnectionHeader header;              // connection header
214     protected ConnectionId remoteId;
215     protected Socket socket = null;                 // connected socket
216     protected DataInputStream in;
217     protected DataOutputStream out;
218 
219     // currently active calls
220     protected final ConcurrentSkipListMap<Integer, Call> calls = new ConcurrentSkipListMap<Integer, Call>();
221     protected final AtomicLong lastActivity = new AtomicLong();// last I/O activity time
222     protected final AtomicBoolean shouldCloseConnection = new AtomicBoolean();  // indicate if the connection is closed
223     protected IOException closeException; // close reason
224 
225     public Connection(ConnectionId remoteId) throws IOException {
226       if (remoteId.getAddress().isUnresolved()) {
227         throw new UnknownHostException("unknown host: " +
228                                        remoteId.getAddress().getHostName());
229       }
230       this.remoteId = remoteId;
231       User ticket = remoteId.getTicket();
232       Class<? extends VersionedProtocol> protocol = remoteId.getProtocol();
233 
234       header = new ConnectionHeader(
235           protocol == null ? null : protocol.getName(), ticket);
236 
237       this.setName("IPC Client (" + socketFactory.hashCode() +") connection to " +
238         remoteId.getAddress().toString() +
239         ((ticket==null)?" from an unknown user": (" from " + ticket.getName())));
240       this.setDaemon(true);
241     }
242 
243     /** Update lastActivity with the current time. */
244     protected void touch() {
245       lastActivity.set(System.currentTimeMillis());
246     }
247 
248     /**
249      * Add a call to this connection's call queue and notify
250      * a listener; synchronized.
251      * Returns false if called during shutdown.
252      * @param call to add
253      * @return true if the call was added.
254      */
255     protected synchronized boolean addCall(Call call) {
256       if (shouldCloseConnection.get())
257         return false;
258       calls.put(call.id, call);
259       notify();
260       return true;
261     }
262 
263     /** This class sends a ping to the remote side when timeout on
264      * reading. If no failure is detected, it retries until at least
265      * a byte is read.
266      */
267     protected class PingInputStream extends FilterInputStream {
268       /* constructor */
269       protected PingInputStream(InputStream in) {
270         super(in);
271       }
272 
273       /* Process timeout exception
274        * if the connection is not going to be closed, send a ping.
275        * otherwise, throw the timeout exception.
276        */
277       private void handleTimeout(SocketTimeoutException e) throws IOException {
278         if (shouldCloseConnection.get() || !running.get() ||
279             remoteId.rpcTimeout > 0) {
280           throw e;
281         }
282         sendPing();
283       }
284 
285       /** Read a byte from the stream.
286        * Send a ping if timeout on read. Retries if no failure is detected
287        * until a byte is read.
288        * @throws IOException for any IO problem other than socket timeout
289        */
290       @Override
291       public int read() throws IOException {
292         do {
293           try {
294             return super.read();
295           } catch (SocketTimeoutException e) {
296             handleTimeout(e);
297           }
298         } while (true);
299       }
300 
301       /** Read bytes into a buffer starting from offset <code>off</code>
302        * Send a ping if timeout on read. Retries if no failure is detected
303        * until a byte is read.
304        *
305        * @return the total number of bytes read; -1 if the connection is closed.
306        */
307       @Override
308       public int read(byte[] buf, int off, int len) throws IOException {
309         do {
310           try {
311             return super.read(buf, off, len);
312           } catch (SocketTimeoutException e) {
313             handleTimeout(e);
314           }
315         } while (true);
316       }
317     }
318 
319     protected synchronized void setupConnection() throws IOException {
320       short ioFailures = 0;
321       short timeoutFailures = 0;
322       while (true) {
323         try {
324           this.socket = socketFactory.createSocket();
325           this.socket.setTcpNoDelay(tcpNoDelay);
326           this.socket.setKeepAlive(tcpKeepAlive);
327           // connection time out is 20s
328           NetUtils.connect(this.socket, remoteId.getAddress(),
329               getSocketTimeout(conf));
330           if (remoteId.rpcTimeout > 0) {
331             pingInterval = remoteId.rpcTimeout; // overwrite pingInterval
332           }
333           this.socket.setSoTimeout(pingInterval);
334           return;
335         } catch (SocketTimeoutException toe) {
336           /* The max number of retries is 45,
337            * which amounts to 20s*45 = 15 minutes retries.
338            */
339           handleConnectionFailure(timeoutFailures++, maxRetries, toe);
340         } catch (IOException ie) {
341           handleConnectionFailure(ioFailures++, maxRetries, ie);
342         }
343       }
344     }
345 
346     /** Connect to the server and set up the I/O streams. It then sends
347      * a header to the server and starts
348      * the connection thread that waits for responses.
349      * @throws java.io.IOException e
350      */
351     protected synchronized void setupIOstreams()
352         throws IOException, InterruptedException {
353 
354       if (socket != null || shouldCloseConnection.get()) {
355         return;
356       }
357 
358       try {
359         if (LOG.isDebugEnabled()) {
360           LOG.debug("Connecting to "+remoteId);
361         }
362         setupConnection();
363         this.in = new DataInputStream(new BufferedInputStream
364             (new PingInputStream(NetUtils.getInputStream(socket))));
365         this.out = new DataOutputStream
366             (new BufferedOutputStream(NetUtils.getOutputStream(socket)));
367         writeHeader();
368 
369         // update last activity time
370         touch();
371 
372         // start the receiver thread after the socket connection has been set up
373         start();
374       } catch (IOException e) {
375         markClosed(e);
376         close();
377 
378         throw e;
379       }
380     }
381 
382     protected void closeConnection() {
383       // close the current connection
384       if (socket != null) {
385         try {
386           socket.close();
387         } catch (IOException e) {
388           LOG.warn("Not able to close a socket", e);
389         }
390       }
391       // set socket to null so that the next call to setupIOstreams
392       // can start the process of connect all over again.
393       socket = null;
394     }
395 
396     /**
397      *  Handle connection failures
398      *
399      * If the current number of retries is equal to the max number of retries,
400      * stop retrying and throw the exception; Otherwise backoff N seconds and
401      * try connecting again.
402      *
403      * This Method is only called from inside setupIOstreams(), which is
404      * synchronized. Hence the sleep is synchronized; the locks will be retained.
405      *
406      * @param curRetries current number of retries
407      * @param maxRetries max number of retries allowed
408      * @param ioe failure reason
409      * @throws IOException if max number of retries is reached
410      */
411     private void handleConnectionFailure(
412         int curRetries, int maxRetries, IOException ioe) throws IOException {
413 
414       closeConnection();
415 
416       // throw the exception if the maximum number of retries is reached
417       if (curRetries >= maxRetries) {
418         throw ioe;
419       }
420 
421       // otherwise back off and retry
422       try {
423         Thread.sleep(failureSleep);
424       } catch (InterruptedException ignored) {}
425 
426       LOG.info("Retrying connect to server: " + remoteId.getAddress() +
427         " after sleeping " + failureSleep + "ms. Already tried " + curRetries +
428         " time(s).");
429     }
430 
431     /* Write the header for each connection
432      * Out is not synchronized because only the first thread does this.
433      */
434     private void writeHeader() throws IOException {
435       out.write(HBaseServer.HEADER.array());
436       out.write(HBaseServer.CURRENT_VERSION);
437       //When there are more fields we can have ConnectionHeader Writable.
438       DataOutputBuffer buf = new DataOutputBuffer();
439       header.write(buf);
440 
441       int bufLen = buf.getLength();
442       out.writeInt(bufLen);
443       out.write(buf.getData(), 0, bufLen);
444     }
445 
446     /* wait till someone signals us to start reading RPC response or
447      * it is idle too long, it is marked as to be closed,
448      * or the client is marked as not running.
449      *
450      * Return true if it is time to read a response; false otherwise.
451      */
452     @SuppressWarnings({"ThrowableInstanceNeverThrown"})
453     protected synchronized boolean waitForWork() {
454       if (calls.isEmpty() && !shouldCloseConnection.get()  && running.get())  {
455         long timeout = maxIdleTime-
456               (System.currentTimeMillis()-lastActivity.get());
457         if (timeout>0) {
458           try {
459             wait(timeout);
460           } catch (InterruptedException ignored) {}
461         }
462       }
463 
464       if (!calls.isEmpty() && !shouldCloseConnection.get() && running.get()) {
465         return true;
466       } else if (shouldCloseConnection.get()) {
467         return false;
468       } else if (calls.isEmpty()) { // idle connection closed or stopped
469         markClosed(null);
470         return false;
471       } else { // get stopped but there are still pending requests
472         markClosed((IOException)new IOException().initCause(
473             new InterruptedException()));
474         return false;
475       }
476     }
477 
478     public InetSocketAddress getRemoteAddress() {
479       return remoteId.getAddress();
480     }
481 
482     /* Send a ping to the server if the time elapsed
483      * since last I/O activity is equal to or greater than the ping interval
484      */
485     protected synchronized void sendPing() throws IOException {
486       long curTime = System.currentTimeMillis();
487       if ( curTime - lastActivity.get() >= pingInterval) {
488         lastActivity.set(curTime);
489         //noinspection SynchronizeOnNonFinalField
490         synchronized (this.out) {
491           out.writeInt(PING_CALL_ID);
492           out.flush();
493         }
494       }
495     }
496 
497     @Override
498     public void run() {
499       if (LOG.isDebugEnabled())
500         LOG.debug(getName() + ": starting, having connections "
501             + connections.size());
502 
503       try {
504         while (waitForWork()) {//wait here for work - read or close connection
505           receiveResponse();
506         }
507       } catch (Throwable t) {
508         LOG.warn("Unexpected exception receiving call responses", t);
509         markClosed(new IOException("Unexpected exception receiving call responses", t));
510       }
511 
512       close();
513 
514       if (LOG.isDebugEnabled())
515         LOG.debug(getName() + ": stopped, remaining connections "
516             + connections.size());
517     }
518 
519     /* Initiates a call by sending the parameter to the remote server.
520      * Note: this is not called from the Connection thread, but by other
521      * threads.
522      */
523     protected void sendParam(Call call) {
524       if (shouldCloseConnection.get()) {
525         return;
526       }
527 
528       // For serializing the data to be written.
529 
530       final DataOutputBuffer d = new DataOutputBuffer();
531       try {
532         if (LOG.isDebugEnabled())
533           LOG.debug(getName() + " sending #" + call.id);
534 
535         d.writeInt(0xdeadbeef); // placeholder for data length
536         d.writeInt(call.id);
537         call.param.write(d);
538         byte[] data = d.getData();
539         int dataLength = d.getLength();
540         // fill in the placeholder
541         Bytes.putInt(data, 0, dataLength - 4);
542         //noinspection SynchronizeOnNonFinalField
543         synchronized (this.out) { // FindBugs IS2_INCONSISTENT_SYNC
544           out.write(data, 0, dataLength);
545           out.flush();
546         }
547       } catch(IOException e) {
548         markClosed(e);
549       } finally {
550         //the buffer is just an in-memory buffer, but it is still polite to
551         // close early
552         IOUtils.closeStream(d);
553       }
554     }
555 
556     /* Receive a response.
557      * Because only one receiver, so no synchronization on in.
558      */
559     protected void receiveResponse() {
560       if (shouldCloseConnection.get()) {
561         return;
562       }
563       touch();
564 
565       try {
566         // See HBaseServer.Call.setResponse for where we write out the response.
567         // It writes the call.id (int), a flag byte, then optionally the length
568         // of the response (int) followed by data.
569 
570         // Read the call id.
571         int id = in.readInt();
572 
573         if (LOG.isDebugEnabled())
574           LOG.debug(getName() + " got value #" + id);
575         Call call = calls.remove(id);
576 
577         // Read the flag byte
578         byte flag = in.readByte();
579         boolean isError = ResponseFlag.isError(flag);
580         if (ResponseFlag.isLength(flag)) {
581           // Currently length if present is unused.
582           in.readInt();
583         }
584         int state = in.readInt(); // Read the state.  Currently unused.
585         if (isError) {
586           //noinspection ThrowableInstanceNeverThrown
587           call.setException(new RemoteException( WritableUtils.readString(in),
588               WritableUtils.readString(in)));
589         } else {
590           Writable value = ReflectionUtils.newInstance(valueClass, conf);
591           value.readFields(in);                 // read value
592           // it's possible that this call may have been cleaned up due to a RPC
593           // timeout, so check if it still exists before setting the value.
594           if (call != null) {
595             call.setValue(value);
596           }
597         }
598       } catch (IOException e) {
599         if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
600           // Clean up open calls but don't treat this as a fatal condition,
601           // since we expect certain responses to not make it by the specified
602           // {@link ConnectionId#rpcTimeout}.
603           closeException = e;
604         } else {
605           // Since the server did not respond within the default ping interval
606           // time, treat this as a fatal condition and close this connection
607           markClosed(e);
608         }
609       } finally {
610         if (remoteId.rpcTimeout > 0) {
611           cleanupCalls(remoteId.rpcTimeout);
612         }
613       }
614     }
615 
616     protected synchronized void markClosed(IOException e) {
617       if (shouldCloseConnection.compareAndSet(false, true)) {
618         closeException = e;
619         notifyAll();
620       }
621     }
622 
623     /** Close the connection. */
624     protected synchronized void close() {
625       if (!shouldCloseConnection.get()) {
626         LOG.error("The connection is not in the closed state");
627         return;
628       }
629 
630       // release the resources
631       // first thing to do;take the connection out of the connection list
632       synchronized (connections) {
633         connections.remove(remoteId, this);
634       }
635 
636       // close the streams and therefore the socket
637       IOUtils.closeStream(out);
638       IOUtils.closeStream(in);
639 
640       // clean up all calls
641       if (closeException == null) {
642         if (!calls.isEmpty()) {
643           LOG.warn(
644               "A connection is closed for no cause and calls are not empty");
645 
646           // clean up calls anyway
647           closeException = new IOException("Unexpected closed connection");
648           cleanupCalls();
649         }
650       } else {
651         // log the info
652         if (LOG.isDebugEnabled()) {
653           LOG.debug("closing ipc connection to " + remoteId.address + ": " +
654               closeException.getMessage(),closeException);
655         }
656 
657         // cleanup calls
658         cleanupCalls();
659       }
660       if (LOG.isDebugEnabled())
661         LOG.debug(getName() + ": closed");
662     }
663 
664     /* Cleanup all calls and mark them as done */
665     protected void cleanupCalls() {
666       cleanupCalls(0);
667     }
668 
669     protected void cleanupCalls(long rpcTimeout) {
670       Iterator<Entry<Integer, Call>> itor = calls.entrySet().iterator();
671       while (itor.hasNext()) {
672         Call c = itor.next().getValue();
673         long waitTime = System.currentTimeMillis() - c.getStartTime();
674         if (waitTime >= rpcTimeout) {
675           c.setException(closeException); // local exception
676           synchronized (c) {
677             c.notifyAll();
678           }
679           itor.remove();
680         } else {
681           break;
682         }
683       }
684       try {
685         if (!calls.isEmpty()) {
686           Call firstCall = calls.get(calls.firstKey());
687           long maxWaitTime = System.currentTimeMillis() - firstCall.getStartTime();
688           if (maxWaitTime < rpcTimeout) {
689             rpcTimeout -= maxWaitTime;
690           }
691         }
692         if (!shouldCloseConnection.get()) {
693           closeException = null;
694           if (socket != null) {
695             socket.setSoTimeout((int) rpcTimeout);
696           }
697         }
698       } catch (SocketException e) {
699         LOG.debug("Couldn't lower timeout, which may result in longer than expected calls");
700       }
701     }
702   }
703 
704   /** Call implementation used for parallel calls. */
705   protected class ParallelCall extends Call {
706     private final ParallelResults results;
707     protected final int index;
708 
709     public ParallelCall(Writable param, ParallelResults results, int index) {
710       super(param);
711       this.results = results;
712       this.index = index;
713     }
714 
715     /** Deliver result to result collector. */
716     @Override
717     protected void callComplete() {
718       results.callComplete(this);
719     }
720   }
721 
722   /** Result collector for parallel calls. */
723   protected static class ParallelResults {
724     protected final Writable[] values;
725     protected int size;
726     protected int count;
727 
728     public ParallelResults(int size) {
729       this.values = new Writable[size];
730       this.size = size;
731     }
732 
733     /*
734      * Collect a result.
735      */
736     synchronized void callComplete(ParallelCall call) {
737       // FindBugs IS2_INCONSISTENT_SYNC
738       values[call.index] = call.value;            // store the value
739       count++;                                    // count it
740       if (count == size)                          // if all values are in
741         notify();                                 // then notify waiting caller
742     }
743   }
744 
745   /**
746    * Construct an IPC client whose values are of the given {@link Writable}
747    * class.
748    * @param valueClass value class
749    * @param conf configuration
750    * @param factory socket factory
751    */
752   public HBaseClient(Class<? extends Writable> valueClass, Configuration conf,
753       SocketFactory factory) {
754     this.valueClass = valueClass;
755     this.maxIdleTime =
756       conf.getInt("hbase.ipc.client.connection.maxidletime", 10000); //10s
757     this.maxRetries = conf.getInt("hbase.ipc.client.connect.max.retries", 0);
758     this.failureSleep = conf.getInt("hbase.client.pause", 1000);
759     this.tcpNoDelay = conf.getBoolean("hbase.ipc.client.tcpnodelay", false);
760     this.tcpKeepAlive = conf.getBoolean("hbase.ipc.client.tcpkeepalive", true);
761     this.pingInterval = getPingInterval(conf);
762     if (LOG.isDebugEnabled()) {
763       LOG.debug("The ping interval is" + this.pingInterval + "ms.");
764     }
765     this.conf = conf;
766     this.socketFactory = factory;
767     this.clusterId = conf.get(HConstants.CLUSTER_ID, "default");
768     this.connections = new PoolMap<ConnectionId, Connection>(
769         getPoolType(conf), getPoolSize(conf));
770   }
771 
772   /**
773    * Construct an IPC client with the default SocketFactory
774    * @param valueClass value class
775    * @param conf configuration
776    */
777   public HBaseClient(Class<? extends Writable> valueClass, Configuration conf) {
778     this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
779   }
780 
781   /**
782    * Return the pool type specified in the configuration, which must be set to
783    * either {@link PoolType#RoundRobin} or {@link PoolType#ThreadLocal},
784    * otherwise default to the former.
785    *
786    * For applications with many user threads, use a small round-robin pool. For
787    * applications with few user threads, you may want to try using a
788    * thread-local pool. In any case, the number of {@link HBaseClient} instances
789    * should not exceed the operating system's hard limit on the number of
790    * connections.
791    *
792    * @param config configuration
793    * @return either a {@link PoolType#RoundRobin} or
794    *         {@link PoolType#ThreadLocal}
795    */
796   protected static PoolType getPoolType(Configuration config) {
797     return PoolType.valueOf(config.get(HConstants.HBASE_CLIENT_IPC_POOL_TYPE),
798         PoolType.RoundRobin, PoolType.ThreadLocal);
799   }
800 
801   /**
802    * Return the pool size specified in the configuration, which is applicable only if
803    * the pool type is {@link PoolType#RoundRobin}.
804    *
805    * @param config
806    * @return the maximum pool size
807    */
808   protected static int getPoolSize(Configuration config) {
809     return config.getInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, 1);
810   }
811 
812   /** Return the socket factory of this client
813    *
814    * @return this client's socket factory
815    */
816   SocketFactory getSocketFactory() {
817     return socketFactory;
818   }
819 
820   /** Stop all threads related to this client.  No further calls may be made
821    * using this client. */
822   public void stop() {
823     if (LOG.isDebugEnabled()) {
824       LOG.debug("Stopping client");
825     }
826 
827     if (!running.compareAndSet(true, false)) {
828       return;
829     }
830 
831     // wake up all connections
832     synchronized (connections) {
833       for (Connection conn : connections.values()) {
834         conn.interrupt();
835       }
836     }
837 
838     // wait until all connections are closed
839     while (!connections.isEmpty()) {
840       try {
841         Thread.sleep(100);
842       } catch (InterruptedException ignored) {
843       }
844     }
845   }
846 
847   /** Make a call, passing <code>param</code>, to the IPC server running at
848    * <code>address</code>, returning the value.  Throws exceptions if there are
849    * network problems or if the remote code threw an exception.
850    * @param param writable parameter
851    * @param address network address
852    * @return Writable
853    * @throws IOException e
854    */
855   public Writable call(Writable param, InetSocketAddress address)
856   throws IOException, InterruptedException {
857       return call(param, address, null, 0);
858   }
859 
860   public Writable call(Writable param, InetSocketAddress addr,
861                        User ticket, int rpcTimeout)
862                        throws IOException, InterruptedException {
863     return call(param, addr, null, ticket, rpcTimeout);
864   }
865 
866   /** Make a call, passing <code>param</code>, to the IPC server running at
867    * <code>address</code> which is servicing the <code>protocol</code> protocol,
868    * with the <code>ticket</code> credentials, returning the value.
869    * Throws exceptions if there are network problems or if the remote code
870    * threw an exception. */
871   public Writable call(Writable param, InetSocketAddress addr,
872                        Class<? extends VersionedProtocol> protocol,
873                        User ticket, int rpcTimeout)
874       throws InterruptedException, IOException {
875     Call call = new Call(param);
876     Connection connection = getConnection(addr, protocol, ticket, rpcTimeout, call);
877     connection.sendParam(call);                 // send the parameter
878     boolean interrupted = false;
879     //noinspection SynchronizationOnLocalVariableOrMethodParameter
880     synchronized (call) {
881       while (!call.done) {
882         try {
883           call.wait();                           // wait for the result
884         } catch (InterruptedException ignored) {
885           // save the fact that we were interrupted
886           interrupted = true;
887         }
888       }
889 
890       if (interrupted) {
891         // set the interrupt flag now that we are done waiting
892         Thread.currentThread().interrupt();
893       }
894 
895       if (call.error != null) {
896         if (call.error instanceof RemoteException) {
897           call.error.fillInStackTrace();
898           throw call.error;
899         }
900         // local exception
901         throw wrapException(addr, call.error);
902       }
903       return call.value;
904     }
905   }
906 
907   /**
908    * Take an IOException and the address we were trying to connect to
909    * and return an IOException with the input exception as the cause.
910    * The new exception provides the stack trace of the place where
911    * the exception is thrown and some extra diagnostics information.
912    * If the exception is ConnectException or SocketTimeoutException,
913    * return a new one of the same type; Otherwise return an IOException.
914    *
915    * @param addr target address
916    * @param exception the relevant exception
917    * @return an exception to throw
918    */
919   @SuppressWarnings({"ThrowableInstanceNeverThrown"})
920   protected IOException wrapException(InetSocketAddress addr,
921                                          IOException exception) {
922     if (exception instanceof ConnectException) {
923       //connection refused; include the host:port in the error
924       return (ConnectException)new ConnectException(
925            "Call to " + addr + " failed on connection exception: " + exception)
926                     .initCause(exception);
927     } else if (exception instanceof SocketTimeoutException) {
928       return (SocketTimeoutException)new SocketTimeoutException(
929            "Call to " + addr + " failed on socket timeout exception: "
930                       + exception).initCause(exception);
931     } else {
932       return (IOException)new IOException(
933            "Call to " + addr + " failed on local exception: " + exception)
934                                  .initCause(exception);
935 
936     }
937   }
938 
939   /** Makes a set of calls in parallel.  Each parameter is sent to the
940    * corresponding address.  When all values are available, or have timed out
941    * or errored, the collected results are returned in an array.  The array
942    * contains nulls for calls that timed out or errored.
943    * @param params writable parameters
944    * @param addresses socket addresses
945    * @return  Writable[]
946    * @throws IOException e
947    * @deprecated Use {@link #call(Writable[], InetSocketAddress[], Class, User)} instead
948    */
949   @Deprecated
950   public Writable[] call(Writable[] params, InetSocketAddress[] addresses)
951     throws IOException, InterruptedException {
952     return call(params, addresses, null, null);
953   }
954 
955   /** Makes a set of calls in parallel.  Each parameter is sent to the
956    * corresponding address.  When all values are available, or have timed out
957    * or errored, the collected results are returned in an array.  The array
958    * contains nulls for calls that timed out or errored.  */
959   public Writable[] call(Writable[] params, InetSocketAddress[] addresses,
960                          Class<? extends VersionedProtocol> protocol,
961                          User ticket)
962       throws IOException, InterruptedException {
963     if (addresses.length == 0) return new Writable[0];
964 
965     ParallelResults results = new ParallelResults(params.length);
966     // TODO this synchronization block doesnt make any sense, we should possibly fix it
967     //noinspection SynchronizationOnLocalVariableOrMethodParameter
968     synchronized (results) {
969       for (int i = 0; i < params.length; i++) {
970         ParallelCall call = new ParallelCall(params[i], results, i);
971         try {
972           Connection connection =
973               getConnection(addresses[i], protocol, ticket, 0, call);
974           connection.sendParam(call);             // send each parameter
975         } catch (IOException e) {
976           // log errors
977           LOG.info("Calling "+addresses[i]+" caught: " +
978                    e.getMessage(),e);
979           results.size--;                         //  wait for one fewer result
980         }
981       }
982       while (results.count != results.size) {
983         try {
984           results.wait();                    // wait for all results
985         } catch (InterruptedException ignored) {}
986       }
987 
988       return results.values;
989     }
990   }
991 
992   /* Get a connection from the pool, or create a new one and add it to the
993    * pool.  Connections to a given host/port are reused. */
994   protected Connection getConnection(InetSocketAddress addr,
995                                    Class<? extends VersionedProtocol> protocol,
996                                    User ticket,
997                                    int rpcTimeout,
998                                    Call call)
999                                    throws IOException, InterruptedException {
1000     if (!running.get()) {
1001       // the client is stopped
1002       throw new IOException("The client is stopped");
1003     }
1004     Connection connection;
1005     /* we could avoid this allocation for each RPC by having a
1006      * connectionsId object and with set() method. We need to manage the
1007      * refs for keys in HashMap properly. For now its ok.
1008      */
1009     ConnectionId remoteId = new ConnectionId(addr, protocol, ticket, rpcTimeout);
1010     do {
1011       synchronized (connections) {
1012         connection = connections.get(remoteId);
1013         if (connection == null) {
1014           connection = new Connection(remoteId);
1015           connections.put(remoteId, connection);
1016         }
1017       }
1018     } while (!connection.addCall(call));
1019 
1020     //we don't invoke the method below inside "synchronized (connections)"
1021     //block above. The reason for that is if the server happens to be slow,
1022     //it will take longer to establish a connection and that will slow the
1023     //entire system down.
1024     connection.setupIOstreams();
1025     return connection;
1026   }
1027 
1028   /**
1029    * This class holds the address and the user ticket. The client connections
1030    * to servers are uniquely identified by <remoteAddress, ticket>
1031    */
1032   protected static class ConnectionId {
1033     final InetSocketAddress address;
1034     final User ticket;
1035     final int rpcTimeout;
1036     Class<? extends VersionedProtocol> protocol;
1037     private static final int PRIME = 16777619;
1038 
1039     ConnectionId(InetSocketAddress address,
1040         Class<? extends VersionedProtocol> protocol,
1041         User ticket,
1042         int rpcTimeout) {
1043       this.protocol = protocol;
1044       this.address = address;
1045       this.ticket = ticket;
1046       this.rpcTimeout = rpcTimeout;
1047     }
1048 
1049     InetSocketAddress getAddress() {
1050       return address;
1051     }
1052 
1053     Class<? extends VersionedProtocol> getProtocol() {
1054       return protocol;
1055     }
1056 
1057     User getTicket() {
1058       return ticket;
1059     }
1060 
1061     @Override
1062     public boolean equals(Object obj) {
1063      if (obj instanceof ConnectionId) {
1064        ConnectionId id = (ConnectionId) obj;
1065        return address.equals(id.address) && protocol == id.protocol &&
1066               ((ticket != null && ticket.equals(id.ticket)) ||
1067                (ticket == id.ticket)) && rpcTimeout == id.rpcTimeout;
1068      }
1069      return false;
1070     }
1071 
1072     @Override  // simply use the default Object#hashcode() ?
1073     public int hashCode() {
1074       return (address.hashCode() + PRIME * (
1075                   PRIME * System.identityHashCode(protocol) ^
1076              (ticket == null ? 0 : ticket.hashCode()) )) ^ rpcTimeout;
1077     }
1078   }
1079 }