View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.io.UnsupportedEncodingException;
26  import java.lang.reflect.Field;
27  import java.math.BigDecimal;
28  import java.math.BigInteger;
29  import java.nio.ByteBuffer;
30  import java.nio.ByteOrder;
31  import java.security.AccessController;
32  import java.security.PrivilegedAction;
33  import java.util.Comparator;
34  import java.util.Iterator;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
40  import org.apache.hadoop.io.RawComparator;
41  import org.apache.hadoop.io.WritableComparator;
42  import org.apache.hadoop.io.WritableUtils;
43  
44  import sun.misc.Unsafe;
45  
46  import com.google.common.annotations.VisibleForTesting;
47  
48  /**
49   * Utility class that handles byte arrays, conversions to/from other types,
50   * comparisons, hash code generation, manufacturing keys for HashMaps or
51   * HashSets, etc.
52   */
53  public class Bytes {
54  
55    private static final Log LOG = LogFactory.getLog(Bytes.class);
56  
57    /**
58     * Size of boolean in bytes
59     */
60    public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
61  
62    /**
63     * Size of byte in bytes
64     */
65    public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
66  
67    /**
68     * Size of char in bytes
69     */
70    public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
71  
72    /**
73     * Size of double in bytes
74     */
75    public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
76  
77    /**
78     * Size of float in bytes
79     */
80    public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
81  
82    /**
83     * Size of int in bytes
84     */
85    public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
86  
87    /**
88     * Size of long in bytes
89     */
90    public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
91  
92    /**
93     * Size of short in bytes
94     */
95    public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
96  
97  
98    /**
99     * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
100    * Estimate based on study of jhat and jprofiler numbers.
101    */
102   // JHat says BU is 56 bytes.
103   // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
104   public static final int ESTIMATED_HEAP_TAX = 16;
105 
106   /**
107    * Byte array comparator class.
108    */
109   public static class ByteArrayComparator implements RawComparator<byte []> {
110     /**
111      * Constructor
112      */
113     public ByteArrayComparator() {
114       super();
115     }
116     public int compare(byte [] left, byte [] right) {
117       return compareTo(left, right);
118     }
119     public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
120       return LexicographicalComparerHolder.BEST_COMPARER.
121         compareTo(b1, s1, l1, b2, s2, l2);
122     }
123   }
124 
125   /**
126    * Pass this to TreeMaps where byte [] are keys.
127    */
128   public static Comparator<byte []> BYTES_COMPARATOR =
129     new ByteArrayComparator();
130 
131   /**
132    * Use comparing byte arrays, byte-by-byte
133    */
134   public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
135     new ByteArrayComparator();
136 
137   /**
138    * Read byte-array written with a WritableableUtils.vint prefix.
139    * @param in Input to read from.
140    * @return byte array read off <code>in</code>
141    * @throws IOException e
142    */
143   public static byte [] readByteArray(final DataInput in)
144   throws IOException {
145     int len = WritableUtils.readVInt(in);
146     if (len < 0) {
147       throw new NegativeArraySizeException(Integer.toString(len));
148     }
149     byte [] result = new byte[len];
150     in.readFully(result, 0, len);
151     return result;
152   }
153 
154   /**
155    * Read byte-array written with a WritableableUtils.vint prefix.
156    * IOException is converted to a RuntimeException.
157    * @param in Input to read from.
158    * @return byte array read off <code>in</code>
159    */
160   public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
161     try {
162       return readByteArray(in);
163     } catch (Exception e) {
164       throw new RuntimeException(e);
165     }
166   }
167 
168   /**
169    * Write byte-array with a WritableableUtils.vint prefix.
170    * @param out output stream to be written to
171    * @param b array to write
172    * @throws IOException e
173    */
174   public static void writeByteArray(final DataOutput out, final byte [] b)
175   throws IOException {
176     if(b == null) {
177       WritableUtils.writeVInt(out, 0);
178     } else {
179       writeByteArray(out, b, 0, b.length);
180     }
181   }
182 
183   /**
184    * Write byte-array to out with a vint length prefix.
185    * @param out output stream
186    * @param b array
187    * @param offset offset into array
188    * @param length length past offset
189    * @throws IOException e
190    */
191   public static void writeByteArray(final DataOutput out, final byte [] b,
192       final int offset, final int length)
193   throws IOException {
194     WritableUtils.writeVInt(out, length);
195     out.write(b, offset, length);
196   }
197 
198   /**
199    * Write byte-array from src to tgt with a vint length prefix.
200    * @param tgt target array
201    * @param tgtOffset offset into target array
202    * @param src source array
203    * @param srcOffset source offset
204    * @param srcLength source length
205    * @return New offset in src array.
206    */
207   public static int writeByteArray(final byte [] tgt, final int tgtOffset,
208       final byte [] src, final int srcOffset, final int srcLength) {
209     byte [] vint = vintToBytes(srcLength);
210     System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
211     int offset = tgtOffset + vint.length;
212     System.arraycopy(src, srcOffset, tgt, offset, srcLength);
213     return offset + srcLength;
214   }
215 
216   /**
217    * Put bytes at the specified byte array position.
218    * @param tgtBytes the byte array
219    * @param tgtOffset position in the array
220    * @param srcBytes array to write out
221    * @param srcOffset source offset
222    * @param srcLength source length
223    * @return incremented offset
224    */
225   public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
226       int srcOffset, int srcLength) {
227     System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
228     return tgtOffset + srcLength;
229   }
230 
231   /**
232    * Write a single byte out to the specified byte array position.
233    * @param bytes the byte array
234    * @param offset position in the array
235    * @param b byte to write out
236    * @return incremented offset
237    */
238   public static int putByte(byte[] bytes, int offset, byte b) {
239     bytes[offset] = b;
240     return offset + 1;
241   }
242 
243   /**
244    * Returns a new byte array, copied from the passed ByteBuffer.
245    * @param bb A ByteBuffer
246    * @return the byte array
247    */
248   public static byte[] toBytes(ByteBuffer bb) {
249     int length = bb.limit();
250     byte [] result = new byte[length];
251     System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length);
252     return result;
253   }
254 
255   /**
256    * @param b Presumed UTF-8 encoded byte array.
257    * @return String made from <code>b</code>
258    */
259   public static String toString(final byte [] b) {
260     if (b == null) {
261       return null;
262     }
263     return toString(b, 0, b.length);
264   }
265 
266   /**
267    * Joins two byte arrays together using a separator.
268    * @param b1 The first byte array.
269    * @param sep The separator to use.
270    * @param b2 The second byte array.
271    */
272   public static String toString(final byte [] b1,
273                                 String sep,
274                                 final byte [] b2) {
275     return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
276   }
277 
278   /**
279    * This method will convert utf8 encoded bytes into a string. If
280    * an UnsupportedEncodingException occurs, this method will eat it
281    * and return null instead.
282    *
283    * @param b Presumed UTF-8 encoded byte array.
284    * @param off offset into array
285    * @param len length of utf-8 sequence
286    * @return String made from <code>b</code> or null
287    */
288   public static String toString(final byte [] b, int off, int len) {
289     if (b == null) {
290       return null;
291     }
292     if (len == 0) {
293       return "";
294     }
295     try {
296       return new String(b, off, len, HConstants.UTF8_ENCODING);
297     } catch (UnsupportedEncodingException e) {
298       LOG.error("UTF-8 not supported?", e);
299       return null;
300     }
301   }
302 
303   /**
304    * Write a printable representation of a byte array.
305    *
306    * @param b byte array
307    * @return string
308    * @see #toStringBinary(byte[], int, int)
309    */
310   public static String toStringBinary(final byte [] b) {
311     if (b == null)
312       return "null";
313     return toStringBinary(b, 0, b.length);
314   }
315   
316   /**
317    * Converts the given byte buffer, from its array offset to its limit, to
318    * a string. The position and the mark are ignored.
319    *
320    * @param buf a byte buffer
321    * @return a string representation of the buffer's binary contents
322    */
323   public static String toStringBinary(ByteBuffer buf) {
324     if (buf == null)
325       return "null";
326     return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
327   }
328 
329   /**
330    * Write a printable representation of a byte array. Non-printable
331    * characters are hex escaped in the format \\x%02X, eg:
332    * \x00 \x05 etc
333    *
334    * @param b array to write out
335    * @param off offset to start at
336    * @param len length to write
337    * @return string output
338    */
339   public static String toStringBinary(final byte [] b, int off, int len) {
340     StringBuilder result = new StringBuilder();
341     try {
342       String first = new String(b, off, len, "ISO-8859-1");
343       for (int i = 0; i < first.length() ; ++i ) {
344         int ch = first.charAt(i) & 0xFF;
345         if ( (ch >= '0' && ch <= '9')
346             || (ch >= 'A' && ch <= 'Z')
347             || (ch >= 'a' && ch <= 'z')
348             || " `~!@#$%^&*()-_=+[]{}\\|;:'\",.<>/?".indexOf(ch) >= 0 ) {
349           result.append(first.charAt(i));
350         } else {
351           result.append(String.format("\\x%02X", ch));
352         }
353       }
354     } catch (UnsupportedEncodingException e) {
355       LOG.error("ISO-8859-1 not supported?", e);
356     }
357     return result.toString();
358   }
359 
360   private static boolean isHexDigit(char c) {
361     return
362         (c >= 'A' && c <= 'F') ||
363         (c >= '0' && c <= '9');
364   }
365 
366   /**
367    * Takes a ASCII digit in the range A-F0-9 and returns
368    * the corresponding integer/ordinal value.
369    * @param ch  The hex digit.
370    * @return The converted hex value as a byte.
371    */
372   public static byte toBinaryFromHex(byte ch) {
373     if ( ch >= 'A' && ch <= 'F' )
374       return (byte) ((byte)10 + (byte) (ch - 'A'));
375     // else
376     return (byte) (ch - '0');
377   }
378 
379   public static byte [] toBytesBinary(String in) {
380     // this may be bigger than we need, but lets be safe.
381     byte [] b = new byte[in.length()];
382     int size = 0;
383     for (int i = 0; i < in.length(); ++i) {
384       char ch = in.charAt(i);
385       if (ch == '\\') {
386         // begin hex escape:
387         char next = in.charAt(i+1);
388         if (next != 'x') {
389           // invalid escape sequence, ignore this one.
390           b[size++] = (byte)ch;
391           continue;
392         }
393         // ok, take next 2 hex digits.
394         char hd1 = in.charAt(i+2);
395         char hd2 = in.charAt(i+3);
396 
397         // they need to be A-F0-9:
398         if (!isHexDigit(hd1) ||
399             !isHexDigit(hd2)) {
400           // bogus escape code, ignore:
401           continue;
402         }
403         // turn hex ASCII digit -> number
404         byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
405 
406         b[size++] = d;
407         i += 3; // skip 3
408       } else {
409         b[size++] = (byte) ch;
410       }
411     }
412     // resize:
413     byte [] b2 = new byte[size];
414     System.arraycopy(b, 0, b2, 0, size);
415     return b2;
416   }
417 
418   /**
419    * Converts a string to a UTF-8 byte array.
420    * @param s string
421    * @return the byte array
422    */
423   public static byte[] toBytes(String s) {
424     try {
425       return s.getBytes(HConstants.UTF8_ENCODING);
426     } catch (UnsupportedEncodingException e) {
427       LOG.error("UTF-8 not supported?", e);
428       return null;
429     }
430   }
431 
432   /**
433    * Convert a boolean to a byte array. True becomes -1
434    * and false becomes 0.
435    *
436    * @param b value
437    * @return <code>b</code> encoded in a byte array.
438    */
439   public static byte [] toBytes(final boolean b) {
440     return new byte[] { b ? (byte) -1 : (byte) 0 };
441   }
442 
443   /**
444    * Reverses {@link #toBytes(boolean)}
445    * @param b array
446    * @return True or false.
447    */
448   public static boolean toBoolean(final byte [] b) {
449     if (b.length != 1) {
450       throw new IllegalArgumentException("Array has wrong size: " + b.length);
451     }
452     return b[0] != (byte) 0;
453   }
454 
455   /**
456    * Convert a long value to a byte array using big-endian.
457    *
458    * @param val value to convert
459    * @return the byte array
460    */
461   public static byte[] toBytes(long val) {
462     byte [] b = new byte[8];
463     for (int i = 7; i > 0; i--) {
464       b[i] = (byte) val;
465       val >>>= 8;
466     }
467     b[0] = (byte) val;
468     return b;
469   }
470 
471   /**
472    * Converts a byte array to a long value. Reverses
473    * {@link #toBytes(long)}
474    * @param bytes array
475    * @return the long value
476    */
477   public static long toLong(byte[] bytes) {
478     return toLong(bytes, 0, SIZEOF_LONG);
479   }
480 
481   /**
482    * Converts a byte array to a long value. Assumes there will be
483    * {@link #SIZEOF_LONG} bytes available.
484    *
485    * @param bytes bytes
486    * @param offset offset
487    * @return the long value
488    */
489   public static long toLong(byte[] bytes, int offset) {
490     return toLong(bytes, offset, SIZEOF_LONG);
491   }
492 
493   /**
494    * Converts a byte array to a long value.
495    *
496    * @param bytes array of bytes
497    * @param offset offset into array
498    * @param length length of data (must be {@link #SIZEOF_LONG})
499    * @return the long value
500    * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
501    * if there's not enough room in the array at the offset indicated.
502    */
503   public static long toLong(byte[] bytes, int offset, final int length) {
504     if (length != SIZEOF_LONG || offset + length > bytes.length) {
505       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
506     }
507     long l = 0;
508     for(int i = offset; i < offset + length; i++) {
509       l <<= 8;
510       l ^= bytes[i] & 0xFF;
511     }
512     return l;
513   }
514 
515   private static IllegalArgumentException
516     explainWrongLengthOrOffset(final byte[] bytes,
517                                final int offset,
518                                final int length,
519                                final int expectedLength) {
520     String reason;
521     if (length != expectedLength) {
522       reason = "Wrong length: " + length + ", expected " + expectedLength;
523     } else {
524      reason = "offset (" + offset + ") + length (" + length + ") exceed the"
525         + " capacity of the array: " + bytes.length;
526     }
527     return new IllegalArgumentException(reason);
528   }
529 
530   /**
531    * Put a long value out to the specified byte array position.
532    * @param bytes the byte array
533    * @param offset position in the array
534    * @param val long to write out
535    * @return incremented offset
536    * @throws IllegalArgumentException if the byte array given doesn't have
537    * enough room at the offset specified.
538    */
539   public static int putLong(byte[] bytes, int offset, long val) {
540     if (bytes.length - offset < SIZEOF_LONG) {
541       throw new IllegalArgumentException("Not enough room to put a long at"
542           + " offset " + offset + " in a " + bytes.length + " byte array");
543     }
544     for(int i = offset + 7; i > offset; i--) {
545       bytes[i] = (byte) val;
546       val >>>= 8;
547     }
548     bytes[offset] = (byte) val;
549     return offset + SIZEOF_LONG;
550   }
551 
552   /**
553    * Presumes float encoded as IEEE 754 floating-point "single format"
554    * @param bytes byte array
555    * @return Float made from passed byte array.
556    */
557   public static float toFloat(byte [] bytes) {
558     return toFloat(bytes, 0);
559   }
560 
561   /**
562    * Presumes float encoded as IEEE 754 floating-point "single format"
563    * @param bytes array to convert
564    * @param offset offset into array
565    * @return Float made from passed byte array.
566    */
567   public static float toFloat(byte [] bytes, int offset) {
568     return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
569   }
570 
571   /**
572    * @param bytes byte array
573    * @param offset offset to write to
574    * @param f float value
575    * @return New offset in <code>bytes</code>
576    */
577   public static int putFloat(byte [] bytes, int offset, float f) {
578     return putInt(bytes, offset, Float.floatToRawIntBits(f));
579   }
580 
581   /**
582    * @param f float value
583    * @return the float represented as byte []
584    */
585   public static byte [] toBytes(final float f) {
586     // Encode it as int
587     return Bytes.toBytes(Float.floatToRawIntBits(f));
588   }
589 
590   /**
591    * @param bytes byte array
592    * @return Return double made from passed bytes.
593    */
594   public static double toDouble(final byte [] bytes) {
595     return toDouble(bytes, 0);
596   }
597 
598   /**
599    * @param bytes byte array
600    * @param offset offset where double is
601    * @return Return double made from passed bytes.
602    */
603   public static double toDouble(final byte [] bytes, final int offset) {
604     return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
605   }
606 
607   /**
608    * @param bytes byte array
609    * @param offset offset to write to
610    * @param d value
611    * @return New offset into array <code>bytes</code>
612    */
613   public static int putDouble(byte [] bytes, int offset, double d) {
614     return putLong(bytes, offset, Double.doubleToLongBits(d));
615   }
616 
617   /**
618    * Serialize a double as the IEEE 754 double format output. The resultant
619    * array will be 8 bytes long.
620    *
621    * @param d value
622    * @return the double represented as byte []
623    */
624   public static byte [] toBytes(final double d) {
625     // Encode it as a long
626     return Bytes.toBytes(Double.doubleToRawLongBits(d));
627   }
628 
629   /**
630    * Convert an int value to a byte array
631    * @param val value
632    * @return the byte array
633    */
634   public static byte[] toBytes(int val) {
635     byte [] b = new byte[4];
636     for(int i = 3; i > 0; i--) {
637       b[i] = (byte) val;
638       val >>>= 8;
639     }
640     b[0] = (byte) val;
641     return b;
642   }
643 
644   /**
645    * Converts a byte array to an int value
646    * @param bytes byte array
647    * @return the int value
648    */
649   public static int toInt(byte[] bytes) {
650     return toInt(bytes, 0, SIZEOF_INT);
651   }
652 
653   /**
654    * Converts a byte array to an int value
655    * @param bytes byte array
656    * @param offset offset into array
657    * @return the int value
658    */
659   public static int toInt(byte[] bytes, int offset) {
660     return toInt(bytes, offset, SIZEOF_INT);
661   }
662 
663   /**
664    * Converts a byte array to an int value
665    * @param bytes byte array
666    * @param offset offset into array
667    * @param length length of int (has to be {@link #SIZEOF_INT})
668    * @return the int value
669    * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or
670    * if there's not enough room in the array at the offset indicated.
671    */
672   public static int toInt(byte[] bytes, int offset, final int length) {
673     if (length != SIZEOF_INT || offset + length > bytes.length) {
674       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
675     }
676     int n = 0;
677     for(int i = offset; i < (offset + length); i++) {
678       n <<= 8;
679       n ^= bytes[i] & 0xFF;
680     }
681     return n;
682   }
683 
684   /**
685    * Put an int value out to the specified byte array position.
686    * @param bytes the byte array
687    * @param offset position in the array
688    * @param val int to write out
689    * @return incremented offset
690    * @throws IllegalArgumentException if the byte array given doesn't have
691    * enough room at the offset specified.
692    */
693   public static int putInt(byte[] bytes, int offset, int val) {
694     if (bytes.length - offset < SIZEOF_INT) {
695       throw new IllegalArgumentException("Not enough room to put an int at"
696           + " offset " + offset + " in a " + bytes.length + " byte array");
697     }
698     for(int i= offset + 3; i > offset; i--) {
699       bytes[i] = (byte) val;
700       val >>>= 8;
701     }
702     bytes[offset] = (byte) val;
703     return offset + SIZEOF_INT;
704   }
705 
706   /**
707    * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
708    * @param val value
709    * @return the byte array
710    */
711   public static byte[] toBytes(short val) {
712     byte[] b = new byte[SIZEOF_SHORT];
713     b[1] = (byte) val;
714     val >>= 8;
715     b[0] = (byte) val;
716     return b;
717   }
718 
719   /**
720    * Converts a byte array to a short value
721    * @param bytes byte array
722    * @return the short value
723    */
724   public static short toShort(byte[] bytes) {
725     return toShort(bytes, 0, SIZEOF_SHORT);
726   }
727 
728   /**
729    * Converts a byte array to a short value
730    * @param bytes byte array
731    * @param offset offset into array
732    * @return the short value
733    */
734   public static short toShort(byte[] bytes, int offset) {
735     return toShort(bytes, offset, SIZEOF_SHORT);
736   }
737 
738   /**
739    * Converts a byte array to a short value
740    * @param bytes byte array
741    * @param offset offset into array
742    * @param length length, has to be {@link #SIZEOF_SHORT}
743    * @return the short value
744    * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT}
745    * or if there's not enough room in the array at the offset indicated.
746    */
747   public static short toShort(byte[] bytes, int offset, final int length) {
748     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
749       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
750     }
751     short n = 0;
752     n ^= bytes[offset] & 0xFF;
753     n <<= 8;
754     n ^= bytes[offset+1] & 0xFF;
755     return n;
756   }
757 
758   /**
759    * This method will get a sequence of bytes from pos -> limit,
760    * but will restore pos after.
761    * @param buf
762    * @return byte array
763    */
764   public static byte[] getBytes(ByteBuffer buf) {
765     int savedPos = buf.position();
766     byte [] newBytes = new byte[buf.remaining()];
767     buf.get(newBytes);
768     buf.position(savedPos);
769     return newBytes;
770   }
771 
772   /**
773    * Put a short value out to the specified byte array position.
774    * @param bytes the byte array
775    * @param offset position in the array
776    * @param val short to write out
777    * @return incremented offset
778    * @throws IllegalArgumentException if the byte array given doesn't have
779    * enough room at the offset specified.
780    */
781   public static int putShort(byte[] bytes, int offset, short val) {
782     if (bytes.length - offset < SIZEOF_SHORT) {
783       throw new IllegalArgumentException("Not enough room to put a short at"
784           + " offset " + offset + " in a " + bytes.length + " byte array");
785     }
786     bytes[offset+1] = (byte) val;
787     val >>= 8;
788     bytes[offset] = (byte) val;
789     return offset + SIZEOF_SHORT;
790   }
791 
792   /**
793    * Convert a BigDecimal value to a byte array
794    *
795    * @param val
796    * @return the byte array
797    */
798   public static byte[] toBytes(BigDecimal val) {
799     byte[] valueBytes = val.unscaledValue().toByteArray();
800     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
801     int offset = putInt(result, 0, val.scale());
802     putBytes(result, offset, valueBytes, 0, valueBytes.length);
803     return result;
804   }
805 
806 
807   /**
808    * Converts a byte array to a BigDecimal
809    *
810    * @param bytes
811    * @return the char value
812    */
813   public static BigDecimal toBigDecimal(byte[] bytes) {
814     return toBigDecimal(bytes, 0, bytes.length);
815   }
816 
817   /**
818    * Converts a byte array to a BigDecimal value
819    *
820    * @param bytes
821    * @param offset
822    * @param length
823    * @return the char value
824    */
825   public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
826     if (bytes == null || length < SIZEOF_INT + 1 ||
827       (offset + length > bytes.length)) {
828       return null;
829     }
830 
831     int scale = toInt(bytes, offset);
832     byte[] tcBytes = new byte[length - SIZEOF_INT];
833     System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
834     return new BigDecimal(new BigInteger(tcBytes), scale);
835   }
836 
837   /**
838    * Put a BigDecimal value out to the specified byte array position.
839    *
840    * @param bytes  the byte array
841    * @param offset position in the array
842    * @param val    BigDecimal to write out
843    * @return incremented offset
844    */
845   public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
846     if (bytes == null) {
847       return offset;
848     }
849 
850     byte[] valueBytes = val.unscaledValue().toByteArray();
851     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
852     offset = putInt(result, offset, val.scale());
853     return putBytes(result, offset, valueBytes, 0, valueBytes.length);
854   }
855   
856   /**
857    * @param vint Integer to make a vint of.
858    * @return Vint as bytes array.
859    */
860   public static byte [] vintToBytes(final long vint) {
861     long i = vint;
862     int size = WritableUtils.getVIntSize(i);
863     byte [] result = new byte[size];
864     int offset = 0;
865     if (i >= -112 && i <= 127) {
866       result[offset] = (byte) i;
867       return result;
868     }
869 
870     int len = -112;
871     if (i < 0) {
872       i ^= -1L; // take one's complement'
873       len = -120;
874     }
875 
876     long tmp = i;
877     while (tmp != 0) {
878       tmp = tmp >> 8;
879       len--;
880     }
881 
882     result[offset++] = (byte) len;
883 
884     len = (len < -120) ? -(len + 120) : -(len + 112);
885 
886     for (int idx = len; idx != 0; idx--) {
887       int shiftbits = (idx - 1) * 8;
888       long mask = 0xFFL << shiftbits;
889       result[offset++] = (byte)((i & mask) >> shiftbits);
890     }
891     return result;
892   }
893 
894   /**
895    * @param buffer buffer to convert
896    * @return vint bytes as an integer.
897    */
898   public static long bytesToVint(final byte [] buffer) {
899     int offset = 0;
900     byte firstByte = buffer[offset++];
901     int len = WritableUtils.decodeVIntSize(firstByte);
902     if (len == 1) {
903       return firstByte;
904     }
905     long i = 0;
906     for (int idx = 0; idx < len-1; idx++) {
907       byte b = buffer[offset++];
908       i = i << 8;
909       i = i | (b & 0xFF);
910     }
911     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
912   }
913 
914   /**
915    * Reads a zero-compressed encoded long from input stream and returns it.
916    * @param buffer Binary array
917    * @param offset Offset into array at which vint begins.
918    * @throws java.io.IOException e
919    * @return deserialized long from stream.
920    */
921   public static long readVLong(final byte [] buffer, final int offset)
922   throws IOException {
923     byte firstByte = buffer[offset];
924     int len = WritableUtils.decodeVIntSize(firstByte);
925     if (len == 1) {
926       return firstByte;
927     }
928     long i = 0;
929     for (int idx = 0; idx < len-1; idx++) {
930       byte b = buffer[offset + 1 + idx];
931       i = i << 8;
932       i = i | (b & 0xFF);
933     }
934     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
935   }
936 
937   /**
938    * @param left left operand
939    * @param right right operand
940    * @return 0 if equal, < 0 if left is less than right, etc.
941    */
942   public static int compareTo(final byte [] left, final byte [] right) {
943     return LexicographicalComparerHolder.BEST_COMPARER.
944       compareTo(left, 0, left.length, right, 0, right.length);
945   }
946 
947   /**
948    * Lexicographically compare two arrays.
949    *
950    * @param buffer1 left operand
951    * @param buffer2 right operand
952    * @param offset1 Where to start comparing in the left buffer
953    * @param offset2 Where to start comparing in the right buffer
954    * @param length1 How much to compare from the left buffer
955    * @param length2 How much to compare from the right buffer
956    * @return 0 if equal, < 0 if left is less than right, etc.
957    */
958   public static int compareTo(byte[] buffer1, int offset1, int length1,
959       byte[] buffer2, int offset2, int length2) {
960     return LexicographicalComparerHolder.BEST_COMPARER.
961       compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
962   }
963   
964   interface Comparer<T> {
965     abstract public int compareTo(T buffer1, int offset1, int length1,
966         T buffer2, int offset2, int length2);
967   }
968 
969   @VisibleForTesting
970   static Comparer<byte[]> lexicographicalComparerJavaImpl() {
971     return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
972   }
973 
974   /**
975    * Provides a lexicographical comparer implementation; either a Java
976    * implementation or a faster implementation based on {@link Unsafe}.
977    *
978    * <p>Uses reflection to gracefully fall back to the Java implementation if
979    * {@code Unsafe} isn't available.
980    */
981   @VisibleForTesting
982   static class LexicographicalComparerHolder {
983     static final String UNSAFE_COMPARER_NAME =
984         LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
985     
986     static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
987     /**
988      * Returns the Unsafe-using Comparer, or falls back to the pure-Java
989      * implementation if unable to do so.
990      */
991     static Comparer<byte[]> getBestComparer() {
992       try {
993         Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
994 
995         // yes, UnsafeComparer does implement Comparer<byte[]>
996         @SuppressWarnings("unchecked")
997         Comparer<byte[]> comparer =
998           (Comparer<byte[]>) theClass.getEnumConstants()[0];
999         return comparer;
1000       } catch (Throwable t) { // ensure we really catch *everything*
1001         return lexicographicalComparerJavaImpl();
1002       }
1003     }
1004     
1005     enum PureJavaComparer implements Comparer<byte[]> {
1006       INSTANCE;
1007 
1008       @Override
1009       public int compareTo(byte[] buffer1, int offset1, int length1,
1010           byte[] buffer2, int offset2, int length2) {
1011         // Short circuit equal case
1012         if (buffer1 == buffer2 &&
1013             offset1 == offset2 &&
1014             length1 == length2) {
1015           return 0;
1016         }
1017         // Bring WritableComparator code local
1018         int end1 = offset1 + length1;
1019         int end2 = offset2 + length2;
1020         for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1021           int a = (buffer1[i] & 0xff);
1022           int b = (buffer2[j] & 0xff);
1023           if (a != b) {
1024             return a - b;
1025           }
1026         }
1027         return length1 - length2;
1028       }
1029     }
1030     
1031     @VisibleForTesting
1032     enum UnsafeComparer implements Comparer<byte[]> {
1033       INSTANCE;
1034 
1035       static final Unsafe theUnsafe;
1036 
1037       /** The offset to the first element in a byte array. */
1038       static final int BYTE_ARRAY_BASE_OFFSET;
1039 
1040       static {
1041         theUnsafe = (Unsafe) AccessController.doPrivileged(
1042             new PrivilegedAction<Object>() {
1043               @Override
1044               public Object run() {
1045                 try {
1046                   Field f = Unsafe.class.getDeclaredField("theUnsafe");
1047                   f.setAccessible(true);
1048                   return f.get(null);
1049                 } catch (NoSuchFieldException e) {
1050                   // It doesn't matter what we throw;
1051                   // it's swallowed in getBestComparer().
1052                   throw new Error();
1053                 } catch (IllegalAccessException e) {
1054                   throw new Error();
1055                 }
1056               }
1057             });
1058 
1059         BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1060 
1061         // sanity check - this should never fail
1062         if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1063           throw new AssertionError();
1064         }
1065       }
1066 
1067       static final boolean littleEndian =
1068         ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1069 
1070       /**
1071        * Returns true if x1 is less than x2, when both values are treated as
1072        * unsigned.
1073        */
1074       static boolean lessThanUnsigned(long x1, long x2) {
1075         return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1076       }
1077 
1078       /**
1079        * Lexicographically compare two arrays.
1080        *
1081        * @param buffer1 left operand
1082        * @param buffer2 right operand
1083        * @param offset1 Where to start comparing in the left buffer
1084        * @param offset2 Where to start comparing in the right buffer
1085        * @param length1 How much to compare from the left buffer
1086        * @param length2 How much to compare from the right buffer
1087        * @return 0 if equal, < 0 if left is less than right, etc.
1088        */
1089       @Override
1090       public int compareTo(byte[] buffer1, int offset1, int length1,
1091           byte[] buffer2, int offset2, int length2) {
1092         // Short circuit equal case
1093         if (buffer1 == buffer2 &&
1094             offset1 == offset2 &&
1095             length1 == length2) {
1096           return 0;
1097         }
1098         int minLength = Math.min(length1, length2);
1099         int minWords = minLength / SIZEOF_LONG;
1100         int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1101         int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1102 
1103         /*
1104          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
1105          * time is no slower than comparing 4 bytes at a time even on 32-bit.
1106          * On the other hand, it is substantially faster on 64-bit.
1107          */
1108         for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1109           long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1110           long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1111           long diff = lw ^ rw;
1112 
1113           if (diff != 0) {
1114             if (!littleEndian) {
1115               return lessThanUnsigned(lw, rw) ? -1 : 1;
1116             }
1117 
1118             // Use binary search
1119             int n = 0;
1120             int y;
1121             int x = (int) diff;
1122             if (x == 0) {
1123               x = (int) (diff >>> 32);
1124               n = 32;
1125             }
1126 
1127             y = x << 16;
1128             if (y == 0) {
1129               n += 16;
1130             } else {
1131               x = y;
1132             }
1133 
1134             y = x << 8;
1135             if (y == 0) {
1136               n += 8;
1137             }
1138             return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1139           }
1140         }
1141 
1142         // The epilogue to cover the last (minLength % 8) elements.
1143         for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1144           int a = (buffer1[offset1 + i] & 0xff);
1145           int b = (buffer2[offset2 + i] & 0xff);
1146           if (a != b) {
1147             return a - b;
1148           }
1149         }
1150         return length1 - length2;
1151       }
1152     }
1153   }
1154 
1155   /**
1156    * @param left left operand
1157    * @param right right operand
1158    * @return True if equal
1159    */
1160   public static boolean equals(final byte [] left, final byte [] right) {
1161     // Could use Arrays.equals?
1162     //noinspection SimplifiableConditionalExpression
1163     if (left == right) return true;
1164     if (left == null || right == null) return false;
1165     if (left.length != right.length) return false;
1166     if (left.length == 0) return true;
1167     
1168     // Since we're often comparing adjacent sorted data,
1169     // it's usual to have equal arrays except for the very last byte
1170     // so check that first
1171     if (left[left.length - 1] != right[right.length - 1]) return false;
1172 
1173     return compareTo(left, right) == 0;
1174   }
1175   
1176   public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1177                                final byte[] right, int rightOffset, int rightLen) {
1178     // short circuit case
1179     if (left == right &&
1180         leftOffset == rightOffset &&
1181         leftLen == rightLen) {
1182       return true;
1183     }
1184     // different lengths fast check
1185     if (leftLen != rightLen) {
1186       return false;
1187     }
1188     if (leftLen == 0) {
1189       return true;
1190     }
1191     
1192     // Since we're often comparing adjacent sorted data,
1193     // it's usual to have equal arrays except for the very last byte
1194     // so check that first
1195     if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1196 
1197     return LexicographicalComparerHolder.BEST_COMPARER.
1198       compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1199   }
1200   
1201 
1202   /**
1203    * Return true if the byte array on the right is a prefix of the byte
1204    * array on the left.
1205    */
1206   public static boolean startsWith(byte[] bytes, byte[] prefix) {
1207     return bytes != null && prefix != null &&
1208       bytes.length >= prefix.length &&
1209       LexicographicalComparerHolder.BEST_COMPARER.
1210         compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;      
1211   }
1212 
1213   /**
1214    * @param b bytes to hash
1215    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1216    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1217    * {@link ImmutableBytesWritable} use calculating hash code.
1218    */
1219   public static int hashCode(final byte [] b) {
1220     return hashCode(b, b.length);
1221   }
1222 
1223   /**
1224    * @param b value
1225    * @param length length of the value
1226    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1227    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1228    * {@link ImmutableBytesWritable} use calculating hash code.
1229    */
1230   public static int hashCode(final byte [] b, final int length) {
1231     return WritableComparator.hashBytes(b, length);
1232   }
1233 
1234   /**
1235    * @param b bytes to hash
1236    * @return A hash of <code>b</code> as an Integer that can be used as key in
1237    * Maps.
1238    */
1239   public static Integer mapKey(final byte [] b) {
1240     return hashCode(b);
1241   }
1242 
1243   /**
1244    * @param b bytes to hash
1245    * @param length length to hash
1246    * @return A hash of <code>b</code> as an Integer that can be used as key in
1247    * Maps.
1248    */
1249   public static Integer mapKey(final byte [] b, final int length) {
1250     return hashCode(b, length);
1251   }
1252 
1253   /**
1254    * @param a lower half
1255    * @param b upper half
1256    * @return New array that has a in lower half and b in upper half.
1257    */
1258   public static byte [] add(final byte [] a, final byte [] b) {
1259     return add(a, b, HConstants.EMPTY_BYTE_ARRAY);
1260   }
1261 
1262   /**
1263    * @param a first third
1264    * @param b second third
1265    * @param c third third
1266    * @return New array made from a, b and c
1267    */
1268   public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1269     byte [] result = new byte[a.length + b.length + c.length];
1270     System.arraycopy(a, 0, result, 0, a.length);
1271     System.arraycopy(b, 0, result, a.length, b.length);
1272     System.arraycopy(c, 0, result, a.length + b.length, c.length);
1273     return result;
1274   }
1275 
1276   /**
1277    * @param a array
1278    * @param length amount of bytes to grab
1279    * @return First <code>length</code> bytes from <code>a</code>
1280    */
1281   public static byte [] head(final byte [] a, final int length) {
1282     if (a.length < length) {
1283       return null;
1284     }
1285     byte [] result = new byte[length];
1286     System.arraycopy(a, 0, result, 0, length);
1287     return result;
1288   }
1289 
1290   /**
1291    * @param a array
1292    * @param length amount of bytes to snarf
1293    * @return Last <code>length</code> bytes from <code>a</code>
1294    */
1295   public static byte [] tail(final byte [] a, final int length) {
1296     if (a.length < length) {
1297       return null;
1298     }
1299     byte [] result = new byte[length];
1300     System.arraycopy(a, a.length - length, result, 0, length);
1301     return result;
1302   }
1303 
1304   /**
1305    * @param a array
1306    * @param length new array size
1307    * @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes
1308    */
1309   public static byte [] padHead(final byte [] a, final int length) {
1310     byte [] padding = new byte[length];
1311     for (int i = 0; i < length; i++) {
1312       padding[i] = 0;
1313     }
1314     return add(padding,a);
1315   }
1316 
1317   /**
1318    * @param a array
1319    * @param length new array size
1320    * @return Value in <code>a</code> plus <code>length</code> appended 0 bytes
1321    */
1322   public static byte [] padTail(final byte [] a, final int length) {
1323     byte [] padding = new byte[length];
1324     for (int i = 0; i < length; i++) {
1325       padding[i] = 0;
1326     }
1327     return add(a,padding);
1328   }
1329 
1330   /**
1331    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1332    * Useful splitting ranges for MapReduce jobs.
1333    * @param a Beginning of range
1334    * @param b End of range
1335    * @param num Number of times to split range.  Pass 1 if you want to split
1336    * the range in two; i.e. one split.
1337    * @return Array of dividing values
1338    */
1339   public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1340     byte[][] ret = new byte[num+2][];
1341     int i = 0;
1342     Iterable<byte[]> iter = iterateOnSplits(a, b, num);
1343     if (iter == null) return null;
1344     for (byte[] elem : iter) {
1345       ret[i++] = elem;
1346     }
1347     return ret;
1348   }
1349   
1350   /**
1351    * Iterate over keys within the passed inclusive range.
1352    */
1353   public static Iterable<byte[]> iterateOnSplits(
1354       final byte[] a, final byte[]b, final int num)
1355   {  
1356     byte [] aPadded;
1357     byte [] bPadded;
1358     if (a.length < b.length) {
1359       aPadded = padTail(a, b.length - a.length);
1360       bPadded = b;
1361     } else if (b.length < a.length) {
1362       aPadded = a;
1363       bPadded = padTail(b, a.length - b.length);
1364     } else {
1365       aPadded = a;
1366       bPadded = b;
1367     }
1368     if (compareTo(aPadded,bPadded) >= 0) {
1369       throw new IllegalArgumentException("b <= a");
1370     }
1371     if (num <= 0) {
1372       throw new IllegalArgumentException("num cannot be < 0");
1373     }
1374     byte [] prependHeader = {1, 0};
1375     final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1376     final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1377     final BigInteger diffBI = stopBI.subtract(startBI);
1378     final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1379     if(diffBI.compareTo(splitsBI) < 0) {
1380       return null;
1381     }
1382     final BigInteger intervalBI;
1383     try {
1384       intervalBI = diffBI.divide(splitsBI);
1385     } catch(Exception e) {
1386       LOG.error("Exception caught during division", e);
1387       return null;
1388     }
1389 
1390     final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1391       private int i = -1;
1392       
1393       @Override
1394       public boolean hasNext() {
1395         return i < num+1;
1396       }
1397 
1398       @Override
1399       public byte[] next() {
1400         i++;
1401         if (i == 0) return a;
1402         if (i == num + 1) return b;
1403         
1404         BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1405         byte [] padded = curBI.toByteArray();
1406         if (padded[1] == 0)
1407           padded = tail(padded, padded.length - 2);
1408         else
1409           padded = tail(padded, padded.length - 1);
1410         return padded;
1411       }
1412 
1413       @Override
1414       public void remove() {
1415         throw new UnsupportedOperationException();
1416       }
1417       
1418     };
1419     
1420     return new Iterable<byte[]>() {
1421       @Override
1422       public Iterator<byte[]> iterator() {
1423         return iterator;
1424       }
1425     };
1426   }
1427 
1428   /**
1429    * @param t operands
1430    * @return Array of byte arrays made from passed array of Text
1431    */
1432   public static byte [][] toByteArrays(final String [] t) {
1433     byte [][] result = new byte[t.length][];
1434     for (int i = 0; i < t.length; i++) {
1435       result[i] = Bytes.toBytes(t[i]);
1436     }
1437     return result;
1438   }
1439 
1440   /**
1441    * @param column operand
1442    * @return A byte array of a byte array where first and only entry is
1443    * <code>column</code>
1444    */
1445   public static byte [][] toByteArrays(final String column) {
1446     return toByteArrays(toBytes(column));
1447   }
1448 
1449   /**
1450    * @param column operand
1451    * @return A byte array of a byte array where first and only entry is
1452    * <code>column</code>
1453    */
1454   public static byte [][] toByteArrays(final byte [] column) {
1455     byte [][] result = new byte[1][];
1456     result[0] = column;
1457     return result;
1458   }
1459 
1460   /**
1461    * Binary search for keys in indexes.
1462    *
1463    * @param arr array of byte arrays to search for
1464    * @param key the key you want to find
1465    * @param offset the offset in the key you want to find
1466    * @param length the length of the key
1467    * @param comparator a comparator to compare.
1468    * @return zero-based index of the key, if the key is present in the array.
1469    *         Otherwise, a value -(i + 1) such that the key is between arr[i -
1470    *         1] and arr[i] non-inclusively, where i is in [0, i], if we define
1471    *         arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above
1472    *         means that this function can return 2N + 1 different values
1473    *         ranging from -(N + 1) to N - 1.
1474    */
1475   public static int binarySearch(byte [][]arr, byte []key, int offset,
1476       int length, RawComparator<byte []> comparator) {
1477     int low = 0;
1478     int high = arr.length - 1;
1479 
1480     while (low <= high) {
1481       int mid = (low+high) >>> 1;
1482       // we have to compare in this order, because the comparator order
1483       // has special logic when the 'left side' is a special key.
1484       int cmp = comparator.compare(key, offset, length,
1485           arr[mid], 0, arr[mid].length);
1486       // key lives above the midpoint
1487       if (cmp > 0)
1488         low = mid + 1;
1489       // key lives below the midpoint
1490       else if (cmp < 0)
1491         high = mid - 1;
1492       // BAM. how often does this really happen?
1493       else
1494         return mid;
1495     }
1496     return - (low+1);
1497   }
1498 
1499   /**
1500    * Bytewise binary increment/deincrement of long contained in byte array
1501    * on given amount.
1502    *
1503    * @param value - array of bytes containing long (length <= SIZEOF_LONG)
1504    * @param amount value will be incremented on (deincremented if negative)
1505    * @return array of bytes containing incremented long (length == SIZEOF_LONG)
1506    */
1507   public static byte [] incrementBytes(byte[] value, long amount)
1508   {
1509     byte[] val = value;
1510     if (val.length < SIZEOF_LONG) {
1511       // Hopefully this doesn't happen too often.
1512       byte [] newvalue;
1513       if (val[0] < 0) {
1514         newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1515       } else {
1516         newvalue = new byte[SIZEOF_LONG];
1517       }
1518       System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1519         val.length);
1520       val = newvalue;
1521     } else if (val.length > SIZEOF_LONG) {
1522       throw new IllegalArgumentException("Increment Bytes - value too big: " +
1523         val.length);
1524     }
1525     if(amount == 0) return val;
1526     if(val[0] < 0){
1527       return binaryIncrementNeg(val, amount);
1528     }
1529     return binaryIncrementPos(val, amount);
1530   }
1531 
1532   /* increment/deincrement for positive value */
1533   private static byte [] binaryIncrementPos(byte [] value, long amount) {
1534     long amo = amount;
1535     int sign = 1;
1536     if (amount < 0) {
1537       amo = -amount;
1538       sign = -1;
1539     }
1540     for(int i=0;i<value.length;i++) {
1541       int cur = ((int)amo % 256) * sign;
1542       amo = (amo >> 8);
1543       int val = value[value.length-i-1] & 0x0ff;
1544       int total = val + cur;
1545       if(total > 255) {
1546         amo += sign;
1547         total %= 256;
1548       } else if (total < 0) {
1549         amo -= sign;
1550       }
1551       value[value.length-i-1] = (byte)total;
1552       if (amo == 0) return value;
1553     }
1554     return value;
1555   }
1556 
1557   /* increment/deincrement for negative value */
1558   private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1559     long amo = amount;
1560     int sign = 1;
1561     if (amount < 0) {
1562       amo = -amount;
1563       sign = -1;
1564     }
1565     for(int i=0;i<value.length;i++) {
1566       int cur = ((int)amo % 256) * sign;
1567       amo = (amo >> 8);
1568       int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1569       int total = cur - val;
1570       if(total >= 0) {
1571         amo += sign;
1572       } else if (total < -256) {
1573         amo -= sign;
1574         total %= 256;
1575       }
1576       value[value.length-i-1] = (byte)total;
1577       if (amo == 0) return value;
1578     }
1579     return value;
1580   }
1581 
1582   /**
1583    * Writes a string as a fixed-size field, padded with zeros.
1584    */
1585   public static void writeStringFixedSize(final DataOutput out, String s,
1586       int size) throws IOException {
1587     byte[] b = toBytes(s);
1588     if (b.length > size) {
1589       throw new IOException("Trying to write " + b.length + " bytes (" +
1590           toStringBinary(b) + ") into a field of length " + size);
1591     }
1592 
1593     out.writeBytes(s);
1594     for (int i = 0; i < size - s.length(); ++i)
1595       out.writeByte(0);
1596   }
1597 
1598   /**
1599    * Reads a fixed-size field and interprets it as a string padded with zeros.
1600    */
1601   public static String readStringFixedSize(final DataInput in, int size) 
1602       throws IOException {
1603     byte[] b = new byte[size];
1604     in.readFully(b);
1605     int n = b.length;
1606     while (n > 0 && b[n - 1] == 0)
1607       --n;
1608 
1609     return toString(b, 0, n);
1610   }
1611 
1612 }