View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.io;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataInput;
24  import java.io.DataOutput;
25  import java.io.IOException;
26  import java.io.ObjectInputStream;
27  import java.io.ObjectOutputStream;
28  import java.io.Serializable;
29  import java.lang.reflect.Array;
30  import java.util.ArrayList;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.NavigableSet;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.conf.Configurable;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.conf.Configured;
41  import org.apache.hadoop.hbase.ClusterStatus;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HServerAddress;
46  import org.apache.hadoop.hbase.HServerInfo;
47  import org.apache.hadoop.hbase.HServerLoad;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.client.Delete;
51  import org.apache.hadoop.hbase.client.Get;
52  import org.apache.hadoop.hbase.client.Increment;
53  import org.apache.hadoop.hbase.client.MultiAction;
54  import org.apache.hadoop.hbase.client.Action;
55  import org.apache.hadoop.hbase.client.MultiResponse;
56  import org.apache.hadoop.hbase.client.Put;
57  import org.apache.hadoop.hbase.client.Result;
58  import org.apache.hadoop.hbase.client.Row;
59  import org.apache.hadoop.hbase.client.Scan;
60  import org.apache.hadoop.hbase.client.coprocessor.Exec;
61  import org.apache.hadoop.hbase.filter.BinaryComparator;
62  import org.apache.hadoop.hbase.filter.BitComparator;
63  import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
64  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
65  import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
66  import org.apache.hadoop.hbase.filter.CompareFilter;
67  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
68  import org.apache.hadoop.hbase.filter.DependentColumnFilter;
69  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
70  import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
71  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
72  import org.apache.hadoop.hbase.filter.PageFilter;
73  import org.apache.hadoop.hbase.filter.PrefixFilter;
74  import org.apache.hadoop.hbase.filter.QualifierFilter;
75  import org.apache.hadoop.hbase.filter.RandomRowFilter;
76  import org.apache.hadoop.hbase.filter.RowFilter;
77  import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
78  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
79  import org.apache.hadoop.hbase.filter.SkipFilter;
80  import org.apache.hadoop.hbase.filter.ValueFilter;
81  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
82  import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
83  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
84  import org.apache.hadoop.hbase.regionserver.HRegion;
85  import org.apache.hadoop.hbase.regionserver.wal.HLog;
86  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
87  import org.apache.hadoop.hbase.util.Bytes;
88  import org.apache.hadoop.io.MapWritable;
89  import org.apache.hadoop.io.ObjectWritable;
90  import org.apache.hadoop.io.Text;
91  import org.apache.hadoop.io.Writable;
92  import org.apache.hadoop.io.WritableFactories;
93  import org.apache.hadoop.io.WritableUtils;
94  
95  /**
96   * This is a customized version of the polymorphic hadoop
97   * {@link ObjectWritable}.  It removes UTF8 (HADOOP-414).
98   * Using {@link Text} intead of UTF-8 saves ~2% CPU between reading and writing
99   * objects running a short sequentialWrite Performance Evaluation test just in
100  * ObjectWritable alone; more when we're doing randomRead-ing.  Other
101  * optimizations include our passing codes for classes instead of the
102  * actual class names themselves.  This makes it so this class needs amendment
103  * if non-Writable classes are introduced -- if passed a Writable for which we
104  * have no code, we just do the old-school passing of the class name, etc. --
105  * but passing codes the  savings are large particularly when cell
106  * data is small (If < a couple of kilobytes, the encoding/decoding of class
107  * name and reflection to instantiate class was costing in excess of the cell
108  * handling).
109  */
110 public class HbaseObjectWritable implements Writable, WritableWithSize, Configurable {
111   protected final static Log LOG = LogFactory.getLog(HbaseObjectWritable.class);
112 
113   // Here we maintain two static maps of classes to code and vice versa.
114   // Add new classes+codes as wanted or figure way to auto-generate these
115   // maps from the HMasterInterface.
116   static final Map<Integer, Class<?>> CODE_TO_CLASS =
117     new HashMap<Integer, Class<?>>();
118   static final Map<Class<?>, Integer> CLASS_TO_CODE =
119     new HashMap<Class<?>, Integer>();
120   // Special code that means 'not-encoded'; in this case we do old school
121   // sending of the class name using reflection, etc.
122   private static final byte NOT_ENCODED = 0;
123   static {
124     int code = NOT_ENCODED + 1;
125     // Primitive types.
126     addToMap(Boolean.TYPE, code++);
127     addToMap(Byte.TYPE, code++);
128     addToMap(Character.TYPE, code++);
129     addToMap(Short.TYPE, code++);
130     addToMap(Integer.TYPE, code++);
131     addToMap(Long.TYPE, code++);
132     addToMap(Float.TYPE, code++);
133     addToMap(Double.TYPE, code++);
134     addToMap(Void.TYPE, code++);
135 
136     // Other java types
137     addToMap(String.class, code++);
138     addToMap(byte [].class, code++);
139     addToMap(byte [][].class, code++);
140 
141     // Hadoop types
142     addToMap(Text.class, code++);
143     addToMap(Writable.class, code++);
144     addToMap(Writable [].class, code++);
145     addToMap(HbaseMapWritable.class, code++);
146     addToMap(NullInstance.class, code++);
147 
148     // Hbase types
149     addToMap(HColumnDescriptor.class, code++);
150     addToMap(HConstants.Modify.class, code++);
151 
152     // We used to have a class named HMsg but its been removed.  Rather than
153     // just axe it, use following random Integer class -- we just chose any
154     // class from java.lang -- instead just so codes that follow stay
155     // in same relative place.
156     addToMap(Integer.class, code++);
157     addToMap(Integer[].class, code++);
158 
159     addToMap(HRegion.class, code++);
160     addToMap(HRegion[].class, code++);
161     addToMap(HRegionInfo.class, code++);
162     addToMap(HRegionInfo[].class, code++);
163     addToMap(HServerAddress.class, code++);
164     addToMap(HServerInfo.class, code++);
165     addToMap(HTableDescriptor.class, code++);
166     addToMap(HTableDescriptor[].class, code++);
167     addToMap(MapWritable.class, code++);
168 
169     //
170     // HBASE-880
171     //
172     addToMap(ClusterStatus.class, code++);
173     addToMap(Delete.class, code++);
174     addToMap(Get.class, code++);
175     addToMap(KeyValue.class, code++);
176     addToMap(KeyValue[].class, code++);
177     addToMap(Put.class, code++);
178     addToMap(Put[].class, code++);
179     addToMap(Result.class, code++);
180     addToMap(Result[].class, code++);
181     addToMap(Scan.class, code++);
182 
183     addToMap(WhileMatchFilter.class, code++);
184     addToMap(PrefixFilter.class, code++);
185     addToMap(PageFilter.class, code++);
186     addToMap(InclusiveStopFilter.class, code++);
187     addToMap(ColumnCountGetFilter.class, code++);
188     addToMap(SingleColumnValueFilter.class, code++);
189     addToMap(SingleColumnValueExcludeFilter.class, code++);
190     addToMap(BinaryComparator.class, code++);
191     addToMap(BitComparator.class, code++);
192     addToMap(CompareFilter.class, code++);
193     addToMap(RowFilter.class, code++);
194     addToMap(ValueFilter.class, code++);
195     addToMap(QualifierFilter.class, code++);
196     addToMap(SkipFilter.class, code++);
197     addToMap(WritableByteArrayComparable.class, code++);
198     addToMap(FirstKeyOnlyFilter.class, code++);
199     addToMap(DependentColumnFilter.class, code++);
200 
201     addToMap(Delete [].class, code++);
202 
203     addToMap(HLog.Entry.class, code++);
204     addToMap(HLog.Entry[].class, code++);
205     addToMap(HLogKey.class, code++);
206 
207     addToMap(List.class, code++);
208 
209     addToMap(NavigableSet.class, code++);
210     addToMap(ColumnPrefixFilter.class, code++);
211 
212     // Multi
213     addToMap(Row.class, code++);
214     addToMap(Action.class, code++);
215     addToMap(MultiAction.class, code++);
216     addToMap(MultiResponse.class, code++);
217 
218     // coprocessor execution
219     addToMap(Exec.class, code++);
220     addToMap(Increment.class, code++);
221 
222     addToMap(KeyOnlyFilter.class, code++);
223     
224     // serializable
225     addToMap(Serializable.class, code++);
226 
227     addToMap(RandomRowFilter.class, code++);
228 
229     addToMap(CompareOp.class, code++);
230 
231     addToMap(ColumnRangeFilter.class, code++);
232 
233     addToMap(HServerLoad.class, code++);
234     
235     addToMap(RegionOpeningState.class, code++);
236     
237   }
238 
239   private Class<?> declaredClass;
240   private Object instance;
241   private Configuration conf;
242 
243   /** default constructor for writable */
244   public HbaseObjectWritable() {
245     super();
246   }
247 
248   /**
249    * @param instance
250    */
251   public HbaseObjectWritable(Object instance) {
252     set(instance);
253   }
254 
255   /**
256    * @param declaredClass
257    * @param instance
258    */
259   public HbaseObjectWritable(Class<?> declaredClass, Object instance) {
260     this.declaredClass = declaredClass;
261     this.instance = instance;
262   }
263 
264   /** @return the instance, or null if none. */
265   public Object get() { return instance; }
266 
267   /** @return the class this is meant to be. */
268   public Class<?> getDeclaredClass() { return declaredClass; }
269 
270   /**
271    * Reset the instance.
272    * @param instance
273    */
274   public void set(Object instance) {
275     this.declaredClass = instance.getClass();
276     this.instance = instance;
277   }
278 
279   /**
280    * @see java.lang.Object#toString()
281    */
282   @Override
283   public String toString() {
284     return "OW[class=" + declaredClass + ",value=" + instance + "]";
285   }
286 
287 
288   public void readFields(DataInput in) throws IOException {
289     readObject(in, this, this.conf);
290   }
291 
292   public void write(DataOutput out) throws IOException {
293     writeObject(out, instance, declaredClass, conf);
294   }
295 
296   public long getWritableSize() {
297     return getWritableSize(instance, declaredClass, conf);
298   }
299 
300   private static class NullInstance extends Configured implements Writable {
301     Class<?> declaredClass;
302     /** default constructor for writable */
303     @SuppressWarnings("unused")
304     public NullInstance() { super(null); }
305 
306     /**
307      * @param declaredClass
308      * @param conf
309      */
310     public NullInstance(Class<?> declaredClass, Configuration conf) {
311       super(conf);
312       this.declaredClass = declaredClass;
313     }
314 
315     public void readFields(DataInput in) throws IOException {
316       this.declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in));
317     }
318 
319     public void write(DataOutput out) throws IOException {
320       writeClassCode(out, this.declaredClass);
321     }
322   }
323 
324   /**
325    * Write out the code byte for passed Class.
326    * @param out
327    * @param c
328    * @throws IOException
329    */
330   static void writeClassCode(final DataOutput out, final Class<?> c)
331   throws IOException {
332     Integer code = CLASS_TO_CODE.get(c);
333     if (code == null ) {
334       if ( List.class.isAssignableFrom(c)) {
335         code = CLASS_TO_CODE.get(List.class);
336       }
337       else if (Writable.class.isAssignableFrom(c)) {
338         code = CLASS_TO_CODE.get(Writable.class);
339       }
340       else if (Serializable.class.isAssignableFrom(c)){
341         code = CLASS_TO_CODE.get(Serializable.class);
342       }
343     }
344     if (code == null) {
345       LOG.error("Unsupported type " + c);
346       StackTraceElement[] els = new Exception().getStackTrace();
347       for(StackTraceElement elem : els) {
348         LOG.error(elem.getMethodName());
349       }
350       throw new UnsupportedOperationException("No code for unexpected " + c);
351     }
352     WritableUtils.writeVInt(out, code);
353   }
354 
355 
356   public static long getWritableSize(Object instance, Class declaredClass,
357                                      Configuration conf) {
358     long size = Bytes.SIZEOF_BYTE; // code
359     if (instance == null) {
360       return 0L;
361     }
362 
363     if (declaredClass.isArray()) {
364       if (declaredClass.equals(Result[].class)) {
365 
366         return size + Result.getWriteArraySize((Result[])instance);
367       }
368     }
369     if (declaredClass.equals(Result.class)) {
370       Result r = (Result) instance;
371       // one extra class code for writable instance.
372       return r.getWritableSize() + size + Bytes.SIZEOF_BYTE;
373     }
374     return 0L; // no hint is the default.
375   }
376   /**
377    * Write a {@link Writable}, {@link String}, primitive type, or an array of
378    * the preceding.
379    * @param out
380    * @param instance
381    * @param declaredClass
382    * @param conf
383    * @throws IOException
384    */
385   @SuppressWarnings("unchecked")
386   public static void writeObject(DataOutput out, Object instance,
387                                  Class declaredClass,
388                                  Configuration conf)
389   throws IOException {
390 
391     Object instanceObj = instance;
392     Class declClass = declaredClass;
393 
394     if (instanceObj == null) {                       // null
395       instanceObj = new NullInstance(declClass, conf);
396       declClass = Writable.class;
397     }
398     writeClassCode(out, declClass);
399     if (declClass.isArray()) {                // array
400       // If bytearray, just dump it out -- avoid the recursion and
401       // byte-at-a-time we were previously doing.
402       if (declClass.equals(byte [].class)) {
403         Bytes.writeByteArray(out, (byte [])instanceObj);
404       } else if(declClass.equals(Result [].class)) {
405         Result.writeArray(out, (Result [])instanceObj);
406       } else {
407         int length = Array.getLength(instanceObj);
408         out.writeInt(length);
409         for (int i = 0; i < length; i++) {
410           writeObject(out, Array.get(instanceObj, i),
411                     declClass.getComponentType(), conf);
412         }
413       }
414     } else if (List.class.isAssignableFrom(declClass)) {
415       List list = (List)instanceObj;
416       int length = list.size();
417       out.writeInt(length);
418       for (int i = 0; i < length; i++) {
419         writeObject(out, list.get(i),
420                   list.get(i).getClass(), conf);
421       }
422     } else if (declClass == String.class) {   // String
423       Text.writeString(out, (String)instanceObj);
424     } else if (declClass.isPrimitive()) {     // primitive type
425       if (declClass == Boolean.TYPE) {        // boolean
426         out.writeBoolean(((Boolean)instanceObj).booleanValue());
427       } else if (declClass == Character.TYPE) { // char
428         out.writeChar(((Character)instanceObj).charValue());
429       } else if (declClass == Byte.TYPE) {    // byte
430         out.writeByte(((Byte)instanceObj).byteValue());
431       } else if (declClass == Short.TYPE) {   // short
432         out.writeShort(((Short)instanceObj).shortValue());
433       } else if (declClass == Integer.TYPE) { // int
434         out.writeInt(((Integer)instanceObj).intValue());
435       } else if (declClass == Long.TYPE) {    // long
436         out.writeLong(((Long)instanceObj).longValue());
437       } else if (declClass == Float.TYPE) {   // float
438         out.writeFloat(((Float)instanceObj).floatValue());
439       } else if (declClass == Double.TYPE) {  // double
440         out.writeDouble(((Double)instanceObj).doubleValue());
441       } else if (declClass == Void.TYPE) {    // void
442       } else {
443         throw new IllegalArgumentException("Not a primitive: "+declClass);
444       }
445     } else if (declClass.isEnum()) {         // enum
446       Text.writeString(out, ((Enum)instanceObj).name());
447     } else if (Writable.class.isAssignableFrom(declClass)) { // Writable
448       Class <?> c = instanceObj.getClass();
449       Integer code = CLASS_TO_CODE.get(c);
450       if (code == null) {
451         out.writeByte(NOT_ENCODED);
452         Text.writeString(out, c.getName());
453       } else {
454         writeClassCode(out, c);
455       }
456       ((Writable)instanceObj).write(out);
457     } else if (Serializable.class.isAssignableFrom(declClass)) {
458       Class <?> c = instanceObj.getClass();
459       Integer code = CLASS_TO_CODE.get(c);
460       if (code == null) {
461         out.writeByte(NOT_ENCODED);
462         Text.writeString(out, c.getName());
463       } else {
464         writeClassCode(out, c);
465       }
466       ByteArrayOutputStream bos = null;
467       ObjectOutputStream oos = null;
468       try{
469         bos = new ByteArrayOutputStream();
470         oos = new ObjectOutputStream(bos);
471         oos.writeObject(instanceObj);
472         byte[] value = bos.toByteArray();
473         out.writeInt(value.length);
474         out.write(value);
475       } finally {
476         if(bos!=null) bos.close();
477         if(oos!=null) oos.close();
478       }
479     } else {
480       throw new IOException("Can't write: "+instanceObj+" as "+declClass);
481     }
482   }
483 
484 
485   /**
486    * Read a {@link Writable}, {@link String}, primitive type, or an array of
487    * the preceding.
488    * @param in
489    * @param conf
490    * @return the object
491    * @throws IOException
492    */
493   public static Object readObject(DataInput in, Configuration conf)
494     throws IOException {
495     return readObject(in, null, conf);
496   }
497 
498   /**
499    * Read a {@link Writable}, {@link String}, primitive type, or an array of
500    * the preceding.
501    * @param in
502    * @param objectWritable
503    * @param conf
504    * @return the object
505    * @throws IOException
506    */
507   @SuppressWarnings("unchecked")
508   public static Object readObject(DataInput in,
509       HbaseObjectWritable objectWritable, Configuration conf)
510   throws IOException {
511     Class<?> declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in));
512     Object instance;
513     if (declaredClass.isPrimitive()) {            // primitive types
514       if (declaredClass == Boolean.TYPE) {             // boolean
515         instance = Boolean.valueOf(in.readBoolean());
516       } else if (declaredClass == Character.TYPE) {    // char
517         instance = Character.valueOf(in.readChar());
518       } else if (declaredClass == Byte.TYPE) {         // byte
519         instance = Byte.valueOf(in.readByte());
520       } else if (declaredClass == Short.TYPE) {        // short
521         instance = Short.valueOf(in.readShort());
522       } else if (declaredClass == Integer.TYPE) {      // int
523         instance = Integer.valueOf(in.readInt());
524       } else if (declaredClass == Long.TYPE) {         // long
525         instance = Long.valueOf(in.readLong());
526       } else if (declaredClass == Float.TYPE) {        // float
527         instance = Float.valueOf(in.readFloat());
528       } else if (declaredClass == Double.TYPE) {       // double
529         instance = Double.valueOf(in.readDouble());
530       } else if (declaredClass == Void.TYPE) {         // void
531         instance = null;
532       } else {
533         throw new IllegalArgumentException("Not a primitive: "+declaredClass);
534       }
535     } else if (declaredClass.isArray()) {              // array
536       if (declaredClass.equals(byte [].class)) {
537         instance = Bytes.readByteArray(in);
538       } else if(declaredClass.equals(Result [].class)) {
539         instance = Result.readArray(in);
540       } else {
541         int length = in.readInt();
542         instance = Array.newInstance(declaredClass.getComponentType(), length);
543         for (int i = 0; i < length; i++) {
544           Array.set(instance, i, readObject(in, conf));
545         }
546       }
547     } else if (List.class.isAssignableFrom(declaredClass)) {            // List
548       int length = in.readInt();
549       instance = new ArrayList(length);
550       for (int i = 0; i < length; i++) {
551         ((ArrayList)instance).add(readObject(in, conf));
552       }
553     } else if (declaredClass == String.class) {        // String
554       instance = Text.readString(in);
555     } else if (declaredClass.isEnum()) {         // enum
556       instance = Enum.valueOf((Class<? extends Enum>) declaredClass,
557         Text.readString(in));
558     } else {                                      // Writable or Serializable
559       Class instanceClass = null;
560       int b = (byte)WritableUtils.readVInt(in);
561       if (b == NOT_ENCODED) {
562         String className = Text.readString(in);
563         try {
564           instanceClass = getClassByName(conf, className);
565         } catch (ClassNotFoundException e) {
566           LOG.error("Can't find class " + className, e);
567           throw new IOException("Can't find class " + className, e);
568         }
569       } else {
570         instanceClass = CODE_TO_CLASS.get(b);
571       }
572       if(Writable.class.isAssignableFrom(instanceClass)){
573         Writable writable = WritableFactories.newInstance(instanceClass, conf);
574         try {
575           writable.readFields(in);
576         } catch (Exception e) {
577           LOG.error("Error in readFields", e);
578           throw new IOException("Error in readFields" , e);
579         }
580         instance = writable;
581         if (instanceClass == NullInstance.class) {  // null
582           declaredClass = ((NullInstance)instance).declaredClass;
583           instance = null;
584         }
585       } else {
586         int length = in.readInt();
587         byte[] objectBytes = new byte[length];
588         in.readFully(objectBytes);
589         ByteArrayInputStream bis = null;
590         ObjectInputStream ois = null;
591         try { 
592           bis = new ByteArrayInputStream(objectBytes);
593           ois = new ObjectInputStream(bis);
594           instance = ois.readObject();
595         } catch (ClassNotFoundException e) {
596           LOG.error("Class not found when attempting to deserialize object", e);
597           throw new IOException("Class not found when attempting to " +
598               "deserialize object", e);
599         } finally {
600           if(bis!=null) bis.close();
601           if(ois!=null) ois.close();
602         }
603       }
604     }
605     if (objectWritable != null) {                 // store values
606       objectWritable.declaredClass = declaredClass;
607       objectWritable.instance = instance;
608     }
609     return instance;
610   }
611 
612   @SuppressWarnings("unchecked")
613   private static Class getClassByName(Configuration conf, String className)
614   throws ClassNotFoundException {
615     if(conf != null) {
616       return conf.getClassByName(className);
617     }
618     ClassLoader cl = Thread.currentThread().getContextClassLoader();
619     if(cl == null) {
620       cl = HbaseObjectWritable.class.getClassLoader();
621     }
622     return Class.forName(className, true, cl);
623   }
624 
625   private static void addToMap(final Class<?> clazz, final int code) {
626     CLASS_TO_CODE.put(clazz, code);
627     CODE_TO_CLASS.put(code, clazz);
628   }
629 
630   public void setConf(Configuration conf) {
631     this.conf = conf;
632   }
633 
634   public Configuration getConf() {
635     return this.conf;
636   }
637 }