1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  
23  import static org.junit.Assert.*;
24  
25  import java.io.ByteArrayOutputStream;
26  import java.io.DataOutputStream;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.NavigableSet;
31  import java.util.Set;
32  
33  import org.apache.hadoop.hbase.client.Delete;
34  import org.apache.hadoop.hbase.client.Get;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.RowLock;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.filter.BinaryComparator;
40  import org.apache.hadoop.hbase.filter.Filter;
41  import org.apache.hadoop.hbase.filter.PrefixFilter;
42  import org.apache.hadoop.hbase.filter.RowFilter;
43  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
44  import org.apache.hadoop.hbase.io.HbaseMapWritable;
45  import org.apache.hadoop.hbase.io.TimeRange;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.Writables;
48  import org.apache.hadoop.io.DataInputBuffer;
49  import org.junit.Test;
50  
51  /**
52   * Test HBase Writables serializations
53   */
54  public class TestSerialization {
55  
56    @Test public void testCompareFilter() throws Exception {
57      Filter f = new RowFilter(CompareOp.EQUAL,
58        new BinaryComparator(Bytes.toBytes("testRowOne-2")));
59      byte [] bytes = Writables.getBytes(f);
60      Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter());
61      assertNotNull(ff);
62    }
63  
64    @Test public void testKeyValue() throws Exception {
65      final String name = "testKeyValue";
66      byte [] row = Bytes.toBytes(name);
67      byte [] family = Bytes.toBytes(name);
68      byte [] qualifier = Bytes.toBytes(name);
69      KeyValue original = new KeyValue(row, family, qualifier);
70      byte [] bytes = Writables.getBytes(original);
71      KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
72      assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
73    }
74  
75    @SuppressWarnings("unchecked")
76    @Test public void testHbaseMapWritable() throws Exception {
77      HbaseMapWritable<byte [], byte []> hmw =
78        new HbaseMapWritable<byte[], byte[]>();
79      hmw.put("key".getBytes(), "value".getBytes());
80      byte [] bytes = Writables.getBytes(hmw);
81      hmw = (HbaseMapWritable<byte[], byte[]>)
82        Writables.getWritable(bytes, new HbaseMapWritable<byte [], byte []>());
83      assertTrue(hmw.size() == 1);
84      assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
85    }
86  
87    @Test public void testTableDescriptor() throws Exception {
88      final String name = "testTableDescriptor";
89      HTableDescriptor htd = createTableDescriptor(name);
90      byte [] mb = Writables.getBytes(htd);
91      HTableDescriptor deserializedHtd =
92        (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
93      assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString());
94    }
95  
96    /**
97     * Test RegionInfo serialization
98     * @throws Exception
99     */
100   @Test public void testRegionInfo() throws Exception {
101     HRegionInfo hri = createRandomRegion("testRegionInfo");
102     byte [] hrib = Writables.getBytes(hri);
103     HRegionInfo deserializedHri =
104       (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo());
105     assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
106     //assertEquals(hri.getTableDesc().getFamilies().size(),
107     //  deserializedHri.getTableDesc().getFamilies().size());
108   }
109 
110   @Test public void testRegionInfos() throws Exception {
111     HRegionInfo hri = createRandomRegion("testRegionInfos");
112     byte [] hrib = Writables.getBytes(hri);
113     byte [] triple = new byte [3 * hrib.length];
114     System.arraycopy(hrib, 0, triple, 0, hrib.length);
115     System.arraycopy(hrib, 0, triple, hrib.length, hrib.length);
116     System.arraycopy(hrib, 0, triple, hrib.length * 2, hrib.length);
117     List<HRegionInfo> regions = Writables.getHRegionInfos(triple, 0, triple.length);
118     assertTrue(regions.size() == 3);
119     assertTrue(regions.get(0).equals(regions.get(1)));
120     assertTrue(regions.get(0).equals(regions.get(2)));
121   }
122 
123   private HRegionInfo createRandomRegion(final String name) {
124     HTableDescriptor htd = new HTableDescriptor(name);
125     String [] families = new String [] {"info", "anchor"};
126     for (int i = 0; i < families.length; i++) {
127       htd.addFamily(new HColumnDescriptor(families[i]));
128     }
129     return new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW,
130       HConstants.EMPTY_END_ROW);
131   }
132 
133   @Test public void testPut() throws Exception{
134     byte[] row = "row".getBytes();
135     byte[] fam = "fam".getBytes();
136     byte[] qf1 = "qf1".getBytes();
137     byte[] qf2 = "qf2".getBytes();
138     byte[] qf3 = "qf3".getBytes();
139     byte[] qf4 = "qf4".getBytes();
140     byte[] qf5 = "qf5".getBytes();
141     byte[] qf6 = "qf6".getBytes();
142     byte[] qf7 = "qf7".getBytes();
143     byte[] qf8 = "qf8".getBytes();
144 
145     long ts = System.currentTimeMillis();
146     byte[] val = "val".getBytes();
147 
148     Put put = new Put(row);
149     put.setWriteToWAL(false);
150     put.add(fam, qf1, ts, val);
151     put.add(fam, qf2, ts, val);
152     put.add(fam, qf3, ts, val);
153     put.add(fam, qf4, ts, val);
154     put.add(fam, qf5, ts, val);
155     put.add(fam, qf6, ts, val);
156     put.add(fam, qf7, ts, val);
157     put.add(fam, qf8, ts, val);
158 
159     byte[] sb = Writables.getBytes(put);
160     Put desPut = (Put)Writables.getWritable(sb, new Put());
161 
162     //Timing test
163 //    long start = System.nanoTime();
164 //    desPut = (Put)Writables.getWritable(sb, new Put());
165 //    long stop = System.nanoTime();
166 //    System.out.println("timer " +(stop-start));
167 
168     assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
169     List<KeyValue> list = null;
170     List<KeyValue> desList = null;
171     for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
172       assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
173       list = entry.getValue();
174       desList = desPut.getFamilyMap().get(entry.getKey());
175       for(int i=0; i<list.size(); i++){
176         assertTrue(list.get(i).equals(desList.get(i)));
177       }
178     }
179   }
180 
181 
182   @Test public void testPut2() throws Exception{
183     byte[] row = "testAbort,,1243116656250".getBytes();
184     byte[] fam = "historian".getBytes();
185     byte[] qf1 = "creation".getBytes();
186 
187     long ts = 9223372036854775807L;
188     byte[] val = "dont-care".getBytes();
189 
190     Put put = new Put(row);
191     put.add(fam, qf1, ts, val);
192 
193     byte[] sb = Writables.getBytes(put);
194     Put desPut = (Put)Writables.getWritable(sb, new Put());
195 
196     assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
197     List<KeyValue> list = null;
198     List<KeyValue> desList = null;
199     for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
200       assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
201       list = entry.getValue();
202       desList = desPut.getFamilyMap().get(entry.getKey());
203       for(int i=0; i<list.size(); i++){
204         assertTrue(list.get(i).equals(desList.get(i)));
205       }
206     }
207   }
208 
209 
210   @Test public void testDelete() throws Exception{
211     byte[] row = "row".getBytes();
212     byte[] fam = "fam".getBytes();
213     byte[] qf1 = "qf1".getBytes();
214 
215     long ts = System.currentTimeMillis();
216 
217     Delete delete = new Delete(row);
218     delete.deleteColumn(fam, qf1, ts);
219 
220     byte[] sb = Writables.getBytes(delete);
221     Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
222 
223     assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
224     List<KeyValue> list = null;
225     List<KeyValue> desList = null;
226     for(Map.Entry<byte[], List<KeyValue>> entry :
227         delete.getFamilyMap().entrySet()){
228       assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
229       list = entry.getValue();
230       desList = desDelete.getFamilyMap().get(entry.getKey());
231       for(int i=0; i<list.size(); i++){
232         assertTrue(list.get(i).equals(desList.get(i)));
233       }
234     }
235   }
236 
237   @Test public void testGet() throws Exception{
238     byte[] row = "row".getBytes();
239     byte[] fam = "fam".getBytes();
240     byte[] qf1 = "qf1".getBytes();
241 
242     long ts = System.currentTimeMillis();
243     int maxVersions = 2;
244     long lockid = 5;
245     RowLock rowLock = new RowLock(lockid);
246 
247     Get get = new Get(row, rowLock);
248     get.addColumn(fam, qf1);
249     get.setTimeRange(ts, ts+1);
250     get.setMaxVersions(maxVersions);
251 
252     byte[] sb = Writables.getBytes(get);
253     Get desGet = (Get)Writables.getWritable(sb, new Get());
254 
255     assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
256     Set<byte[]> set = null;
257     Set<byte[]> desSet = null;
258 
259     for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
260         get.getFamilyMap().entrySet()){
261       assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
262       set = entry.getValue();
263       desSet = desGet.getFamilyMap().get(entry.getKey());
264       for(byte [] qualifier : set){
265         assertTrue(desSet.contains(qualifier));
266       }
267     }
268 
269     assertEquals(get.getLockId(), desGet.getLockId());
270     assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
271     TimeRange tr = get.getTimeRange();
272     TimeRange desTr = desGet.getTimeRange();
273     assertEquals(tr.getMax(), desTr.getMax());
274     assertEquals(tr.getMin(), desTr.getMin());
275   }
276 
277 
278   @Test public void testScan() throws Exception {
279 
280     byte[] startRow = "startRow".getBytes();
281     byte[] stopRow  = "stopRow".getBytes();
282     byte[] fam = "fam".getBytes();
283     byte[] qf1 = "qf1".getBytes();
284 
285     long ts = System.currentTimeMillis();
286     int maxVersions = 2;
287 
288     Scan scan = new Scan(startRow, stopRow);
289     scan.addColumn(fam, qf1);
290     scan.setTimeRange(ts, ts+1);
291     scan.setMaxVersions(maxVersions);
292 
293     byte[] sb = Writables.getBytes(scan);
294     Scan desScan = (Scan)Writables.getWritable(sb, new Scan());
295 
296     assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
297     assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
298     assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
299     Set<byte[]> set = null;
300     Set<byte[]> desSet = null;
301 
302     for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
303         scan.getFamilyMap().entrySet()){
304       assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
305       set = entry.getValue();
306       desSet = desScan.getFamilyMap().get(entry.getKey());
307       for(byte[] column : set){
308         assertTrue(desSet.contains(column));
309       }
310 
311       // Test filters are serialized properly.
312       scan = new Scan(startRow);
313       final String name = "testScan";
314       byte [] prefix = Bytes.toBytes(name);
315       scan.setFilter(new PrefixFilter(prefix));
316       sb = Writables.getBytes(scan);
317       desScan = (Scan)Writables.getWritable(sb, new Scan());
318       Filter f = desScan.getFilter();
319       assertTrue(f instanceof PrefixFilter);
320     }
321 
322     assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
323     TimeRange tr = scan.getTimeRange();
324     TimeRange desTr = desScan.getTimeRange();
325     assertEquals(tr.getMax(), desTr.getMax());
326     assertEquals(tr.getMin(), desTr.getMin());
327   }
328 
329   @Test public void testResultEmpty() throws Exception {
330     List<KeyValue> keys = new ArrayList<KeyValue>();
331     Result r = new Result(keys);
332     assertTrue(r.isEmpty());
333     byte [] rb = Writables.getBytes(r);
334     Result deserializedR = (Result)Writables.getWritable(rb, new Result());
335     assertTrue(deserializedR.isEmpty());
336   }
337 
338 
339   @Test public void testResult() throws Exception {
340     byte [] rowA = Bytes.toBytes("rowA");
341     byte [] famA = Bytes.toBytes("famA");
342     byte [] qfA = Bytes.toBytes("qfA");
343     byte [] valueA = Bytes.toBytes("valueA");
344 
345     byte [] rowB = Bytes.toBytes("rowB");
346     byte [] famB = Bytes.toBytes("famB");
347     byte [] qfB = Bytes.toBytes("qfB");
348     byte [] valueB = Bytes.toBytes("valueB");
349 
350     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
351     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
352 
353     Result result = new Result(new KeyValue[]{kvA, kvB});
354 
355     byte [] rb = Writables.getBytes(result);
356     Result deResult = (Result)Writables.getWritable(rb, new Result());
357 
358     assertTrue("results are not equivalent, first key mismatch",
359         result.raw()[0].equals(deResult.raw()[0]));
360 
361     assertTrue("results are not equivalent, second key mismatch",
362         result.raw()[1].equals(deResult.raw()[1]));
363 
364     // Test empty Result
365     Result r = new Result();
366     byte [] b = Writables.getBytes(r);
367     Result deserialized = (Result)Writables.getWritable(b, new Result());
368     assertEquals(r.size(), deserialized.size());
369   }
370 
371   @Test public void testResultDynamicBuild() throws Exception {
372     byte [] rowA = Bytes.toBytes("rowA");
373     byte [] famA = Bytes.toBytes("famA");
374     byte [] qfA = Bytes.toBytes("qfA");
375     byte [] valueA = Bytes.toBytes("valueA");
376 
377     byte [] rowB = Bytes.toBytes("rowB");
378     byte [] famB = Bytes.toBytes("famB");
379     byte [] qfB = Bytes.toBytes("qfB");
380     byte [] valueB = Bytes.toBytes("valueB");
381 
382     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
383     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
384 
385     Result result = new Result(new KeyValue[]{kvA, kvB});
386 
387     byte [] rb = Writables.getBytes(result);
388 
389 
390     // Call getRow() first
391     Result deResult = (Result)Writables.getWritable(rb, new Result());
392     byte [] row = deResult.getRow();
393     assertTrue(Bytes.equals(row, rowA));
394 
395     // Call sorted() first
396     deResult = (Result)Writables.getWritable(rb, new Result());
397     assertTrue("results are not equivalent, first key mismatch",
398         result.raw()[0].equals(deResult.raw()[0]));
399     assertTrue("results are not equivalent, second key mismatch",
400         result.raw()[1].equals(deResult.raw()[1]));
401 
402     // Call raw() first
403     deResult = (Result)Writables.getWritable(rb, new Result());
404     assertTrue("results are not equivalent, first key mismatch",
405         result.raw()[0].equals(deResult.raw()[0]));
406     assertTrue("results are not equivalent, second key mismatch",
407         result.raw()[1].equals(deResult.raw()[1]));
408 
409 
410   }
411 
412   @Test public void testResultArray() throws Exception {
413     byte [] rowA = Bytes.toBytes("rowA");
414     byte [] famA = Bytes.toBytes("famA");
415     byte [] qfA = Bytes.toBytes("qfA");
416     byte [] valueA = Bytes.toBytes("valueA");
417 
418     byte [] rowB = Bytes.toBytes("rowB");
419     byte [] famB = Bytes.toBytes("famB");
420     byte [] qfB = Bytes.toBytes("qfB");
421     byte [] valueB = Bytes.toBytes("valueB");
422 
423     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
424     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
425 
426 
427     Result result1 = new Result(new KeyValue[]{kvA, kvB});
428     Result result2 = new Result(new KeyValue[]{kvB});
429     Result result3 = new Result(new KeyValue[]{kvB});
430 
431     Result [] results = new Result [] {result1, result2, result3};
432 
433     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
434     DataOutputStream out = new DataOutputStream(byteStream);
435     Result.writeArray(out, results);
436 
437     byte [] rb = byteStream.toByteArray();
438 
439     DataInputBuffer in = new DataInputBuffer();
440     in.reset(rb, 0, rb.length);
441 
442     Result [] deResults = Result.readArray(in);
443 
444     assertTrue(results.length == deResults.length);
445 
446     for(int i=0;i<results.length;i++) {
447       KeyValue [] keysA = results[i].raw();
448       KeyValue [] keysB = deResults[i].raw();
449       assertTrue(keysA.length == keysB.length);
450       for(int j=0;j<keysA.length;j++) {
451         assertTrue("Expected equivalent keys but found:\n" +
452             "KeyA : " + keysA[j].toString() + "\n" +
453             "KeyB : " + keysB[j].toString() + "\n" +
454             keysA.length + " total keys, " + i + "th so far"
455             ,keysA[j].equals(keysB[j]));
456       }
457     }
458 
459   }
460 
461   @Test public void testResultArrayEmpty() throws Exception {
462     List<KeyValue> keys = new ArrayList<KeyValue>();
463     Result r = new Result(keys);
464     Result [] results = new Result [] {r};
465 
466     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
467     DataOutputStream out = new DataOutputStream(byteStream);
468 
469     Result.writeArray(out, results);
470 
471     results = null;
472 
473     byteStream = new ByteArrayOutputStream();
474     out = new DataOutputStream(byteStream);
475     Result.writeArray(out, results);
476 
477     byte [] rb = byteStream.toByteArray();
478 
479     DataInputBuffer in = new DataInputBuffer();
480     in.reset(rb, 0, rb.length);
481 
482     Result [] deResults = Result.readArray(in);
483 
484     assertTrue(deResults.length == 0);
485 
486     results = new Result[0];
487 
488     byteStream = new ByteArrayOutputStream();
489     out = new DataOutputStream(byteStream);
490     Result.writeArray(out, results);
491 
492     rb = byteStream.toByteArray();
493 
494     in = new DataInputBuffer();
495     in.reset(rb, 0, rb.length);
496 
497     deResults = Result.readArray(in);
498 
499     assertTrue(deResults.length == 0);
500 
501   }
502 
503   @Test public void testTimeRange() throws Exception{
504     TimeRange tr = new TimeRange(0,5);
505     byte [] mb = Writables.getBytes(tr);
506     TimeRange deserializedTr =
507       (TimeRange)Writables.getWritable(mb, new TimeRange());
508 
509     assertEquals(tr.getMax(), deserializedTr.getMax());
510     assertEquals(tr.getMin(), deserializedTr.getMin());
511 
512   }
513 
514   @Test public void testKeyValue2() throws Exception {
515     final String name = "testKeyValue2";
516     byte[] row = name.getBytes();
517     byte[] fam = "fam".getBytes();
518     byte[] qf = "qf".getBytes();
519     long ts = System.currentTimeMillis();
520     byte[] val = "val".getBytes();
521 
522     KeyValue kv = new KeyValue(row, fam, qf, ts, val);
523 
524     byte [] mb = Writables.getBytes(kv);
525     KeyValue deserializedKv =
526       (KeyValue)Writables.getWritable(mb, new KeyValue());
527     assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
528     assertEquals(kv.getOffset(), deserializedKv.getOffset());
529     assertEquals(kv.getLength(), deserializedKv.getLength());
530   }
531 
532   protected static final int MAXVERSIONS = 3;
533   protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
534   protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
535   protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
536   protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
537 
538   /**
539    * Create a table of name <code>name</code> with {@link COLUMNS} for
540    * families.
541    * @param name Name to give table.
542    * @return Column descriptor.
543    */
544   protected HTableDescriptor createTableDescriptor(final String name) {
545     return createTableDescriptor(name, MAXVERSIONS);
546   }
547 
548   /**
549    * Create a table of name <code>name</code> with {@link COLUMNS} for
550    * families.
551    * @param name Name to give table.
552    * @param versions How many versions to allow per column.
553    * @return Column descriptor.
554    */
555   protected HTableDescriptor createTableDescriptor(final String name,
556       final int versions) {
557     HTableDescriptor htd = new HTableDescriptor(name);
558     htd.addFamily(new HColumnDescriptor(fam1, versions,
559       HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
560       HColumnDescriptor.DEFAULT_BLOCKSIZE, HConstants.FOREVER,
561       HColumnDescriptor.DEFAULT_BLOOMFILTER,
562       HConstants.REPLICATION_SCOPE_LOCAL));
563     htd.addFamily(new HColumnDescriptor(fam2, versions,
564         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
565         HColumnDescriptor.DEFAULT_BLOCKSIZE, HConstants.FOREVER,
566         HColumnDescriptor.DEFAULT_BLOOMFILTER,
567         HConstants.REPLICATION_SCOPE_LOCAL));
568     htd.addFamily(new HColumnDescriptor(fam3, versions,
569         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
570         HColumnDescriptor.DEFAULT_BLOCKSIZE,  HConstants.FOREVER,
571         HColumnDescriptor.DEFAULT_BLOOMFILTER,
572         HConstants.REPLICATION_SCOPE_LOCAL));
573     return htd;
574   }
575 }