1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22
23 import static org.junit.Assert.*;
24
25 import java.io.ByteArrayOutputStream;
26 import java.io.DataOutputStream;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.NavigableSet;
31 import java.util.Set;
32
33 import org.apache.hadoop.hbase.client.Delete;
34 import org.apache.hadoop.hbase.client.Get;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.client.RowLock;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.filter.BinaryComparator;
40 import org.apache.hadoop.hbase.filter.Filter;
41 import org.apache.hadoop.hbase.filter.PrefixFilter;
42 import org.apache.hadoop.hbase.filter.RowFilter;
43 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
44 import org.apache.hadoop.hbase.io.HbaseMapWritable;
45 import org.apache.hadoop.hbase.io.TimeRange;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.apache.hadoop.io.DataInputBuffer;
49 import org.junit.Test;
50
51
52
53
54 public class TestSerialization {
55
56 @Test public void testCompareFilter() throws Exception {
57 Filter f = new RowFilter(CompareOp.EQUAL,
58 new BinaryComparator(Bytes.toBytes("testRowOne-2")));
59 byte [] bytes = Writables.getBytes(f);
60 Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter());
61 assertNotNull(ff);
62 }
63
64 @Test public void testKeyValue() throws Exception {
65 final String name = "testKeyValue";
66 byte [] row = Bytes.toBytes(name);
67 byte [] family = Bytes.toBytes(name);
68 byte [] qualifier = Bytes.toBytes(name);
69 KeyValue original = new KeyValue(row, family, qualifier);
70 byte [] bytes = Writables.getBytes(original);
71 KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
72 assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
73 }
74
75 @SuppressWarnings("unchecked")
76 @Test public void testHbaseMapWritable() throws Exception {
77 HbaseMapWritable<byte [], byte []> hmw =
78 new HbaseMapWritable<byte[], byte[]>();
79 hmw.put("key".getBytes(), "value".getBytes());
80 byte [] bytes = Writables.getBytes(hmw);
81 hmw = (HbaseMapWritable<byte[], byte[]>)
82 Writables.getWritable(bytes, new HbaseMapWritable<byte [], byte []>());
83 assertTrue(hmw.size() == 1);
84 assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
85 }
86
87 @Test public void testTableDescriptor() throws Exception {
88 final String name = "testTableDescriptor";
89 HTableDescriptor htd = createTableDescriptor(name);
90 byte [] mb = Writables.getBytes(htd);
91 HTableDescriptor deserializedHtd =
92 (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
93 assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString());
94 }
95
96
97
98
99
100 @Test public void testRegionInfo() throws Exception {
101 HRegionInfo hri = createRandomRegion("testRegionInfo");
102 byte [] hrib = Writables.getBytes(hri);
103 HRegionInfo deserializedHri =
104 (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo());
105 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
106
107
108 }
109
110 @Test public void testRegionInfos() throws Exception {
111 HRegionInfo hri = createRandomRegion("testRegionInfos");
112 byte [] hrib = Writables.getBytes(hri);
113 byte [] triple = new byte [3 * hrib.length];
114 System.arraycopy(hrib, 0, triple, 0, hrib.length);
115 System.arraycopy(hrib, 0, triple, hrib.length, hrib.length);
116 System.arraycopy(hrib, 0, triple, hrib.length * 2, hrib.length);
117 List<HRegionInfo> regions = Writables.getHRegionInfos(triple, 0, triple.length);
118 assertTrue(regions.size() == 3);
119 assertTrue(regions.get(0).equals(regions.get(1)));
120 assertTrue(regions.get(0).equals(regions.get(2)));
121 }
122
123 private HRegionInfo createRandomRegion(final String name) {
124 HTableDescriptor htd = new HTableDescriptor(name);
125 String [] families = new String [] {"info", "anchor"};
126 for (int i = 0; i < families.length; i++) {
127 htd.addFamily(new HColumnDescriptor(families[i]));
128 }
129 return new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW,
130 HConstants.EMPTY_END_ROW);
131 }
132
133 @Test public void testPut() throws Exception{
134 byte[] row = "row".getBytes();
135 byte[] fam = "fam".getBytes();
136 byte[] qf1 = "qf1".getBytes();
137 byte[] qf2 = "qf2".getBytes();
138 byte[] qf3 = "qf3".getBytes();
139 byte[] qf4 = "qf4".getBytes();
140 byte[] qf5 = "qf5".getBytes();
141 byte[] qf6 = "qf6".getBytes();
142 byte[] qf7 = "qf7".getBytes();
143 byte[] qf8 = "qf8".getBytes();
144
145 long ts = System.currentTimeMillis();
146 byte[] val = "val".getBytes();
147
148 Put put = new Put(row);
149 put.setWriteToWAL(false);
150 put.add(fam, qf1, ts, val);
151 put.add(fam, qf2, ts, val);
152 put.add(fam, qf3, ts, val);
153 put.add(fam, qf4, ts, val);
154 put.add(fam, qf5, ts, val);
155 put.add(fam, qf6, ts, val);
156 put.add(fam, qf7, ts, val);
157 put.add(fam, qf8, ts, val);
158
159 byte[] sb = Writables.getBytes(put);
160 Put desPut = (Put)Writables.getWritable(sb, new Put());
161
162
163
164
165
166
167
168 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
169 List<KeyValue> list = null;
170 List<KeyValue> desList = null;
171 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
172 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
173 list = entry.getValue();
174 desList = desPut.getFamilyMap().get(entry.getKey());
175 for(int i=0; i<list.size(); i++){
176 assertTrue(list.get(i).equals(desList.get(i)));
177 }
178 }
179 }
180
181
182 @Test public void testPut2() throws Exception{
183 byte[] row = "testAbort,,1243116656250".getBytes();
184 byte[] fam = "historian".getBytes();
185 byte[] qf1 = "creation".getBytes();
186
187 long ts = 9223372036854775807L;
188 byte[] val = "dont-care".getBytes();
189
190 Put put = new Put(row);
191 put.add(fam, qf1, ts, val);
192
193 byte[] sb = Writables.getBytes(put);
194 Put desPut = (Put)Writables.getWritable(sb, new Put());
195
196 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
197 List<KeyValue> list = null;
198 List<KeyValue> desList = null;
199 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
200 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
201 list = entry.getValue();
202 desList = desPut.getFamilyMap().get(entry.getKey());
203 for(int i=0; i<list.size(); i++){
204 assertTrue(list.get(i).equals(desList.get(i)));
205 }
206 }
207 }
208
209
210 @Test public void testDelete() throws Exception{
211 byte[] row = "row".getBytes();
212 byte[] fam = "fam".getBytes();
213 byte[] qf1 = "qf1".getBytes();
214
215 long ts = System.currentTimeMillis();
216
217 Delete delete = new Delete(row);
218 delete.deleteColumn(fam, qf1, ts);
219
220 byte[] sb = Writables.getBytes(delete);
221 Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
222
223 assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
224 List<KeyValue> list = null;
225 List<KeyValue> desList = null;
226 for(Map.Entry<byte[], List<KeyValue>> entry :
227 delete.getFamilyMap().entrySet()){
228 assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
229 list = entry.getValue();
230 desList = desDelete.getFamilyMap().get(entry.getKey());
231 for(int i=0; i<list.size(); i++){
232 assertTrue(list.get(i).equals(desList.get(i)));
233 }
234 }
235 }
236
237 @Test public void testGet() throws Exception{
238 byte[] row = "row".getBytes();
239 byte[] fam = "fam".getBytes();
240 byte[] qf1 = "qf1".getBytes();
241
242 long ts = System.currentTimeMillis();
243 int maxVersions = 2;
244 long lockid = 5;
245 RowLock rowLock = new RowLock(lockid);
246
247 Get get = new Get(row, rowLock);
248 get.addColumn(fam, qf1);
249 get.setTimeRange(ts, ts+1);
250 get.setMaxVersions(maxVersions);
251
252 byte[] sb = Writables.getBytes(get);
253 Get desGet = (Get)Writables.getWritable(sb, new Get());
254
255 assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
256 Set<byte[]> set = null;
257 Set<byte[]> desSet = null;
258
259 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
260 get.getFamilyMap().entrySet()){
261 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
262 set = entry.getValue();
263 desSet = desGet.getFamilyMap().get(entry.getKey());
264 for(byte [] qualifier : set){
265 assertTrue(desSet.contains(qualifier));
266 }
267 }
268
269 assertEquals(get.getLockId(), desGet.getLockId());
270 assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
271 TimeRange tr = get.getTimeRange();
272 TimeRange desTr = desGet.getTimeRange();
273 assertEquals(tr.getMax(), desTr.getMax());
274 assertEquals(tr.getMin(), desTr.getMin());
275 }
276
277
278 @Test public void testScan() throws Exception {
279
280 byte[] startRow = "startRow".getBytes();
281 byte[] stopRow = "stopRow".getBytes();
282 byte[] fam = "fam".getBytes();
283 byte[] qf1 = "qf1".getBytes();
284
285 long ts = System.currentTimeMillis();
286 int maxVersions = 2;
287
288 Scan scan = new Scan(startRow, stopRow);
289 scan.addColumn(fam, qf1);
290 scan.setTimeRange(ts, ts+1);
291 scan.setMaxVersions(maxVersions);
292
293 byte[] sb = Writables.getBytes(scan);
294 Scan desScan = (Scan)Writables.getWritable(sb, new Scan());
295
296 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
297 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
298 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
299 Set<byte[]> set = null;
300 Set<byte[]> desSet = null;
301
302 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
303 scan.getFamilyMap().entrySet()){
304 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
305 set = entry.getValue();
306 desSet = desScan.getFamilyMap().get(entry.getKey());
307 for(byte[] column : set){
308 assertTrue(desSet.contains(column));
309 }
310
311
312 scan = new Scan(startRow);
313 final String name = "testScan";
314 byte [] prefix = Bytes.toBytes(name);
315 scan.setFilter(new PrefixFilter(prefix));
316 sb = Writables.getBytes(scan);
317 desScan = (Scan)Writables.getWritable(sb, new Scan());
318 Filter f = desScan.getFilter();
319 assertTrue(f instanceof PrefixFilter);
320 }
321
322 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
323 TimeRange tr = scan.getTimeRange();
324 TimeRange desTr = desScan.getTimeRange();
325 assertEquals(tr.getMax(), desTr.getMax());
326 assertEquals(tr.getMin(), desTr.getMin());
327 }
328
329 @Test public void testResultEmpty() throws Exception {
330 List<KeyValue> keys = new ArrayList<KeyValue>();
331 Result r = new Result(keys);
332 assertTrue(r.isEmpty());
333 byte [] rb = Writables.getBytes(r);
334 Result deserializedR = (Result)Writables.getWritable(rb, new Result());
335 assertTrue(deserializedR.isEmpty());
336 }
337
338
339 @Test public void testResult() throws Exception {
340 byte [] rowA = Bytes.toBytes("rowA");
341 byte [] famA = Bytes.toBytes("famA");
342 byte [] qfA = Bytes.toBytes("qfA");
343 byte [] valueA = Bytes.toBytes("valueA");
344
345 byte [] rowB = Bytes.toBytes("rowB");
346 byte [] famB = Bytes.toBytes("famB");
347 byte [] qfB = Bytes.toBytes("qfB");
348 byte [] valueB = Bytes.toBytes("valueB");
349
350 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
351 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
352
353 Result result = new Result(new KeyValue[]{kvA, kvB});
354
355 byte [] rb = Writables.getBytes(result);
356 Result deResult = (Result)Writables.getWritable(rb, new Result());
357
358 assertTrue("results are not equivalent, first key mismatch",
359 result.raw()[0].equals(deResult.raw()[0]));
360
361 assertTrue("results are not equivalent, second key mismatch",
362 result.raw()[1].equals(deResult.raw()[1]));
363
364
365 Result r = new Result();
366 byte [] b = Writables.getBytes(r);
367 Result deserialized = (Result)Writables.getWritable(b, new Result());
368 assertEquals(r.size(), deserialized.size());
369 }
370
371 @Test public void testResultDynamicBuild() throws Exception {
372 byte [] rowA = Bytes.toBytes("rowA");
373 byte [] famA = Bytes.toBytes("famA");
374 byte [] qfA = Bytes.toBytes("qfA");
375 byte [] valueA = Bytes.toBytes("valueA");
376
377 byte [] rowB = Bytes.toBytes("rowB");
378 byte [] famB = Bytes.toBytes("famB");
379 byte [] qfB = Bytes.toBytes("qfB");
380 byte [] valueB = Bytes.toBytes("valueB");
381
382 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
383 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
384
385 Result result = new Result(new KeyValue[]{kvA, kvB});
386
387 byte [] rb = Writables.getBytes(result);
388
389
390
391 Result deResult = (Result)Writables.getWritable(rb, new Result());
392 byte [] row = deResult.getRow();
393 assertTrue(Bytes.equals(row, rowA));
394
395
396 deResult = (Result)Writables.getWritable(rb, new Result());
397 assertTrue("results are not equivalent, first key mismatch",
398 result.raw()[0].equals(deResult.raw()[0]));
399 assertTrue("results are not equivalent, second key mismatch",
400 result.raw()[1].equals(deResult.raw()[1]));
401
402
403 deResult = (Result)Writables.getWritable(rb, new Result());
404 assertTrue("results are not equivalent, first key mismatch",
405 result.raw()[0].equals(deResult.raw()[0]));
406 assertTrue("results are not equivalent, second key mismatch",
407 result.raw()[1].equals(deResult.raw()[1]));
408
409
410 }
411
412 @Test public void testResultArray() throws Exception {
413 byte [] rowA = Bytes.toBytes("rowA");
414 byte [] famA = Bytes.toBytes("famA");
415 byte [] qfA = Bytes.toBytes("qfA");
416 byte [] valueA = Bytes.toBytes("valueA");
417
418 byte [] rowB = Bytes.toBytes("rowB");
419 byte [] famB = Bytes.toBytes("famB");
420 byte [] qfB = Bytes.toBytes("qfB");
421 byte [] valueB = Bytes.toBytes("valueB");
422
423 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
424 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
425
426
427 Result result1 = new Result(new KeyValue[]{kvA, kvB});
428 Result result2 = new Result(new KeyValue[]{kvB});
429 Result result3 = new Result(new KeyValue[]{kvB});
430
431 Result [] results = new Result [] {result1, result2, result3};
432
433 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
434 DataOutputStream out = new DataOutputStream(byteStream);
435 Result.writeArray(out, results);
436
437 byte [] rb = byteStream.toByteArray();
438
439 DataInputBuffer in = new DataInputBuffer();
440 in.reset(rb, 0, rb.length);
441
442 Result [] deResults = Result.readArray(in);
443
444 assertTrue(results.length == deResults.length);
445
446 for(int i=0;i<results.length;i++) {
447 KeyValue [] keysA = results[i].raw();
448 KeyValue [] keysB = deResults[i].raw();
449 assertTrue(keysA.length == keysB.length);
450 for(int j=0;j<keysA.length;j++) {
451 assertTrue("Expected equivalent keys but found:\n" +
452 "KeyA : " + keysA[j].toString() + "\n" +
453 "KeyB : " + keysB[j].toString() + "\n" +
454 keysA.length + " total keys, " + i + "th so far"
455 ,keysA[j].equals(keysB[j]));
456 }
457 }
458
459 }
460
461 @Test public void testResultArrayEmpty() throws Exception {
462 List<KeyValue> keys = new ArrayList<KeyValue>();
463 Result r = new Result(keys);
464 Result [] results = new Result [] {r};
465
466 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
467 DataOutputStream out = new DataOutputStream(byteStream);
468
469 Result.writeArray(out, results);
470
471 results = null;
472
473 byteStream = new ByteArrayOutputStream();
474 out = new DataOutputStream(byteStream);
475 Result.writeArray(out, results);
476
477 byte [] rb = byteStream.toByteArray();
478
479 DataInputBuffer in = new DataInputBuffer();
480 in.reset(rb, 0, rb.length);
481
482 Result [] deResults = Result.readArray(in);
483
484 assertTrue(deResults.length == 0);
485
486 results = new Result[0];
487
488 byteStream = new ByteArrayOutputStream();
489 out = new DataOutputStream(byteStream);
490 Result.writeArray(out, results);
491
492 rb = byteStream.toByteArray();
493
494 in = new DataInputBuffer();
495 in.reset(rb, 0, rb.length);
496
497 deResults = Result.readArray(in);
498
499 assertTrue(deResults.length == 0);
500
501 }
502
503 @Test public void testTimeRange() throws Exception{
504 TimeRange tr = new TimeRange(0,5);
505 byte [] mb = Writables.getBytes(tr);
506 TimeRange deserializedTr =
507 (TimeRange)Writables.getWritable(mb, new TimeRange());
508
509 assertEquals(tr.getMax(), deserializedTr.getMax());
510 assertEquals(tr.getMin(), deserializedTr.getMin());
511
512 }
513
514 @Test public void testKeyValue2() throws Exception {
515 final String name = "testKeyValue2";
516 byte[] row = name.getBytes();
517 byte[] fam = "fam".getBytes();
518 byte[] qf = "qf".getBytes();
519 long ts = System.currentTimeMillis();
520 byte[] val = "val".getBytes();
521
522 KeyValue kv = new KeyValue(row, fam, qf, ts, val);
523
524 byte [] mb = Writables.getBytes(kv);
525 KeyValue deserializedKv =
526 (KeyValue)Writables.getWritable(mb, new KeyValue());
527 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
528 assertEquals(kv.getOffset(), deserializedKv.getOffset());
529 assertEquals(kv.getLength(), deserializedKv.getLength());
530 }
531
532 protected static final int MAXVERSIONS = 3;
533 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
534 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
535 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
536 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
537
538
539
540
541
542
543
544 protected HTableDescriptor createTableDescriptor(final String name) {
545 return createTableDescriptor(name, MAXVERSIONS);
546 }
547
548
549
550
551
552
553
554
555 protected HTableDescriptor createTableDescriptor(final String name,
556 final int versions) {
557 HTableDescriptor htd = new HTableDescriptor(name);
558 htd.addFamily(new HColumnDescriptor(fam1, versions,
559 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
560 HColumnDescriptor.DEFAULT_BLOCKSIZE, HConstants.FOREVER,
561 HColumnDescriptor.DEFAULT_BLOOMFILTER,
562 HConstants.REPLICATION_SCOPE_LOCAL));
563 htd.addFamily(new HColumnDescriptor(fam2, versions,
564 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
565 HColumnDescriptor.DEFAULT_BLOCKSIZE, HConstants.FOREVER,
566 HColumnDescriptor.DEFAULT_BLOOMFILTER,
567 HConstants.REPLICATION_SCOPE_LOCAL));
568 htd.addFamily(new HColumnDescriptor(fam3, versions,
569 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
570 HColumnDescriptor.DEFAULT_BLOCKSIZE, HConstants.FOREVER,
571 HColumnDescriptor.DEFAULT_BLOOMFILTER,
572 HConstants.REPLICATION_SCOPE_LOCAL));
573 return htd;
574 }
575 }