1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.management.ManagementFactory;
24 import java.lang.management.MemoryMXBean;
25 import java.rmi.UnexpectedException;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.List;
29 import java.util.concurrent.atomic.AtomicReference;
30
31 import junit.framework.TestCase;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.HBaseConfiguration;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.KeyValueTestUtil;
40 import org.apache.hadoop.hbase.client.Scan;
41 import org.apache.hadoop.hbase.util.Bytes;
42
43 import com.google.common.base.Joiner;
44 import com.google.common.collect.Iterables;
45 import com.google.common.collect.Lists;
46
47
48 public class TestMemStore extends TestCase {
49 private final Log LOG = LogFactory.getLog(this.getClass());
50 private MemStore memstore;
51 private static final int ROW_COUNT = 10;
52 private static final int QUALIFIER_COUNT = ROW_COUNT;
53 private static final byte [] FAMILY = Bytes.toBytes("column");
54 private static final byte [] CONTENTS = Bytes.toBytes("contents");
55 private static final byte [] BASIC = Bytes.toBytes("basic");
56 private static final String CONTENTSTR = "contentstr";
57 private MultiVersionConsistencyControl mvcc;
58
59 @Override
60 public void setUp() throws Exception {
61 super.setUp();
62 this.mvcc = new MultiVersionConsistencyControl();
63 this.memstore = new MemStore();
64 }
65
66 public void testPutSameKey() {
67 byte [] bytes = Bytes.toBytes(getName());
68 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
69 this.memstore.add(kv);
70 byte [] other = Bytes.toBytes("somethingelse");
71 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
72 this.memstore.add(samekey);
73 KeyValue found = this.memstore.kvset.first();
74 assertEquals(1, this.memstore.kvset.size());
75 assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
76 found.getValue()));
77 }
78
79
80
81
82
83 public void testScanAcrossSnapshot() throws IOException {
84 int rowCount = addRows(this.memstore);
85 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
86 Scan scan = new Scan();
87 List<KeyValue> result = new ArrayList<KeyValue>();
88 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
89 StoreScanner s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
90 this.memstore.comparator, null, memstorescanners);
91 int count = 0;
92 try {
93 while (s.next(result)) {
94 LOG.info(result);
95 count++;
96
97 assertEquals(rowCount, result.size());
98 result.clear();
99 }
100 } finally {
101 s.close();
102 }
103 assertEquals(rowCount, count);
104 for (KeyValueScanner scanner : memstorescanners) {
105 scanner.close();
106 }
107
108 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
109 memstorescanners = this.memstore.getScanners();
110
111 s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
112 this.memstore.comparator, null, memstorescanners);
113 count = 0;
114 try {
115 while (s.next(result)) {
116 LOG.info(result);
117
118 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
119 count++;
120
121 assertEquals(rowCount, result.size());
122 if (count == 2) {
123 this.memstore.snapshot();
124 LOG.info("Snapshotted");
125 }
126 result.clear();
127 }
128 } finally {
129 s.close();
130 }
131 assertEquals(rowCount, count);
132 for (KeyValueScanner scanner : memstorescanners) {
133 scanner.close();
134 }
135 memstorescanners = this.memstore.getScanners();
136
137 long ts = System.currentTimeMillis();
138 s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
139 this.memstore.comparator, null, memstorescanners);
140 count = 0;
141 int snapshotIndex = 5;
142 try {
143 while (s.next(result)) {
144 LOG.info(result);
145
146 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
147
148 assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
149 count++;
150 if (count == snapshotIndex) {
151 this.memstore.snapshot();
152 this.memstore.clearSnapshot(this.memstore.getSnapshot());
153
154 addRows(this.memstore, ts);
155 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
156 }
157 result.clear();
158 }
159 } finally {
160 s.close();
161 }
162 assertEquals(rowCount, count);
163 }
164
165
166
167
168
169 public void testScanAcrossSnapshot2() throws IOException {
170
171
172 final byte[] one = Bytes.toBytes(1);
173 final byte[] two = Bytes.toBytes(2);
174 final byte[] f = Bytes.toBytes("f");
175 final byte[] q = Bytes.toBytes("q");
176 final byte[] v = Bytes.toBytes(3);
177
178 final KeyValue kv1 = new KeyValue(one, f, q, v);
179 final KeyValue kv2 = new KeyValue(two, f, q, v);
180
181
182 this.memstore.add(kv1.clone());
183 this.memstore.add(kv2.clone());
184 verifyScanAcrossSnapshot2(kv1, kv2);
185
186
187 this.memstore.snapshot();
188 verifyScanAcrossSnapshot2(kv1, kv2);
189
190
191 this.memstore = new MemStore();
192 this.memstore.add(kv1.clone());
193 this.memstore.snapshot();
194 this.memstore.add(kv2.clone());
195 verifyScanAcrossSnapshot2(kv1, kv2);
196 }
197
198 private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
199 throws IOException {
200 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
201 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
202 assertEquals(1, memstorescanners.size());
203 final KeyValueScanner scanner = memstorescanners.get(0);
204 scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
205 assertEquals(kv1, scanner.next());
206 assertEquals(kv2, scanner.next());
207 assertNull(scanner.next());
208 }
209
210 private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
211 throws IOException {
212 scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
213 List<KeyValue> returned = Lists.newArrayList();
214
215 while (true) {
216 KeyValue next = scanner.next();
217 if (next == null) break;
218 returned.add(next);
219 }
220
221 assertTrue(
222 "Got:\n" + Joiner.on("\n").join(returned) +
223 "\nExpected:\n" + Joiner.on("\n").join(expected),
224 Iterables.elementsEqual(Arrays.asList(expected), returned));
225 assertNull(scanner.peek());
226 }
227
228 public void testMemstoreConcurrentControl() throws IOException {
229 final byte[] row = Bytes.toBytes(1);
230 final byte[] f = Bytes.toBytes("family");
231 final byte[] q1 = Bytes.toBytes("q1");
232 final byte[] q2 = Bytes.toBytes("q2");
233 final byte[] v = Bytes.toBytes("value");
234
235 MultiVersionConsistencyControl.WriteEntry w =
236 mvcc.beginMemstoreInsert();
237
238 KeyValue kv1 = new KeyValue(row, f, q1, v);
239 kv1.setMemstoreTS(w.getWriteNumber());
240 memstore.add(kv1);
241
242 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
243 KeyValueScanner s = this.memstore.getScanners().get(0);
244 assertScannerResults(s, new KeyValue[]{});
245
246 mvcc.completeMemstoreInsert(w);
247
248 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
249 s = this.memstore.getScanners().get(0);
250 assertScannerResults(s, new KeyValue[]{kv1});
251
252 w = mvcc.beginMemstoreInsert();
253 KeyValue kv2 = new KeyValue(row, f, q2, v);
254 kv2.setMemstoreTS(w.getWriteNumber());
255 memstore.add(kv2);
256
257 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
258 s = this.memstore.getScanners().get(0);
259 assertScannerResults(s, new KeyValue[]{kv1});
260
261 mvcc.completeMemstoreInsert(w);
262
263 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
264 s = this.memstore.getScanners().get(0);
265 assertScannerResults(s, new KeyValue[]{kv1, kv2});
266 }
267
268
269
270
271
272
273
274 public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
275 final byte[] row = Bytes.toBytes(1);
276 final byte[] f = Bytes.toBytes("family");
277 final byte[] q1 = Bytes.toBytes("q1");
278 final byte[] q2 = Bytes.toBytes("q2");
279 final byte[] v1 = Bytes.toBytes("value1");
280 final byte[] v2 = Bytes.toBytes("value2");
281
282
283 MultiVersionConsistencyControl.WriteEntry w =
284 mvcc.beginMemstoreInsert();
285
286 KeyValue kv11 = new KeyValue(row, f, q1, v1);
287 kv11.setMemstoreTS(w.getWriteNumber());
288 memstore.add(kv11);
289
290 KeyValue kv12 = new KeyValue(row, f, q2, v1);
291 kv12.setMemstoreTS(w.getWriteNumber());
292 memstore.add(kv12);
293 mvcc.completeMemstoreInsert(w);
294
295
296 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
297 KeyValueScanner s = this.memstore.getScanners().get(0);
298 assertScannerResults(s, new KeyValue[]{kv11, kv12});
299
300
301 w = mvcc.beginMemstoreInsert();
302 KeyValue kv21 = new KeyValue(row, f, q1, v2);
303 kv21.setMemstoreTS(w.getWriteNumber());
304 memstore.add(kv21);
305
306 KeyValue kv22 = new KeyValue(row, f, q2, v2);
307 kv22.setMemstoreTS(w.getWriteNumber());
308 memstore.add(kv22);
309
310
311 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
312 s = this.memstore.getScanners().get(0);
313 assertScannerResults(s, new KeyValue[]{kv11, kv12});
314
315
316 mvcc.completeMemstoreInsert(w);
317
318
319
320
321 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
322 s = this.memstore.getScanners().get(0);
323 assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
324 }
325
326
327
328
329
330
331 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
332 final byte[] row = Bytes.toBytes(1);
333 final byte[] f = Bytes.toBytes("family");
334 final byte[] q1 = Bytes.toBytes("q1");
335 final byte[] q2 = Bytes.toBytes("q2");
336 final byte[] v1 = Bytes.toBytes("value1");
337
338 MultiVersionConsistencyControl.WriteEntry w =
339 mvcc.beginMemstoreInsert();
340
341 KeyValue kv11 = new KeyValue(row, f, q1, v1);
342 kv11.setMemstoreTS(w.getWriteNumber());
343 memstore.add(kv11);
344
345 KeyValue kv12 = new KeyValue(row, f, q2, v1);
346 kv12.setMemstoreTS(w.getWriteNumber());
347 memstore.add(kv12);
348 mvcc.completeMemstoreInsert(w);
349
350
351 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
352 KeyValueScanner s = this.memstore.getScanners().get(0);
353 assertScannerResults(s, new KeyValue[]{kv11, kv12});
354
355
356 w = mvcc.beginMemstoreInsert();
357 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
358 KeyValue.Type.DeleteColumn);
359 kvDel.setMemstoreTS(w.getWriteNumber());
360 memstore.add(kvDel);
361
362
363 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
364 s = this.memstore.getScanners().get(0);
365 assertScannerResults(s, new KeyValue[]{kv11, kv12});
366
367
368 mvcc.completeMemstoreInsert(w);
369
370
371 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
372 s = this.memstore.getScanners().get(0);
373 assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
374 }
375
376
377 private static class ReadOwnWritesTester extends Thread {
378 static final int NUM_TRIES = 1000;
379
380 final byte[] row;
381
382 final byte[] f = Bytes.toBytes("family");
383 final byte[] q1 = Bytes.toBytes("q1");
384
385 final MultiVersionConsistencyControl mvcc;
386 final MemStore memstore;
387
388 AtomicReference<Throwable> caughtException;
389
390
391 public ReadOwnWritesTester(int id,
392 MemStore memstore,
393 MultiVersionConsistencyControl mvcc,
394 AtomicReference<Throwable> caughtException)
395 {
396 this.mvcc = mvcc;
397 this.memstore = memstore;
398 this.caughtException = caughtException;
399 row = Bytes.toBytes(id);
400 }
401
402 public void run() {
403 try {
404 internalRun();
405 } catch (Throwable t) {
406 caughtException.compareAndSet(null, t);
407 }
408 }
409
410 private void internalRun() throws IOException {
411 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
412 MultiVersionConsistencyControl.WriteEntry w =
413 mvcc.beginMemstoreInsert();
414
415
416 byte[] v = Bytes.toBytes(i);
417
418 KeyValue kv = new KeyValue(row, f, q1, i, v);
419 kv.setMemstoreTS(w.getWriteNumber());
420 memstore.add(kv);
421 mvcc.completeMemstoreInsert(w);
422
423
424 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
425
426 KeyValueScanner s = this.memstore.getScanners().get(0);
427 s.seek(kv);
428
429 KeyValue ret = s.next();
430 assertNotNull("Didnt find own write at all", ret);
431 assertEquals("Didnt read own writes",
432 kv.getTimestamp(), ret.getTimestamp());
433 }
434 }
435 }
436
437 public void testReadOwnWritesUnderConcurrency() throws Throwable {
438
439 int NUM_THREADS = 8;
440
441 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
442 AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
443
444 for (int i = 0; i < NUM_THREADS; i++) {
445 threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
446 threads[i].start();
447 }
448
449 for (int i = 0; i < NUM_THREADS; i++) {
450 threads[i].join();
451 }
452
453 if (caught.get() != null) {
454 throw caught.get();
455 }
456 }
457
458
459
460
461
462 public void testSnapshotting() throws IOException {
463 final int snapshotCount = 5;
464
465 for (int i = 0; i < snapshotCount; i++) {
466 addRows(this.memstore);
467 runSnapshot(this.memstore);
468 KeyValueSkipListSet ss = this.memstore.getSnapshot();
469 assertEquals("History not being cleared", 0, ss.size());
470 }
471 }
472
473 public void testMultipleVersionsSimple() throws Exception {
474 MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
475 byte [] row = Bytes.toBytes("testRow");
476 byte [] family = Bytes.toBytes("testFamily");
477 byte [] qf = Bytes.toBytes("testQualifier");
478 long [] stamps = {1,2,3};
479 byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
480 Bytes.toBytes("value2")};
481 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
482 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
483 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
484
485 m.add(key0);
486 m.add(key1);
487 m.add(key2);
488
489 assertTrue("Expected memstore to hold 3 values, actually has " +
490 m.kvset.size(), m.kvset.size() == 3);
491 }
492
493 public void testBinary() throws IOException {
494 MemStore mc = new MemStore(new Configuration(), KeyValue.ROOT_COMPARATOR);
495 final int start = 43;
496 final int end = 46;
497 for (int k = start; k <= end; k++) {
498 byte [] kk = Bytes.toBytes(k);
499 byte [] row =
500 Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
501 KeyValue key = new KeyValue(row, CONTENTS, BASIC,
502 System.currentTimeMillis(),
503 (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
504 mc.add(key);
505 System.out.println(key);
506
507
508
509
510
511 }
512 int index = start;
513 for (KeyValue kv: mc.kvset) {
514 System.out.println(kv);
515 byte [] b = kv.getRow();
516
517 String str = Bytes.toString(b, 13, 4);
518 byte [] bb = Bytes.toBytes(index);
519 String bbStr = Bytes.toString(bb);
520 assertEquals(str, bbStr);
521 index++;
522 }
523 }
524
525
526
527
528
529
530
531
532 public void testGetNextRow() throws Exception {
533 MultiVersionConsistencyControl.resetThreadReadPoint();
534 addRows(this.memstore);
535
536 Thread.sleep(1);
537 addRows(this.memstore);
538 KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
539 assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
540 new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
541 for (int i = 0; i < ROW_COUNT; i++) {
542 KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
543 System.currentTimeMillis()));
544 if (i + 1 == ROW_COUNT) {
545 assertEquals(nr, null);
546 } else {
547 assertTrue(KeyValue.COMPARATOR.compareRows(nr,
548 new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
549 }
550 }
551
552 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
553 InternalScanner scanner =
554 new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY,
555 Integer.MAX_VALUE, this.memstore.comparator, null,
556 memstore.getScanners());
557 List<KeyValue> results = new ArrayList<KeyValue>();
558 for (int i = 0; scanner.next(results); i++) {
559 int rowId = startRowId + i;
560 assertTrue("Row name",
561 KeyValue.COMPARATOR.compareRows(results.get(0),
562 Bytes.toBytes(rowId)) == 0);
563 assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
564 List<KeyValue> row = new ArrayList<KeyValue>();
565 for (KeyValue kv : results) {
566 row.add(kv);
567 }
568 isExpectedRowWithoutTimestamps(rowId, row);
569
570 results.clear();
571 }
572 }
573 }
574
575 public void testGet_memstoreAndSnapShot() throws IOException {
576 byte [] row = Bytes.toBytes("testrow");
577 byte [] fam = Bytes.toBytes("testfamily");
578 byte [] qf1 = Bytes.toBytes("testqualifier1");
579 byte [] qf2 = Bytes.toBytes("testqualifier2");
580 byte [] qf3 = Bytes.toBytes("testqualifier3");
581 byte [] qf4 = Bytes.toBytes("testqualifier4");
582 byte [] qf5 = Bytes.toBytes("testqualifier5");
583 byte [] val = Bytes.toBytes("testval");
584
585
586 memstore.add(new KeyValue(row, fam ,qf1, val));
587 memstore.add(new KeyValue(row, fam ,qf2, val));
588 memstore.add(new KeyValue(row, fam ,qf3, val));
589
590 memstore.snapshot();
591 assertEquals(3, memstore.snapshot.size());
592
593 assertEquals(0, memstore.kvset.size());
594 memstore.add(new KeyValue(row, fam ,qf4, val));
595 memstore.add(new KeyValue(row, fam ,qf5, val));
596 assertEquals(2, memstore.kvset.size());
597 }
598
599
600
601
602 public void testGetWithDelete() throws IOException {
603 byte [] row = Bytes.toBytes("testrow");
604 byte [] fam = Bytes.toBytes("testfamily");
605 byte [] qf1 = Bytes.toBytes("testqualifier");
606 byte [] val = Bytes.toBytes("testval");
607
608 long ts1 = System.nanoTime();
609 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
610 long ts2 = ts1 + 1;
611 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
612 long ts3 = ts2 +1;
613 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
614 memstore.add(put1);
615 memstore.add(put2);
616 memstore.add(put3);
617
618 assertEquals(3, memstore.kvset.size());
619
620 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
621 memstore.delete(del2);
622
623 List<KeyValue> expected = new ArrayList<KeyValue>();
624 expected.add(put3);
625 expected.add(del2);
626 expected.add(put2);
627 expected.add(put1);
628
629 assertEquals(4, memstore.kvset.size());
630 int i = 0;
631 for(KeyValue kv : memstore.kvset) {
632 assertEquals(expected.get(i++), kv);
633 }
634 }
635
636 public void testGetWithDeleteColumn() throws IOException {
637 byte [] row = Bytes.toBytes("testrow");
638 byte [] fam = Bytes.toBytes("testfamily");
639 byte [] qf1 = Bytes.toBytes("testqualifier");
640 byte [] val = Bytes.toBytes("testval");
641
642 long ts1 = System.nanoTime();
643 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
644 long ts2 = ts1 + 1;
645 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
646 long ts3 = ts2 +1;
647 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
648 memstore.add(put1);
649 memstore.add(put2);
650 memstore.add(put3);
651
652 assertEquals(3, memstore.kvset.size());
653
654 KeyValue del2 =
655 new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
656 memstore.delete(del2);
657
658 List<KeyValue> expected = new ArrayList<KeyValue>();
659 expected.add(put3);
660 expected.add(del2);
661 expected.add(put2);
662 expected.add(put1);
663
664
665 assertEquals(4, memstore.kvset.size());
666 int i = 0;
667 for (KeyValue kv: memstore.kvset) {
668 assertEquals(expected.get(i++), kv);
669 }
670 }
671
672
673 public void testGetWithDeleteFamily() throws IOException {
674 byte [] row = Bytes.toBytes("testrow");
675 byte [] fam = Bytes.toBytes("testfamily");
676 byte [] qf1 = Bytes.toBytes("testqualifier1");
677 byte [] qf2 = Bytes.toBytes("testqualifier2");
678 byte [] qf3 = Bytes.toBytes("testqualifier3");
679 byte [] val = Bytes.toBytes("testval");
680 long ts = System.nanoTime();
681
682 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
683 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
684 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
685 KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
686
687 memstore.add(put1);
688 memstore.add(put2);
689 memstore.add(put3);
690 memstore.add(put4);
691
692 KeyValue del =
693 new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
694 memstore.delete(del);
695
696 List<KeyValue> expected = new ArrayList<KeyValue>();
697 expected.add(del);
698 expected.add(put1);
699 expected.add(put2);
700 expected.add(put4);
701 expected.add(put3);
702
703
704
705 assertEquals(5, memstore.kvset.size());
706 int i = 0;
707 for (KeyValue kv: memstore.kvset) {
708 assertEquals(expected.get(i++), kv);
709 }
710 }
711
712 public void testKeepDeleteInmemstore() {
713 byte [] row = Bytes.toBytes("testrow");
714 byte [] fam = Bytes.toBytes("testfamily");
715 byte [] qf = Bytes.toBytes("testqualifier");
716 byte [] val = Bytes.toBytes("testval");
717 long ts = System.nanoTime();
718 memstore.add(new KeyValue(row, fam, qf, ts, val));
719 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
720 memstore.delete(delete);
721 assertEquals(2, memstore.kvset.size());
722 assertEquals(delete, memstore.kvset.first());
723 }
724
725 public void testRetainsDeleteVersion() throws IOException {
726
727 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
728
729
730 KeyValue delete = KeyValueTestUtil.create(
731 "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
732 memstore.delete(delete);
733
734 assertEquals(2, memstore.kvset.size());
735 assertEquals(delete, memstore.kvset.first());
736 }
737 public void testRetainsDeleteColumn() throws IOException {
738
739 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
740
741
742 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
743 KeyValue.Type.DeleteColumn, "dont-care");
744 memstore.delete(delete);
745
746 assertEquals(2, memstore.kvset.size());
747 assertEquals(delete, memstore.kvset.first());
748 }
749 public void testRetainsDeleteFamily() throws IOException {
750
751 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
752
753
754 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
755 KeyValue.Type.DeleteFamily, "dont-care");
756 memstore.delete(delete);
757
758 assertEquals(2, memstore.kvset.size());
759 assertEquals(delete, memstore.kvset.first());
760 }
761
762
763
764
765
766
767
768
769 public void testMultipleTimestamps() throws IOException {
770 long[] timestamps = new long[] {20,10,5,1};
771 Scan scan = new Scan();
772
773 for (long timestamp: timestamps)
774 addRows(memstore,timestamp);
775
776 scan.setTimeRange(0, 2);
777 assertTrue(memstore.shouldSeek(scan));
778
779 scan.setTimeRange(20, 82);
780 assertTrue(memstore.shouldSeek(scan));
781
782 scan.setTimeRange(10, 20);
783 assertTrue(memstore.shouldSeek(scan));
784
785 scan.setTimeRange(8, 12);
786 assertTrue(memstore.shouldSeek(scan));
787
788
789
790
791
792 }
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811 public void testUpsertMSLAB() throws Exception {
812 Configuration conf = HBaseConfiguration.create();
813 conf.setBoolean(MemStore.USEMSLAB_KEY, true);
814 memstore = new MemStore(conf, KeyValue.COMPARATOR);
815
816 int ROW_SIZE = 2048;
817 byte[] qualifier = new byte[ROW_SIZE - 4];
818
819 MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
820 for (int i = 0; i < 3; i++) { System.gc(); }
821 long usageBefore = bean.getHeapMemoryUsage().getUsed();
822
823 long size = 0;
824 long ts=0;
825
826 for (int newValue = 0; newValue < 1000; newValue++) {
827 for (int row = newValue; row < newValue + 1000; row++) {
828 byte[] rowBytes = Bytes.toBytes(row);
829 size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
830 }
831 }
832 System.out.println("Wrote " + ts + " vals");
833 for (int i = 0; i < 3; i++) { System.gc(); }
834 long usageAfter = bean.getHeapMemoryUsage().getUsed();
835 System.out.println("Memory used: " + (usageAfter - usageBefore)
836 + " (heapsize: " + memstore.heapSize() +
837 " size: " + size + ")");
838 }
839
840
841
842
843 private static byte [] makeQualifier(final int i1, final int i2){
844 return Bytes.toBytes(Integer.toString(i1) + ";" +
845 Integer.toString(i2));
846 }
847
848
849
850
851
852
853
854 private int addRows(final MemStore hmc) {
855 return addRows(hmc, HConstants.LATEST_TIMESTAMP);
856 }
857
858
859
860
861
862
863
864 private int addRows(final MemStore hmc, final long ts) {
865 for (int i = 0; i < ROW_COUNT; i++) {
866 long timestamp = ts == HConstants.LATEST_TIMESTAMP?
867 System.currentTimeMillis(): ts;
868 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
869 byte [] row = Bytes.toBytes(i);
870 byte [] qf = makeQualifier(i, ii);
871 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
872 }
873 }
874 return ROW_COUNT;
875 }
876
877 private void runSnapshot(final MemStore hmc) throws UnexpectedException {
878
879 int oldHistorySize = hmc.getSnapshot().size();
880 hmc.snapshot();
881 KeyValueSkipListSet ss = hmc.getSnapshot();
882
883 assertTrue("History size has not increased", oldHistorySize < ss.size());
884 hmc.clearSnapshot(ss);
885 }
886
887 private void isExpectedRowWithoutTimestamps(final int rowIndex,
888 List<KeyValue> kvs) {
889 int i = 0;
890 for (KeyValue kv: kvs) {
891 String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
892 String colnameStr = Bytes.toString(kv.getQualifier());
893 assertEquals("Column name", colnameStr, expectedColname);
894
895
896
897
898 String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
899 kv.getValueLength());
900 assertEquals("Content", colnameStr, colvalueStr);
901 }
902 }
903
904 private KeyValue getDeleteKV(byte [] row) {
905 return new KeyValue(row, Bytes.toBytes("test_col"), null,
906 HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
907 }
908
909 private KeyValue getKV(byte [] row, byte [] value) {
910 return new KeyValue(row, Bytes.toBytes("test_col"), null,
911 HConstants.LATEST_TIMESTAMP, value);
912 }
913 private static void addRows(int count, final MemStore mem) {
914 long nanos = System.nanoTime();
915
916 for (int i = 0 ; i < count ; i++) {
917 if (i % 1000 == 0) {
918
919 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
920 nanos = System.nanoTime();
921 }
922 long timestamp = System.currentTimeMillis();
923
924 for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
925 byte [] row = Bytes.toBytes(i);
926 byte [] qf = makeQualifier(i, ii);
927 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
928 }
929 }
930 }
931
932
933 static void doScan(MemStore ms, int iteration) throws IOException {
934 long nanos = System.nanoTime();
935 KeyValueScanner s = ms.getScanners().get(0);
936 s.seek(KeyValue.createFirstOnRow(new byte[]{}));
937
938 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
939 int cnt=0;
940 while(s.next() != null) ++cnt;
941
942 System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
943
944 }
945
946 public static void main(String [] args) throws IOException {
947 MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
948 MemStore ms = new MemStore();
949
950 long n1 = System.nanoTime();
951 addRows(25000, ms);
952 System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
953
954
955 System.out.println("foo");
956
957 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
958
959 for (int i = 0 ; i < 50 ; i++)
960 doScan(ms, i);
961
962 }
963
964
965 }