1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.MemoryMXBean;
25  import java.rmi.UnexpectedException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.List;
29  import java.util.concurrent.atomic.AtomicReference;
30  
31  import junit.framework.TestCase;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.HBaseConfiguration;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueTestUtil;
40  import org.apache.hadoop.hbase.client.Scan;
41  import org.apache.hadoop.hbase.util.Bytes;
42  
43  import com.google.common.base.Joiner;
44  import com.google.common.collect.Iterables;
45  import com.google.common.collect.Lists;
46  
47  /** memstore test case */
48  public class TestMemStore extends TestCase {
49    private final Log LOG = LogFactory.getLog(this.getClass());
50    private MemStore memstore;
51    private static final int ROW_COUNT = 10;
52    private static final int QUALIFIER_COUNT = ROW_COUNT;
53    private static final byte [] FAMILY = Bytes.toBytes("column");
54    private static final byte [] CONTENTS = Bytes.toBytes("contents");
55    private static final byte [] BASIC = Bytes.toBytes("basic");
56    private static final String CONTENTSTR = "contentstr";
57    private MultiVersionConsistencyControl mvcc;
58  
59    @Override
60    public void setUp() throws Exception {
61      super.setUp();
62      this.mvcc = new MultiVersionConsistencyControl();
63      this.memstore = new MemStore();
64    }
65  
66    public void testPutSameKey() {
67      byte [] bytes = Bytes.toBytes(getName());
68      KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
69      this.memstore.add(kv);
70      byte [] other = Bytes.toBytes("somethingelse");
71      KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
72      this.memstore.add(samekey);
73      KeyValue found = this.memstore.kvset.first();
74      assertEquals(1, this.memstore.kvset.size());
75      assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
76        found.getValue()));
77    }
78  
79    /**
80     * Test memstore snapshot happening while scanning.
81     * @throws IOException
82     */
83    public void testScanAcrossSnapshot() throws IOException {
84      int rowCount = addRows(this.memstore);
85      List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
86      Scan scan = new Scan();
87      List<KeyValue> result = new ArrayList<KeyValue>();
88      MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
89      StoreScanner s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
90        this.memstore.comparator, null, memstorescanners);
91      int count = 0;
92      try {
93        while (s.next(result)) {
94          LOG.info(result);
95          count++;
96          // Row count is same as column count.
97          assertEquals(rowCount, result.size());
98          result.clear();
99        }
100     } finally {
101       s.close();
102     }
103     assertEquals(rowCount, count);
104     for (KeyValueScanner scanner : memstorescanners) {
105       scanner.close();
106     }
107 
108     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
109     memstorescanners = this.memstore.getScanners();
110     // Now assert can count same number even if a snapshot mid-scan.
111     s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
112       this.memstore.comparator, null, memstorescanners);
113     count = 0;
114     try {
115       while (s.next(result)) {
116         LOG.info(result);
117         // Assert the stuff is coming out in right order.
118         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
119         count++;
120         // Row count is same as column count.
121         assertEquals(rowCount, result.size());
122         if (count == 2) {
123           this.memstore.snapshot();
124           LOG.info("Snapshotted");
125         }
126         result.clear();
127       }
128     } finally {
129       s.close();
130     }
131     assertEquals(rowCount, count);
132     for (KeyValueScanner scanner : memstorescanners) {
133       scanner.close();
134     }
135     memstorescanners = this.memstore.getScanners();
136     // Assert that new values are seen in kvset as we scan.
137     long ts = System.currentTimeMillis();
138     s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
139       this.memstore.comparator, null, memstorescanners);
140     count = 0;
141     int snapshotIndex = 5;
142     try {
143       while (s.next(result)) {
144         LOG.info(result);
145         // Assert the stuff is coming out in right order.
146         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
147         // Row count is same as column count.
148         assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
149         count++;
150         if (count == snapshotIndex) {
151           this.memstore.snapshot();
152           this.memstore.clearSnapshot(this.memstore.getSnapshot());
153           // Added more rows into kvset.  But the scanner wont see these rows.
154           addRows(this.memstore, ts);
155           LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
156         }
157         result.clear();
158       }
159     } finally {
160       s.close();
161     }
162     assertEquals(rowCount, count);
163   }
164 
165   /**
166    * A simple test which verifies the 3 possible states when scanning across snapshot.
167    * @throws IOException
168    */
169   public void testScanAcrossSnapshot2() throws IOException {
170     // we are going to the scanning across snapshot with two kvs
171     // kv1 should always be returned before kv2
172     final byte[] one = Bytes.toBytes(1);
173     final byte[] two = Bytes.toBytes(2);
174     final byte[] f = Bytes.toBytes("f");
175     final byte[] q = Bytes.toBytes("q");
176     final byte[] v = Bytes.toBytes(3);
177 
178     final KeyValue kv1 = new KeyValue(one, f, q, v);
179     final KeyValue kv2 = new KeyValue(two, f, q, v);
180 
181     // use case 1: both kvs in kvset
182     this.memstore.add(kv1.clone());
183     this.memstore.add(kv2.clone());
184     verifyScanAcrossSnapshot2(kv1, kv2);
185 
186     // use case 2: both kvs in snapshot
187     this.memstore.snapshot();
188     verifyScanAcrossSnapshot2(kv1, kv2);
189 
190     // use case 3: first in snapshot second in kvset
191     this.memstore = new MemStore();
192     this.memstore.add(kv1.clone());
193     this.memstore.snapshot();
194     this.memstore.add(kv2.clone());
195     verifyScanAcrossSnapshot2(kv1, kv2);
196   }
197 
198   private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
199       throws IOException {
200     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
201     List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
202     assertEquals(1, memstorescanners.size());
203     final KeyValueScanner scanner = memstorescanners.get(0);
204     scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
205     assertEquals(kv1, scanner.next());
206     assertEquals(kv2, scanner.next());
207     assertNull(scanner.next());
208   }
209 
210   private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
211       throws IOException {
212     scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
213     List<KeyValue> returned = Lists.newArrayList();
214 
215     while (true) {
216       KeyValue next = scanner.next();
217       if (next == null) break;
218       returned.add(next);
219     }
220 
221     assertTrue(
222         "Got:\n" + Joiner.on("\n").join(returned) +
223         "\nExpected:\n" + Joiner.on("\n").join(expected),
224         Iterables.elementsEqual(Arrays.asList(expected), returned));
225     assertNull(scanner.peek());
226   }
227 
228   public void testMemstoreConcurrentControl() throws IOException {
229     final byte[] row = Bytes.toBytes(1);
230     final byte[] f = Bytes.toBytes("family");
231     final byte[] q1 = Bytes.toBytes("q1");
232     final byte[] q2 = Bytes.toBytes("q2");
233     final byte[] v = Bytes.toBytes("value");
234 
235     MultiVersionConsistencyControl.WriteEntry w =
236         mvcc.beginMemstoreInsert();
237 
238     KeyValue kv1 = new KeyValue(row, f, q1, v);
239     kv1.setMemstoreTS(w.getWriteNumber());
240     memstore.add(kv1);
241 
242     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
243     KeyValueScanner s = this.memstore.getScanners().get(0);
244     assertScannerResults(s, new KeyValue[]{});
245 
246     mvcc.completeMemstoreInsert(w);
247 
248     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
249     s = this.memstore.getScanners().get(0);
250     assertScannerResults(s, new KeyValue[]{kv1});
251 
252     w = mvcc.beginMemstoreInsert();
253     KeyValue kv2 = new KeyValue(row, f, q2, v);
254     kv2.setMemstoreTS(w.getWriteNumber());
255     memstore.add(kv2);
256 
257     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
258     s = this.memstore.getScanners().get(0);
259     assertScannerResults(s, new KeyValue[]{kv1});
260 
261     mvcc.completeMemstoreInsert(w);
262 
263     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
264     s = this.memstore.getScanners().get(0);
265     assertScannerResults(s, new KeyValue[]{kv1, kv2});
266   }
267 
268   /**
269    * Regression test for HBASE-2616, HBASE-2670.
270    * When we insert a higher-memstoreTS version of a cell but with
271    * the same timestamp, we still need to provide consistent reads
272    * for the same scanner.
273    */
274   public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
275     final byte[] row = Bytes.toBytes(1);
276     final byte[] f = Bytes.toBytes("family");
277     final byte[] q1 = Bytes.toBytes("q1");
278     final byte[] q2 = Bytes.toBytes("q2");
279     final byte[] v1 = Bytes.toBytes("value1");
280     final byte[] v2 = Bytes.toBytes("value2");
281 
282     // INSERT 1: Write both columns val1
283     MultiVersionConsistencyControl.WriteEntry w =
284         mvcc.beginMemstoreInsert();
285 
286     KeyValue kv11 = new KeyValue(row, f, q1, v1);
287     kv11.setMemstoreTS(w.getWriteNumber());
288     memstore.add(kv11);
289 
290     KeyValue kv12 = new KeyValue(row, f, q2, v1);
291     kv12.setMemstoreTS(w.getWriteNumber());
292     memstore.add(kv12);
293     mvcc.completeMemstoreInsert(w);
294 
295     // BEFORE STARTING INSERT 2, SEE FIRST KVS
296     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
297     KeyValueScanner s = this.memstore.getScanners().get(0);
298     assertScannerResults(s, new KeyValue[]{kv11, kv12});
299 
300     // START INSERT 2: Write both columns val2
301     w = mvcc.beginMemstoreInsert();
302     KeyValue kv21 = new KeyValue(row, f, q1, v2);
303     kv21.setMemstoreTS(w.getWriteNumber());
304     memstore.add(kv21);
305 
306     KeyValue kv22 = new KeyValue(row, f, q2, v2);
307     kv22.setMemstoreTS(w.getWriteNumber());
308     memstore.add(kv22);
309 
310     // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
311     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
312     s = this.memstore.getScanners().get(0);
313     assertScannerResults(s, new KeyValue[]{kv11, kv12});
314 
315     // COMPLETE INSERT 2
316     mvcc.completeMemstoreInsert(w);
317 
318     // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
319     // See HBASE-1485 for discussion about what we should do with
320     // the duplicate-TS inserts
321     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
322     s = this.memstore.getScanners().get(0);
323     assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
324   }
325 
326   /**
327    * When we insert a higher-memstoreTS deletion of a cell but with
328    * the same timestamp, we still need to provide consistent reads
329    * for the same scanner.
330    */
331   public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
332     final byte[] row = Bytes.toBytes(1);
333     final byte[] f = Bytes.toBytes("family");
334     final byte[] q1 = Bytes.toBytes("q1");
335     final byte[] q2 = Bytes.toBytes("q2");
336     final byte[] v1 = Bytes.toBytes("value1");
337     // INSERT 1: Write both columns val1
338     MultiVersionConsistencyControl.WriteEntry w =
339         mvcc.beginMemstoreInsert();
340 
341     KeyValue kv11 = new KeyValue(row, f, q1, v1);
342     kv11.setMemstoreTS(w.getWriteNumber());
343     memstore.add(kv11);
344 
345     KeyValue kv12 = new KeyValue(row, f, q2, v1);
346     kv12.setMemstoreTS(w.getWriteNumber());
347     memstore.add(kv12);
348     mvcc.completeMemstoreInsert(w);
349 
350     // BEFORE STARTING INSERT 2, SEE FIRST KVS
351     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
352     KeyValueScanner s = this.memstore.getScanners().get(0);
353     assertScannerResults(s, new KeyValue[]{kv11, kv12});
354 
355     // START DELETE: Insert delete for one of the columns
356     w = mvcc.beginMemstoreInsert();
357     KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
358         KeyValue.Type.DeleteColumn);
359     kvDel.setMemstoreTS(w.getWriteNumber());
360     memstore.add(kvDel);
361 
362     // BEFORE COMPLETING DELETE, SEE FIRST KVS
363     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
364     s = this.memstore.getScanners().get(0);
365     assertScannerResults(s, new KeyValue[]{kv11, kv12});
366 
367     // COMPLETE DELETE
368     mvcc.completeMemstoreInsert(w);
369 
370     // NOW WE SHOULD SEE DELETE
371     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
372     s = this.memstore.getScanners().get(0);
373     assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
374   }
375 
376 
377   private static class ReadOwnWritesTester extends Thread {
378     static final int NUM_TRIES = 1000;
379 
380     final byte[] row;
381 
382     final byte[] f = Bytes.toBytes("family");
383     final byte[] q1 = Bytes.toBytes("q1");
384 
385     final MultiVersionConsistencyControl mvcc;
386     final MemStore memstore;
387 
388     AtomicReference<Throwable> caughtException;
389 
390 
391     public ReadOwnWritesTester(int id,
392                                MemStore memstore,
393                                MultiVersionConsistencyControl mvcc,
394                                AtomicReference<Throwable> caughtException)
395     {
396       this.mvcc = mvcc;
397       this.memstore = memstore;
398       this.caughtException = caughtException;
399       row = Bytes.toBytes(id);
400     }
401 
402     public void run() {
403       try {
404         internalRun();
405       } catch (Throwable t) {
406         caughtException.compareAndSet(null, t);
407       }
408     }
409 
410     private void internalRun() throws IOException {
411       for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
412         MultiVersionConsistencyControl.WriteEntry w =
413           mvcc.beginMemstoreInsert();
414 
415         // Insert the sequence value (i)
416         byte[] v = Bytes.toBytes(i);
417 
418         KeyValue kv = new KeyValue(row, f, q1, i, v);
419         kv.setMemstoreTS(w.getWriteNumber());
420         memstore.add(kv);
421         mvcc.completeMemstoreInsert(w);
422 
423         // Assert that we can read back
424         MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
425 
426         KeyValueScanner s = this.memstore.getScanners().get(0);
427         s.seek(kv);
428 
429         KeyValue ret = s.next();
430         assertNotNull("Didnt find own write at all", ret);
431         assertEquals("Didnt read own writes",
432                      kv.getTimestamp(), ret.getTimestamp());
433       }
434     }
435   }
436 
437   public void testReadOwnWritesUnderConcurrency() throws Throwable {
438 
439     int NUM_THREADS = 8;
440 
441     ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
442     AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
443 
444     for (int i = 0; i < NUM_THREADS; i++) {
445       threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
446       threads[i].start();
447     }
448 
449     for (int i = 0; i < NUM_THREADS; i++) {
450       threads[i].join();
451     }
452 
453     if (caught.get() != null) {
454       throw caught.get();
455     }
456   }
457 
458   /**
459    * Test memstore snapshots
460    * @throws IOException
461    */
462   public void testSnapshotting() throws IOException {
463     final int snapshotCount = 5;
464     // Add some rows, run a snapshot. Do it a few times.
465     for (int i = 0; i < snapshotCount; i++) {
466       addRows(this.memstore);
467       runSnapshot(this.memstore);
468       KeyValueSkipListSet ss = this.memstore.getSnapshot();
469       assertEquals("History not being cleared", 0, ss.size());
470     }
471   }
472 
473   public void testMultipleVersionsSimple() throws Exception {
474     MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
475     byte [] row = Bytes.toBytes("testRow");
476     byte [] family = Bytes.toBytes("testFamily");
477     byte [] qf = Bytes.toBytes("testQualifier");
478     long [] stamps = {1,2,3};
479     byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
480         Bytes.toBytes("value2")};
481     KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
482     KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
483     KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
484 
485     m.add(key0);
486     m.add(key1);
487     m.add(key2);
488 
489     assertTrue("Expected memstore to hold 3 values, actually has " +
490         m.kvset.size(), m.kvset.size() == 3);
491   }
492 
493   public void testBinary() throws IOException {
494     MemStore mc = new MemStore(new Configuration(), KeyValue.ROOT_COMPARATOR);
495     final int start = 43;
496     final int end = 46;
497     for (int k = start; k <= end; k++) {
498       byte [] kk = Bytes.toBytes(k);
499       byte [] row =
500         Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
501       KeyValue key = new KeyValue(row, CONTENTS, BASIC,
502         System.currentTimeMillis(),
503         (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
504       mc.add(key);
505       System.out.println(key);
506 //      key = new KeyValue(row, Bytes.toBytes(ANCHORNUM + k),
507 //        System.currentTimeMillis(),
508 //        (ANCHORSTR + k).getBytes(HConstants.UTF8_ENCODING));
509 //      mc.add(key);
510 //      System.out.println(key);
511     }
512     int index = start;
513     for (KeyValue kv: mc.kvset) {
514       System.out.println(kv);
515       byte [] b = kv.getRow();
516       // Hardcoded offsets into String
517       String str = Bytes.toString(b, 13, 4);
518       byte [] bb = Bytes.toBytes(index);
519       String bbStr = Bytes.toString(bb);
520       assertEquals(str, bbStr);
521       index++;
522     }
523   }
524 
525   //////////////////////////////////////////////////////////////////////////////
526   // Get tests
527   //////////////////////////////////////////////////////////////////////////////
528 
529   /** Test getNextRow from memstore
530    * @throws InterruptedException
531    */
532   public void testGetNextRow() throws Exception {
533     MultiVersionConsistencyControl.resetThreadReadPoint();
534     addRows(this.memstore);
535     // Add more versions to make it a little more interesting.
536     Thread.sleep(1);
537     addRows(this.memstore);
538     KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
539     assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
540       new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
541     for (int i = 0; i < ROW_COUNT; i++) {
542       KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
543         System.currentTimeMillis()));
544       if (i + 1 == ROW_COUNT) {
545         assertEquals(nr, null);
546       } else {
547         assertTrue(KeyValue.COMPARATOR.compareRows(nr,
548           new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
549       }
550     }
551     //starting from each row, validate results should contain the starting row
552     for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
553       InternalScanner scanner =
554           new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY,
555               Integer.MAX_VALUE, this.memstore.comparator, null,
556               memstore.getScanners());
557       List<KeyValue> results = new ArrayList<KeyValue>();
558       for (int i = 0; scanner.next(results); i++) {
559         int rowId = startRowId + i;
560         assertTrue("Row name",
561           KeyValue.COMPARATOR.compareRows(results.get(0),
562           Bytes.toBytes(rowId)) == 0);
563         assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
564         List<KeyValue> row = new ArrayList<KeyValue>();
565         for (KeyValue kv : results) {
566           row.add(kv);
567         }
568         isExpectedRowWithoutTimestamps(rowId, row);
569         // Clear out set.  Otherwise row results accumulate.
570         results.clear();
571       }
572     }
573   }
574 
575   public void testGet_memstoreAndSnapShot() throws IOException {
576     byte [] row = Bytes.toBytes("testrow");
577     byte [] fam = Bytes.toBytes("testfamily");
578     byte [] qf1 = Bytes.toBytes("testqualifier1");
579     byte [] qf2 = Bytes.toBytes("testqualifier2");
580     byte [] qf3 = Bytes.toBytes("testqualifier3");
581     byte [] qf4 = Bytes.toBytes("testqualifier4");
582     byte [] qf5 = Bytes.toBytes("testqualifier5");
583     byte [] val = Bytes.toBytes("testval");
584 
585     //Setting up memstore
586     memstore.add(new KeyValue(row, fam ,qf1, val));
587     memstore.add(new KeyValue(row, fam ,qf2, val));
588     memstore.add(new KeyValue(row, fam ,qf3, val));
589     //Creating a snapshot
590     memstore.snapshot();
591     assertEquals(3, memstore.snapshot.size());
592     //Adding value to "new" memstore
593     assertEquals(0, memstore.kvset.size());
594     memstore.add(new KeyValue(row, fam ,qf4, val));
595     memstore.add(new KeyValue(row, fam ,qf5, val));
596     assertEquals(2, memstore.kvset.size());
597   }
598 
599   //////////////////////////////////////////////////////////////////////////////
600   // Delete tests
601   //////////////////////////////////////////////////////////////////////////////
602   public void testGetWithDelete() throws IOException {
603     byte [] row = Bytes.toBytes("testrow");
604     byte [] fam = Bytes.toBytes("testfamily");
605     byte [] qf1 = Bytes.toBytes("testqualifier");
606     byte [] val = Bytes.toBytes("testval");
607 
608     long ts1 = System.nanoTime();
609     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
610     long ts2 = ts1 + 1;
611     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
612     long ts3 = ts2 +1;
613     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
614     memstore.add(put1);
615     memstore.add(put2);
616     memstore.add(put3);
617 
618     assertEquals(3, memstore.kvset.size());
619 
620     KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
621     memstore.delete(del2);
622 
623     List<KeyValue> expected = new ArrayList<KeyValue>();
624     expected.add(put3);
625     expected.add(del2);
626     expected.add(put2);
627     expected.add(put1);
628 
629     assertEquals(4, memstore.kvset.size());
630     int i = 0;
631     for(KeyValue kv : memstore.kvset) {
632       assertEquals(expected.get(i++), kv);
633     }
634   }
635 
636   public void testGetWithDeleteColumn() throws IOException {
637     byte [] row = Bytes.toBytes("testrow");
638     byte [] fam = Bytes.toBytes("testfamily");
639     byte [] qf1 = Bytes.toBytes("testqualifier");
640     byte [] val = Bytes.toBytes("testval");
641 
642     long ts1 = System.nanoTime();
643     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
644     long ts2 = ts1 + 1;
645     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
646     long ts3 = ts2 +1;
647     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
648     memstore.add(put1);
649     memstore.add(put2);
650     memstore.add(put3);
651 
652     assertEquals(3, memstore.kvset.size());
653 
654     KeyValue del2 =
655       new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
656     memstore.delete(del2);
657 
658     List<KeyValue> expected = new ArrayList<KeyValue>();
659     expected.add(put3);
660     expected.add(del2);
661     expected.add(put2);
662     expected.add(put1);
663 
664 
665     assertEquals(4, memstore.kvset.size());
666     int i = 0;
667     for (KeyValue kv: memstore.kvset) {
668       assertEquals(expected.get(i++), kv);
669     }
670   }
671 
672 
673   public void testGetWithDeleteFamily() throws IOException {
674     byte [] row = Bytes.toBytes("testrow");
675     byte [] fam = Bytes.toBytes("testfamily");
676     byte [] qf1 = Bytes.toBytes("testqualifier1");
677     byte [] qf2 = Bytes.toBytes("testqualifier2");
678     byte [] qf3 = Bytes.toBytes("testqualifier3");
679     byte [] val = Bytes.toBytes("testval");
680     long ts = System.nanoTime();
681 
682     KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
683     KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
684     KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
685     KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
686 
687     memstore.add(put1);
688     memstore.add(put2);
689     memstore.add(put3);
690     memstore.add(put4);
691 
692     KeyValue del =
693       new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
694     memstore.delete(del);
695 
696     List<KeyValue> expected = new ArrayList<KeyValue>();
697     expected.add(del);
698     expected.add(put1);
699     expected.add(put2);
700     expected.add(put4);
701     expected.add(put3);
702 
703 
704 
705     assertEquals(5, memstore.kvset.size());
706     int i = 0;
707     for (KeyValue kv: memstore.kvset) {
708       assertEquals(expected.get(i++), kv);
709     }
710   }
711 
712   public void testKeepDeleteInmemstore() {
713     byte [] row = Bytes.toBytes("testrow");
714     byte [] fam = Bytes.toBytes("testfamily");
715     byte [] qf = Bytes.toBytes("testqualifier");
716     byte [] val = Bytes.toBytes("testval");
717     long ts = System.nanoTime();
718     memstore.add(new KeyValue(row, fam, qf, ts, val));
719     KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
720     memstore.delete(delete);
721     assertEquals(2, memstore.kvset.size());
722     assertEquals(delete, memstore.kvset.first());
723   }
724 
725   public void testRetainsDeleteVersion() throws IOException {
726     // add a put to memstore
727     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
728 
729     // now process a specific delete:
730     KeyValue delete = KeyValueTestUtil.create(
731         "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
732     memstore.delete(delete);
733 
734     assertEquals(2, memstore.kvset.size());
735     assertEquals(delete, memstore.kvset.first());
736   }
737   public void testRetainsDeleteColumn() throws IOException {
738     // add a put to memstore
739     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
740 
741     // now process a specific delete:
742     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
743         KeyValue.Type.DeleteColumn, "dont-care");
744     memstore.delete(delete);
745 
746     assertEquals(2, memstore.kvset.size());
747     assertEquals(delete, memstore.kvset.first());
748   }
749   public void testRetainsDeleteFamily() throws IOException {
750     // add a put to memstore
751     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
752 
753     // now process a specific delete:
754     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
755         KeyValue.Type.DeleteFamily, "dont-care");
756     memstore.delete(delete);
757 
758     assertEquals(2, memstore.kvset.size());
759     assertEquals(delete, memstore.kvset.first());
760   }
761 
762   ////////////////////////////////////
763   //Test for timestamps
764   ////////////////////////////////////
765 
766   /**
767    * Test to ensure correctness when using Memstore with multiple timestamps
768    */
769   public void testMultipleTimestamps() throws IOException {
770     long[] timestamps = new long[] {20,10,5,1};
771     Scan scan = new Scan();
772 
773     for (long timestamp: timestamps)
774       addRows(memstore,timestamp);
775 
776     scan.setTimeRange(0, 2);
777     assertTrue(memstore.shouldSeek(scan));
778 
779     scan.setTimeRange(20, 82);
780     assertTrue(memstore.shouldSeek(scan));
781 
782     scan.setTimeRange(10, 20);
783     assertTrue(memstore.shouldSeek(scan));
784 
785     scan.setTimeRange(8, 12);
786     assertTrue(memstore.shouldSeek(scan));
787 
788     /*This test is not required for correctness but it should pass when
789      * timestamp range optimization is on*/
790     //scan.setTimeRange(28, 42);
791     //assertTrue(!memstore.shouldSeek(scan));
792   }
793 
794   ////////////////////////////////////
795   //Test for upsert with MSLAB
796   ////////////////////////////////////
797   
798   /**
799    * Test a pathological pattern that shows why we can't currently
800    * use the MSLAB for upsert workloads. This test inserts data
801    * in the following pattern:
802    * 
803    * - row0001 through row1000 (fills up one 2M Chunk)
804    * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
805    *   to the first chunk
806    * - row0003 through row1002 (another chunk, another dangling reference)
807    * 
808    * This causes OOME pretty quickly if we use MSLAB for upsert
809    * since each 2M chunk is held onto by a single reference.
810    */
811   public void testUpsertMSLAB() throws Exception {
812     Configuration conf = HBaseConfiguration.create();
813     conf.setBoolean(MemStore.USEMSLAB_KEY, true);
814     memstore = new MemStore(conf, KeyValue.COMPARATOR);
815     
816     int ROW_SIZE = 2048;
817     byte[] qualifier = new byte[ROW_SIZE - 4];
818     
819     MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
820     for (int i = 0; i < 3; i++) { System.gc(); }
821     long usageBefore = bean.getHeapMemoryUsage().getUsed();
822     
823     long size = 0;
824     long ts=0;
825     
826     for (int newValue = 0; newValue < 1000; newValue++) {
827       for (int row = newValue; row < newValue + 1000; row++) {
828         byte[] rowBytes = Bytes.toBytes(row);
829         size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
830       }
831     }
832     System.out.println("Wrote " + ts + " vals");
833     for (int i = 0; i < 3; i++) { System.gc(); }
834     long usageAfter = bean.getHeapMemoryUsage().getUsed();
835     System.out.println("Memory used: " + (usageAfter - usageBefore)
836         + " (heapsize: " + memstore.heapSize() + 
837         " size: " + size + ")");
838   }
839   
840   //////////////////////////////////////////////////////////////////////////////
841   // Helpers
842   //////////////////////////////////////////////////////////////////////////////
843   private static byte [] makeQualifier(final int i1, final int i2){
844     return Bytes.toBytes(Integer.toString(i1) + ";" +
845         Integer.toString(i2));
846   }
847 
848   /**
849    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
850    * @param hmc Instance to add rows to.
851    * @return How many rows we added.
852    * @throws IOException
853    */
854   private int addRows(final MemStore hmc) {
855     return addRows(hmc, HConstants.LATEST_TIMESTAMP);
856   }
857 
858   /**
859    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
860    * @param hmc Instance to add rows to.
861    * @return How many rows we added.
862    * @throws IOException
863    */
864   private int addRows(final MemStore hmc, final long ts) {
865     for (int i = 0; i < ROW_COUNT; i++) {
866       long timestamp = ts == HConstants.LATEST_TIMESTAMP?
867         System.currentTimeMillis(): ts;
868       for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
869         byte [] row = Bytes.toBytes(i);
870         byte [] qf = makeQualifier(i, ii);
871         hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
872       }
873     }
874     return ROW_COUNT;
875   }
876 
877   private void runSnapshot(final MemStore hmc) throws UnexpectedException {
878     // Save off old state.
879     int oldHistorySize = hmc.getSnapshot().size();
880     hmc.snapshot();
881     KeyValueSkipListSet ss = hmc.getSnapshot();
882     // Make some assertions about what just happened.
883     assertTrue("History size has not increased", oldHistorySize < ss.size());
884     hmc.clearSnapshot(ss);
885   }
886 
887   private void isExpectedRowWithoutTimestamps(final int rowIndex,
888       List<KeyValue> kvs) {
889     int i = 0;
890     for (KeyValue kv: kvs) {
891       String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
892       String colnameStr = Bytes.toString(kv.getQualifier());
893       assertEquals("Column name", colnameStr, expectedColname);
894       // Value is column name as bytes.  Usually result is
895       // 100 bytes in size at least. This is the default size
896       // for BytesWriteable.  For comparison, convert bytes to
897       // String and trim to remove trailing null bytes.
898       String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
899         kv.getValueLength());
900       assertEquals("Content", colnameStr, colvalueStr);
901     }
902   }
903 
904   private KeyValue getDeleteKV(byte [] row) {
905     return new KeyValue(row, Bytes.toBytes("test_col"), null,
906       HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
907   }
908 
909   private KeyValue getKV(byte [] row, byte [] value) {
910     return new KeyValue(row, Bytes.toBytes("test_col"), null,
911       HConstants.LATEST_TIMESTAMP, value);
912   }
913   private static void addRows(int count, final MemStore mem) {
914     long nanos = System.nanoTime();
915 
916     for (int i = 0 ; i < count ; i++) {
917       if (i % 1000 == 0) {
918 
919         System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
920         nanos = System.nanoTime();
921       }
922       long timestamp = System.currentTimeMillis();
923 
924       for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
925         byte [] row = Bytes.toBytes(i);
926         byte [] qf = makeQualifier(i, ii);
927         mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
928       }
929     }
930   }
931 
932 
933   static void doScan(MemStore ms, int iteration) throws IOException {
934     long nanos = System.nanoTime();
935     KeyValueScanner s = ms.getScanners().get(0);
936     s.seek(KeyValue.createFirstOnRow(new byte[]{}));
937 
938     System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
939     int cnt=0;
940     while(s.next() != null) ++cnt;
941 
942     System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
943 
944   }
945 
946   public static void main(String [] args) throws IOException {
947     MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
948     MemStore ms = new MemStore();
949 
950     long n1 = System.nanoTime();
951     addRows(25000, ms);
952     System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
953 
954 
955     System.out.println("foo");
956 
957     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
958 
959     for (int i = 0 ; i < 50 ; i++)
960       doScan(ms, i);
961 
962   }
963 
964 
965 }