1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.hadoop.hbase.HBaseTestCase;
26  import org.apache.hadoop.hbase.HTableDescriptor;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.client.Delete;
29  import org.apache.hadoop.hbase.client.Get;
30  import org.apache.hadoop.hbase.client.Put;
31  import org.apache.hadoop.hbase.client.Result;
32  import org.apache.hadoop.hbase.filter.TimestampsFilter;
33  import org.apache.hadoop.hbase.util.Bytes;
34  
35  /**
36   * Test Minimum Versions feature (HBASE-4071).
37   */
38  public class TestMinVersions extends HBaseTestCase {
39    private final byte[] T0 = Bytes.toBytes("0");
40    private final byte[] T1 = Bytes.toBytes("1");
41    private final byte[] T2 = Bytes.toBytes("2");
42    private final byte[] T3 = Bytes.toBytes("3");
43    private final byte[] T4 = Bytes.toBytes("4");
44    private final byte[] T5 = Bytes.toBytes("5");
45  
46    private final byte[] c0 = COLUMNS[0];
47  
48    /**
49     * Verify behavior of getClosestBefore(...)
50     */
51    public void testGetClosestBefore() throws Exception {
52      HTableDescriptor htd = createTableDescriptor(getName(), 1, 1000, 1);
53      HRegion region = createNewHRegion(htd, null, null);
54  
55      long ts = System.currentTimeMillis() - 2000; // 2s in the past
56  
57      Put p = new Put(T1, ts);
58      p.add(c0, c0, T1);
59      region.put(p);
60  
61      p = new Put(T1, ts+1);
62      p.add(c0, c0, T4);
63      region.put(p);
64  
65      p = new Put(T3, ts);
66      p.add(c0, c0, T3);
67      region.put(p);
68  
69      // now make sure that getClosestBefore(...) get can
70      // rows that would be expired without minVersion.
71      // also make sure it gets the latest version
72      Result r = region.getClosestRowBefore(T1, c0);
73      checkResult(r, c0, T4);
74  
75      r = region.getClosestRowBefore(T2, c0);
76      checkResult(r, c0, T4);
77  
78      // now flush/compact
79      region.flushcache();
80      region.compactStores(true);
81  
82      r = region.getClosestRowBefore(T1, c0);
83      checkResult(r, c0, T4);
84  
85      r = region.getClosestRowBefore(T2, c0);
86      checkResult(r, c0, T4);
87    }
88  
89    /**
90     * Test mixed memstore and storefile scanning
91     * with minimum versions.
92     */
93    public void testStoreMemStore() throws Exception {
94      // keep 3 versions minimum
95      HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1);
96      HRegion region = createNewHRegion(htd, null, null);
97  
98      long ts = System.currentTimeMillis() - 2000; // 2s in the past
99  
100     Put p = new Put(T1, ts-1);
101     p.add(c0, c0, T2);
102     region.put(p);
103 
104     p = new Put(T1, ts-3);
105     p.add(c0, c0, T0);
106     region.put(p);
107 
108     // now flush/compact
109     region.flushcache();
110     region.compactStores(true);
111 
112     p = new Put(T1, ts);
113     p.add(c0, c0, T3);
114     region.put(p);
115 
116     p = new Put(T1, ts-2);
117     p.add(c0, c0, T1);
118     region.put(p);
119 
120     p = new Put(T1, ts-3);
121     p.add(c0, c0, T0);
122     region.put(p);
123 
124     // newest version in the memstore
125     // the 2nd oldest in the store file
126     // and the 3rd, 4th oldest also in the memstore
127 
128     Get g = new Get(T1);
129     g.setMaxVersions();
130     Result r = region.get(g, null); // this'll use ScanWildcardColumnTracker
131     checkResult(r, c0, T3,T2,T1);
132 
133     g = new Get(T1);
134     g.setMaxVersions();
135     g.addColumn(c0, c0);
136     r = region.get(g, null);  // this'll use ExplicitColumnTracker
137     checkResult(r, c0, T3,T2,T1);
138   }
139 
140   /**
141    * Make sure the Deletes behave as expected with minimum versions
142    */
143   public void testDelete() throws Exception {
144     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1);
145     HRegion region = createNewHRegion(htd, null, null);
146 
147     long ts = System.currentTimeMillis() - 2000; // 2s in the past
148 
149     Put p = new Put(T1, ts-2);
150     p.add(c0, c0, T1);
151     region.put(p);
152 
153     p = new Put(T1, ts-1);
154     p.add(c0, c0, T2);
155     region.put(p);
156 
157     p = new Put(T1, ts);
158     p.add(c0, c0, T3);
159     region.put(p);
160 
161     Delete d = new Delete(T1, ts-1, null);
162     region.delete(d, null, true);
163 
164     Get g = new Get(T1);
165     g.setMaxVersions();
166     Result r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
167     checkResult(r, c0, T3);
168 
169     g = new Get(T1);
170     g.setMaxVersions();
171     g.addColumn(c0, c0);
172     r = region.get(g, null);  // this'll use ExplicitColumnTracker
173     checkResult(r, c0, T3);
174 
175     // now flush/compact
176     region.flushcache();
177     region.compactStores(true);
178 
179     // try again
180     g = new Get(T1);
181     g.setMaxVersions();
182     r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
183     checkResult(r, c0, T3);
184 
185     g = new Get(T1);
186     g.setMaxVersions();
187     g.addColumn(c0, c0);
188     r = region.get(g, null);  // this'll use ExplicitColumnTracker
189     checkResult(r, c0, T3);
190   }
191 
192   /**
193    * Make sure the memstor behaves correctly with minimum versions
194    */
195   public void testMemStore() throws Exception {
196     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1);
197     HRegion region = createNewHRegion(htd, null, null);
198 
199     long ts = System.currentTimeMillis() - 2000; // 2s in the past
200 
201     // 2nd version
202     Put p = new Put(T1, ts-2);
203     p.add(c0, c0, T2);
204     region.put(p);
205 
206     // 3rd version
207     p = new Put(T1, ts-1);
208     p.add(c0, c0, T3);
209     region.put(p);
210 
211     // 4th version
212     p = new Put(T1, ts);
213     p.add(c0, c0, T4);
214     region.put(p);
215 
216     // now flush/compact
217     region.flushcache();
218     region.compactStores(true);
219 
220     // now put the first version (backdated)
221     p = new Put(T1, ts-3);
222     p.add(c0, c0, T1);
223     region.put(p);
224 
225     // now the latest change is in the memstore,
226     // but it is not the latest version
227 
228     Result r = region.get(new Get(T1), null);
229     checkResult(r, c0, T4);
230 
231     Get g = new Get(T1);
232     g.setMaxVersions();
233     r = region.get(g, null); // this'll use ScanWildcardColumnTracker
234     checkResult(r, c0, T4,T3);
235 
236     g = new Get(T1);
237     g.setMaxVersions();
238     g.addColumn(c0, c0);
239     r = region.get(g, null);  // this'll use ExplicitColumnTracker
240     checkResult(r, c0, T4,T3);
241 
242     p = new Put(T1, ts+1);
243     p.add(c0, c0, T5);
244     region.put(p);
245 
246     // now the latest version is in the memstore
247 
248     g = new Get(T1);
249     g.setMaxVersions();
250     r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
251     checkResult(r, c0, T5,T4);
252 
253     g = new Get(T1);
254     g.setMaxVersions();
255     g.addColumn(c0, c0);
256     r = region.get(g, null);  // this'll use ExplicitColumnTracker
257     checkResult(r, c0, T5,T4);
258   }
259 
260   /**
261    * Verify basic minimum versions functionality
262    */
263   public void testBaseCase() throws Exception {
264     // 1 version minimum, 1000 versions maximum, ttl = 1s
265     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1);
266     HRegion region = createNewHRegion(htd, null, null);
267 
268     long ts = System.currentTimeMillis() - 2000; // 2s in the past
269 
270      // 1st version
271     Put p = new Put(T1, ts-3);
272     p.add(c0, c0, T1);
273     region.put(p);
274 
275     // 2nd version
276     p = new Put(T1, ts-2);
277     p.add(c0, c0, T2);
278     region.put(p);
279 
280     // 3rd version
281     p = new Put(T1, ts-1);
282     p.add(c0, c0, T3);
283     region.put(p);
284 
285     // 4th version
286     p = new Put(T1, ts);
287     p.add(c0, c0, T4);
288     region.put(p);
289 
290     Result r = region.get(new Get(T1), null);
291     checkResult(r, c0, T4);
292 
293     Get g = new Get(T1);
294     g.setTimeRange(0L, ts+1);
295     r = region.get(g, null);
296     checkResult(r, c0, T4);
297 
298     // oldest version still exists
299     g.setTimeRange(0L, ts-2);
300     r = region.get(g, null);
301     checkResult(r, c0, T1);
302 
303     // gets see only available versions
304     // even before compactions
305     g = new Get(T1);
306     g.setMaxVersions();
307     r = region.get(g, null); // this'll use ScanWildcardColumnTracker
308     checkResult(r, c0, T4,T3);
309 
310     g = new Get(T1);
311     g.setMaxVersions();
312     g.addColumn(c0, c0);
313     r = region.get(g, null);  // this'll use ExplicitColumnTracker
314     checkResult(r, c0, T4,T3);
315 
316     // now flush
317     region.flushcache();
318 
319     // with HBASE-4241 a flush will eliminate the expired rows
320     g = new Get(T1);
321     g.setTimeRange(0L, ts-2);
322     r = region.get(g, null);
323     assertTrue(r.isEmpty());
324 
325     // major compaction
326     region.compactStores(true);
327 
328     // after compaction the 4th version is still available
329     g = new Get(T1);
330     g.setTimeRange(0L, ts+1);
331     r = region.get(g, null);
332     checkResult(r, c0, T4);
333 
334     // so is the 3rd
335     g.setTimeRange(0L, ts);
336     r = region.get(g, null);
337     checkResult(r, c0, T3);
338 
339     // but the 2nd and earlier versions are gone
340     g.setTimeRange(0L, ts-1);
341     r = region.get(g, null);
342     assertTrue(r.isEmpty());
343   }
344 
345   /**
346    * Verify that basic filters still behave correctly with
347    * minimum versions enabled.
348    */
349   public void testFilters() throws Exception {
350     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1);
351     HRegion region = createNewHRegion(htd, null, null);
352     final byte [] c1 = COLUMNS[1];
353 
354     long ts = System.currentTimeMillis() - 2000; // 2s in the past
355 
356     Put p = new Put(T1, ts-3);
357     p.add(c0, c0, T0);
358     p.add(c1, c1, T0);
359     region.put(p);
360 
361     p = new Put(T1, ts-2);
362     p.add(c0, c0, T1);
363     p.add(c1, c1, T1);
364     region.put(p);
365 
366     p = new Put(T1, ts-1);
367     p.add(c0, c0, T2);
368     p.add(c1, c1, T2);
369     region.put(p);
370 
371     p = new Put(T1, ts);
372     p.add(c0, c0, T3);
373     p.add(c1, c1, T3);
374     region.put(p);
375 
376     List<Long> tss = new ArrayList<Long>();
377     tss.add(ts-1);
378     tss.add(ts-2);
379 
380     Get g = new Get(T1);
381     g.addColumn(c1,c1);
382     g.setFilter(new TimestampsFilter(tss));
383     g.setMaxVersions();
384     Result r = region.get(g, null);
385     checkResult(r, c1, T2,T1);
386 
387     g = new Get(T1);
388     g.addColumn(c0,c0);
389     g.setFilter(new TimestampsFilter(tss));
390     g.setMaxVersions();
391     r = region.get(g, null);
392     checkResult(r, c0, T2,T1);
393 
394     // now flush/compact
395     region.flushcache();
396     region.compactStores(true);
397 
398     g = new Get(T1);
399     g.addColumn(c1,c1);
400     g.setFilter(new TimestampsFilter(tss));
401     g.setMaxVersions();
402     r = region.get(g, null);
403     checkResult(r, c1, T2);
404 
405     g = new Get(T1);
406     g.addColumn(c0,c0);
407     g.setFilter(new TimestampsFilter(tss));
408     g.setMaxVersions();
409     r = region.get(g, null);
410     checkResult(r, c0, T2);
411 }
412 
413   private void checkResult(Result r, byte[] col, byte[] ... vals) {
414     assertEquals(r.size(), vals.length);
415     List<KeyValue> kvs = r.getColumn(col, col);
416     assertEquals(kvs.size(), vals.length);
417     for (int i=0;i<vals.length;i++) {
418       assertEquals(kvs.get(i).getValue(), vals[i]);
419     }
420   }
421 }