1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.*;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.junit.After;
35  import org.junit.AfterClass;
36  import org.junit.Before;
37  import org.junit.BeforeClass;
38  import org.junit.Test;
39  
40  /**
41   * Run tests related to {@link TimestampsFilter} using HBase client APIs.
42   * Sets up the HBase mini cluster once at start. Each creates a table
43   * named for the method and does its stuff against that.
44   */
45  public class TestMultipleTimestamps {
46    final Log LOG = LogFactory.getLog(getClass());
47    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
48  
49    /**
50     * @throws java.lang.Exception
51     */
52    @BeforeClass
53    public static void setUpBeforeClass() throws Exception {
54      TEST_UTIL.startMiniCluster();
55    }
56  
57    /**
58     * @throws java.lang.Exception
59     */
60    @AfterClass
61    public static void tearDownAfterClass() throws Exception {
62      TEST_UTIL.shutdownMiniCluster();
63    }
64  
65    /**
66     * @throws java.lang.Exception
67     */
68    @Before
69    public void setUp() throws Exception {
70      // Nothing to do.
71    }
72  
73    /**
74     * @throws java.lang.Exception
75     */
76    @After
77    public void tearDown() throws Exception {
78      // Nothing to do.
79    }
80  
81    @Test
82    public void testReseeksWithOneColumnMiltipleTimestamp() throws IOException {
83      byte [] TABLE = Bytes.toBytes("testReseeksWithOne" +
84      "ColumnMiltipleTimestamps");
85      byte [] FAMILY = Bytes.toBytes("event_log");
86      byte [][] FAMILIES = new byte[][] { FAMILY };
87  
88      // create table; set versions to max...
89      HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
90  
91      Integer[] putRows = new Integer[] {1, 3, 5, 7};
92      Integer[] putColumns = new Integer[] { 1, 3, 5};
93      Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
94  
95      Integer[] scanRows = new Integer[] {3, 5};
96      Integer[] scanColumns = new Integer[] {3};
97      Long[] scanTimestamps = new Long[] {3L, 4L};
98      int scanMaxVersions = 2;
99  
100     put(ht, FAMILY, putRows, putColumns, putTimestamps);
101 
102     flush(TABLE);
103 
104     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
105         scanTimestamps, scanMaxVersions);
106 
107     KeyValue[] kvs;
108 
109     kvs = scanner.next().raw();
110     assertEquals(2, kvs.length);
111     checkOneCell(kvs[0], FAMILY, 3, 3, 4);
112     checkOneCell(kvs[1], FAMILY, 3, 3, 3);
113     kvs = scanner.next().raw();
114     assertEquals(2, kvs.length);
115     checkOneCell(kvs[0], FAMILY, 5, 3, 4);
116     checkOneCell(kvs[1], FAMILY, 5, 3, 3);
117   }
118 
119   @Test
120   public void testReseeksWithMultipleColumnOneTimestamp() throws IOException {
121     LOG.info("testReseeksWithMultipleColumnOneTimestamp");
122     byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" +
123     "ColumnOneTimestamps");
124     byte [] FAMILY = Bytes.toBytes("event_log");
125     byte [][] FAMILIES = new byte[][] { FAMILY };
126 
127     // create table; set versions to max...
128     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
129 
130     Integer[] putRows = new Integer[] {1, 3, 5, 7};
131     Integer[] putColumns = new Integer[] { 1, 3, 5};
132     Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
133 
134     Integer[] scanRows = new Integer[] {3, 5};
135     Integer[] scanColumns = new Integer[] {3,4};
136     Long[] scanTimestamps = new Long[] {3L};
137     int scanMaxVersions = 2;
138 
139     put(ht, FAMILY, putRows, putColumns, putTimestamps);
140 
141     flush(TABLE);
142 
143     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
144         scanTimestamps, scanMaxVersions);
145 
146     KeyValue[] kvs;
147 
148     kvs = scanner.next().raw();
149     assertEquals(1, kvs.length);
150     checkOneCell(kvs[0], FAMILY, 3, 3, 3);
151     kvs = scanner.next().raw();
152     assertEquals(1, kvs.length);
153     checkOneCell(kvs[0], FAMILY, 5, 3, 3);
154   }
155 
156   @Test
157   public void testReseeksWithMultipleColumnMultipleTimestamp() throws
158   IOException {
159     LOG.info("testReseeksWithMultipleColumnMultipleTimestamp");
160 
161     byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" +
162     "ColumnMiltipleTimestamps");
163     byte [] FAMILY = Bytes.toBytes("event_log");
164     byte [][] FAMILIES = new byte[][] { FAMILY };
165 
166     // create table; set versions to max...
167     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
168 
169     Integer[] putRows = new Integer[] {1, 3, 5, 7};
170     Integer[] putColumns = new Integer[] { 1, 3, 5};
171     Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
172 
173     Integer[] scanRows = new Integer[] {5, 7};
174     Integer[] scanColumns = new Integer[] {3, 4, 5};
175     Long[] scanTimestamps = new Long[] {2l, 3L};
176     int scanMaxVersions = 2;
177 
178     put(ht, FAMILY, putRows, putColumns, putTimestamps);
179 
180     flush(TABLE);
181 
182     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
183         scanTimestamps, scanMaxVersions);
184 
185     KeyValue[] kvs;
186 
187     kvs = scanner.next().raw();
188     assertEquals(4, kvs.length);
189     checkOneCell(kvs[0], FAMILY, 5, 3, 3);
190     checkOneCell(kvs[1], FAMILY, 5, 3, 2);
191     checkOneCell(kvs[2], FAMILY, 5, 5, 3);
192     checkOneCell(kvs[3], FAMILY, 5, 5, 2);
193     kvs = scanner.next().raw();
194     assertEquals(4, kvs.length);
195     checkOneCell(kvs[0], FAMILY, 7, 3, 3);
196     checkOneCell(kvs[1], FAMILY, 7, 3, 2);
197     checkOneCell(kvs[2], FAMILY, 7, 5, 3);
198     checkOneCell(kvs[3], FAMILY, 7, 5, 2);
199   }
200 
201   @Test
202   public void testReseeksWithMultipleFiles() throws IOException {
203     LOG.info("testReseeksWithMultipleFiles");
204     byte [] TABLE = Bytes.toBytes("testReseeksWithMultipleFiles");
205     byte [] FAMILY = Bytes.toBytes("event_log");
206     byte [][] FAMILIES = new byte[][] { FAMILY };
207 
208     // create table; set versions to max...
209     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
210 
211     Integer[] putRows1 = new Integer[] {1, 2, 3};
212     Integer[] putColumns1 = new Integer[] { 2, 5, 6};
213     Long[] putTimestamps1 = new Long[] {1L, 2L, 5L};
214 
215     Integer[] putRows2 = new Integer[] {6, 7};
216     Integer[] putColumns2 = new Integer[] {3, 6};
217     Long[] putTimestamps2 = new Long[] {4L, 5L};
218 
219     Integer[] putRows3 = new Integer[] {2, 3, 5};
220     Integer[] putColumns3 = new Integer[] {1, 2, 3};
221     Long[] putTimestamps3 = new Long[] {4L,8L};
222 
223 
224     Integer[] scanRows = new Integer[] {3, 5, 7};
225     Integer[] scanColumns = new Integer[] {3, 4, 5};
226     Long[] scanTimestamps = new Long[] {2l, 4L};
227     int scanMaxVersions = 5;
228 
229     put(ht, FAMILY, putRows1, putColumns1, putTimestamps1);
230     flush(TABLE);
231     put(ht, FAMILY, putRows2, putColumns2, putTimestamps2);
232     flush(TABLE);
233     put(ht, FAMILY, putRows3, putColumns3, putTimestamps3);
234 
235     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
236         scanTimestamps, scanMaxVersions);
237 
238     KeyValue[] kvs;
239 
240     kvs = scanner.next().raw();
241     assertEquals(2, kvs.length);
242     checkOneCell(kvs[0], FAMILY, 3, 3, 4);
243     checkOneCell(kvs[1], FAMILY, 3, 5, 2);
244 
245     kvs = scanner.next().raw();
246     assertEquals(1, kvs.length);
247     checkOneCell(kvs[0], FAMILY, 5, 3, 4);
248 
249     kvs = scanner.next().raw();
250     assertEquals(1, kvs.length);
251     checkOneCell(kvs[0], FAMILY, 6, 3, 4);
252 
253     kvs = scanner.next().raw();
254     assertEquals(1, kvs.length);
255     checkOneCell(kvs[0], FAMILY, 7, 3, 4);
256   }
257 
258   @Test
259   public void testWithVersionDeletes() throws Exception {
260 
261     // first test from memstore (without flushing).
262     testWithVersionDeletes(false);
263 
264     // run same test against HFiles (by forcing a flush).
265     testWithVersionDeletes(true);
266   }
267 
268   public void testWithVersionDeletes(boolean flushTables) throws IOException {
269     LOG.info("testWithVersionDeletes_"+
270         (flushTables ? "flush" : "noflush"));
271 
272     byte [] TABLE = Bytes.toBytes("testWithVersionDeletes_" +
273         (flushTables ? "flush" : "noflush"));
274 
275     byte [] FAMILY = Bytes.toBytes("event_log");
276     byte [][] FAMILIES = new byte[][] { FAMILY };
277 
278     // create table; set versions to max...
279     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
280 
281     // For row:0, col:0: insert versions 1 through 5.
282     putNVersions(ht, FAMILY, 0, 0, 1, 5);
283 
284     if (flushTables) {
285       flush(TABLE);
286     }
287 
288     // delete version 4.
289     deleteOneVersion(ht, FAMILY, 0, 0, 4);
290 
291     // request a bunch of versions including the deleted version. We should
292     // only get back entries for the versions that exist.
293     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0,
294         Arrays.asList(2L, 3L, 4L, 5L));
295     assertEquals(3, kvs.length);
296     checkOneCell(kvs[0], FAMILY, 0, 0, 5);
297     checkOneCell(kvs[1], FAMILY, 0, 0, 3);
298     checkOneCell(kvs[2], FAMILY, 0, 0, 2);
299   }
300 
301   @Test
302   public void testWithMultipleVersionDeletes() throws IOException {
303     LOG.info("testWithMultipleVersionDeletes");
304 
305     byte [] TABLE = Bytes.toBytes("testWithMultipleVersionDeletes");
306     byte [] FAMILY = Bytes.toBytes("event_log");
307     byte [][] FAMILIES = new byte[][] { FAMILY };
308 
309     // create table; set versions to max...
310     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
311 
312     // For row:0, col:0: insert versions 1 through 5.
313     putNVersions(ht, FAMILY, 0, 0, 1, 5);
314 
315     flush(TABLE);
316 
317     // delete all versions before 4.
318     deleteAllVersionsBefore(ht, FAMILY, 0, 0, 4);
319 
320     // request a bunch of versions including the deleted version. We should
321     // only get back entries for the versions that exist.
322     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
323     assertEquals(0, kvs.length);
324   }
325 
326   @Test
327   public void testWithColumnDeletes() throws IOException {
328     byte [] TABLE = Bytes.toBytes("testWithColumnDeletes");
329     byte [] FAMILY = Bytes.toBytes("event_log");
330     byte [][] FAMILIES = new byte[][] { FAMILY };
331 
332     // create table; set versions to max...
333     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
334 
335     // For row:0, col:0: insert versions 1 through 5.
336     putNVersions(ht, FAMILY, 0, 0, 1, 5);
337 
338     flush(TABLE);
339 
340     // delete all versions before 4.
341     deleteColumn(ht, FAMILY, 0, 0);
342 
343     // request a bunch of versions including the deleted version. We should
344     // only get back entries for the versions that exist.
345     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
346     assertEquals(0, kvs.length);
347   }
348 
349   @Test
350   public void testWithFamilyDeletes() throws IOException {
351     byte [] TABLE = Bytes.toBytes("testWithFamilyDeletes");
352     byte [] FAMILY = Bytes.toBytes("event_log");
353     byte [][] FAMILIES = new byte[][] { FAMILY };
354 
355     // create table; set versions to max...
356     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
357 
358     // For row:0, col:0: insert versions 1 through 5.
359     putNVersions(ht, FAMILY, 0, 0, 1, 5);
360 
361     flush(TABLE);
362 
363     // delete all versions before 4.
364     deleteFamily(ht, FAMILY, 0);
365 
366     // request a bunch of versions including the deleted version. We should
367     // only get back entries for the versions that exist.
368     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
369     assertEquals(0, kvs.length);
370   }
371 
372   // Flush tables. Since flushing is asynchronous, sleep for a bit.
373   private void flush(byte [] tableName) throws IOException {
374     TEST_UTIL.flush(tableName);
375     try {
376       Thread.sleep(3000);
377     } catch (InterruptedException i) {
378       // ignore
379     }
380   }
381 
382   /**
383    * Assert that the passed in KeyValue has expected contents for the
384    * specified row, column & timestamp.
385    */
386   private void checkOneCell(KeyValue kv, byte[] cf,
387       int rowIdx, int colIdx, long ts) {
388 
389     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
390 
391     assertEquals("Row mismatch which checking: " + ctx,
392         "row:"+ rowIdx, Bytes.toString(kv.getRow()));
393 
394     assertEquals("ColumnFamily mismatch while checking: " + ctx,
395         Bytes.toString(cf), Bytes.toString(kv.getFamily()));
396 
397     assertEquals("Column qualifier mismatch while checking: " + ctx,
398         "column:" + colIdx,
399         Bytes.toString(kv.getQualifier()));
400 
401     assertEquals("Timestamp mismatch while checking: " + ctx,
402         ts, kv.getTimestamp());
403 
404     assertEquals("Value mismatch while checking: " + ctx,
405         "value-version-" + ts, Bytes.toString(kv.getValue()));
406   }
407 
408   /**
409    * Uses the TimestampFilter on a Get to request a specified list of
410    * versions for the row/column specified by rowIdx & colIdx.
411    *
412    */
413   private  KeyValue[] getNVersions(HTable ht, byte[] cf, int rowIdx,
414       int colIdx, List<Long> versions)
415   throws IOException {
416     byte row[] = Bytes.toBytes("row:" + rowIdx);
417     byte column[] = Bytes.toBytes("column:" + colIdx);
418     Get get = new Get(row);
419     get.addColumn(cf, column);
420     get.setMaxVersions();
421     get.setTimeRange(Collections.min(versions), Collections.max(versions)+1);
422     Result result = ht.get(get);
423 
424     return result.raw();
425   }
426 
427   private  ResultScanner scan(HTable ht, byte[] cf,
428       Integer[] rowIndexes, Integer[] columnIndexes,
429       Long[] versions, int maxVersions)
430   throws IOException {
431     Arrays.asList(rowIndexes);
432     byte startRow[] = Bytes.toBytes("row:" +
433         Collections.min( Arrays.asList(rowIndexes)));
434     byte endRow[] = Bytes.toBytes("row:" +
435         Collections.max( Arrays.asList(rowIndexes))+1);
436     Scan scan = new Scan(startRow, endRow);
437     for (Integer colIdx: columnIndexes) {
438       byte column[] = Bytes.toBytes("column:" + colIdx);
439       scan.addColumn(cf, column);
440     }
441     scan.setMaxVersions(maxVersions);
442     scan.setTimeRange(Collections.min(Arrays.asList(versions)),
443         Collections.max(Arrays.asList(versions))+1);
444     ResultScanner scanner = ht.getScanner(scan);
445     return scanner;
446   }
447 
448   private void put(HTable ht, byte[] cf, Integer[] rowIndexes,
449       Integer[] columnIndexes, Long[] versions)
450   throws IOException {
451     for (int rowIdx: rowIndexes) {
452       byte row[] = Bytes.toBytes("row:" + rowIdx);
453       Put put = new Put(row);
454       put.setWriteToWAL(false);
455       for(int colIdx: columnIndexes) {
456         byte column[] = Bytes.toBytes("column:" + colIdx);
457         for (long version: versions) {
458           put.add(cf, column, version, Bytes.toBytes("value-version-" +
459               version));
460         }
461       }
462       ht.put(put);
463     }
464   }
465 
466   /**
467    * Insert in specific row/column versions with timestamps
468    * versionStart..versionEnd.
469    */
470   private void putNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx,
471       long versionStart, long versionEnd)
472   throws IOException {
473     byte row[] = Bytes.toBytes("row:" + rowIdx);
474     byte column[] = Bytes.toBytes("column:" + colIdx);
475     Put put = new Put(row);
476     put.setWriteToWAL(false);
477 
478     for (long idx = versionStart; idx <= versionEnd; idx++) {
479       put.add(cf, column, idx, Bytes.toBytes("value-version-" + idx));
480     }
481 
482     ht.put(put);
483   }
484 
485   /**
486    * For row/column specified by rowIdx/colIdx, delete the cell
487    * corresponding to the specified version.
488    */
489   private void deleteOneVersion(HTable ht, byte[] cf, int rowIdx,
490       int colIdx, long version)
491   throws IOException {
492     byte row[] = Bytes.toBytes("row:" + rowIdx);
493     byte column[] = Bytes.toBytes("column:" + colIdx);
494     Delete del = new Delete(row);
495     del.deleteColumn(cf, column, version);
496     ht.delete(del);
497   }
498 
499   /**
500    * For row/column specified by rowIdx/colIdx, delete all cells
501    * preceeding the specified version.
502    */
503   private void deleteAllVersionsBefore(HTable ht, byte[] cf, int rowIdx,
504       int colIdx, long version)
505   throws IOException {
506     byte row[] = Bytes.toBytes("row:" + rowIdx);
507     byte column[] = Bytes.toBytes("column:" + colIdx);
508     Delete del = new Delete(row);
509     del.deleteColumns(cf, column, version);
510     ht.delete(del);
511   }
512 
513   private void deleteColumn(HTable ht, byte[] cf, int rowIdx, int colIdx) throws IOException {
514     byte row[] = Bytes.toBytes("row:" + rowIdx);
515     byte column[] = Bytes.toBytes("column:" + colIdx);
516     Delete del = new Delete(row);
517     del.deleteColumns(cf, column);
518     ht.delete(del);
519   }
520 
521   private void deleteFamily(HTable ht, byte[] cf, int rowIdx) throws IOException {
522     byte row[] = Bytes.toBytes("row:" + rowIdx);
523     Delete del = new Delete(row);
524     del.deleteFamily(cf);
525     ht.delete(del);
526   }
527 }
528