1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.regionserver;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HBaseTestCase;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.client.Delete;
39  import org.apache.hadoop.hbase.client.Get;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.io.hfile.BlockCache;
43  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
46  import org.junit.Test;
47  
48  public class TestBlocksRead extends HBaseTestCase {
49    static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
50  
51    private static BlockCache blockCache;
52  
53    private HBaseConfiguration getConf() {
54      HBaseConfiguration conf = new HBaseConfiguration();
55  
56      // disable compactions in this test.
57      conf.setInt("hbase.hstore.compactionThreshold", 10000);
58      return conf;
59    }
60  
61    HRegion region = null;
62    private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63    private final String DIR = TEST_UTIL.getDataTestDir("TestBlocksRead").toString();
64  
65    /**
66     * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
67     */
68    @SuppressWarnings("deprecation")
69    @Override
70    protected void setUp() throws Exception {
71      super.setUp();
72    }
73  
74    @SuppressWarnings("deprecation")
75    @Override
76    protected void tearDown() throws Exception {
77      super.tearDown();
78      EnvironmentEdgeManagerTestHelper.reset();
79    }
80  
81    private void initHRegion (byte [] tableName, String callingMethod,
82        HBaseConfiguration conf, byte [] ... families)
83      throws IOException {
84      HTableDescriptor htd = new HTableDescriptor(tableName);
85      for(byte [] family : families) {
86          HColumnDescriptor familyDesc =  new HColumnDescriptor(
87            family,
88            HColumnDescriptor.DEFAULT_VERSIONS,
89            HColumnDescriptor.DEFAULT_COMPRESSION,
90            HColumnDescriptor.DEFAULT_IN_MEMORY,
91            HColumnDescriptor.DEFAULT_BLOCKCACHE,
92            1, // small block size deliberate; each kv on its own block
93            HColumnDescriptor.DEFAULT_TTL,
94            HColumnDescriptor.DEFAULT_BLOOMFILTER,
95            HColumnDescriptor.DEFAULT_REPLICATION_SCOPE);
96        htd.addFamily(familyDesc);
97      }
98      HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
99      Path path = new Path(DIR + callingMethod);
100     region = HRegion.createHRegion(info, path, conf, htd);
101     blockCache = new CacheConfig(conf).getBlockCache();
102   }
103 
104   private void putData(byte[] cf, String row, String col, long version)
105   throws IOException {
106     putData(cf, row, col, version, version);
107   }
108 
109   // generates a value to put for a row/col/version.
110   private static byte[] genValue(String row, String col, long version) {
111     return Bytes.toBytes("Value:" + row + "#" + col + "#" + version);
112   }
113 
114   private void putData(byte[] cf, String row, String col,
115                        long versionStart, long versionEnd)
116   throws IOException {
117     byte columnBytes[] = Bytes.toBytes(col);
118     Put put = new Put(Bytes.toBytes(row));
119     put.setWriteToWAL(false);
120 
121     for (long version = versionStart; version <= versionEnd; version++) {
122       put.add(cf, columnBytes, version, genValue(row, col, version));
123     }
124     region.put(put);
125   }
126 
127   private KeyValue[] getData(byte[] cf, String row, List<String> columns,
128                              int expBlocks)
129     throws IOException {
130 
131     long blocksStart = getBlkAccessCount(cf);
132     Get get = new Get(Bytes.toBytes(row));
133 
134     for (String column : columns) {
135       get.addColumn(cf, Bytes.toBytes(column));
136     }
137 
138     KeyValue[] kvs = region.get(get, null).raw();
139     long blocksEnd = getBlkAccessCount(cf);
140     if (expBlocks != -1) {
141       assertEquals("Blocks Read Check", expBlocks, blocksEnd - blocksStart);
142     }
143     System.out.println("Blocks Read = " + (blocksEnd - blocksStart) +
144                        "Expected = " + expBlocks);
145     return kvs;
146   }
147 
148   private KeyValue[] getData(byte[] cf, String row, String column,
149                              int expBlocks)
150     throws IOException {
151     return getData(cf, row, Arrays.asList(column), expBlocks);
152   }
153 
154   private void deleteFamily(byte[] cf, String row, long version)
155     throws IOException {
156     Delete del = new Delete(Bytes.toBytes(row));
157     del.deleteFamily(cf, version);
158     region.delete(del, null, true);
159   }
160 
161   private void deleteFamily(byte[] cf, String row, String column, long version)
162     throws IOException {
163     Delete del = new Delete(Bytes.toBytes(row));
164     del.deleteColumns(cf, Bytes.toBytes(column), version);
165     region.delete(del, null, true);
166   }
167 
168   private static void verifyData(KeyValue kv, String expectedRow,
169                                  String expectedCol, long expectedVersion) {
170     assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow()));
171     assertEquals("ColumnCheck", expectedCol, Bytes.toString(kv.getQualifier()));
172     assertEquals("TSCheck", expectedVersion, kv.getTimestamp());
173     assertEquals("ValueCheck",
174                  Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)),
175                  Bytes.toString(kv.getValue()));
176   }
177 
178   private static long getBlkAccessCount(byte[] cf) {
179       return blockCache.getStats().getRequestCount();
180   }
181 
182   private static long getBlkCount() {
183     return blockCache.getBlockCount();
184   }
185 
186   /**
187    * Test # of blocks read for some simple seek cases.
188    * @throws Exception
189    */
190   @Test
191   public void testBlocksRead() throws Exception {
192     byte [] TABLE = Bytes.toBytes("testBlocksRead");
193     byte [] FAMILY = Bytes.toBytes("cf1");
194     byte [][] FAMILIES = new byte[][] { FAMILY };
195     KeyValue kvs[];
196 
197     HBaseConfiguration conf = getConf();
198     initHRegion(TABLE, getName(), conf, FAMILIES);
199 
200     putData(FAMILY, "row", "col1", 1);
201     putData(FAMILY, "row", "col2", 2);
202     putData(FAMILY, "row", "col3", 3);
203     putData(FAMILY, "row", "col4", 4);
204     putData(FAMILY, "row", "col5", 5);
205     putData(FAMILY, "row", "col6", 6);
206     putData(FAMILY, "row", "col7", 7);
207     region.flushcache();
208 
209     // Expected block reads: 1
210     kvs = getData(FAMILY, "row", "col1", 1);
211     assertEquals(1, kvs.length);
212     verifyData(kvs[0], "row", "col1", 1);
213 
214     // Expected block reads: 2
215     kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
216     assertEquals(2, kvs.length);
217     verifyData(kvs[0], "row", "col1", 1);
218     verifyData(kvs[1], "row", "col2", 2);
219 
220     // Expected block reads: 3
221     kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3);
222     assertEquals(2, kvs.length);
223     verifyData(kvs[0], "row", "col2", 2);
224     verifyData(kvs[1], "row", "col3", 3);
225 
226     // Expected block reads: 3
227     kvs = getData(FAMILY, "row", Arrays.asList("col5"), 3);
228     assertEquals(1, kvs.length);
229     verifyData(kvs[0], "row", "col5", 5);
230   }
231 
232   /**
233    * Test # of blocks read (targetted at some of the cases Lazy Seek optimizes).
234    * @throws Exception
235    */
236   @Test
237   public void testLazySeekBlocksRead() throws Exception {
238     byte [] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
239     byte [] FAMILY = Bytes.toBytes("cf1");
240     byte [][] FAMILIES = new byte[][] { FAMILY };
241     KeyValue kvs[];
242 
243     HBaseConfiguration conf = getConf();
244     initHRegion(TABLE, getName(), conf, FAMILIES);
245 
246     // File 1
247     putData(FAMILY, "row", "col1", 1);
248     putData(FAMILY, "row", "col2", 2);
249     region.flushcache();
250 
251     // File 2
252     putData(FAMILY, "row", "col1", 3);
253     putData(FAMILY, "row", "col2", 4);
254     region.flushcache();
255 
256     // Baseline expected blocks read: 2
257     kvs = getData(FAMILY, "row", Arrays.asList("col1"), 2);
258     assertEquals(1, kvs.length);
259     verifyData(kvs[0], "row", "col1", 3);
260 
261     // Baseline expected blocks read: 4
262     kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 4);
263     assertEquals(2, kvs.length);
264     verifyData(kvs[0], "row", "col1", 3);
265     verifyData(kvs[1], "row", "col2", 4);
266 
267     // File 3: Add another column
268     putData(FAMILY, "row", "col3", 5);
269     region.flushcache();
270 
271     // Baseline expected blocks read: 5
272     kvs = getData(FAMILY, "row", "col3", 5);
273     assertEquals(1, kvs.length);
274     verifyData(kvs[0], "row", "col3", 5);
275 
276     // Get a column from older file.
277     // Baseline expected blocks read: 3
278     kvs = getData(FAMILY, "row", Arrays.asList("col1"), 3);
279     assertEquals(1, kvs.length);
280     verifyData(kvs[0], "row", "col1", 3);
281 
282     // File 4: Delete the entire row.
283     deleteFamily(FAMILY, "row", 6);
284     region.flushcache();
285 
286     // Baseline expected blocks read: 6.
287     kvs = getData(FAMILY, "row", "col1", 6);
288     assertEquals(0, kvs.length);
289     kvs = getData(FAMILY, "row", "col2", 6);
290     assertEquals(0, kvs.length);
291     kvs = getData(FAMILY, "row", "col3", 6);
292     assertEquals(0, kvs.length);
293     kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 6);
294     assertEquals(0, kvs.length);
295 
296     // File 5: Delete
297     deleteFamily(FAMILY, "row", 10);
298     region.flushcache();
299 
300     // File 6: some more puts, but with timestamps older than the
301     // previous delete.
302     putData(FAMILY, "row", "col1", 7);
303     putData(FAMILY, "row", "col2", 8);
304     putData(FAMILY, "row", "col3", 9);
305     region.flushcache();
306 
307     // Baseline expected blocks read: 10
308     kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 10);
309     assertEquals(0, kvs.length);
310 
311     // File 7: Put back new data
312     putData(FAMILY, "row", "col1", 11);
313     putData(FAMILY, "row", "col2", 12);
314     putData(FAMILY, "row", "col3", 13);
315     region.flushcache();
316 
317     // Baseline expected blocks read: 13
318     kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 13);
319     assertEquals(3, kvs.length);
320     verifyData(kvs[0], "row", "col1", 11);
321     verifyData(kvs[1], "row", "col2", 12);
322     verifyData(kvs[2], "row", "col3", 13);
323   }
324 
325   /**
326    * Test # of blocks read to ensure disabling cache-fill on Scan works.
327    * @throws Exception
328    */
329   @Test
330   public void testBlocksStoredWhenCachingDisabled() throws Exception {
331     byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled");
332     byte [] FAMILY = Bytes.toBytes("cf1");
333     byte [][] FAMILIES = new byte[][] { FAMILY };
334 
335     HBaseConfiguration conf = getConf();
336     initHRegion(TABLE, getName(), conf, FAMILIES);
337 
338     putData(FAMILY, "row", "col1", 1);
339     putData(FAMILY, "row", "col2", 2);
340     region.flushcache();
341 
342     // Execute a scan with caching turned off
343     // Expected blocks stored: 0
344     long blocksStart = getBlkCount();
345     Scan scan = new Scan();
346     scan.setCacheBlocks(false);
347     RegionScanner rs = region.getScanner(scan);
348     List<KeyValue> result = new ArrayList<KeyValue>(2);
349     rs.next(result);
350     assertEquals(2, result.size());
351     rs.close();
352     long blocksEnd = getBlkCount();
353 
354     assertEquals(blocksStart, blocksEnd);
355 
356     // Execute with caching turned on
357     // Expected blocks stored: 2
358     blocksStart = blocksEnd;
359     scan.setCacheBlocks(true);
360     rs = region.getScanner(scan);
361     result = new ArrayList<KeyValue>(2);
362     rs.next(result);
363     assertEquals(2, result.size());
364     rs.close();
365     blocksEnd = getBlkCount();
366     
367     assertEquals(2, blocksEnd - blocksStart);
368   }
369 }