1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.regionserver;
22  
23  import static org.junit.Assert.*;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.HashMap;
29  import java.util.HashSet;
30  import java.util.List;
31  import java.util.Set;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.KeyValueTestUtil;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.junit.Test;
45  
46  public class TestColumnSeeking {
47  
48    private final static HBaseTestingUtility TEST_UTIL =
49        new HBaseTestingUtility();
50  
51    static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
52  
53    @SuppressWarnings("unchecked")
54    @Test
55    public void testDuplicateVersions() throws IOException {
56      String family = "Family";
57      byte[] familyBytes = Bytes.toBytes("Family");
58      String table = "TestDuplicateVersions";
59  
60      HColumnDescriptor hcd =
61          new HColumnDescriptor(familyBytes, 1000,
62              HColumnDescriptor.DEFAULT_COMPRESSION,
63              HColumnDescriptor.DEFAULT_IN_MEMORY,
64              HColumnDescriptor.DEFAULT_BLOCKCACHE,
65              HColumnDescriptor.DEFAULT_TTL,
66              HColumnDescriptor.DEFAULT_BLOOMFILTER);
67      HTableDescriptor htd = new HTableDescriptor(table);
68      htd.addFamily(hcd);
69      HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
70      HRegion region =
71          HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
72              .getConfiguration(), htd);
73  
74      List<String> rows = generateRandomWords(10, "row");
75      List<String> allColumns = generateRandomWords(10, "column");
76      List<String> values = generateRandomWords(100, "value");
77  
78      long maxTimestamp = 2;
79      double selectPercent = 0.5;
80      int numberOfTests = 5;
81      double flushPercentage = 0.2;
82      double minorPercentage = 0.2;
83      double majorPercentage = 0.2;
84      double putPercentage = 0.2;
85  
86      HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
87  
88      HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
89      ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
90  
91      for (int i = 0; i < numberOfTests; i++) {
92        kvMaps[i] = new HashMap<String, KeyValue>();
93        columnLists[i] = new ArrayList<String>();
94        for (String column : allColumns) {
95          if (Math.random() < selectPercent) {
96            columnLists[i].add(column);
97          }
98        }
99      }
100 
101     for (String value : values) {
102       for (String row : rows) {
103         Put p = new Put(Bytes.toBytes(row));
104         p.setWriteToWAL(false);
105         for (String column : allColumns) {
106           for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
107             KeyValue kv =
108                 KeyValueTestUtil.create(row, family, column, timestamp, value);
109             if (Math.random() < putPercentage) {
110               p.add(kv);
111               allKVMap.put(kv.getKeyString(), kv);
112               for (int i = 0; i < numberOfTests; i++) {
113                 if (columnLists[i].contains(column)) {
114                   kvMaps[i].put(kv.getKeyString(), kv);
115                 }
116               }
117             }
118           }
119         }
120         region.put(p);
121         if (Math.random() < flushPercentage) {
122           LOG.info("Flushing... ");
123           region.flushcache();
124         }
125 
126         if (Math.random() < minorPercentage) {
127           LOG.info("Minor compacting... ");
128           region.compactStores(false);
129         }
130 
131         if (Math.random() < majorPercentage) {
132           LOG.info("Major compacting... ");
133           region.compactStores(true);
134         }
135       }
136     }
137 
138     for (int i = 0; i < numberOfTests + 1; i++) {
139       Collection<KeyValue> kvSet;
140       Scan scan = new Scan();
141       scan.setMaxVersions();
142       if (i < numberOfTests) {
143         kvSet = kvMaps[i].values();
144         for (String column : columnLists[i]) {
145           scan.addColumn(familyBytes, Bytes.toBytes(column));
146         }
147         LOG.info("ExplicitColumns scanner");
148         LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
149             + kvSet.size());
150       } else {
151         kvSet = allKVMap.values();
152         LOG.info("Wildcard scanner");
153         LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
154 
155       }
156       InternalScanner scanner = region.getScanner(scan);
157       List<KeyValue> results = new ArrayList<KeyValue>();
158       while (scanner.next(results))
159         ;
160       assertEquals(kvSet.size(), results.size());
161       assertTrue(results.containsAll(kvSet));
162     }
163   }
164 
165   @SuppressWarnings("unchecked")
166   @Test
167   public void testReseeking() throws IOException {
168     String family = "Family";
169     byte[] familyBytes = Bytes.toBytes("Family");
170     String table = "TestSingleVersions";
171 
172     HTableDescriptor htd = new HTableDescriptor(table);
173     htd.addFamily(new HColumnDescriptor(family));
174 
175     HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
176     HRegion region =
177         HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
178             .getConfiguration(), htd);
179 
180     List<String> rows = generateRandomWords(10, "row");
181     List<String> allColumns = generateRandomWords(100, "column");
182 
183     long maxTimestamp = 2;
184     double selectPercent = 0.5;
185     int numberOfTests = 5;
186     double flushPercentage = 0.2;
187     double minorPercentage = 0.2;
188     double majorPercentage = 0.2;
189     double putPercentage = 0.2;
190 
191     HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
192 
193     HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
194     ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
195     String valueString = "Value";
196 
197     for (int i = 0; i < numberOfTests; i++) {
198       kvMaps[i] = new HashMap<String, KeyValue>();
199       columnLists[i] = new ArrayList<String>();
200       for (String column : allColumns) {
201         if (Math.random() < selectPercent) {
202           columnLists[i].add(column);
203         }
204       }
205     }
206 
207     for (String row : rows) {
208       Put p = new Put(Bytes.toBytes(row));
209       p.setWriteToWAL(false);
210       for (String column : allColumns) {
211         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
212           KeyValue kv =
213               KeyValueTestUtil.create(row, family, column, timestamp,
214                   valueString);
215           if (Math.random() < putPercentage) {
216             p.add(kv);
217             allKVMap.put(kv.getKeyString(), kv);
218             for (int i = 0; i < numberOfTests; i++) {
219               if (columnLists[i].contains(column)) {
220                 kvMaps[i].put(kv.getKeyString(), kv);
221               }
222             }
223           }
224 
225         }
226       }
227       region.put(p);
228       if (Math.random() < flushPercentage) {
229         LOG.info("Flushing... ");
230         region.flushcache();
231       }
232 
233       if (Math.random() < minorPercentage) {
234         LOG.info("Minor compacting... ");
235         region.compactStores(false);
236       }
237 
238       if (Math.random() < majorPercentage) {
239         LOG.info("Major compacting... ");
240         region.compactStores(true);
241       }
242     }
243 
244     for (int i = 0; i < numberOfTests + 1; i++) {
245       Collection<KeyValue> kvSet;
246       Scan scan = new Scan();
247       scan.setMaxVersions();
248       if (i < numberOfTests) {
249         kvSet = kvMaps[i].values();
250         for (String column : columnLists[i]) {
251           scan.addColumn(familyBytes, Bytes.toBytes(column));
252         }
253         LOG.info("ExplicitColumns scanner");
254         LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
255             + kvSet.size());
256       } else {
257         kvSet = allKVMap.values();
258         LOG.info("Wildcard scanner");
259         LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
260 
261       }
262       InternalScanner scanner = region.getScanner(scan);
263       List<KeyValue> results = new ArrayList<KeyValue>();
264       while (scanner.next(results))
265         ;
266       assertEquals(kvSet.size(), results.size());
267       assertTrue(results.containsAll(kvSet));
268     }
269   }
270 
271   List<String> generateRandomWords(int numberOfWords, String suffix) {
272     Set<String> wordSet = new HashSet<String>();
273     for (int i = 0; i < numberOfWords; i++) {
274       int lengthOfWords = (int) (Math.random() * 5) + 1;
275       char[] wordChar = new char[lengthOfWords];
276       for (int j = 0; j < wordChar.length; j++) {
277         wordChar[j] = (char) (Math.random() * 26 + 97);
278       }
279       String word;
280       if (suffix == null) {
281         word = new String(wordChar);
282       } else {
283         word = new String(wordChar) + suffix;
284       }
285       wordSet.add(word);
286     }
287     List<String> wordList = new ArrayList<String>(wordSet);
288     return wordList;
289   }
290 }