1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import static org.junit.Assert.*;
24
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.List;
31 import java.util.Set;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.KeyValue;
40 import org.apache.hadoop.hbase.KeyValueTestUtil;
41 import org.apache.hadoop.hbase.client.Put;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.Test;
45
46 public class TestColumnSeeking {
47
48 private final static HBaseTestingUtility TEST_UTIL =
49 new HBaseTestingUtility();
50
51 static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
52
53 @SuppressWarnings("unchecked")
54 @Test
55 public void testDuplicateVersions() throws IOException {
56 String family = "Family";
57 byte[] familyBytes = Bytes.toBytes("Family");
58 String table = "TestDuplicateVersions";
59
60 HColumnDescriptor hcd =
61 new HColumnDescriptor(familyBytes, 1000,
62 HColumnDescriptor.DEFAULT_COMPRESSION,
63 HColumnDescriptor.DEFAULT_IN_MEMORY,
64 HColumnDescriptor.DEFAULT_BLOCKCACHE,
65 HColumnDescriptor.DEFAULT_TTL,
66 HColumnDescriptor.DEFAULT_BLOOMFILTER);
67 HTableDescriptor htd = new HTableDescriptor(table);
68 htd.addFamily(hcd);
69 HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
70 HRegion region =
71 HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
72 .getConfiguration(), htd);
73
74 List<String> rows = generateRandomWords(10, "row");
75 List<String> allColumns = generateRandomWords(10, "column");
76 List<String> values = generateRandomWords(100, "value");
77
78 long maxTimestamp = 2;
79 double selectPercent = 0.5;
80 int numberOfTests = 5;
81 double flushPercentage = 0.2;
82 double minorPercentage = 0.2;
83 double majorPercentage = 0.2;
84 double putPercentage = 0.2;
85
86 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
87
88 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
89 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
90
91 for (int i = 0; i < numberOfTests; i++) {
92 kvMaps[i] = new HashMap<String, KeyValue>();
93 columnLists[i] = new ArrayList<String>();
94 for (String column : allColumns) {
95 if (Math.random() < selectPercent) {
96 columnLists[i].add(column);
97 }
98 }
99 }
100
101 for (String value : values) {
102 for (String row : rows) {
103 Put p = new Put(Bytes.toBytes(row));
104 p.setWriteToWAL(false);
105 for (String column : allColumns) {
106 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
107 KeyValue kv =
108 KeyValueTestUtil.create(row, family, column, timestamp, value);
109 if (Math.random() < putPercentage) {
110 p.add(kv);
111 allKVMap.put(kv.getKeyString(), kv);
112 for (int i = 0; i < numberOfTests; i++) {
113 if (columnLists[i].contains(column)) {
114 kvMaps[i].put(kv.getKeyString(), kv);
115 }
116 }
117 }
118 }
119 }
120 region.put(p);
121 if (Math.random() < flushPercentage) {
122 LOG.info("Flushing... ");
123 region.flushcache();
124 }
125
126 if (Math.random() < minorPercentage) {
127 LOG.info("Minor compacting... ");
128 region.compactStores(false);
129 }
130
131 if (Math.random() < majorPercentage) {
132 LOG.info("Major compacting... ");
133 region.compactStores(true);
134 }
135 }
136 }
137
138 for (int i = 0; i < numberOfTests + 1; i++) {
139 Collection<KeyValue> kvSet;
140 Scan scan = new Scan();
141 scan.setMaxVersions();
142 if (i < numberOfTests) {
143 kvSet = kvMaps[i].values();
144 for (String column : columnLists[i]) {
145 scan.addColumn(familyBytes, Bytes.toBytes(column));
146 }
147 LOG.info("ExplicitColumns scanner");
148 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
149 + kvSet.size());
150 } else {
151 kvSet = allKVMap.values();
152 LOG.info("Wildcard scanner");
153 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
154
155 }
156 InternalScanner scanner = region.getScanner(scan);
157 List<KeyValue> results = new ArrayList<KeyValue>();
158 while (scanner.next(results))
159 ;
160 assertEquals(kvSet.size(), results.size());
161 assertTrue(results.containsAll(kvSet));
162 }
163 }
164
165 @SuppressWarnings("unchecked")
166 @Test
167 public void testReseeking() throws IOException {
168 String family = "Family";
169 byte[] familyBytes = Bytes.toBytes("Family");
170 String table = "TestSingleVersions";
171
172 HTableDescriptor htd = new HTableDescriptor(table);
173 htd.addFamily(new HColumnDescriptor(family));
174
175 HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
176 HRegion region =
177 HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
178 .getConfiguration(), htd);
179
180 List<String> rows = generateRandomWords(10, "row");
181 List<String> allColumns = generateRandomWords(100, "column");
182
183 long maxTimestamp = 2;
184 double selectPercent = 0.5;
185 int numberOfTests = 5;
186 double flushPercentage = 0.2;
187 double minorPercentage = 0.2;
188 double majorPercentage = 0.2;
189 double putPercentage = 0.2;
190
191 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
192
193 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
194 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
195 String valueString = "Value";
196
197 for (int i = 0; i < numberOfTests; i++) {
198 kvMaps[i] = new HashMap<String, KeyValue>();
199 columnLists[i] = new ArrayList<String>();
200 for (String column : allColumns) {
201 if (Math.random() < selectPercent) {
202 columnLists[i].add(column);
203 }
204 }
205 }
206
207 for (String row : rows) {
208 Put p = new Put(Bytes.toBytes(row));
209 p.setWriteToWAL(false);
210 for (String column : allColumns) {
211 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
212 KeyValue kv =
213 KeyValueTestUtil.create(row, family, column, timestamp,
214 valueString);
215 if (Math.random() < putPercentage) {
216 p.add(kv);
217 allKVMap.put(kv.getKeyString(), kv);
218 for (int i = 0; i < numberOfTests; i++) {
219 if (columnLists[i].contains(column)) {
220 kvMaps[i].put(kv.getKeyString(), kv);
221 }
222 }
223 }
224
225 }
226 }
227 region.put(p);
228 if (Math.random() < flushPercentage) {
229 LOG.info("Flushing... ");
230 region.flushcache();
231 }
232
233 if (Math.random() < minorPercentage) {
234 LOG.info("Minor compacting... ");
235 region.compactStores(false);
236 }
237
238 if (Math.random() < majorPercentage) {
239 LOG.info("Major compacting... ");
240 region.compactStores(true);
241 }
242 }
243
244 for (int i = 0; i < numberOfTests + 1; i++) {
245 Collection<KeyValue> kvSet;
246 Scan scan = new Scan();
247 scan.setMaxVersions();
248 if (i < numberOfTests) {
249 kvSet = kvMaps[i].values();
250 for (String column : columnLists[i]) {
251 scan.addColumn(familyBytes, Bytes.toBytes(column));
252 }
253 LOG.info("ExplicitColumns scanner");
254 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
255 + kvSet.size());
256 } else {
257 kvSet = allKVMap.values();
258 LOG.info("Wildcard scanner");
259 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
260
261 }
262 InternalScanner scanner = region.getScanner(scan);
263 List<KeyValue> results = new ArrayList<KeyValue>();
264 while (scanner.next(results))
265 ;
266 assertEquals(kvSet.size(), results.size());
267 assertTrue(results.containsAll(kvSet));
268 }
269 }
270
271 List<String> generateRandomWords(int numberOfWords, String suffix) {
272 Set<String> wordSet = new HashSet<String>();
273 for (int i = 0; i < numberOfWords; i++) {
274 int lengthOfWords = (int) (Math.random() * 5) + 1;
275 char[] wordChar = new char[lengthOfWords];
276 for (int j = 0; j < wordChar.length; j++) {
277 wordChar[j] = (char) (Math.random() * 26 + 97);
278 }
279 String word;
280 if (suffix == null) {
281 word = new String(wordChar);
282 } else {
283 word = new String(wordChar) + suffix;
284 }
285 wordSet.add(word);
286 }
287 List<String> wordList = new ArrayList<String>(wordSet);
288 return wordList;
289 }
290 }