1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.Random;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.HBaseTestCase;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.HRegionInfo;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.KeyValue;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.io.hfile.Compression;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hdfs.MiniDFSCluster;
42
43 public class TestWideScanner extends HBaseTestCase {
44 private final Log LOG = LogFactory.getLog(this.getClass());
45
46 static final byte[] A = Bytes.toBytes("A");
47 static final byte[] B = Bytes.toBytes("B");
48 static final byte[] C = Bytes.toBytes("C");
49 static byte[][] COLUMNS = { A, B, C };
50 static final Random rng = new Random();
51 static final HTableDescriptor TESTTABLEDESC =
52 new HTableDescriptor("testwidescan");
53 static {
54 TESTTABLEDESC.addFamily(new HColumnDescriptor(A,
55 100,
56 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
57 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
58 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
59 TESTTABLEDESC.addFamily(new HColumnDescriptor(B,
60 100,
61 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
62 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
63 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
64 TESTTABLEDESC.addFamily(new HColumnDescriptor(C,
65 100,
66 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
67 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
68 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
69 }
70
71
72 HRegion r;
73
74 private int addWideContent(HRegion region) throws IOException {
75 int count = 0;
76 for (char c = 'a'; c <= 'c'; c++) {
77 byte[] row = Bytes.toBytes("ab" + c);
78 int i, j;
79 long ts = System.currentTimeMillis();
80 for (i = 0; i < 100; i++) {
81 byte[] b = Bytes.toBytes(String.format("%10d", i));
82 for (j = 0; j < 100; j++) {
83 Put put = new Put(row);
84 put.setWriteToWAL(false);
85 put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, ++ts, b);
86 region.put(put);
87 count++;
88 }
89 }
90 }
91 return count;
92 }
93
94 public void testWideScanBatching() throws IOException {
95 final int batch = 256;
96 try {
97 this.r = createNewHRegion(TESTTABLEDESC, null, null);
98 int inserted = addWideContent(this.r);
99 List<KeyValue> results = new ArrayList<KeyValue>();
100 Scan scan = new Scan();
101 scan.addFamily(A);
102 scan.addFamily(B);
103 scan.addFamily(C);
104 scan.setMaxVersions(100);
105 scan.setBatch(batch);
106 InternalScanner s = r.getScanner(scan);
107 int total = 0;
108 int i = 0;
109 boolean more;
110 do {
111 more = s.next(results);
112 i++;
113 LOG.info("iteration #" + i + ", results.size=" + results.size());
114
115
116 assertTrue(results.size() <= batch);
117
118 total += results.size();
119
120 if (results.size() > 0) {
121
122 byte[] row = results.get(0).getRow();
123 for (KeyValue kv: results) {
124 assertTrue(Bytes.equals(row, kv.getRow()));
125 }
126 }
127
128 results.clear();
129
130
131 Iterator<KeyValueScanner> scanners =
132 ((HRegion.RegionScannerImpl)s).storeHeap.getHeap().iterator();
133 while (scanners.hasNext()) {
134 StoreScanner ss = (StoreScanner)scanners.next();
135 ss.updateReaders();
136 }
137 } while (more);
138
139
140 LOG.info("inserted " + inserted + ", scanned " + total);
141 assertEquals(total, inserted);
142
143 s.close();
144 } finally {
145 this.r.close();
146 this.r.getLog().closeAndDelete();
147 }
148 }
149 }