1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapred;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Matchers.anyObject;
26 import static org.mockito.Mockito.doAnswer;
27 import static org.mockito.Mockito.doReturn;
28 import static org.mockito.Mockito.doThrow;
29 import static org.mockito.Mockito.mock;
30 import static org.mockito.Mockito.spy;
31
32 import java.io.IOException;
33 import java.util.Arrays;
34 import java.util.Map;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.hbase.DoNotRetryIOException;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.UnknownScannerException;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.client.ResultScanner;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.junit.AfterClass;
49 import org.junit.Before;
50 import org.junit.BeforeClass;
51 import org.junit.Test;
52 import org.mockito.invocation.InvocationOnMock;
53 import org.mockito.stubbing.Answer;
54
55
56
57
58
59 public class TestTableInputFormat {
60
61 private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
62
63 private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
64 static final byte[] FAMILY = Bytes.toBytes("family");
65
66 private static final byte[][] columns = new byte[][] { FAMILY };
67
68 @BeforeClass
69 public static void beforeClass() throws Exception {
70 UTIL.startMiniCluster();
71 }
72
73 @AfterClass
74 public static void afterClass() throws Exception {
75 UTIL.shutdownMiniCluster();
76 }
77
78 @Before
79 public void before() throws IOException {
80 LOG.info("before");
81 UTIL.ensureSomeRegionServersAvailable(1);
82 LOG.info("before done");
83 }
84
85
86
87
88
89
90
91
92 public static HTable createTable(byte[] tableName) throws IOException {
93 HTable table = UTIL.createTable(tableName, FAMILY);
94 Put p = new Put("aaa".getBytes());
95 p.add(FAMILY, null, "value aaa".getBytes());
96 table.put(p);
97 p = new Put("bbb".getBytes());
98 p.add(FAMILY, null, "value bbb".getBytes());
99 table.put(p);
100 return table;
101 }
102
103
104
105
106
107
108
109
110
111
112 static boolean checkResult(Result r, ImmutableBytesWritable key,
113 byte[] expectedKey, byte[] expectedValue) {
114 assertEquals(0, key.compareTo(expectedKey));
115 Map<byte[], byte[]> vals = r.getFamilyMap(FAMILY);
116 byte[] value = vals.values().iterator().next();
117 assertTrue(Arrays.equals(value, expectedValue));
118 return true;
119 }
120
121
122
123
124
125
126
127
128 static void runTestMapred(HTable table) throws IOException {
129 org.apache.hadoop.hbase.mapred.TableRecordReader trr =
130 new org.apache.hadoop.hbase.mapred.TableRecordReader();
131 trr.setStartRow("aaa".getBytes());
132 trr.setEndRow("zzz".getBytes());
133 trr.setHTable(table);
134 trr.setInputColumns(columns);
135
136 trr.init();
137 Result r = new Result();
138 ImmutableBytesWritable key = new ImmutableBytesWritable();
139
140 boolean more = trr.next(key, r);
141 assertTrue(more);
142 checkResult(r, key, "aaa".getBytes(), "value aaa".getBytes());
143
144 more = trr.next(key, r);
145 assertTrue(more);
146 checkResult(r, key, "bbb".getBytes(), "value bbb".getBytes());
147
148
149 more = trr.next(key, r);
150 assertFalse(more);
151 }
152
153
154
155
156
157
158
159
160
161 static void runTestMapreduce(HTable table) throws IOException,
162 InterruptedException {
163 org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl trr =
164 new org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl();
165 Scan s = new Scan();
166 s.setStartRow("aaa".getBytes());
167 s.setStopRow("zzz".getBytes());
168 s.addFamily(FAMILY);
169 trr.setScan(s);
170 trr.setHTable(table);
171
172 trr.init();
173 Result r = new Result();
174 ImmutableBytesWritable key = new ImmutableBytesWritable();
175
176 boolean more = trr.nextKeyValue();
177 assertTrue(more);
178 key = trr.getCurrentKey();
179 r = trr.getCurrentValue();
180 checkResult(r, key, "aaa".getBytes(), "value aaa".getBytes());
181
182 more = trr.nextKeyValue();
183 assertTrue(more);
184 key = trr.getCurrentKey();
185 r = trr.getCurrentValue();
186 checkResult(r, key, "bbb".getBytes(), "value bbb".getBytes());
187
188
189 more = trr.nextKeyValue();
190 assertFalse(more);
191 }
192
193
194
195
196
197
198 static HTable createIOEScannerTable(byte[] name) throws IOException {
199
200 Answer<ResultScanner> a = new Answer<ResultScanner>() {
201 boolean first = true;
202
203 @Override
204 public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
205
206 if (first) {
207 first = false;
208
209 Scan scan = mock(Scan.class);
210 doReturn("bogus".getBytes()).when(scan).getStartRow();
211 ResultScanner scanner = mock(ResultScanner.class);
212
213 doThrow(new IOException("Injected exception")).when(scanner).next();
214 return scanner;
215 }
216
217
218 return (ResultScanner) invocation.callRealMethod();
219 }
220 };
221
222 HTable htable = spy(createTable(name));
223 doAnswer(a).when(htable).getScanner((Scan) anyObject());
224 return htable;
225 }
226
227
228
229
230
231
232
233 static HTable createDNRIOEScannerTable(byte[] name) throws IOException {
234
235 Answer<ResultScanner> a = new Answer<ResultScanner>() {
236 boolean first = true;
237
238 @Override
239 public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
240
241 if (first) {
242 first = false;
243
244 Scan scan = mock(Scan.class);
245 doReturn("bogus".getBytes()).when(scan).getStartRow();
246 ResultScanner scanner = mock(ResultScanner.class);
247
248 invocation.callRealMethod();
249 doThrow(
250 new UnknownScannerException("Injected simulated TimeoutException"))
251 .when(scanner).next();
252 return scanner;
253 }
254
255
256 return (ResultScanner) invocation.callRealMethod();
257 }
258 };
259
260 HTable htable = spy(createTable(name));
261 doAnswer(a).when(htable).getScanner((Scan) anyObject());
262 return htable;
263 }
264
265
266
267
268
269
270 @Test
271 public void testTableRecordReader() throws IOException {
272 HTable table = createTable("table1".getBytes());
273 runTestMapred(table);
274 }
275
276
277
278
279
280
281 @Test
282 public void testTableRecordReaderScannerFail() throws IOException {
283 HTable htable = createIOEScannerTable("table2".getBytes());
284 runTestMapred(htable);
285 }
286
287
288
289
290
291
292
293 @Test(expected = DoNotRetryIOException.class)
294 public void testTableRecordReaderScannerTimeout() throws IOException {
295 HTable htable = createDNRIOEScannerTable("table3".getBytes());
296 runTestMapred(htable);
297 }
298
299
300
301
302
303
304
305 @Test
306 public void testTableRecordReaderMapreduce() throws IOException,
307 InterruptedException {
308 HTable table = createTable("table1-mr".getBytes());
309 runTestMapreduce(table);
310 }
311
312
313
314
315
316
317
318 @Test
319 public void testTableRecordReaderScannerFailMapreduce() throws IOException,
320 InterruptedException {
321 HTable htable = createIOEScannerTable("table2-mr".getBytes());
322 runTestMapreduce(htable);
323 }
324
325
326
327
328
329
330
331
332 @Test(expected = DoNotRetryIOException.class)
333 public void testTableRecordReaderScannerTimeoutMapreduce()
334 throws IOException, InterruptedException {
335 HTable htable = createDNRIOEScannerTable("table3-mr".getBytes());
336 runTestMapreduce(htable);
337 }
338 }