1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.lang.reflect.Field;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.concurrent.ThreadPoolExecutor;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.hbase.util.JVMClusterUtil;
34 import org.junit.AfterClass;
35 import org.junit.Assert;
36 import org.junit.Before;
37 import org.junit.BeforeClass;
38 import org.junit.Test;
39
40 import static org.junit.Assert.*;
41
42 public class TestMultiParallel {
43 private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
44 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
45 private static final byte[] VALUE = Bytes.toBytes("value");
46 private static final byte[] QUALIFIER = Bytes.toBytes("qual");
47 private static final String FAMILY = "family";
48 private static final String TEST_TABLE = "multi_test_table";
49 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
50 private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
51 private static final byte [][] KEYS = makeKeys();
52
53 private static final int slaves = 2;
54
55 @BeforeClass public static void beforeClass() throws Exception {
56 UTIL.startMiniCluster(slaves);
57 HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
58 UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
59 }
60
61 @AfterClass public static void afterClass() throws Exception {
62 UTIL.shutdownMiniCluster();
63 }
64
65 @Before public void before() throws IOException {
66 LOG.info("before");
67 if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
68
69 UTIL.getMiniHBaseCluster().getMaster().balance();
70 }
71 LOG.info("before done");
72 }
73
74 private static byte[][] makeKeys() {
75 byte [][] starterKeys = HBaseTestingUtility.KEYS;
76
77
78
79
80
81 int numKeys = (int) ((float) starterKeys.length * 10.33F);
82
83 List<byte[]> keys = new ArrayList<byte[]>();
84 for (int i = 0; i < numKeys; i++) {
85 int kIdx = i % starterKeys.length;
86 byte[] k = starterKeys[kIdx];
87 byte[] cp = new byte[k.length + 1];
88 System.arraycopy(k, 0, cp, 0, k.length);
89 cp[k.length] = new Integer(i % 256).byteValue();
90 keys.add(cp);
91 }
92
93
94
95
96
97 for (int i = 0; i < 100; i++) {
98 int kIdx = i % starterKeys.length;
99 byte[] k = starterKeys[kIdx];
100 byte[] cp = new byte[k.length + 1];
101 System.arraycopy(k, 0, cp, 0, k.length);
102 cp[k.length] = new Integer(i % 256).byteValue();
103 keys.add(cp);
104 }
105 return keys.toArray(new byte [][] {new byte [] {}});
106 }
107
108
109
110
111
112
113
114
115
116
117
118
119 @Test(timeout=300000)
120 public void testActiveThreadsCount() throws Exception{
121 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
122 List<Row> puts = constructPutRequests();
123 table.batch(puts);
124 Field poolField = table.getClass().getDeclaredField("pool");
125 poolField.setAccessible(true);
126 ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
127 assertEquals(slaves, tExecutor.getLargestPoolSize());
128 }
129
130 @Test(timeout=300000)
131 public void testBatchWithGet() throws Exception {
132 LOG.info("test=testBatchWithGet");
133 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
134
135
136 List<Row> puts = constructPutRequests();
137 table.batch(puts);
138
139
140 List<Row> gets = new ArrayList<Row>();
141 for (byte[] k : KEYS) {
142 Get get = new Get(k);
143 get.addColumn(BYTES_FAMILY, QUALIFIER);
144 gets.add(get);
145 }
146 Result[] multiRes = new Result[gets.size()];
147 table.batch(gets, multiRes);
148
149
150 List<Result> singleRes = new ArrayList<Result>();
151 for (Row get : gets) {
152 singleRes.add(table.get((Get) get));
153 }
154
155
156 Assert.assertEquals(singleRes.size(), multiRes.length);
157 for (int i = 0; i < singleRes.size(); i++) {
158 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
159 KeyValue[] singleKvs = singleRes.get(i).raw();
160 KeyValue[] multiKvs = multiRes[i].raw();
161 for (int j = 0; j < singleKvs.length; j++) {
162 Assert.assertEquals(singleKvs[j], multiKvs[j]);
163 Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
164 .getValue()));
165 }
166 }
167 }
168
169 @Test
170 public void testBadFam() throws Exception {
171 LOG.info("test=testBadFam");
172 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
173
174 List<Row> actions = new ArrayList<Row>();
175 Put p = new Put(Bytes.toBytes("row1"));
176 p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
177 actions.add(p);
178 p = new Put(Bytes.toBytes("row2"));
179 p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
180 actions.add(p);
181
182
183
184 Object [] r = new Object[actions.size()];
185 try {
186 table.batch(actions, r);
187 fail();
188 } catch (RetriesExhaustedWithDetailsException ex) {
189 LOG.debug(ex);
190
191 assertFalse(ex.mayHaveClusterIssues());
192 }
193 assertEquals(2, r.length);
194 assertTrue(r[0] instanceof Throwable);
195 assertTrue(r[1] instanceof Result);
196 }
197
198
199
200
201
202
203
204 @Test (timeout=300000)
205 public void testFlushCommitsWithAbort() throws Exception {
206 LOG.info("test=testFlushCommitsWithAbort");
207 doTestFlushCommits(true);
208 }
209
210 @Test (timeout=300000)
211 public void testFlushCommitsNoAbort() throws Exception {
212 LOG.info("test=testFlushCommitsNoAbort");
213 doTestFlushCommits(false);
214 }
215
216 private void doTestFlushCommits(boolean doAbort) throws Exception {
217
218 LOG.info("get new table");
219 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
220 table.setAutoFlush(false);
221 table.setWriteBufferSize(10 * 1024 * 1024);
222
223 LOG.info("constructPutRequests");
224 List<Row> puts = constructPutRequests();
225 for (Row put : puts) {
226 table.put((Put) put);
227 }
228 LOG.info("puts");
229 table.flushCommits();
230 if (doAbort) {
231 LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
232
233
234
235 puts = constructPutRequests();
236 for (Row put : puts) {
237 table.put((Put) put);
238 }
239
240 table.flushCommits();
241 }
242
243 LOG.info("validating loaded data");
244 validateLoadedData(table);
245
246
247 List<JVMClusterUtil.RegionServerThread> liveRSs =
248 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
249 int count = 0;
250 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
251 count++;
252 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
253 }
254 LOG.info("Count=" + count);
255 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
256 (doAbort ? 1 : 2), count);
257 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
258 int regions = t.getRegionServer().getOnlineRegions().size();
259 Assert.assertTrue("Count of regions=" + regions, regions > 10);
260 }
261 LOG.info("done");
262 }
263
264 @Test (timeout=300000)
265 public void testBatchWithPut() throws Exception {
266 LOG.info("test=testBatchWithPut");
267 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
268
269
270 List<Row> puts = constructPutRequests();
271
272 Object[] results = table.batch(puts);
273 validateSizeAndEmpty(results, KEYS.length);
274
275 if (true) {
276 UTIL.getMiniHBaseCluster().abortRegionServer(0);
277
278 puts = constructPutRequests();
279 results = table.batch(puts);
280 validateSizeAndEmpty(results, KEYS.length);
281 }
282
283 validateLoadedData(table);
284 }
285
286 @Test(timeout=300000)
287 public void testBatchWithDelete() throws Exception {
288 LOG.info("test=testBatchWithDelete");
289 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
290
291
292 List<Row> puts = constructPutRequests();
293 Object[] results = table.batch(puts);
294 validateSizeAndEmpty(results, KEYS.length);
295
296
297 List<Row> deletes = new ArrayList<Row>();
298 for (int i = 0; i < KEYS.length; i++) {
299 Delete delete = new Delete(KEYS[i]);
300 delete.deleteFamily(BYTES_FAMILY);
301 deletes.add(delete);
302 }
303 results = table.batch(deletes);
304 validateSizeAndEmpty(results, KEYS.length);
305
306
307 for (byte[] k : KEYS) {
308 Get get = new Get(k);
309 get.addColumn(BYTES_FAMILY, QUALIFIER);
310 Assert.assertFalse(table.exists(get));
311 }
312
313 }
314
315 @Test(timeout=300000)
316 public void testHTableDeleteWithList() throws Exception {
317 LOG.info("test=testHTableDeleteWithList");
318 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
319
320
321 List<Row> puts = constructPutRequests();
322 Object[] results = table.batch(puts);
323 validateSizeAndEmpty(results, KEYS.length);
324
325
326 ArrayList<Delete> deletes = new ArrayList<Delete>();
327 for (int i = 0; i < KEYS.length; i++) {
328 Delete delete = new Delete(KEYS[i]);
329 delete.deleteFamily(BYTES_FAMILY);
330 deletes.add(delete);
331 }
332 table.delete(deletes);
333 Assert.assertTrue(deletes.isEmpty());
334
335
336 for (byte[] k : KEYS) {
337 Get get = new Get(k);
338 get.addColumn(BYTES_FAMILY, QUALIFIER);
339 Assert.assertFalse(table.exists(get));
340 }
341
342 }
343
344 @Test(timeout=300000)
345 public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
346 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
347 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
348
349 List<Row> puts = new ArrayList<Row>();
350 for (int i = 0; i < 100; i++) {
351 Put put = new Put(ONE_ROW);
352 byte[] qual = Bytes.toBytes("column" + i);
353 put.add(BYTES_FAMILY, qual, VALUE);
354 puts.add(put);
355 }
356 Object[] results = table.batch(puts);
357
358
359 validateSizeAndEmpty(results, 100);
360
361
362 List<Row> gets = new ArrayList<Row>();
363 for (int i = 0; i < 100; i++) {
364 Get get = new Get(ONE_ROW);
365 byte[] qual = Bytes.toBytes("column" + i);
366 get.addColumn(BYTES_FAMILY, qual);
367 gets.add(get);
368 }
369
370 Object[] multiRes = table.batch(gets);
371
372 int idx = 0;
373 for (Object r : multiRes) {
374 byte[] qual = Bytes.toBytes("column" + idx);
375 validateResult(r, qual, VALUE);
376 idx++;
377 }
378
379 }
380
381 @Test(timeout=300000)
382 public void testBatchWithMixedActions() throws Exception {
383 LOG.info("test=testBatchWithMixedActions");
384 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
385
386
387 Object[] results = table.batch(constructPutRequests());
388 validateSizeAndEmpty(results, KEYS.length);
389
390
391
392 List<Row> actions = new ArrayList<Row>();
393
394 byte[] qual2 = Bytes.toBytes("qual2");
395 byte[] val2 = Bytes.toBytes("putvalue2");
396
397
398 Get get = new Get(KEYS[10]);
399 get.addColumn(BYTES_FAMILY, QUALIFIER);
400 actions.add(get);
401
402
403 get = new Get(KEYS[11]);
404 get.addColumn(BYTES_FAMILY, QUALIFIER);
405 actions.add(get);
406
407
408 Put put = new Put(KEYS[10]);
409 put.add(BYTES_FAMILY, qual2, val2);
410 actions.add(put);
411
412
413 Delete delete = new Delete(KEYS[20]);
414 delete.deleteFamily(BYTES_FAMILY);
415 actions.add(delete);
416
417
418 get = new Get(KEYS[30]);
419 get.addColumn(BYTES_FAMILY, QUALIFIER);
420 actions.add(get);
421
422
423
424
425
426
427 put = new Put(KEYS[40]);
428 put.add(BYTES_FAMILY, qual2, val2);
429 actions.add(put);
430
431 results = table.batch(actions);
432
433
434
435 validateResult(results[0]);
436 validateResult(results[1]);
437 validateEmpty(results[2]);
438 validateEmpty(results[3]);
439 validateResult(results[4]);
440 validateEmpty(results[5]);
441
442
443 get = new Get(KEYS[40]);
444 get.addColumn(BYTES_FAMILY, qual2);
445 Result r = table.get(get);
446 validateResult(r, qual2, val2);
447 }
448
449
450
451 private void validateResult(Object r) {
452 validateResult(r, QUALIFIER, VALUE);
453 }
454
455 private void validateResult(Object r1, byte[] qual, byte[] val) {
456
457 Result r = (Result)r1;
458 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
459 Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
460 }
461
462 private List<Row> constructPutRequests() {
463 List<Row> puts = new ArrayList<Row>();
464 for (byte[] k : KEYS) {
465 Put put = new Put(k);
466 put.add(BYTES_FAMILY, QUALIFIER, VALUE);
467 puts.add(put);
468 }
469 return puts;
470 }
471
472 private void validateLoadedData(HTable table) throws IOException {
473
474 for (byte[] k : KEYS) {
475 Get get = new Get(k);
476 get.addColumn(BYTES_FAMILY, QUALIFIER);
477 Result r = table.get(get);
478 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
479 Assert.assertEquals(0, Bytes.compareTo(VALUE, r
480 .getValue(BYTES_FAMILY, QUALIFIER)));
481 }
482 }
483
484 private void validateEmpty(Object r1) {
485 Result result = (Result)r1;
486 Assert.assertTrue(result != null);
487 Assert.assertTrue(result.getRow() == null);
488 Assert.assertEquals(0, result.raw().length);
489 }
490
491 private void validateSizeAndEmpty(Object[] results, int expectedSize) {
492
493 Assert.assertEquals(expectedSize, results.length);
494 for (Object result : results) {
495 validateEmpty(result);
496 }
497 }
498 }