1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.wal;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.security.PrivilegedExceptionAction;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.concurrent.atomic.AtomicInteger;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.HBaseConfiguration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.client.Get;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.io.hfile.HFile;
47 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
48 import org.apache.hadoop.hbase.regionserver.FlushRequester;
49 import org.apache.hadoop.hbase.regionserver.HRegion;
50 import org.apache.hadoop.hbase.regionserver.Store;
51 import org.apache.hadoop.hbase.security.User;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.EnvironmentEdge;
54 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
55 import org.apache.hadoop.hbase.util.Pair;
56 import org.junit.After;
57 import org.junit.AfterClass;
58 import org.junit.Before;
59 import org.junit.BeforeClass;
60 import org.junit.Test;
61 import org.mockito.Mockito;
62
63
64
65
66 public class TestWALReplay {
67 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
68 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
69 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
70 private Path hbaseRootDir = null;
71 private Path oldLogDir;
72 private Path logDir;
73 private FileSystem fs;
74 private Configuration conf;
75
76 @BeforeClass
77 public static void setUpBeforeClass() throws Exception {
78 Configuration conf = TEST_UTIL.getConfiguration();
79 conf.setBoolean("dfs.support.append", true);
80
81 conf.setInt("dfs.client.block.recovery.retries", 2);
82 TEST_UTIL.startMiniDFSCluster(3);
83 Path hbaseRootDir =
84 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
85 LOG.info("hbase.rootdir=" + hbaseRootDir);
86 conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
87 }
88
89 @AfterClass
90 public static void tearDownAfterClass() throws Exception {
91 TEST_UTIL.shutdownMiniDFSCluster();
92 }
93
94 @Before
95 public void setUp() throws Exception {
96 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
97 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
98 this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
99 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
100 this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
101 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
102 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
103 }
104 }
105
106 @After
107 public void tearDown() throws Exception {
108 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
109 }
110
111
112
113
114 private void deleteDir(final Path p) throws IOException {
115 if (this.fs.exists(p)) {
116 if (!this.fs.delete(p, true)) {
117 throw new IOException("Failed remove of " + p);
118 }
119 }
120 }
121
122
123
124
125
126
127 @Test
128 public void test2727() throws Exception {
129
130
131 final String tableNameStr = "test2727";
132 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
133 Path basedir = new Path(hbaseRootDir, tableNameStr);
134 deleteDir(basedir);
135 fs.mkdirs(new Path(basedir, hri.getEncodedName()));
136
137 HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
138 HRegion region2 = HRegion.createHRegion(hri,
139 hbaseRootDir, this.conf, htd);
140
141 final byte [] tableName = Bytes.toBytes(tableNameStr);
142 final byte [] rowName = tableName;
143
144 HLog wal1 = createWAL(this.conf);
145
146 final int countPerFamily = 1000;
147 for (HColumnDescriptor hcd: htd.getFamilies()) {
148 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
149 wal1, htd);
150 }
151 wal1.close();
152 runWALSplit(this.conf);
153
154 HLog wal2 = createWAL(this.conf);
155
156 wal2.setSequenceNumber(wal1.getSequenceNumber());
157
158 for (HColumnDescriptor hcd: htd.getFamilies()) {
159 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
160 ee, wal2, htd);
161 }
162 wal2.close();
163 runWALSplit(this.conf);
164
165 HLog wal3 = createWAL(this.conf);
166 wal3.setSequenceNumber(wal2.getSequenceNumber());
167 try {
168 final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri,
169 htd, null);
170 long seqid = region.initialize();
171 assertTrue(seqid > wal3.getSequenceNumber());
172
173
174 region.close();
175 } finally {
176 wal3.closeAndDelete();
177 }
178 }
179
180
181
182
183
184
185
186
187
188
189 @Test
190 public void testRegionMadeOfBulkLoadedFilesOnly()
191 throws IOException, SecurityException, IllegalArgumentException,
192 NoSuchFieldException, IllegalAccessException, InterruptedException {
193 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
194 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
195 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
196 deleteDir(basedir);
197 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
198 HRegion region2 = HRegion.createHRegion(hri,
199 hbaseRootDir, this.conf, htd);
200 HLog wal = createWAL(this.conf);
201 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
202 Path f = new Path(basedir, "hfile");
203 HFile.Writer writer =
204 HFile.getWriterFactory(conf).createWriter(this.fs, f);
205 byte [] family = htd.getFamilies().iterator().next().getName();
206 byte [] row = Bytes.toBytes(tableNameStr);
207 writer.append(new KeyValue(row, family, family, row));
208 writer.close();
209 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
210 hfs.add(Pair.newPair(family, f.toString()));
211 region.bulkLoadHFiles(hfs);
212
213 region.put((new Put(row)).add(family, family, family));
214 wal.sync();
215
216
217 final Configuration newConf = HBaseConfiguration.create(this.conf);
218 User user = HBaseTestingUtility.getDifferentUser(newConf,
219 tableNameStr);
220 user.runAs(new PrivilegedExceptionAction() {
221 public Object run() throws Exception {
222 runWALSplit(newConf);
223 HLog wal2 = createWAL(newConf);
224 HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
225 newConf, hri, htd, null);
226 long seqid2 = region2.initialize();
227 assertTrue(seqid2 > -1);
228
229
230 region2.close();
231 wal2.closeAndDelete();
232 return null;
233 }
234 });
235 }
236
237
238
239
240
241
242
243
244
245
246 @Test
247 public void testReplayEditsWrittenViaHRegion()
248 throws IOException, SecurityException, IllegalArgumentException,
249 NoSuchFieldException, IllegalAccessException, InterruptedException {
250 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
251 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
252 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
253 deleteDir(basedir);
254 final byte[] rowName = Bytes.toBytes(tableNameStr);
255 final int countPerFamily = 10;
256 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
257 HRegion region3 = HRegion.createHRegion(hri,
258 hbaseRootDir, this.conf, htd);
259
260
261
262
263 HLog wal = createWAL(this.conf);
264 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
265 long seqid = region.initialize();
266
267 wal.setSequenceNumber(seqid);
268 boolean first = true;
269 for (HColumnDescriptor hcd: htd.getFamilies()) {
270 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
271 if (first ) {
272
273 region.flushcache();
274 first = false;
275 }
276 }
277
278 final Get g = new Get(rowName);
279 Result result = region.get(g, null);
280 assertEquals(countPerFamily * htd.getFamilies().size(),
281 result.size());
282
283
284
285 region.close(true);
286 wal.close();
287 runWALSplit(this.conf);
288 HLog wal2 = createWAL(this.conf);
289 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null);
290 long seqid2 = region2.initialize();
291
292 wal2.setSequenceNumber(seqid2);
293 assertTrue(seqid + result.size() < seqid2);
294 final Result result1b = region2.get(g, null);
295 assertEquals(result.size(), result1b.size());
296
297
298
299
300 for (HColumnDescriptor hcd: htd.getFamilies()) {
301 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
302 }
303
304 final Result result2 = region2.get(g, null);
305 assertEquals(2 * result.size(), result2.size());
306 wal2.sync();
307
308
309 HBaseTestingUtility.setMaxRecoveryErrorCount(wal2.getOutputStream(), 1);
310 final Configuration newConf = HBaseConfiguration.create(this.conf);
311 User user = HBaseTestingUtility.getDifferentUser(newConf,
312 tableNameStr);
313 user.runAs(new PrivilegedExceptionAction() {
314 public Object run() throws Exception {
315 runWALSplit(newConf);
316 FileSystem newFS = FileSystem.get(newConf);
317
318 HLog wal3 = createWAL(newConf);
319 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
320 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
321 @Override
322 protected boolean restoreEdit(Store s, KeyValue kv) {
323 boolean b = super.restoreEdit(s, kv);
324 countOfRestoredEdits.incrementAndGet();
325 return b;
326 }
327 };
328 long seqid3 = region3.initialize();
329
330 wal3.setSequenceNumber(seqid3);
331 Result result3 = region3.get(g, null);
332
333 assertEquals(result2.size(), result3.size());
334 assertEquals(htd.getFamilies().size() * countPerFamily,
335 countOfRestoredEdits.get());
336
337
338 region3.close();
339 wal3.closeAndDelete();
340 return null;
341 }
342 });
343 }
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363 @Test
364 public void testReplayEditsAfterPartialFlush()
365 throws IOException, SecurityException, IllegalArgumentException,
366 NoSuchFieldException, IllegalAccessException, InterruptedException {
367 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
368 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
369 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
370 deleteDir(basedir);
371 final byte[] rowName = Bytes.toBytes(tableNameStr);
372 final int countPerFamily = 10;
373 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
374 HRegion region3 = HRegion.createHRegion(hri,
375 hbaseRootDir, this.conf, htd);
376
377
378
379
380 HLog wal = createWAL(this.conf);
381 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
382 long seqid = region.initialize();
383
384 wal.setSequenceNumber(seqid);
385 for (HColumnDescriptor hcd: htd.getFamilies()) {
386 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
387 }
388
389
390 final Get g = new Get(rowName);
391 Result result = region.get(g, null);
392 assertEquals(countPerFamily * htd.getFamilies().size(),
393 result.size());
394
395
396 region.flushcache();
397 region.close(true);
398 wal.close();
399
400
401
402
403
404 int cf_count = 0;
405 for (HColumnDescriptor hcd: htd.getFamilies()) {
406 cf_count++;
407 if (cf_count == 2) {
408 this.fs.delete(new Path(region.getRegionDir(), Bytes.toString(hcd.getName()))
409 , true);
410 }
411 }
412
413
414
415 runWALSplit(this.conf);
416 HLog wal2 = createWAL(this.conf);
417 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null);
418 long seqid2 = region2.initialize();
419
420 wal2.setSequenceNumber(seqid2);
421 assertTrue(seqid + result.size() < seqid2);
422
423 final Result result1b = region2.get(g, null);
424 assertEquals(result.size(), result1b.size());
425 }
426
427
428
429
430
431
432 @Test
433 public void testReplayEditsWrittenIntoWAL() throws Exception {
434 final String tableNameStr = "testReplayEditsWrittenIntoWAL";
435 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
436 final Path basedir = new Path(hbaseRootDir, tableNameStr);
437 deleteDir(basedir);
438 fs.mkdirs(new Path(basedir, hri.getEncodedName()));
439 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
440 HRegion region2 = HRegion.createHRegion(hri,
441 hbaseRootDir, this.conf, htd);
442
443 final HLog wal = createWAL(this.conf);
444 final byte[] tableName = Bytes.toBytes(tableNameStr);
445 final byte[] rowName = tableName;
446 final byte[] regionName = hri.getEncodedNameAsBytes();
447
448
449 final int countPerFamily = 1000;
450 for (HColumnDescriptor hcd: htd.getFamilies()) {
451 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
452 ee, wal, htd);
453 }
454
455
456 long logSeqId = wal.startCacheFlush(regionName);
457 wal.completeCacheFlush(regionName, tableName, logSeqId, hri.isMetaRegion());
458
459
460 WALEdit edit = new WALEdit();
461 long now = ee.currentTimeMillis();
462 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
463 now, rowName));
464 wal.append(hri, tableName, edit, now, htd);
465
466
467 edit = new WALEdit();
468 now = ee.currentTimeMillis();
469 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
470 KeyValue.Type.DeleteFamily));
471 wal.append(hri, tableName, edit, now, htd);
472
473
474 wal.sync();
475
476
477 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
478
479
480 final Configuration newConf = HBaseConfiguration.create(this.conf);
481 User user = HBaseTestingUtility.getDifferentUser(newConf,
482 ".replay.wal.secondtime");
483 user.runAs(new PrivilegedExceptionAction() {
484 public Object run() throws Exception {
485 runWALSplit(newConf);
486 FileSystem newFS = FileSystem.get(newConf);
487
488 newConf.setInt("hbase.hregion.memstore.flush.size", 1024 * 100);
489
490 HLog newWal = createWAL(newConf);
491 final AtomicInteger flushcount = new AtomicInteger(0);
492 try {
493 final HRegion region =
494 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
495 protected boolean internalFlushcache(
496 final HLog wal, final long myseqid, MonitoredTask status)
497 throws IOException {
498 LOG.info("InternalFlushCache Invoked");
499 boolean b = super.internalFlushcache(wal, myseqid,
500 Mockito.mock(MonitoredTask.class));
501 flushcount.incrementAndGet();
502 return b;
503 };
504 };
505 long seqid = region.initialize();
506
507 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
508 assertTrue(seqid > wal.getSequenceNumber());
509
510 Get get = new Get(rowName);
511 Result result = region.get(get, -1);
512
513 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
514 result.size());
515 region.close();
516 } finally {
517 newWal.closeAndDelete();
518 }
519 return null;
520 }
521 });
522 }
523
524
525
526 class TestFlusher implements FlushRequester {
527 private int count = 0;
528 private HRegion r;
529
530 @Override
531 public void requestFlush(HRegion region) {
532 count++;
533 try {
534 r.flushcache();
535 } catch (IOException e) {
536 throw new RuntimeException("Exception flushing", e);
537 }
538 }
539 }
540
541 private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
542 final byte [] rowName, final byte [] family,
543 final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
544 throws IOException {
545 String familyStr = Bytes.toString(family);
546 for (int j = 0; j < count; j++) {
547 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
548 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
549 WALEdit edit = new WALEdit();
550 edit.add(new KeyValue(rowName, family, qualifierBytes,
551 ee.currentTimeMillis(), columnBytes));
552 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
553 }
554 }
555
556 private void addRegionEdits (final byte [] rowName, final byte [] family,
557 final int count, EnvironmentEdge ee, final HRegion r,
558 final String qualifierPrefix)
559 throws IOException {
560 for (int j = 0; j < count; j++) {
561 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
562 Put p = new Put(rowName);
563 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
564 r.put(p);
565 }
566 }
567
568
569
570
571
572
573 private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
574 return new HRegionInfo(Bytes.toBytes(tableName), null, null, false);
575 }
576
577
578
579
580
581
582
583 private Path runWALSplit(final Configuration c) throws IOException {
584 FileSystem fs = FileSystem.get(c);
585 HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c,
586 this.hbaseRootDir, this.logDir, this.oldLogDir, fs);
587 List<Path> splits = logSplitter.splitLog();
588
589 assertEquals("splits=" + splits, 1, splits.size());
590
591 assertTrue(fs.exists(splits.get(0)));
592 LOG.info("Split file=" + splits.get(0));
593 return splits.get(0);
594 }
595
596
597
598
599
600
601 private HLog createWAL(final Configuration c) throws IOException {
602 HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c);
603
604
605 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
606 return wal;
607 }
608
609 private HTableDescriptor createBasic3FamilyHTD(final String tableName) {
610 HTableDescriptor htd = new HTableDescriptor(tableName);
611 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
612 htd.addFamily(a);
613 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
614 htd.addFamily(b);
615 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
616 htd.addFamily(c);
617 return htd;
618 }
619 }