1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.HBaseConfiguration;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.regionserver.HRegion;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.Coprocessor;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.regionserver.wal.HLog;
39  import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
40  import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
41  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
42  import org.apache.hadoop.hbase.security.User;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.EnvironmentEdge;
45  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46  import org.apache.hadoop.hdfs.MiniDFSCluster;
47  import org.junit.After;
48  import org.junit.AfterClass;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  
53  import java.io.IOException;
54  import java.security.PrivilegedExceptionAction;
55  import java.util.Arrays;
56  import java.util.List;
57  import java.util.Map;
58  
59  import static org.junit.Assert.*;
60  
61  /**
62   * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver}
63   * interface hooks at all appropriate times during normal HMaster operations.
64   */
65  public class TestWALObserver {
66    private static final Log LOG = LogFactory.getLog(TestWALObserver.class);
67    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
68  
69    private static byte[] TEST_TABLE = Bytes.toBytes("observedTable");
70    private static byte[][] TEST_FAMILY = { Bytes.toBytes("fam1"),
71      Bytes.toBytes("fam2"),
72      Bytes.toBytes("fam3"),
73    };
74    private static byte[][] TEST_QUALIFIER = { Bytes.toBytes("q1"),
75      Bytes.toBytes("q2"),
76      Bytes.toBytes("q3"),
77    };
78    private static byte[][] TEST_VALUE = { Bytes.toBytes("v1"),
79      Bytes.toBytes("v2"),
80      Bytes.toBytes("v3"),
81    };
82    private static byte[] TEST_ROW = Bytes.toBytes("testRow");
83  
84    private Configuration conf;
85    private FileSystem fs;
86    private Path dir;
87    private MiniDFSCluster cluster;
88    private Path hbaseRootDir;
89    private Path oldLogDir;
90    private Path logDir;
91  
92    @BeforeClass
93    public static void setupBeforeClass() throws Exception {
94      Configuration conf = TEST_UTIL.getConfiguration();
95      conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
96          SampleRegionWALObserver.class.getName());
97      conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
98          SampleRegionWALObserver.class.getName());
99      conf.setBoolean("dfs.support.append", true);
100     conf.setInt("dfs.client.block.recovery.retries", 2);
101 
102     TEST_UTIL.startMiniCluster(1);
103     Path hbaseRootDir =
104       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
105     LOG.info("hbase.rootdir=" + hbaseRootDir);
106     conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
107   }
108 
109   @AfterClass
110   public static void teardownAfterClass() throws Exception {
111     TEST_UTIL.shutdownMiniCluster();
112   }
113 
114   @Before
115   public void setUp() throws Exception {
116     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
117     //this.cluster = TEST_UTIL.getDFSCluster();
118     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
119     this.hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR));
120     this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName());
121     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
122     this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
123 
124     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
125       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
126     }
127   }
128 
129   @After
130   public void tearDown() throws Exception {
131     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
132   }
133 
134   /**
135    * Test WAL write behavior with WALObserver. The coprocessor monitors
136    * a WALEdit written to WAL, and ignore, modify, and add KeyValue's for the
137    * WALEdit.
138    */
139   @Test
140   public void testWALObserverWriteToWAL() throws Exception {
141 
142     HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
143     final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
144     HRegion region2 = HRegion.createHRegion(hri,
145             hbaseRootDir, this.conf, htd);
146 
147     Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
148     deleteDir(basedir);
149     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
150 
151     HLog log = new HLog(this.fs, this.dir, this.oldLogDir, this.conf);
152     SampleRegionWALObserver cp = getCoprocessor(log);
153 
154     // TEST_FAMILY[0] shall be removed from WALEdit.
155     // TEST_FAMILY[1] value shall be changed.
156     // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put.
157     cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0],
158         TEST_FAMILY[1], TEST_QUALIFIER[1],
159         TEST_FAMILY[2], TEST_QUALIFIER[2]);
160 
161     assertFalse(cp.isPreWALWriteCalled());
162     assertFalse(cp.isPostWALWriteCalled());
163 
164     // TEST_FAMILY[2] is not in the put, however it shall be added by the tested
165     // coprocessor.
166     // Use a Put to create familyMap.
167     Put p = creatPutWith2Families(TEST_ROW);
168 
169     Map<byte [], List<KeyValue>> familyMap = p.getFamilyMap();
170     WALEdit edit = new WALEdit();
171     addFamilyMapToWALEdit(familyMap, edit);
172 
173     boolean foundFamily0 = false;
174     boolean foundFamily2 = false;
175     boolean modifiedFamily1 = false;
176 
177     List<KeyValue> kvs = edit.getKeyValues();
178 
179     for (KeyValue kv : kvs) {
180       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) {
181         foundFamily0 = true;
182       }
183       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[2])) {
184         foundFamily2 = true;
185       }
186       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[1])) {
187         if (!Arrays.equals(kv.getValue(), TEST_VALUE[1])) {
188           modifiedFamily1 = true;
189         }
190       }
191     }
192     assertTrue(foundFamily0);
193     assertFalse(foundFamily2);
194     assertFalse(modifiedFamily1);
195 
196     // it's where WAL write cp should occur.
197     long now = EnvironmentEdgeManager.currentTimeMillis();
198     log.append(hri, hri.getTableName(), edit, now, htd);
199 
200     // the edit shall have been change now by the coprocessor.
201     foundFamily0 = false;
202     foundFamily2 = false;
203     modifiedFamily1 = false;
204     for (KeyValue kv : kvs) {
205       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) {
206         foundFamily0 = true;
207       }
208       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[2])) {
209         foundFamily2 = true;
210       }
211       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[1])) {
212         if (!Arrays.equals(kv.getValue(), TEST_VALUE[1])) {
213           modifiedFamily1 = true;
214         }
215       }
216     }
217     assertFalse(foundFamily0);
218     assertTrue(foundFamily2);
219     assertTrue(modifiedFamily1);
220 
221     assertTrue(cp.isPreWALWriteCalled());
222     assertTrue(cp.isPostWALWriteCalled());
223   }
224 
225   /**
226    * Test WAL replay behavior with WALObserver.
227    */
228   @Test
229   public void testWALCoprocessorReplay() throws Exception {
230     // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
231     // ultimately called by HRegion::initialize()
232     byte[] tableName = Bytes.toBytes("testWALCoprocessorReplay");
233     final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(Bytes.toString(tableName));
234     //final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
235     //final HRegionInfo hri1 = createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
236     final HRegionInfo hri = new HRegionInfo(tableName, null, null);
237 
238     final Path basedir = new Path(this.hbaseRootDir, Bytes.toString(tableName));
239     deleteDir(basedir);
240     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
241 
242     final Configuration newConf = HBaseConfiguration.create(this.conf);
243 
244     HRegion region2 = HRegion.createHRegion(hri,
245         hbaseRootDir, newConf,htd);
246 
247 
248     //HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf);
249     HLog wal = createWAL(this.conf);
250     //Put p = creatPutWith2Families(TEST_ROW);
251     WALEdit edit = new WALEdit();
252     long now = EnvironmentEdgeManager.currentTimeMillis();
253     //addFamilyMapToWALEdit(p.getFamilyMap(), edit);
254     final int countPerFamily = 1000;
255     //for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
256     for (HColumnDescriptor hcd: htd.getFamilies()) {
257           //addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
258           //EnvironmentEdgeManager.getDelegate(), wal);
259       addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
260       EnvironmentEdgeManager.getDelegate(), wal, htd);
261     }
262     wal.append(hri, tableName, edit, now, htd);
263     // sync to fs.
264     wal.sync();
265 
266     User user = HBaseTestingUtility.getDifferentUser(newConf,
267         ".replay.wal.secondtime");
268     user.runAs(new PrivilegedExceptionAction() {
269       public Object run() throws Exception {
270         Path p = runWALSplit(newConf);
271         LOG.info("WALSplit path == " + p);
272         FileSystem newFS = FileSystem.get(newConf);
273         // Make a new wal for new region open.
274         HLog wal2 = createWAL(newConf);
275         Path tableDir =
276           HTableDescriptor.getTableDir(hbaseRootDir, hri.getTableName());
277         HRegion region = new HRegion(tableDir, wal2, FileSystem.get(newConf),
278           newConf, hri, htd, TEST_UTIL.getHBaseCluster().getRegionServer(0));
279 
280         long seqid2 = region.initialize();
281         SampleRegionWALObserver cp2 =
282           (SampleRegionWALObserver)region.getCoprocessorHost().findCoprocessor(
283               SampleRegionWALObserver.class.getName());
284         // TODO: asserting here is problematic.
285         assertNotNull(cp2);
286         assertTrue(cp2.isPreWALRestoreCalled());
287         assertTrue(cp2.isPostWALRestoreCalled());
288         region.close();
289         wal2.closeAndDelete();
290         return null;
291       }
292     });
293   }
294 
295   /**
296    * Test to see CP loaded successfully or not. There is a duplication
297    * at TestHLog, but the purpose of that one is to see whether the loaded
298    * CP will impact existing HLog tests or not.
299    */
300   @Test
301   public void testWALObserverLoaded() throws Exception {
302     HLog log = new HLog(fs, dir, oldLogDir, conf);
303     assertNotNull(getCoprocessor(log));
304   }
305 
306   private SampleRegionWALObserver getCoprocessor(HLog wal) throws Exception {
307     WALCoprocessorHost host = wal.getCoprocessorHost();
308     Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName());
309     return (SampleRegionWALObserver)c;
310   }
311 
312   /*
313    * Creates an HRI around an HTD that has <code>tableName</code> and three
314    * column families named.
315    * @param tableName Name of table to use when we create HTableDescriptor.
316    */
317   private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
318     HTableDescriptor htd = new HTableDescriptor(tableName);
319 
320     for (int i = 0; i < TEST_FAMILY.length; i++ ) {
321       HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]);
322       htd.addFamily(a);
323     }
324     return new HRegionInfo(htd.getName(), null, null, false);
325   }
326 
327   /*
328    * @param p Directory to cleanup
329    */
330   private void deleteDir(final Path p) throws IOException {
331     if (this.fs.exists(p)) {
332       if (!this.fs.delete(p, true)) {
333         throw new IOException("Failed remove of " + p);
334       }
335     }
336   }
337 
338   private Put creatPutWith2Families(byte[] row) throws IOException {
339     Put p = new Put(row);
340     for (int i = 0; i < TEST_FAMILY.length-1; i++ ) {
341       p.add(TEST_FAMILY[i], TEST_QUALIFIER[i],
342           TEST_VALUE[i]);
343     }
344     return p;
345   }
346 
347   /**
348    * Copied from HRegion.
349    *
350    * @param familyMap map of family->edits
351    * @param walEdit the destination entry to append into
352    */
353   private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap,
354       WALEdit walEdit) {
355     for (List<KeyValue> edits : familyMap.values()) {
356       for (KeyValue kv : edits) {
357         walEdit.add(kv);
358       }
359     }
360   }
361   private Path runWALSplit(final Configuration c) throws IOException {
362     FileSystem fs = FileSystem.get(c);
363     HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c,
364         this.hbaseRootDir, this.logDir, this.oldLogDir, fs);
365     List<Path> splits = logSplitter.splitLog();
366     // Split should generate only 1 file since there's only 1 region
367     assertEquals(1, splits.size());
368     // Make sure the file exists
369     assertTrue(fs.exists(splits.get(0)));
370     LOG.info("Split file=" + splits.get(0));
371     return splits.get(0);
372   }
373   private HLog createWAL(final Configuration c) throws IOException {
374     HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c);
375     return wal;
376   }
377   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
378       final byte [] rowName, final byte [] family,
379       final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
380   throws IOException {
381     String familyStr = Bytes.toString(family);
382     for (int j = 0; j < count; j++) {
383       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
384       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
385       WALEdit edit = new WALEdit();
386       edit.add(new KeyValue(rowName, family, qualifierBytes,
387         ee.currentTimeMillis(), columnBytes));
388       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
389     }
390   }
391   private HTableDescriptor getBasic3FamilyHTableDescriptor(
392       final String tableName) {
393     HTableDescriptor htd = new HTableDescriptor(tableName);
394 
395     for (int i = 0; i < TEST_FAMILY.length; i++ ) {
396       HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]);
397       htd.addFamily(a);
398     }
399     return htd;
400   }
401 
402   private HTableDescriptor createBasic3FamilyHTD(final String tableName) {
403     HTableDescriptor htd = new HTableDescriptor(tableName);
404     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
405     htd.addFamily(a);
406     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
407     htd.addFamily(b);
408     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
409     htd.addFamily(c);
410     return htd;
411   }
412 
413 }
414