1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import static org.junit.Assert.assertEquals;
23  
24  import java.net.URLEncoder;
25  import java.util.ArrayList;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseConfiguration;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HColumnDescriptor;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.Server;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.catalog.CatalogTracker;
43  import org.apache.hadoop.hbase.regionserver.wal.HLog;
44  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
45  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
46  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
47  import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
50  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
51  import org.junit.After;
52  import org.junit.AfterClass;
53  import org.junit.Before;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  
57  public class TestReplicationSourceManager {
58  
59    private static final Log LOG =
60        LogFactory.getLog(TestReplicationSourceManager.class);
61  
62    private static Configuration conf;
63  
64    private static HBaseTestingUtility utility;
65  
66    private static Replication replication;
67  
68    private static ReplicationSourceManager manager;
69  
70    private static ZooKeeperWatcher zkw;
71  
72    private static HTableDescriptor htd;
73  
74    private static HRegionInfo hri;
75  
76    private static final byte[] r1 = Bytes.toBytes("r1");
77  
78    private static final byte[] r2 = Bytes.toBytes("r2");
79  
80    private static final byte[] f1 = Bytes.toBytes("f1");
81  
82    private static final byte[] test = Bytes.toBytes("test");
83  
84    private static final String slaveId = "1";
85  
86    private static FileSystem fs;
87  
88    private static Path oldLogDir;
89  
90    private static Path logDir;
91  
92  
93    @BeforeClass
94    public static void setUpBeforeClass() throws Exception {
95  
96      conf = HBaseConfiguration.create();
97      conf.set("replication.replicationsource.implementation",
98          ReplicationSourceDummy.class.getCanonicalName());
99      conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
100     utility = new HBaseTestingUtility(conf);
101     utility.startMiniZKCluster();
102 
103     zkw = new ZooKeeperWatcher(conf, "test", null);
104     ZKUtil.createWithParents(zkw, "/hbase/replication");
105     ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
106     ZKUtil.setData(zkw, "/hbase/replication/peers/1",Bytes.toBytes(
107           conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
108           conf.get("hbase.zookeeper.property.clientPort")+":/1"));
109     ZKUtil.createWithParents(zkw, "/hbase/replication/state");
110     ZKUtil.setData(zkw, "/hbase/replication/state", Bytes.toBytes("true"));
111 
112     replication = new Replication(new DummyServer(), fs, logDir, oldLogDir);
113     manager = replication.getReplicationManager();
114     fs = FileSystem.get(conf);
115     oldLogDir = new Path(utility.getDataTestDir(),
116         HConstants.HREGION_OLDLOGDIR_NAME);
117     logDir = new Path(utility.getDataTestDir(),
118         HConstants.HREGION_LOGDIR_NAME);
119 
120     manager.addSource(slaveId);
121 
122     htd = new HTableDescriptor(test);
123     HColumnDescriptor col = new HColumnDescriptor("f1");
124     col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
125     htd.addFamily(col);
126     col = new HColumnDescriptor("f2");
127     col.setScope(HConstants.REPLICATION_SCOPE_LOCAL);
128     htd.addFamily(col);
129 
130     hri = new HRegionInfo(htd.getName(), r1, r2);
131 
132 
133   }
134 
135   @AfterClass
136   public static void tearDownAfterClass() throws Exception {
137     manager.join();
138     utility.shutdownMiniCluster();
139   }
140 
141   @Before
142   public void setUp() throws Exception {
143     fs.delete(logDir, true);
144     fs.delete(oldLogDir, true);
145   }
146 
147   @After
148   public void tearDown() throws Exception {
149     setUp();
150   }
151 
152   @Test
153   public void testLogRoll() throws Exception {
154     long seq = 0;
155     long baseline = 1000;
156     long time = baseline;
157     KeyValue kv = new KeyValue(r1, f1, r1);
158     WALEdit edit = new WALEdit();
159     edit.add(kv);
160 
161     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
162     listeners.add(replication);
163     HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners,
164       URLEncoder.encode("regionserver:60020", "UTF8"));
165 
166     manager.init();
167     HTableDescriptor htd = new HTableDescriptor();
168     htd.addFamily(new HColumnDescriptor(f1));
169     // Testing normal log rolling every 20
170     for(long i = 1; i < 101; i++) {
171       if(i > 1 && i % 20 == 0) {
172         hlog.rollWriter();
173       }
174       LOG.info(i);
175       HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
176           System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
177       hlog.append(hri, key, edit, htd);
178     }
179 
180     // Simulate a rapid insert that's followed
181     // by a report that's still not totally complete (missing last one)
182     LOG.info(baseline + " and " + time);
183     baseline += 101;
184     time = baseline;
185     LOG.info(baseline + " and " + time);
186 
187     for (int i = 0; i < 3; i++) {
188       HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
189           System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
190       hlog.append(hri, key, edit, htd);
191     }
192 
193     assertEquals(6, manager.getHLogs().get(slaveId).size());
194 
195     hlog.rollWriter();
196 
197     manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
198         "1", 0, false);
199 
200     HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
201         System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
202     hlog.append(hri, key, edit, htd);
203 
204     assertEquals(1, manager.getHLogs().size());
205 
206 
207     // TODO Need a case with only 2 HLogs and we only want to delete the first one
208   }
209 
210   static class DummyServer implements Server {
211 
212     @Override
213     public Configuration getConfiguration() {
214       return conf;
215     }
216 
217     @Override
218     public ZooKeeperWatcher getZooKeeper() {
219       return zkw;
220     }
221 
222     @Override
223     public CatalogTracker getCatalogTracker() {
224       return null;  //To change body of implemented methods use File | Settings | File Templates.
225     }
226 
227     @Override
228     public ServerName getServerName() {
229       return new ServerName("hostname.example.org", 1234, -1L);
230     }
231 
232     @Override
233     public void abort(String why, Throwable e) {
234       //To change body of implemented methods use File | Settings | File Templates.
235     }
236     
237     @Override
238     public boolean isAborted() {
239       return false;
240     }
241 
242     @Override
243     public void stop(String why) {
244       //To change body of implemented methods use File | Settings | File Templates.
245     }
246 
247     @Override
248     public boolean isStopped() {
249       return false;  //To change body of implemented methods use File | Settings | File Templates.
250     }
251   }
252 
253 }