1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.concurrent.atomic.AtomicBoolean;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.MiniHBaseCluster;
34  import org.apache.hadoop.hbase.client.HTable;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.ResultScanner;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.executor.EventHandler;
40  import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
41  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
42  import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
43  import org.apache.hadoop.hbase.regionserver.HRegionServer;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.Threads;
46  import org.apache.hadoop.hbase.util.Writables;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  
53  import static org.junit.Assert.assertEquals;
54  import static org.junit.Assert.assertTrue;
55  
56  /**
57   * Test open and close of regions using zk.
58   */
59  public class TestZKBasedOpenCloseRegion {
60    private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
61    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
63    private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
64      Bytes.toBytes("b"), Bytes.toBytes("c")};
65    private static int countOfRegions;
66  
67    @BeforeClass public static void beforeAllTests() throws Exception {
68      Configuration c = TEST_UTIL.getConfiguration();
69      c.setBoolean("dfs.support.append", true);
70      c.setInt("hbase.regionserver.info.port", 0);
71      TEST_UTIL.startMiniCluster(2);
72      TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
73      HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
74      countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
75      waitUntilAllRegionsAssigned();
76      addToEachStartKey(countOfRegions);
77    }
78  
79    @AfterClass public static void afterAllTests() throws Exception {
80      TEST_UTIL.shutdownMiniCluster();
81    }
82  
83    @Before public void setup() throws IOException {
84      if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
85        // Need at least two servers.
86        LOG.info("Started new server=" +
87          TEST_UTIL.getHBaseCluster().startRegionServer());
88  
89      }
90      waitUntilAllRegionsAssigned();
91    }
92  
93    /**
94     * Test we reopen a region once closed.
95     * @throws Exception
96     */
97    @Test (timeout=300000) public void testReOpenRegion()
98    throws Exception {
99      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
100     LOG.info("Number of region servers = " +
101       cluster.getLiveRegionServerThreads().size());
102 
103     int rsIdx = 0;
104     HRegionServer regionServer =
105       TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
106     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
107     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
108 
109     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
110     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
111 
112     EventHandlerListener closeListener =
113       new ReopenEventListener(hri.getRegionNameAsString(),
114           closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
115     cluster.getMaster().executorService.
116       registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
117 
118     EventHandlerListener openListener =
119       new ReopenEventListener(hri.getRegionNameAsString(),
120           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
121     cluster.getMaster().executorService.
122       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
123 
124     LOG.info("Unassign " + hri.getRegionNameAsString());
125     cluster.getMaster().assignmentManager.unassign(hri);
126 
127     while (!closeEventProcessed.get()) {
128       Threads.sleep(100);
129     }
130 
131     while (!reopenEventProcessed.get()) {
132       Threads.sleep(100);
133     }
134 
135     LOG.info("Done with testReOpenRegion");
136   }
137 
138   private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
139     HRegionInfo hri = null;
140     for (HRegionInfo i: regions) {
141       LOG.info(i.getRegionNameAsString());
142       if (!i.isMetaRegion()) {
143         hri = i;
144         break;
145       }
146     }
147     return hri;
148   }
149 
150   public static class ReopenEventListener implements EventHandlerListener {
151     private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
152     String regionName;
153     AtomicBoolean eventProcessed;
154     EventType eventType;
155 
156     public ReopenEventListener(String regionName,
157         AtomicBoolean eventProcessed, EventType eventType) {
158       this.regionName = regionName;
159       this.eventProcessed = eventProcessed;
160       this.eventType = eventType;
161     }
162 
163     @Override
164     public void beforeProcess(EventHandler event) {
165       if(event.getEventType() == eventType) {
166         LOG.info("Received " + eventType + " and beginning to process it");
167       }
168     }
169 
170     @Override
171     public void afterProcess(EventHandler event) {
172       LOG.info("afterProcess(" + event + ")");
173       if(event.getEventType() == eventType) {
174         LOG.info("Finished processing " + eventType);
175         String regionName = "";
176         if(eventType == EventType.RS_ZK_REGION_OPENED) {
177           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
178           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
179         } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
180           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
181           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
182         }
183         if(this.regionName.equals(regionName)) {
184           eventProcessed.set(true);
185         }
186         synchronized(eventProcessed) {
187           eventProcessed.notifyAll();
188         }
189       }
190     }
191   }
192 
193   public static class CloseRegionEventListener implements EventHandlerListener {
194     private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
195     String regionToClose;
196     AtomicBoolean closeEventProcessed;
197 
198     public CloseRegionEventListener(String regionToClose,
199         AtomicBoolean closeEventProcessed) {
200       this.regionToClose = regionToClose;
201       this.closeEventProcessed = closeEventProcessed;
202     }
203 
204     @Override
205     public void afterProcess(EventHandler event) {
206       LOG.info("afterProcess(" + event + ")");
207       if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) {
208         LOG.info("Finished processing CLOSE REGION");
209         TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
210         if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
211           LOG.info("Setting closeEventProcessed flag");
212           closeEventProcessed.set(true);
213         } else {
214           LOG.info("Region to close didn't match");
215         }
216       }
217     }
218 
219     @Override
220     public void beforeProcess(EventHandler event) {
221       if(event.getEventType() == EventType.M_RS_CLOSE_REGION) {
222         LOG.info("Received CLOSE RPC and beginning to process it");
223       }
224     }
225   }
226 
227   /**
228    * This test shows how a region won't be able to be assigned to a RS
229    * if it's already "processing" it.
230    * @throws Exception
231    */
232   @Test
233   public void testRSAlreadyProcessingRegion() throws Exception {
234     LOG.info("starting testRSAlreadyProcessingRegion");
235     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
236 
237     HRegionServer hr0 =
238         cluster.getLiveRegionServerThreads().get(0).getRegionServer();
239     HRegionServer hr1 =
240         cluster.getLiveRegionServerThreads().get(1).getRegionServer();
241     HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
242 
243     // fake that hr1 is processing the region
244     hr1.getRegionsInTransitionInRS().putIfAbsent(hri.getEncodedNameAsBytes(), true);
245 
246     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
247     EventHandlerListener openListener =
248       new ReopenEventListener(hri.getRegionNameAsString(),
249           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
250     cluster.getMaster().executorService.
251       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
252 
253     // now ask the master to move the region to hr1, will fail
254     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
255         Bytes.toBytes(hr1.getServerName().toString()));
256 
257     // make sure the region came back
258     assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null);
259 
260     // remove the block and reset the boolean
261     hr1.getRegionsInTransitionInRS().remove(hri.getEncodedNameAsBytes());
262     reopenEventProcessed.set(false);
263     
264     // now try moving a region when there is no region in transition.
265     hri = getNonMetaRegion(hr1.getOnlineRegions());
266 
267     openListener =
268       new ReopenEventListener(hri.getRegionNameAsString(),
269           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
270 
271     cluster.getMaster().executorService.
272       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
273     
274     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
275         Bytes.toBytes(hr0.getServerName().toString()));
276 
277     while (!reopenEventProcessed.get()) {
278       Threads.sleep(100);
279     }
280 
281     // make sure the region has moved from the original RS
282     assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
283 
284   }
285 
286   @Test (timeout=300000) public void testCloseRegion()
287   throws Exception {
288     LOG.info("Running testCloseRegion");
289     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
290     LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
291 
292     int rsIdx = 0;
293     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
294     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
295     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
296 
297     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
298     EventHandlerListener listener =
299       new CloseRegionEventListener(hri.getRegionNameAsString(),
300           closeEventProcessed);
301     cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener);
302 
303     cluster.getMaster().assignmentManager.unassign(hri);
304 
305     while (!closeEventProcessed.get()) {
306       Threads.sleep(100);
307     }
308     LOG.info("Done with testCloseRegion");
309   }
310 
311   private static void waitUntilAllRegionsAssigned()
312   throws IOException {
313     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
314       HConstants.META_TABLE_NAME);
315     while (true) {
316       int rows = 0;
317       Scan scan = new Scan();
318       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
319       ResultScanner s = meta.getScanner(scan);
320       for (Result r = null; (r = s.next()) != null;) {
321         byte [] b =
322           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
323         if (b == null || b.length <= 0) {
324           break;
325         }
326         rows++;
327       }
328       s.close();
329       // If I get to here and all rows have a Server, then all have been assigned.
330       if (rows >= countOfRegions) {
331         break;
332       }
333       LOG.info("Found=" + rows);
334       Threads.sleep(1000);
335     }
336   }
337 
338   /*
339    * Add to each of the regions in .META. a value.  Key is the startrow of the
340    * region (except its 'aaa' for first region).  Actual value is the row name.
341    * @param expected
342    * @return
343    * @throws IOException
344    */
345   private static int addToEachStartKey(final int expected) throws IOException {
346     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
347     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
348         HConstants.META_TABLE_NAME);
349     int rows = 0;
350     Scan scan = new Scan();
351     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
352     ResultScanner s = meta.getScanner(scan);
353     for (Result r = null; (r = s.next()) != null;) {
354       byte [] b =
355         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
356       if (b == null || b.length <= 0) {
357         break;
358       }
359       HRegionInfo hri = Writables.getHRegionInfo(b);
360       // If start key, add 'aaa'.
361       byte [] row = getStartKey(hri);
362       Put p = new Put(row);
363       p.setWriteToWAL(false);
364       p.add(getTestFamily(), getTestQualifier(), row);
365       t.put(p);
366       rows++;
367     }
368     s.close();
369     Assert.assertEquals(expected, rows);
370     return rows;
371   }
372 
373   private static byte [] getStartKey(final HRegionInfo hri) {
374     return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
375         Bytes.toBytes("aaa"): hri.getStartKey();
376   }
377 
378   private static byte [] getTestFamily() {
379     return FAMILIES[0];
380   }
381 
382   private static byte [] getTestQualifier() {
383     return getTestFamily();
384   }
385 
386   public static void main(String args[]) throws Exception {
387     TestZKBasedOpenCloseRegion.beforeAllTests();
388 
389     TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
390     test.setup();
391     test.testCloseRegion();
392 
393     TestZKBasedOpenCloseRegion.afterAllTests();
394   }
395 }