1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertTrue;
27  
28  import java.io.IOException;
29  import java.util.Collection;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.NavigableMap;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.CoprocessorEnvironment;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HServerAddress;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.MiniHBaseCluster;
44  import org.apache.hadoop.hbase.ServerName;
45  import org.apache.hadoop.hbase.client.HBaseAdmin;
46  import org.apache.hadoop.hbase.client.HTable;
47  import org.apache.hadoop.hbase.master.AssignmentManager;
48  import org.apache.hadoop.hbase.master.HMaster;
49  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
50  import org.apache.hadoop.hbase.regionserver.HRegionServer;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.junit.AfterClass;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  
56  /**
57   * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver}
58   * interface hooks at all appropriate times during normal HMaster operations.
59   */
60  public class TestMasterObserver {
61    private static final Log LOG = LogFactory.getLog(TestMasterObserver.class);
62  
63    public static class CPMasterObserver implements MasterObserver {
64  
65      private boolean bypass = false;
66      private boolean preCreateTableCalled;
67      private boolean postCreateTableCalled;
68      private boolean preDeleteTableCalled;
69      private boolean postDeleteTableCalled;
70      private boolean preModifyTableCalled;
71      private boolean postModifyTableCalled;
72      private boolean preAddColumnCalled;
73      private boolean postAddColumnCalled;
74      private boolean preModifyColumnCalled;
75      private boolean postModifyColumnCalled;
76      private boolean preDeleteColumnCalled;
77      private boolean postDeleteColumnCalled;
78      private boolean preEnableTableCalled;
79      private boolean postEnableTableCalled;
80      private boolean preDisableTableCalled;
81      private boolean postDisableTableCalled;
82      private boolean preMoveCalled;
83      private boolean postMoveCalled;
84      private boolean preAssignCalled;
85      private boolean postAssignCalled;
86      private boolean preUnassignCalled;
87      private boolean postUnassignCalled;
88      private boolean preBalanceCalled;
89      private boolean postBalanceCalled;
90      private boolean preBalanceSwitchCalled;
91      private boolean postBalanceSwitchCalled;
92      private boolean preShutdownCalled;
93      private boolean preStopMasterCalled;
94      private boolean postStartMasterCalled;
95      private boolean startCalled;
96      private boolean stopCalled;
97  
98      public void enableBypass(boolean bypass) {
99        this.bypass = bypass;
100     }
101 
102     public void resetStates() {
103       preCreateTableCalled = false;
104       postCreateTableCalled = false;
105       preDeleteTableCalled = false;
106       postDeleteTableCalled = false;
107       preModifyTableCalled = false;
108       postModifyTableCalled = false;
109       preAddColumnCalled = false;
110       postAddColumnCalled = false;
111       preModifyColumnCalled = false;
112       postModifyColumnCalled = false;
113       preDeleteColumnCalled = false;
114       postDeleteColumnCalled = false;
115       preEnableTableCalled = false;
116       postEnableTableCalled = false;
117       preDisableTableCalled = false;
118       postDisableTableCalled = false;
119       preMoveCalled= false;
120       postMoveCalled = false;
121       preAssignCalled = false;
122       postAssignCalled = false;
123       preUnassignCalled = false;
124       postUnassignCalled = false;
125       preBalanceCalled = false;
126       postBalanceCalled = false;
127       preBalanceSwitchCalled = false;
128       postBalanceSwitchCalled = false;
129     }
130 
131     @Override
132     public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
133         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
134       if (bypass) {
135         env.bypass();
136       }
137       preCreateTableCalled = true;
138     }
139 
140     @Override
141     public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
142         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
143       postCreateTableCalled = true;
144     }
145 
146     public boolean wasCreateTableCalled() {
147       return preCreateTableCalled && postCreateTableCalled;
148     }
149 
150     public boolean preCreateTableCalledOnly() {
151       return preCreateTableCalled && !postCreateTableCalled;
152     }
153 
154     @Override
155     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> env,
156         byte[] tableName) throws IOException {
157       if (bypass) {
158         env.bypass();
159       }
160       preDeleteTableCalled = true;
161     }
162 
163     @Override
164     public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> env,
165         byte[] tableName) throws IOException {
166       postDeleteTableCalled = true;
167     }
168 
169     public boolean wasDeleteTableCalled() {
170       return preDeleteTableCalled && postDeleteTableCalled;
171     }
172 
173     public boolean preDeleteTableCalledOnly() {
174       return preDeleteTableCalled && !postDeleteTableCalled;
175     }
176 
177     @Override
178     public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
179         byte[] tableName, HTableDescriptor htd) throws IOException {
180       if (bypass) {
181         env.bypass();
182       }
183       preModifyTableCalled = true;
184     }
185 
186     @Override
187     public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
188         byte[] tableName, HTableDescriptor htd) throws IOException {
189       postModifyTableCalled = true;
190     }
191 
192     public boolean wasModifyTableCalled() {
193       return preModifyTableCalled && postModifyTableCalled;
194     }
195 
196     public boolean preModifyTableCalledOnly() {
197       return preModifyTableCalled && !postModifyTableCalled;
198     }
199 
200     @Override
201     public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> env,
202         byte[] tableName, HColumnDescriptor column) throws IOException {
203       if (bypass) {
204         env.bypass();
205       }
206       preAddColumnCalled = true;
207     }
208 
209     @Override
210     public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> env,
211         byte[] tableName, HColumnDescriptor column) throws IOException {
212       postAddColumnCalled = true;
213     }
214 
215     public boolean wasAddColumnCalled() {
216       return preAddColumnCalled && postAddColumnCalled;
217     }
218 
219     public boolean preAddColumnCalledOnly() {
220       return preAddColumnCalled && !postAddColumnCalled;
221     }
222 
223     @Override
224     public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> env,
225         byte[] tableName, HColumnDescriptor descriptor) throws IOException {
226       if (bypass) {
227         env.bypass();
228       }
229       preModifyColumnCalled = true;
230     }
231 
232     @Override
233     public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> env,
234         byte[] tableName, HColumnDescriptor descriptor) throws IOException {
235       postModifyColumnCalled = true;
236     }
237 
238     public boolean wasModifyColumnCalled() {
239       return preModifyColumnCalled && postModifyColumnCalled;
240     }
241 
242     public boolean preModifyColumnCalledOnly() {
243       return preModifyColumnCalled && !postModifyColumnCalled;
244     }
245 
246     @Override
247     public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> env,
248         byte[] tableName, byte[] c) throws IOException {
249       if (bypass) {
250         env.bypass();
251       }
252       preDeleteColumnCalled = true;
253     }
254 
255     @Override
256     public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> env,
257         byte[] tableName, byte[] c) throws IOException {
258       postDeleteColumnCalled = true;
259     }
260 
261     public boolean wasDeleteColumnCalled() {
262       return preDeleteColumnCalled && postDeleteColumnCalled;
263     }
264 
265     public boolean preDeleteColumnCalledOnly() {
266       return preDeleteColumnCalled && !postDeleteColumnCalled;
267     }
268 
269     @Override
270     public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> env,
271         byte[] tableName) throws IOException {
272       if (bypass) {
273         env.bypass();
274       }
275       preEnableTableCalled = true;
276     }
277 
278     @Override
279     public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> env,
280         byte[] tableName) throws IOException {
281       postEnableTableCalled = true;
282     }
283 
284     public boolean wasEnableTableCalled() {
285       return preEnableTableCalled && postEnableTableCalled;
286     }
287 
288     public boolean preEnableTableCalledOnly() {
289       return preEnableTableCalled && !postEnableTableCalled;
290     }
291 
292     @Override
293     public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> env,
294         byte[] tableName) throws IOException {
295       if (bypass) {
296         env.bypass();
297       }
298       preDisableTableCalled = true;
299     }
300 
301     @Override
302     public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> env,
303         byte[] tableName) throws IOException {
304       postDisableTableCalled = true;
305     }
306 
307     public boolean wasDisableTableCalled() {
308       return preDisableTableCalled && postDisableTableCalled;
309     }
310 
311     public boolean preDisableTableCalledOnly() {
312       return preDisableTableCalled && !postDisableTableCalled;
313     }
314 
315     @Override
316     public void preMove(ObserverContext<MasterCoprocessorEnvironment> env,
317         HRegionInfo region, ServerName srcServer, ServerName destServer)
318     throws IOException {
319       if (bypass) {
320         env.bypass();
321       }
322       preMoveCalled = true;
323     }
324 
325     @Override
326     public void postMove(ObserverContext<MasterCoprocessorEnvironment> env, HRegionInfo region,
327         ServerName srcServer, ServerName destServer)
328     throws IOException {
329       postMoveCalled = true;
330     }
331 
332     public boolean wasMoveCalled() {
333       return preMoveCalled && postMoveCalled;
334     }
335 
336     public boolean preMoveCalledOnly() {
337       return preMoveCalled && !postMoveCalled;
338     }
339     
340     @Override
341     public void preAssign(ObserverContext<MasterCoprocessorEnvironment> env,
342         final HRegionInfo regionInfo) throws IOException {
343       if (bypass) {
344         env.bypass();
345       }
346       preAssignCalled = true;
347     }
348 
349     @Override
350     public void postAssign(ObserverContext<MasterCoprocessorEnvironment> env,
351         final HRegionInfo regionInfo) throws IOException {
352       postAssignCalled = true;
353     }
354     
355     public boolean wasAssignCalled() {
356       return preAssignCalled && postAssignCalled;
357     }
358 
359     public boolean preAssignCalledOnly() {
360       return preAssignCalled && !postAssignCalled;
361     }
362 
363     @Override
364     public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> env,
365         final HRegionInfo regionInfo, final boolean force) throws IOException {
366       if (bypass) {
367         env.bypass();
368       }
369       preUnassignCalled = true;
370     }
371 
372     @Override
373     public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> env,
374         final HRegionInfo regionInfo, final boolean force) throws IOException {
375       postUnassignCalled = true;
376     }
377 
378     public boolean wasUnassignCalled() {
379       return preUnassignCalled && postUnassignCalled;
380     }
381 
382     public boolean preUnassignCalledOnly() {
383       return preUnassignCalled && !postUnassignCalled;
384     }
385 
386     @Override
387     public void preBalance(ObserverContext<MasterCoprocessorEnvironment> env)
388         throws IOException {
389       if (bypass) {
390         env.bypass();
391       }
392       preBalanceCalled = true;
393     }
394 
395     @Override
396     public void postBalance(ObserverContext<MasterCoprocessorEnvironment> env)
397         throws IOException {
398       postBalanceCalled = true;
399     }
400 
401     public boolean wasBalanceCalled() {
402       return preBalanceCalled && postBalanceCalled;
403     }
404 
405     public boolean preBalanceCalledOnly() {
406       return preBalanceCalled && !postBalanceCalled;
407     }
408 
409     @Override
410     public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> env, boolean b)
411         throws IOException {
412       if (bypass) {
413         env.bypass();
414       }
415       preBalanceSwitchCalled = true;
416       return b;
417     }
418 
419     @Override
420     public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> env,
421         boolean oldValue, boolean newValue) throws IOException {
422       postBalanceSwitchCalled = true;
423     }
424 
425     public boolean wasBalanceSwitchCalled() {
426       return preBalanceSwitchCalled && postBalanceSwitchCalled;
427     }
428 
429     public boolean preBalanceSwitchCalledOnly() {
430       return preBalanceSwitchCalled && !postBalanceSwitchCalled;
431     }
432 
433     @Override
434     public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> env)
435         throws IOException {
436       preShutdownCalled = true;
437     }
438 
439     @Override
440     public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> env)
441         throws IOException {
442       preStopMasterCalled = true;
443     }
444 
445     @Override
446     public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
447         throws IOException {
448       postStartMasterCalled = true;
449     }
450 
451     public boolean wasStartMasterCalled() {
452       return postStartMasterCalled;
453     }
454 
455     @Override
456     public void start(CoprocessorEnvironment env) throws IOException {
457       startCalled = true;
458     }
459 
460     @Override
461     public void stop(CoprocessorEnvironment env) throws IOException {
462       stopCalled = true;
463     }
464 
465     public boolean wasStarted() { return startCalled; }
466 
467     public boolean wasStopped() { return stopCalled; }
468   }
469 
470   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
471   private static byte[] TEST_TABLE = Bytes.toBytes("observed_table");
472   private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
473   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
474 
475   @BeforeClass
476   public static void setupBeforeClass() throws Exception {
477     Configuration conf = UTIL.getConfiguration();
478     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
479         CPMasterObserver.class.getName());
480     // We need more than one data server on this test
481     UTIL.startMiniCluster(2);
482   }
483 
484   @AfterClass
485   public static void tearDownAfterClass() throws Exception {
486     UTIL.shutdownMiniCluster();
487   }
488 
489   @Test
490   public void testStarted() throws Exception {
491     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
492 
493     HMaster master = cluster.getMaster();
494     assertTrue("Master should be active", master.isActiveMaster());
495     MasterCoprocessorHost host = master.getCoprocessorHost();
496     assertNotNull("CoprocessorHost should not be null", host);
497     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
498         CPMasterObserver.class.getName());
499     assertNotNull("CPMasterObserver coprocessor not found or not installed!", cp);
500 
501     // check basic lifecycle
502     assertTrue("MasterObserver should have been started", cp.wasStarted());
503     assertTrue("postStartMaster() hook should have been called",
504         cp.wasStartMasterCalled());
505   }
506 
507   @Test
508   public void testTableOperations() throws Exception {
509     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
510 
511     HMaster master = cluster.getMaster();
512     MasterCoprocessorHost host = master.getCoprocessorHost();
513     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
514         CPMasterObserver.class.getName());
515     cp.enableBypass(true);
516     cp.resetStates();
517     assertFalse("No table created yet", cp.wasCreateTableCalled());
518 
519     // create a table
520     HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
521     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
522     HBaseAdmin admin = UTIL.getHBaseAdmin();
523 
524     admin.createTable(htd);
525     // preCreateTable can't bypass default action.
526     assertTrue("Test table should be created", cp.wasCreateTableCalled());
527 
528     admin.disableTable(TEST_TABLE);
529     assertTrue(admin.isTableDisabled(TEST_TABLE));
530     // preDisableTable can't bypass default action.
531     assertTrue("Coprocessor should have been called on table disable",
532       cp.wasDisableTableCalled());
533 
534     // enable
535     assertFalse(cp.wasEnableTableCalled());
536     admin.enableTable(TEST_TABLE);
537     assertTrue(admin.isTableEnabled(TEST_TABLE));
538     // preEnableTable can't bypass default action.
539     assertTrue("Coprocessor should have been called on table enable",
540       cp.wasEnableTableCalled());
541 
542     admin.disableTable(TEST_TABLE);
543     assertTrue(admin.isTableDisabled(TEST_TABLE));
544 
545     // modify table
546     htd.setMaxFileSize(512 * 1024 * 1024);
547     admin.modifyTable(TEST_TABLE, htd);
548     // preModifyTable can't bypass default action.
549     assertTrue("Test table should have been modified",
550       cp.wasModifyTableCalled());
551 
552     // add a column family
553     admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
554     assertTrue("New column family shouldn't have been added to test table",
555       cp.preAddColumnCalledOnly());
556 
557     // modify a column family
558     HColumnDescriptor hcd1 = new HColumnDescriptor(TEST_FAMILY2);
559     hcd1.setMaxVersions(25);
560     admin.modifyColumn(TEST_TABLE, hcd1);
561     assertTrue("Second column family should be modified",
562       cp.preModifyColumnCalledOnly());
563 
564     // delete table
565     admin.deleteTable(TEST_TABLE);
566     assertFalse("Test table should have been deleted",
567         admin.tableExists(TEST_TABLE));
568     // preDeleteTable can't bypass default action.
569     assertTrue("Coprocessor should have been called on table delete",
570       cp.wasDeleteTableCalled());
571 
572 
573     // turn off bypass, run the tests again
574     cp.enableBypass(false);
575     cp.resetStates();
576 
577     admin.createTable(htd);
578     assertTrue("Test table should be created", cp.wasCreateTableCalled());
579 
580     // disable
581     assertFalse(cp.wasDisableTableCalled());
582 
583     admin.disableTable(TEST_TABLE);
584     assertTrue(admin.isTableDisabled(TEST_TABLE));
585     assertTrue("Coprocessor should have been called on table disable",
586       cp.wasDisableTableCalled());
587 
588     // modify table
589     htd.setMaxFileSize(512 * 1024 * 1024);
590     admin.modifyTable(TEST_TABLE, htd);
591     assertTrue("Test table should have been modified",
592         cp.wasModifyTableCalled());
593 
594     // add a column family
595     admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
596     assertTrue("New column family should have been added to test table",
597         cp.wasAddColumnCalled());
598 
599     // modify a column family
600     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
601     hcd.setMaxVersions(25);
602     admin.modifyColumn(TEST_TABLE, hcd);
603     assertTrue("Second column family should be modified",
604         cp.wasModifyColumnCalled());
605 
606     // enable
607     assertFalse(cp.wasEnableTableCalled());
608     admin.enableTable(TEST_TABLE);
609     assertTrue(admin.isTableEnabled(TEST_TABLE));
610     assertTrue("Coprocessor should have been called on table enable",
611         cp.wasEnableTableCalled());
612 
613     // disable again
614     admin.disableTable(TEST_TABLE);
615     assertTrue(admin.isTableDisabled(TEST_TABLE));
616 
617     // delete column
618     assertFalse("No column family deleted yet", cp.wasDeleteColumnCalled());
619     admin.deleteColumn(TEST_TABLE, TEST_FAMILY2);
620     HTableDescriptor tableDesc = admin.getTableDescriptor(TEST_TABLE);
621     assertNull("'"+Bytes.toString(TEST_FAMILY2)+"' should have been removed",
622         tableDesc.getFamily(TEST_FAMILY2));
623     assertTrue("Coprocessor should have been called on column delete",
624         cp.wasDeleteColumnCalled());
625 
626     // delete table
627     assertFalse("No table deleted yet", cp.wasDeleteTableCalled());
628     admin.deleteTable(TEST_TABLE);
629     assertFalse("Test table should have been deleted",
630         admin.tableExists(TEST_TABLE));
631     assertTrue("Coprocessor should have been called on table delete",
632         cp.wasDeleteTableCalled());
633   }
634 
635   @Test
636   public void testRegionTransitionOperations() throws Exception {
637     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
638 
639     HMaster master = cluster.getMaster();
640     MasterCoprocessorHost host = master.getCoprocessorHost();
641     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
642         CPMasterObserver.class.getName());
643     cp.enableBypass(false);
644     cp.resetStates();
645 
646     HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
647     int countOfRegions = UTIL.createMultiRegions(table, TEST_FAMILY);
648     UTIL.waitUntilAllRegionsAssigned(countOfRegions);
649     
650     NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
651     Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
652     for (Map.Entry<HRegionInfo, ServerName> e: regions.entrySet()) {
653       if (e.getValue() != null) {
654         firstGoodPair = e;
655         break;
656       }
657     }
658     assertNotNull("Found a non-null entry", firstGoodPair);
659     LOG.info("Found " + firstGoodPair.toString());
660     // Try to force a move
661     Collection<ServerName> servers = master.getClusterStatus().getServers();
662     String destName = null;
663     String firstRegionHostnamePortStr = firstGoodPair.getValue().toString();
664     LOG.info("firstRegionHostnamePortStr=" + firstRegionHostnamePortStr);
665     boolean found = false;
666     // Find server that is NOT carrying the first region
667     for (ServerName info : servers) {
668       LOG.info("ServerName=" + info);
669       if (!firstRegionHostnamePortStr.equals(info.getHostAndPort())) {
670         destName = info.toString();
671         found = true;
672         break;
673       }
674     }
675     assertTrue("Found server", found);
676     LOG.info("Found " + destName);
677     master.move(firstGoodPair.getKey().getEncodedNameAsBytes(),
678       Bytes.toBytes(destName));
679     assertTrue("Coprocessor should have been called on region move",
680       cp.wasMoveCalled());
681 
682     // make sure balancer is on
683     master.balanceSwitch(true);
684     assertTrue("Coprocessor should have been called on balance switch",
685         cp.wasBalanceSwitchCalled());
686 
687     // force region rebalancing
688     master.balanceSwitch(false);
689     // move half the open regions from RS 0 to RS 1
690     HRegionServer rs = cluster.getRegionServer(0);
691     byte[] destRS = Bytes.toBytes(cluster.getRegionServer(1).getServerName().toString());
692     List<HRegionInfo> openRegions = rs.getOnlineRegions();
693     int moveCnt = openRegions.size()/2;
694     for (int i=0; i<moveCnt; i++) {
695       HRegionInfo info = openRegions.get(i);
696       if (!info.isMetaTable()) {
697         master.move(openRegions.get(i).getEncodedNameAsBytes(), destRS);
698       }
699     }
700 
701     // wait for assignments to finish
702     AssignmentManager mgr = master.getAssignmentManager();
703     Collection<AssignmentManager.RegionState> transRegions =
704         mgr.getRegionsInTransition().values();
705     for (AssignmentManager.RegionState state : transRegions) {
706       mgr.waitOnRegionToClearRegionsInTransition(state.getRegion());
707     }
708 
709     // now trigger a balance
710     master.balanceSwitch(true);
711     boolean balanceRun = master.balance();
712     assertTrue("Coprocessor should be called on region rebalancing",
713         cp.wasBalanceCalled());
714   }
715 }