View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import java.io.FileNotFoundException;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.Comparator;
29  import java.util.HashMap;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.NavigableMap;
33  import java.util.Random;
34  import java.util.Set;
35  import java.util.TreeMap;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.fs.FileSystem;
41  import org.apache.hadoop.hbase.ClusterStatus;
42  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.TableExistsException;
47  import org.apache.hadoop.hbase.regionserver.HRegion;
48  import org.apache.hadoop.hbase.util.Bytes;
49  
50  import com.google.common.base.Joiner;
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.MinMaxPriorityQueue;
53  import com.google.common.collect.Sets;
54  
55  /**
56   * Makes decisions about the placement and movement of Regions across
57   * RegionServers.
58   *
59   * <p>Cluster-wide load balancing will occur only when there are no regions in
60   * transition and according to a fixed period of a time using {@link #balanceCluster(Map)}.
61   *
62   * <p>Inline region placement with {@link #immediateAssignment} can be used when
63   * the Master needs to handle closed regions that it currently does not have
64   * a destination set for.  This can happen during master failover.
65   *
66   * <p>On cluster startup, bulk assignment can be used to determine
67   * locations for all Regions in a cluster.
68   *
69   * <p>This classes produces plans for the {@link AssignmentManager} to execute.
70   */
71  public class DefaultLoadBalancer implements LoadBalancer {
72    private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
73    private static final Random RANDOM = new Random(System.currentTimeMillis());
74    // slop for regions
75    private float slop;
76    private Configuration config;
77    private ClusterStatus status;
78    private MasterServices services;
79  
80    public void setClusterStatus(ClusterStatus st) {
81      this.status = st;
82    }
83  
84    public void setMasterServices(MasterServices masterServices) {
85      this.services = masterServices;
86    }
87  
88    @Override
89    public void setConf(Configuration conf) {
90      this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2);
91      if (slop < 0) slop = 0;
92      else if (slop > 1) slop = 1;
93      this.config = conf;
94    }
95  
96    @Override
97    public Configuration getConf() {
98      return this.config;
99    }
100 
101   /*
102   * The following comparator assumes that RegionId from HRegionInfo can
103   * represent the age of the region - larger RegionId means the region
104   * is younger.
105   * This comparator is used in balanceCluster() to account for the out-of-band
106   * regions which were assigned to the server after some other region server
107   * crashed.
108   */
109    private class RegionInfoComparator implements Comparator<HRegionInfo> {
110        @Override
111        public int compare(HRegionInfo l, HRegionInfo r) {
112           long diff = r.getRegionId() - l.getRegionId();
113           if (diff < 0) return -1;
114           if (diff > 0) return 1;
115           return 0;
116        }
117    }
118 
119 
120    RegionInfoComparator riComparator = new RegionInfoComparator();
121    
122    private class RegionPlanComparator implements Comparator<RegionPlan> {
123     @Override
124     public int compare(RegionPlan l, RegionPlan r) {
125       long diff = r.getRegionInfo().getRegionId() - l.getRegionInfo().getRegionId();
126       if (diff < 0) return -1;
127       if (diff > 0) return 1;
128       return 0;
129     }
130   }
131 
132   RegionPlanComparator rpComparator = new RegionPlanComparator();
133 
134   /**
135    * Generate a global load balancing plan according to the specified map of
136    * server information to the most loaded regions of each server.
137    *
138    * The load balancing invariant is that all servers are within 1 region of the
139    * average number of regions per server.  If the average is an integer number,
140    * all servers will be balanced to the average.  Otherwise, all servers will
141    * have either floor(average) or ceiling(average) regions.
142    *
143    * HBASE-3609 Modeled regionsToMove using Guava's MinMaxPriorityQueue so that
144    *   we can fetch from both ends of the queue. 
145    * At the beginning, we check whether there was empty region server 
146    *   just discovered by Master. If so, we alternately choose new / old
147    *   regions from head / tail of regionsToMove, respectively. This alternation
148    *   avoids clustering young regions on the newly discovered region server.
149    *   Otherwise, we choose new regions from head of regionsToMove.
150    *   
151    * Another improvement from HBASE-3609 is that we assign regions from
152    *   regionsToMove to underloaded servers in round-robin fashion.
153    *   Previously one underloaded server would be filled before we move onto
154    *   the next underloaded server, leading to clustering of young regions.
155    *   
156    * Finally, we randomly shuffle underloaded servers so that they receive
157    *   offloaded regions relatively evenly across calls to balanceCluster().
158    *         
159    * The algorithm is currently implemented as such:
160    *
161    * <ol>
162    * <li>Determine the two valid numbers of regions each server should have,
163    *     <b>MIN</b>=floor(average) and <b>MAX</b>=ceiling(average).
164    *
165    * <li>Iterate down the most loaded servers, shedding regions from each so
166    *     each server hosts exactly <b>MAX</b> regions.  Stop once you reach a
167    *     server that already has &lt;= <b>MAX</b> regions.
168    *     <p>
169    *     Order the regions to move from most recent to least.
170    *
171    * <li>Iterate down the least loaded servers, assigning regions so each server
172    *     has exactly </b>MIN</b> regions.  Stop once you reach a server that
173    *     already has &gt;= <b>MIN</b> regions.
174    *
175    *     Regions being assigned to underloaded servers are those that were shed
176    *     in the previous step.  It is possible that there were not enough
177    *     regions shed to fill each underloaded server to <b>MIN</b>.  If so we
178    *     end up with a number of regions required to do so, <b>neededRegions</b>.
179    *
180    *     It is also possible that we were able to fill each underloaded but ended
181    *     up with regions that were unassigned from overloaded servers but that
182    *     still do not have assignment.
183    *
184    *     If neither of these conditions hold (no regions needed to fill the
185    *     underloaded servers, no regions leftover from overloaded servers),
186    *     we are done and return.  Otherwise we handle these cases below.
187    *
188    * <li>If <b>neededRegions</b> is non-zero (still have underloaded servers),
189    *     we iterate the most loaded servers again, shedding a single server from
190    *     each (this brings them from having <b>MAX</b> regions to having
191    *     <b>MIN</b> regions).
192    *
193    * <li>We now definitely have more regions that need assignment, either from
194    *     the previous step or from the original shedding from overloaded servers.
195    *     Iterate the least loaded servers filling each to <b>MIN</b>.
196    *
197    * <li>If we still have more regions that need assignment, again iterate the
198    *     least loaded servers, this time giving each one (filling them to
199    *     </b>MAX</b>) until we run out.
200    *
201    * <li>All servers will now either host <b>MIN</b> or <b>MAX</b> regions.
202    *
203    *     In addition, any server hosting &gt;= <b>MAX</b> regions is guaranteed
204    *     to end up with <b>MAX</b> regions at the end of the balancing.  This
205    *     ensures the minimal number of regions possible are moved.
206    * </ol>
207    *
208    * TODO: We can at-most reassign the number of regions away from a particular
209    *       server to be how many they report as most loaded.
210    *       Should we just keep all assignment in memory?  Any objections?
211    *       Does this mean we need HeapSize on HMaster?  Or just careful monitor?
212    *       (current thinking is we will hold all assignments in memory)
213    *
214    * @param clusterState Map of regionservers and their load/region information to
215    *                   a list of their most loaded regions
216    * @return a list of regions to be moved, including source and destination,
217    *         or null if cluster is already balanced
218    */
219   public List<RegionPlan> balanceCluster(
220       Map<ServerName, List<HRegionInfo>> clusterState) {
221     boolean emptyRegionServerPresent = false;
222     long startTime = System.currentTimeMillis();
223 
224     int numServers = clusterState.size();
225     if (numServers == 0) {
226       LOG.debug("numServers=0 so skipping load balancing");
227       return null;
228     }
229     NavigableMap<ServerAndLoad, List<HRegionInfo>> serversByLoad =
230       new TreeMap<ServerAndLoad, List<HRegionInfo>>();
231     int numRegions = 0;
232     // Iterate so we can count regions as we build the map
233     for (Map.Entry<ServerName, List<HRegionInfo>> server: clusterState.entrySet()) {
234       List<HRegionInfo> regions = server.getValue();
235       int sz = regions.size();
236       if (sz == 0) emptyRegionServerPresent = true;
237       numRegions += sz;
238       serversByLoad.put(new ServerAndLoad(server.getKey(), sz), regions);
239     }
240     // Check if we even need to do any load balancing
241     float average = (float)numRegions / numServers; // for logging
242     // HBASE-3681 check sloppiness first
243     int floor = (int) Math.floor(average * (1 - slop));
244     int ceiling = (int) Math.ceil(average * (1 + slop));
245     if (serversByLoad.lastKey().getLoad() <= ceiling &&
246        serversByLoad.firstKey().getLoad() >= floor) {
247       // Skipped because no server outside (min,max) range
248       LOG.info("Skipping load balancing because balanced cluster; " +
249         "servers=" + numServers + " " +
250         "regions=" + numRegions + " average=" + average + " " +
251         "mostloaded=" + serversByLoad.lastKey().getLoad() +
252         " leastloaded=" + serversByLoad.firstKey().getLoad());
253       return null;
254     }
255     int min = numRegions / numServers;
256     int max = numRegions % numServers == 0 ? min : min + 1;
257 
258     // Using to check balance result.
259     StringBuilder strBalanceParam = new StringBuilder();
260     strBalanceParam.append("Balance parameter: numRegions=").append(numRegions)
261         .append(", numServers=").append(numServers).append(", max=").append(max)
262         .append(", min=").append(min);
263     LOG.debug(strBalanceParam.toString());
264 
265     // Balance the cluster
266     // TODO: Look at data block locality or a more complex load to do this
267     MinMaxPriorityQueue<RegionPlan> regionsToMove =
268       MinMaxPriorityQueue.orderedBy(rpComparator).create();
269     List<RegionPlan> regionsToReturn = new ArrayList<RegionPlan>();
270 
271     // Walk down most loaded, pruning each to the max
272     int serversOverloaded = 0;
273     // flag used to fetch regions from head and tail of list, alternately
274     boolean fetchFromTail = false;
275     Map<ServerName, BalanceInfo> serverBalanceInfo =
276       new TreeMap<ServerName, BalanceInfo>();
277     for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server:
278         serversByLoad.descendingMap().entrySet()) {
279       ServerAndLoad sal = server.getKey();
280       int regionCount = sal.getLoad();
281       if (regionCount <= max) {
282         serverBalanceInfo.put(sal.getServerName(), new BalanceInfo(0, 0));
283         break;
284       }
285       serversOverloaded++;
286       List<HRegionInfo> regions = server.getValue();
287       int numToOffload = Math.min(regionCount - max, regions.size());
288       // account for the out-of-band regions which were assigned to this server
289       // after some other region server crashed 
290       Collections.sort(regions, riComparator);
291       int numTaken = 0;
292       for (int i = 0; i <= numToOffload; ) {
293         HRegionInfo hri = regions.get(i); // fetch from head
294         if (fetchFromTail) {
295           hri = regions.get(regions.size() - 1 - i);
296         }
297         i++;
298         // Don't rebalance meta regions.
299         if (hri.isMetaRegion()) continue;
300         regionsToMove.add(new RegionPlan(hri, sal.getServerName(), null));
301         numTaken++;
302         if (numTaken >= numToOffload) break;
303         // fetch in alternate order if there is new region server
304         if (emptyRegionServerPresent) {
305           fetchFromTail = !fetchFromTail;
306         }
307       }
308       serverBalanceInfo.put(sal.getServerName(),
309         new BalanceInfo(numToOffload, (-1)*numTaken));
310     }
311     int totalNumMoved = regionsToMove.size();
312 
313     // Walk down least loaded, filling each to the min
314     int neededRegions = 0; // number of regions needed to bring all up to min
315     fetchFromTail = false;
316 
317     Map<ServerName, Integer> underloadedServers = new HashMap<ServerName, Integer>();
318     for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server:
319         serversByLoad.entrySet()) {
320       int regionCount = server.getKey().getLoad();
321       if (regionCount >= min) {
322         break;
323       }
324       underloadedServers.put(server.getKey().getServerName(), min - regionCount);
325     }
326     // number of servers that get new regions
327     int serversUnderloaded = underloadedServers.size();
328     int incr = 1;
329     List<ServerName> sns =
330       Arrays.asList(underloadedServers.keySet().toArray(new ServerName[serversUnderloaded]));
331     Collections.shuffle(sns, RANDOM);
332     while (regionsToMove.size() > 0) {
333       int cnt = 0;
334       int i = incr > 0 ? 0 : underloadedServers.size()-1;
335       for (; i >= 0 && i < underloadedServers.size(); i += incr) {
336         if (regionsToMove.isEmpty()) break;
337         ServerName si = sns.get(i);
338         int numToTake = underloadedServers.get(si);
339         if (numToTake == 0) continue;
340 
341         addRegionPlan(regionsToMove, fetchFromTail, si, regionsToReturn);
342         if (emptyRegionServerPresent) {
343           fetchFromTail = !fetchFromTail;
344         }
345 
346         underloadedServers.put(si, numToTake-1);
347         cnt++;
348         BalanceInfo bi = serverBalanceInfo.get(si);
349         if (bi == null) {
350           bi = new BalanceInfo(0, 0);
351           serverBalanceInfo.put(si, bi);
352         }
353         bi.setNumRegionsAdded(bi.getNumRegionsAdded()+1);
354       }
355       if (cnt == 0) break;
356       // iterates underloadedServers in the other direction
357       incr = -incr;
358     }
359     for (Integer i : underloadedServers.values()) {
360       // If we still want to take some, increment needed
361       neededRegions += i;
362     }
363 
364     // If none needed to fill all to min and none left to drain all to max,
365     // we are done
366     if (neededRegions == 0 && regionsToMove.isEmpty()) {
367       long endTime = System.currentTimeMillis();
368       LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
369           "Moving " + totalNumMoved + " regions off of " +
370           serversOverloaded + " overloaded servers onto " +
371           serversUnderloaded + " less loaded servers");
372       return regionsToReturn;
373     }
374 
375     // Need to do a second pass.
376     // Either more regions to assign out or servers that are still underloaded
377 
378     // If we need more to fill min, grab one from each most loaded until enough
379     if (neededRegions != 0) {
380       // Walk down most loaded, grabbing one from each until we get enough
381       for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server :
382         serversByLoad.descendingMap().entrySet()) {
383         BalanceInfo balanceInfo =
384           serverBalanceInfo.get(server.getKey().getServerName());
385         int idx =
386           balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
387         if (idx >= server.getValue().size()) break;
388         HRegionInfo region = server.getValue().get(idx);
389         if (region.isMetaRegion()) continue; // Don't move meta regions.
390         regionsToMove.add(new RegionPlan(region, server.getKey().getServerName(), null));
391         totalNumMoved++;
392         if (--neededRegions == 0) {
393           // No more regions needed, done shedding
394           break;
395         }
396       }
397     }
398 
399     // Now we have a set of regions that must be all assigned out
400     // Assign each underloaded up to the min, then if leftovers, assign to max
401 
402     // Walk down least loaded, assigning to each to fill up to min
403     for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server :
404         serversByLoad.entrySet()) {
405       int regionCount = server.getKey().getLoad();
406       if (regionCount >= min) break;
407       BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey().getServerName());
408       if(balanceInfo != null) {
409         regionCount += balanceInfo.getNumRegionsAdded();
410       }
411       if(regionCount >= min) {
412         continue;
413       }
414       int numToTake = min - regionCount;
415       int numTaken = 0;
416       while(numTaken < numToTake && 0 < regionsToMove.size()) {
417         addRegionPlan(regionsToMove, fetchFromTail,
418           server.getKey().getServerName(), regionsToReturn);
419         numTaken++;
420         if (emptyRegionServerPresent) {
421           fetchFromTail = !fetchFromTail;
422         }
423       }
424     }
425 
426     // If we still have regions to dish out, assign underloaded to max
427     if (0 < regionsToMove.size()) {
428       for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server :
429         serversByLoad.entrySet()) {
430         int regionCount = server.getKey().getLoad();
431         if(regionCount >= max) {
432           break;
433         }
434         addRegionPlan(regionsToMove, fetchFromTail,
435           server.getKey().getServerName(), regionsToReturn);
436         if (emptyRegionServerPresent) {
437           fetchFromTail = !fetchFromTail;
438         }
439         if (regionsToMove.isEmpty()) {
440           break;
441         }
442       }
443     }
444 
445     long endTime = System.currentTimeMillis();
446 
447     if (!regionsToMove.isEmpty() || neededRegions != 0) {
448       // Emit data so can diagnose how balancer went astray.
449       LOG.warn("regionsToMove=" + totalNumMoved +
450         ", numServers=" + numServers + ", serversOverloaded=" + serversOverloaded +
451         ", serversUnderloaded=" + serversUnderloaded);
452       StringBuilder sb = new StringBuilder();
453       for (Map.Entry<ServerName, List<HRegionInfo>> e: clusterState.entrySet()) {
454         if (sb.length() > 0) sb.append(", ");
455         sb.append(e.getKey().toString());
456         sb.append(" ");
457         sb.append(e.getValue().size());
458       }
459       LOG.warn("Input " + sb.toString());
460     }
461 
462     // All done!
463     LOG.info("Done. Calculated a load balance in " + (endTime-startTime) + "ms. " +
464         "Moving " + totalNumMoved + " regions off of " +
465         serversOverloaded + " overloaded servers onto " +
466         serversUnderloaded + " less loaded servers");
467 
468     return regionsToReturn;
469   }
470 
471   /**
472    * Add a region from the head or tail to the List of regions to return.
473    */
474   void addRegionPlan(final MinMaxPriorityQueue<RegionPlan> regionsToMove,
475       final boolean fetchFromTail, final ServerName sn, List<RegionPlan> regionsToReturn) {
476     RegionPlan rp = null;
477     if (!fetchFromTail) rp = regionsToMove.remove();
478     else rp = regionsToMove.removeLast();
479     rp.setDestination(sn);
480     regionsToReturn.add(rp);
481   }
482 
483   /**
484    * Stores additional per-server information about the regions added/removed
485    * during the run of the balancing algorithm.
486    *
487    * For servers that shed regions, we need to track which regions we have
488    * already shed.  <b>nextRegionForUnload</b> contains the index in the list
489    * of regions on the server that is the next to be shed.
490    */
491   private static class BalanceInfo {
492 
493     private final int nextRegionForUnload;
494     private int numRegionsAdded;
495 
496     public BalanceInfo(int nextRegionForUnload, int numRegionsAdded) {
497       this.nextRegionForUnload = nextRegionForUnload;
498       this.numRegionsAdded = numRegionsAdded;
499     }
500 
501     public int getNextRegionForUnload() {
502       return nextRegionForUnload;
503     }
504 
505     public int getNumRegionsAdded() {
506       return numRegionsAdded;
507     }
508 
509     public void setNumRegionsAdded(int numAdded) {
510       this.numRegionsAdded = numAdded;
511     }
512   }
513 
514   /**
515    * Generates a bulk assignment plan to be used on cluster startup using a
516    * simple round-robin assignment.
517    * <p>
518    * Takes a list of all the regions and all the servers in the cluster and
519    * returns a map of each server to the regions that it should be assigned.
520    * <p>
521    * Currently implemented as a round-robin assignment.  Same invariant as
522    * load balancing, all servers holding floor(avg) or ceiling(avg).
523    *
524    * TODO: Use block locations from HDFS to place regions with their blocks
525    *
526    * @param regions all regions
527    * @param servers all servers
528    * @return map of server to the regions it should take, or null if no
529    *         assignment is possible (ie. no regions or no servers)
530    */
531   public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(
532       List<HRegionInfo> regions, List<ServerName> servers) {
533     if (regions.isEmpty() || servers.isEmpty()) {
534       return null;
535     }
536     Map<ServerName, List<HRegionInfo>> assignments =
537       new TreeMap<ServerName,List<HRegionInfo>>();
538     int numRegions = regions.size();
539     int numServers = servers.size();
540     int max = (int)Math.ceil((float)numRegions/numServers);
541     int serverIdx = 0;
542     if (numServers > 1) {
543       serverIdx = RANDOM.nextInt(numServers);
544     }
545     int regionIdx = 0;
546     for (int j = 0; j < numServers; j++) {
547       ServerName server = servers.get((j + serverIdx) % numServers);
548       List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max);
549       for (int i=regionIdx; i<numRegions; i += numServers) {
550         serverRegions.add(regions.get(i % numRegions));
551       }
552       assignments.put(server, serverRegions);
553       regionIdx++;
554     }
555     return assignments;
556   }
557 
558   /**
559    * Generates a bulk assignment startup plan, attempting to reuse the existing
560    * assignment information from META, but adjusting for the specified list of
561    * available/online servers available for assignment.
562    * <p>
563    * Takes a map of all regions to their existing assignment from META.  Also
564    * takes a list of online servers for regions to be assigned to.  Attempts to
565    * retain all assignment, so in some instances initial assignment will not be
566    * completely balanced.
567    * <p>
568    * Any leftover regions without an existing server to be assigned to will be
569    * assigned randomly to available servers.
570    * @param regions regions and existing assignment from meta
571    * @param servers available servers
572    * @return map of servers and regions to be assigned to them
573    */
574   public Map<ServerName, List<HRegionInfo>> retainAssignment(
575       Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
576     // Group all of the old assignments by their hostname.
577     // We can't group directly by ServerName since the servers all have
578     // new start-codes.
579     
580     // Group the servers by their hostname. It's possible we have multiple
581     // servers on the same host on different ports.
582     ArrayListMultimap<String, ServerName> serversByHostname =
583         ArrayListMultimap.create();
584     for (ServerName server : servers) {
585       serversByHostname.put(server.getHostname(), server);
586     }
587     
588     // Now come up with new assignments
589     Map<ServerName, List<HRegionInfo>> assignments =
590       new TreeMap<ServerName, List<HRegionInfo>>();
591     
592     for (ServerName server : servers) {
593       assignments.put(server, new ArrayList<HRegionInfo>());
594     }
595     
596     // Collection of the hostnames that used to have regions
597     // assigned, but for which we no longer have any RS running
598     // after the cluster restart.
599     Set<String> oldHostsNoLongerPresent = Sets.newTreeSet();
600     
601     int numRandomAssignments = 0;
602     int numRetainedAssigments = 0;
603     for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
604       HRegionInfo region = entry.getKey();
605       ServerName oldServerName = entry.getValue();
606       List<ServerName> localServers = new ArrayList<ServerName>();
607       if (oldServerName != null) {
608         localServers = serversByHostname.get(oldServerName.getHostname());
609       }
610       if (localServers.isEmpty()) {
611         // No servers on the new cluster match up with this hostname,
612         // assign randomly.
613         ServerName randomServer = servers.get(RANDOM.nextInt(servers.size()));
614         assignments.get(randomServer).add(region);
615         numRandomAssignments++;
616         if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname());
617       } else if (localServers.size() == 1) {
618         // the usual case - one new server on same host
619         assignments.get(localServers.get(0)).add(region);
620         numRetainedAssigments++;
621       } else {
622         // multiple new servers in the cluster on this same host
623         int size = localServers.size();
624         ServerName target = localServers.get(RANDOM.nextInt(size));
625         assignments.get(target).add(region);
626         numRetainedAssigments++;
627       }
628     }
629     
630     String randomAssignMsg = "";
631     if (numRandomAssignments > 0) {
632       randomAssignMsg = numRandomAssignments + " regions were assigned " +
633       		"to random hosts, since the old hosts for these regions are no " +
634       		"longer present in the cluster. These hosts were:\n  " +
635           Joiner.on("\n  ").join(oldHostsNoLongerPresent);
636     }
637     
638     LOG.info("Reassigned " + regions.size() + " regions. " +
639         numRetainedAssigments + " retained the pre-restart assignment. " +
640         randomAssignMsg);
641     return assignments;
642   }
643 
644   /**
645    * Returns an ordered list of hosts that are hosting the blocks for this
646    * region.  The weight of each host is the sum of the block lengths of all
647    * files on that host, so the first host in the list is the server which
648    * holds the most bytes of the given region's HFiles.
649    *
650    * @param fs the filesystem
651    * @param region region
652    * @return ordered list of hosts holding blocks of the specified region
653    */
654   @SuppressWarnings("unused")
655   private List<ServerName> getTopBlockLocations(FileSystem fs,
656     HRegionInfo region) {
657     List<ServerName> topServerNames = null;
658     try {
659       HTableDescriptor tableDescriptor = getTableDescriptor(
660         region.getTableName());
661       if (tableDescriptor != null) {
662         HDFSBlocksDistribution blocksDistribution =
663           HRegion.computeHDFSBlocksDistribution(config, tableDescriptor,
664           region.getEncodedName());
665         List<String> topHosts = blocksDistribution.getTopHosts();
666         topServerNames = mapHostNameToServerName(topHosts);
667       }
668     } catch (IOException ioe) {
669       LOG.debug("IOException during HDFSBlocksDistribution computation. for " +
670         "region = " + region.getEncodedName() , ioe);
671     }
672     
673     return topServerNames;
674   }
675 
676   /**
677    * return HTableDescriptor for a given tableName
678    * @param tableName the table name
679    * @return HTableDescriptor
680    * @throws IOException
681    */
682   private HTableDescriptor getTableDescriptor(byte[] tableName)
683     throws IOException {
684     HTableDescriptor tableDescriptor = null;
685     try {
686       if ( this.services != null)
687       {
688         tableDescriptor = this.services.getTableDescriptors().
689           get(Bytes.toString(tableName));
690     }
691     } catch (TableExistsException tee) {
692       LOG.debug("TableExistsException during getTableDescriptors." +
693         " Current table name = " + tableName , tee);
694     } catch (FileNotFoundException fnfe) {
695       LOG.debug("FileNotFoundException during getTableDescriptors." +
696         " Current table name = " + tableName , fnfe);
697     }
698 
699     return tableDescriptor;
700     }
701 
702   /**
703    * Map hostname to ServerName, The output ServerName list will have the same
704    * order as input hosts.
705    * @param hosts the list of hosts
706    * @return ServerName list
707    */  
708   private List<ServerName> mapHostNameToServerName(List<String> hosts) {
709     if ( hosts == null || status == null) {
710       return null;
711     }
712 
713     List<ServerName> topServerNames = new ArrayList<ServerName>();
714     Collection<ServerName> regionServers = status.getServers();
715 
716     // create a mapping from hostname to ServerName for fast lookup
717     HashMap<String, ServerName> hostToServerName =
718       new HashMap<String, ServerName>();
719     for (ServerName sn : regionServers) {
720       hostToServerName.put(sn.getHostname(), sn);
721         }
722 
723     for (String host : hosts ) {
724       ServerName sn = hostToServerName.get(host);
725       // it is possible that HDFS is up ( thus host is valid ),
726       // but RS is down ( thus sn is null )
727       if (sn != null) {
728         topServerNames.add(sn);
729       }
730     }
731     return topServerNames;
732   }
733 
734 
735   /**
736    * Generates an immediate assignment plan to be used by a new master for
737    * regions in transition that do not have an already known destination.
738    *
739    * Takes a list of regions that need immediate assignment and a list of
740    * all available servers.  Returns a map of regions to the server they
741    * should be assigned to.
742    *
743    * This method will return quickly and does not do any intelligent
744    * balancing.  The goal is to make a fast decision not the best decision
745    * possible.
746    *
747    * Currently this is random.
748    *
749    * @param regions
750    * @param servers
751    * @return map of regions to the server it should be assigned to
752    */
753   public Map<HRegionInfo, ServerName> immediateAssignment(
754       List<HRegionInfo> regions, List<ServerName> servers) {
755     Map<HRegionInfo,ServerName> assignments =
756       new TreeMap<HRegionInfo,ServerName>();
757     for(HRegionInfo region : regions) {
758       assignments.put(region, servers.get(RANDOM.nextInt(servers.size())));
759     }
760     return assignments;
761   }
762 
763   public ServerName randomAssignment(List<ServerName> servers) {
764     if (servers == null || servers.isEmpty()) {
765       LOG.warn("Wanted to do random assignment but no servers to assign to");
766       return null;
767     }
768     return servers.get(RANDOM.nextInt(servers.size()));
769   }
770 
771 }