1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.handler;
21
22 import java.io.IOException;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.NavigableMap;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.Server;
32 import org.apache.hadoop.hbase.ServerName;
33 import org.apache.hadoop.hbase.catalog.CatalogTracker;
34 import org.apache.hadoop.hbase.catalog.MetaEditor;
35 import org.apache.hadoop.hbase.catalog.MetaReader;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.executor.EventHandler;
38 import org.apache.hadoop.hbase.master.AssignmentManager;
39 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
40 import org.apache.hadoop.hbase.master.DeadServer;
41 import org.apache.hadoop.hbase.master.MasterServices;
42 import org.apache.hadoop.hbase.master.ServerManager;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.zookeeper.KeeperException;
45
46
47
48
49
50
51 public class ServerShutdownHandler extends EventHandler {
52 private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
53 private final ServerName serverName;
54 private final MasterServices services;
55 private final DeadServer deadServers;
56 private final boolean shouldSplitHlog;
57
58 public ServerShutdownHandler(final Server server, final MasterServices services,
59 final DeadServer deadServers, final ServerName serverName,
60 final boolean shouldSplitHlog) {
61 this(server, services, deadServers, serverName, EventType.M_SERVER_SHUTDOWN,
62 shouldSplitHlog);
63 }
64
65 ServerShutdownHandler(final Server server, final MasterServices services,
66 final DeadServer deadServers, final ServerName serverName, EventType type,
67 final boolean shouldSplitHlog) {
68 super(server, type);
69 this.serverName = serverName;
70 this.server = server;
71 this.services = services;
72 this.deadServers = deadServers;
73 if (!this.deadServers.contains(this.serverName)) {
74 LOG.warn(this.serverName + " is NOT in deadservers; it should be!");
75 }
76 this.shouldSplitHlog = shouldSplitHlog;
77 }
78
79 @Override
80 public String getInformativeName() {
81 if (serverName != null) {
82 return this.getClass().getSimpleName() + " for " + serverName;
83 } else {
84 return super.getInformativeName();
85 }
86 }
87
88
89
90
91
92
93
94
95
96
97
98
99 private void verifyAndAssignRoot()
100 throws InterruptedException, IOException, KeeperException {
101 long timeout = this.server.getConfiguration().
102 getLong("hbase.catalog.verification.timeout", 1000);
103 if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
104 this.services.getAssignmentManager().assignRoot();
105 }
106 }
107
108
109
110
111
112 private void verifyAndAssignRootWithRetries() throws IOException {
113 int iTimes = this.server.getConfiguration().getInt(
114 "hbase.catalog.verification.retries", 10);
115
116 long waitTime = this.server.getConfiguration().getLong(
117 "hbase.catalog.verification.timeout", 1000);
118
119 int iFlag = 0;
120 while (true) {
121 try {
122 verifyAndAssignRoot();
123 break;
124 } catch (KeeperException e) {
125 this.server.abort("In server shutdown processing, assigning root", e);
126 throw new IOException("Aborting", e);
127 } catch (Exception e) {
128 if (iFlag >= iTimes) {
129 this.server.abort("verifyAndAssignRoot failed after" + iTimes
130 + " times retries, aborting", e);
131 throw new IOException("Aborting", e);
132 }
133 try {
134 Thread.sleep(waitTime);
135 } catch (InterruptedException e1) {
136 LOG.warn("Interrupted when is the thread sleep", e1);
137 Thread.currentThread().interrupt();
138 throw new IOException("Interrupted", e1);
139 }
140 iFlag++;
141 }
142 }
143 }
144
145
146
147
148 boolean isCarryingRoot() {
149 return false;
150 }
151
152
153
154
155 boolean isCarryingMeta() {
156 return false;
157 }
158
159 @Override
160 public String toString() {
161 String name = "UnknownServerName";
162 if(server != null && server.getServerName() != null) {
163 name = server.getServerName().toString();
164 }
165 return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
166 }
167
168 @Override
169 public void process() throws IOException {
170 final ServerName serverName = this.serverName;
171
172 try {
173
174 try {
175 LOG.info("Splitting logs for " + serverName);
176 this.services.getMasterFileSystem().splitLog(serverName);
177 } catch (IOException ioe) {
178 this.services.getExecutorService().submit(this);
179 this.deadServers.add(serverName);
180 throw new IOException("failed log splitting for " +
181 serverName + ", will retry", ioe);
182 }
183
184
185 if (isCarryingRoot()) {
186 LOG.info("Server " + serverName +
187 " was carrying ROOT. Trying to assign.");
188 this.services.getAssignmentManager().
189 regionOffline(HRegionInfo.ROOT_REGIONINFO);
190 verifyAndAssignRootWithRetries();
191 }
192
193
194 if (isCarryingMeta()) {
195 LOG.info("Server " + serverName +
196 " was carrying META. Trying to assign.");
197 this.services.getAssignmentManager().
198 regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
199 this.services.getAssignmentManager().assignMeta();
200 }
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219 if (isCarryingRoot() || isCarryingMeta()) {
220 this.services.getExecutorService().submit(new ServerShutdownHandler(
221 this.server, this.services, this.deadServers, serverName, false));
222 this.deadServers.add(serverName);
223 return;
224 }
225
226
227
228
229
230 List<RegionState> regionsInTransition =
231 this.services.getAssignmentManager()
232 .processServerShutdown(this.serverName);
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250 NavigableMap<HRegionInfo, Result> hris = null;
251 while (!this.server.isStopped()) {
252 try {
253 this.server.getCatalogTracker().waitForMeta();
254 hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
255 this.serverName);
256 break;
257 } catch (InterruptedException e) {
258 Thread.currentThread().interrupt();
259 throw new IOException("Interrupted", e);
260 } catch (IOException ioe) {
261 LOG.info("Received exception accessing META during server shutdown of " +
262 serverName + ", retrying META read", ioe);
263 }
264 }
265
266
267 for (RegionState rit : regionsInTransition) {
268 if (!rit.isClosing() && !rit.isPendingClose()) {
269 LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
270 " from list of regions to assign because in RIT" + " region state: "
271 + rit.getState());
272 if (hris != null) hris.remove(rit.getRegion());
273 }
274 }
275
276 assert regionsInTransition != null;
277 LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) +
278 " region(s) that " + (serverName == null? "null": serverName) +
279 " was carrying (skipping " +
280 regionsInTransition.size() +
281 " regions(s) that are already in transition)");
282
283
284 if (hris != null) {
285 for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
286 if (processDeadRegion(e.getKey(), e.getValue(),
287 this.services.getAssignmentManager(),
288 this.server.getCatalogTracker())) {
289 this.services.getAssignmentManager().assign(e.getKey(), true);
290 }
291 }
292 }
293 } finally {
294 this.deadServers.finish(serverName);
295 }
296 LOG.info("Finished processing of shutdown of " + serverName);
297 }
298
299
300
301
302
303
304
305
306
307
308
309 public static boolean processDeadRegion(HRegionInfo hri, Result result,
310 AssignmentManager assignmentManager, CatalogTracker catalogTracker)
311 throws IOException {
312
313 boolean disabled = assignmentManager.getZKTable().isDisabledTable(
314 hri.getTableNameAsString());
315 if (disabled) return false;
316 if (hri.isOffline() && hri.isSplit()) {
317 LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
318 "; checking daughter presence");
319 fixupDaughters(result, assignmentManager, catalogTracker);
320 return false;
321 }
322 return true;
323 }
324
325
326
327
328
329
330
331 static void fixupDaughters(final Result result,
332 final AssignmentManager assignmentManager,
333 final CatalogTracker catalogTracker)
334 throws IOException {
335 fixupDaughter(result, HConstants.SPLITA_QUALIFIER, assignmentManager,
336 catalogTracker);
337 fixupDaughter(result, HConstants.SPLITB_QUALIFIER, assignmentManager,
338 catalogTracker);
339 }
340
341
342
343
344
345
346
347 static void fixupDaughter(final Result result, final byte [] qualifier,
348 final AssignmentManager assignmentManager,
349 final CatalogTracker catalogTracker)
350 throws IOException {
351 HRegionInfo daughter =
352 MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier);
353 if (daughter == null) return;
354 if (isDaughterMissing(catalogTracker, daughter)) {
355 LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
356 MetaEditor.addDaughter(catalogTracker, daughter, null);
357
358
359
360
361
362
363 assignmentManager.assign(daughter, true);
364 } else {
365 LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
366 }
367 }
368
369
370
371
372
373
374
375
376 private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
377 final HRegionInfo daughter) throws IOException {
378 FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
379
380
381
382
383
384
385 byte [] startrow = daughter.getRegionName();
386 MetaReader.fullScan(catalogTracker, visitor, startrow);
387 return !visitor.foundDaughter();
388 }
389
390
391
392
393
394 static class FindDaughterVisitor implements MetaReader.Visitor {
395 private final HRegionInfo daughter;
396 private boolean found = false;
397
398 FindDaughterVisitor(final HRegionInfo daughter) {
399 this.daughter = daughter;
400 }
401
402
403
404
405 boolean foundDaughter() {
406 return this.found;
407 }
408
409 @Override
410 public boolean visit(Result r) throws IOException {
411 HRegionInfo hri =
412 MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
413 if (hri == null) {
414 LOG.warn("No serialized HRegionInfo in " + r);
415 return true;
416 }
417 byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
418 HConstants.SERVER_QUALIFIER);
419
420 if (value == null) return false;
421
422
423 if (!Bytes.equals(daughter.getTableName(),
424 hri.getTableName())) {
425
426 return false;
427 }
428
429 if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
430 return false;
431 }
432
433
434
435 this.found = true;
436 return false;
437 }
438 }
439 }