View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.zookeeper;
21  
22  import java.io.IOException;
23  import java.io.UnsupportedEncodingException;
24  import java.lang.reflect.Field;
25  import java.net.URLDecoder;
26  import java.net.URLEncoder;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  import java.util.concurrent.atomic.AtomicLong;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.master.SplitLogManager;
38  import org.apache.hadoop.hbase.regionserver.SplitLogWorker;
39  import org.apache.hadoop.hbase.util.Bytes;
40  
41  /**
42   * Common methods and attributes used by {@link SplitLogManager} and
43   * {@link SplitLogWorker}
44   */
45  public class ZKSplitLog {
46    private static final Log LOG = LogFactory.getLog(ZKSplitLog.class);
47  
48    public static final int DEFAULT_TIMEOUT = 25000; // 25 sec
49    public static final int DEFAULT_ZK_RETRIES = 3;
50    public static final int DEFAULT_MAX_RESUBMIT = 3;
51    public static final int DEFAULT_UNASSIGNED_TIMEOUT = (3 * 60 * 1000); //3 min
52  
53    /**
54     * Gets the full path node name for the log file being split.
55     * This method will url encode the filename.
56     * @param zkw zk reference
57     * @param filename log file name (only the basename)
58     */
59    public static String getEncodedNodeName(ZooKeeperWatcher zkw,
60        String filename) {
61        return ZKUtil.joinZNode(zkw.splitLogZNode, encode(filename));
62    }
63  
64    public static String getFileName(String node) {
65      String basename = node.substring(node.lastIndexOf('/') + 1);
66      return decode(basename);
67    }
68  
69  
70    public static String encode(String s) {
71      try {
72        return URLEncoder.encode(s, "UTF-8");
73      } catch (UnsupportedEncodingException e) {
74        throw new RuntimeException("URLENCODER doesn't support UTF-8");
75      }
76    }
77  
78    public static String decode(String s) {
79      try {
80        return URLDecoder.decode(s, "UTF-8");
81      } catch (UnsupportedEncodingException e) {
82        throw new RuntimeException("URLDecoder doesn't support UTF-8");
83      }
84    }
85  
86    public static String getRescanNode(ZooKeeperWatcher zkw) {
87      return ZKUtil.joinZNode(zkw.splitLogZNode, "RESCAN");
88    }
89  
90    public static boolean isRescanNode(ZooKeeperWatcher zkw, String path) {
91      String prefix = getRescanNode(zkw);
92      if (path.length() <= prefix.length()) {
93        return false;
94      }
95      for (int i = 0; i < prefix.length(); i++) {
96        if (prefix.charAt(i) != path.charAt(i)) {
97          return false;
98        }
99      }
100     return true;
101   }
102 
103   public static boolean isTaskPath(ZooKeeperWatcher zkw, String path) {
104     String dirname = path.substring(0, path.lastIndexOf('/'));
105     return dirname.equals(zkw.splitLogZNode);
106   }
107 
108   public static enum TaskState {
109     TASK_UNASSIGNED("unassigned"),
110     TASK_OWNED("owned"),
111     TASK_RESIGNED("resigned"),
112     TASK_DONE("done"),
113     TASK_ERR("err");
114 
115     private final byte[] state;
116     private TaskState(String s) {
117       state = s.getBytes();
118     }
119 
120     public byte[] get(String serverName) {
121       return (Bytes.add(state, " ".getBytes(), serverName.getBytes()));
122     }
123 
124     public String getWriterName(byte[] data) {
125       String str = Bytes.toString(data);
126       return str.substring(str.indexOf(' ') + 1);
127     }
128 
129 
130     /**
131      * @param s
132      * @return True if {@link #state} is a prefix of s. False otherwise.
133      */
134     public boolean equals(byte[] s) {
135       if (s.length < state.length) {
136         return (false);
137       }
138       for (int i = 0; i < state.length; i++) {
139         if (state[i] != s[i]) {
140           return (false);
141         }
142       }
143       return (true);
144     }
145 
146     public boolean equals(byte[] s, String serverName) {
147       return (Arrays.equals(s, get(serverName)));
148     }
149     @Override
150     public String toString() {
151       return new String(state);
152     }
153   }
154 
155   public static Path getSplitLogDir(Path rootdir, String tmpname) {
156     return new Path(new Path(rootdir, HConstants.SPLIT_LOGDIR_NAME), tmpname);
157   }
158 
159   public static Path stripSplitLogTempDir(Path rootdir, Path file) {
160     int skipDepth = rootdir.depth() + 2;
161     List<String> components = new ArrayList<String>(10);
162     do {
163       components.add(file.getName());
164       file = file.getParent();
165     } while (file.depth() > skipDepth);
166     Path ret = rootdir;
167     for (int i = components.size() - 1; i >= 0; i--) {
168       ret = new Path(ret, components.get(i));
169     }
170     return ret;
171   }
172 
173   public static String getSplitLogDirTmpComponent(String worker, String file) {
174     return (worker + "_" + ZKSplitLog.encode(file));
175   }
176 
177   public static void markCorrupted(Path rootdir, String tmpname,
178       FileSystem fs) {
179     Path file = new Path(getSplitLogDir(rootdir, tmpname), "corrupt");
180     try {
181       fs.createNewFile(file);
182     } catch (IOException e) {
183       LOG.warn("Could not flag a log file as corrupted. Failed to create " +
184           file, e);
185     }
186   }
187 
188   public static boolean isCorrupted(Path rootdir, String tmpname,
189       FileSystem fs) throws IOException {
190     Path file = new Path(getSplitLogDir(rootdir, tmpname), "corrupt");
191     boolean isCorrupt;
192     isCorrupt = fs.exists(file);
193     return isCorrupt;
194   }
195 
196   public static boolean isCorruptFlagFile(Path file) {
197     return file.getName().equals("corrupt");
198   }
199 
200 
201   public static class Counters {
202     //SplitLogManager counters
203     public static AtomicLong tot_mgr_log_split_batch_start = new AtomicLong(0);
204     public static AtomicLong tot_mgr_log_split_batch_success =
205       new AtomicLong(0);
206     public static AtomicLong tot_mgr_log_split_batch_err = new AtomicLong(0);
207     public static AtomicLong tot_mgr_new_unexpected_hlogs = new AtomicLong(0);
208     public static AtomicLong tot_mgr_log_split_start = new AtomicLong(0);
209     public static AtomicLong tot_mgr_log_split_success = new AtomicLong(0);
210     public static AtomicLong tot_mgr_log_split_err = new AtomicLong(0);
211     public static AtomicLong tot_mgr_node_create_queued = new AtomicLong(0);
212     public static AtomicLong tot_mgr_node_create_result = new AtomicLong(0);
213     public static AtomicLong tot_mgr_node_already_exists = new AtomicLong(0);
214     public static AtomicLong tot_mgr_node_create_err = new AtomicLong(0);
215     public static AtomicLong tot_mgr_node_create_retry = new AtomicLong(0);
216     public static AtomicLong tot_mgr_get_data_queued = new AtomicLong(0);
217     public static AtomicLong tot_mgr_get_data_result = new AtomicLong(0);
218     public static AtomicLong tot_mgr_get_data_err = new AtomicLong(0);
219     public static AtomicLong tot_mgr_get_data_retry = new AtomicLong(0);
220     public static AtomicLong tot_mgr_node_delete_queued = new AtomicLong(0);
221     public static AtomicLong tot_mgr_node_delete_result = new AtomicLong(0);
222     public static AtomicLong tot_mgr_node_delete_err = new AtomicLong(0);
223     public static AtomicLong tot_mgr_resubmit = new AtomicLong(0);
224     public static AtomicLong tot_mgr_resubmit_failed = new AtomicLong(0);
225     public static AtomicLong tot_mgr_null_data = new AtomicLong(0);
226     public static AtomicLong tot_mgr_orphan_task_acquired = new AtomicLong(0);
227     public static AtomicLong tot_mgr_unacquired_orphan_done = new AtomicLong(0);
228     public static AtomicLong tot_mgr_resubmit_threshold_reached =
229       new AtomicLong(0);
230     public static AtomicLong tot_mgr_missing_state_in_delete =
231       new AtomicLong(0);
232     public static AtomicLong tot_mgr_heartbeat = new AtomicLong(0);
233     public static AtomicLong tot_mgr_rescan = new AtomicLong(0);
234     public static AtomicLong tot_mgr_rescan_deleted = new AtomicLong(0);
235     public static AtomicLong tot_mgr_task_deleted = new AtomicLong(0);
236     public static AtomicLong tot_mgr_resubmit_unassigned = new AtomicLong(0);
237     public static AtomicLong tot_mgr_relist_logdir = new AtomicLong(0);
238     public static AtomicLong tot_mgr_resubmit_dead_server_task =
239       new AtomicLong(0);
240 
241 
242 
243     // SplitLogWorker counters
244     public static AtomicLong tot_wkr_failed_to_grab_task_no_data =
245       new AtomicLong(0);
246     public static AtomicLong tot_wkr_failed_to_grab_task_exception =
247       new AtomicLong(0);
248     public static AtomicLong tot_wkr_failed_to_grab_task_owned =
249       new AtomicLong(0);
250     public static AtomicLong tot_wkr_failed_to_grab_task_lost_race =
251       new AtomicLong(0);
252     public static AtomicLong tot_wkr_task_acquired = new AtomicLong(0);
253     public static AtomicLong tot_wkr_task_resigned = new AtomicLong(0);
254     public static AtomicLong tot_wkr_task_done = new AtomicLong(0);
255     public static AtomicLong tot_wkr_task_err = new AtomicLong(0);
256     public static AtomicLong tot_wkr_task_heartbeat = new AtomicLong(0);
257     public static AtomicLong tot_wkr_task_acquired_rescan = new AtomicLong(0);
258     public static AtomicLong tot_wkr_get_data_queued = new AtomicLong(0);
259     public static AtomicLong tot_wkr_get_data_result = new AtomicLong(0);
260     public static AtomicLong tot_wkr_get_data_retry = new AtomicLong(0);
261     public static AtomicLong tot_wkr_preempt_task = new AtomicLong(0);
262     public static AtomicLong tot_wkr_task_heartbeat_failed = new AtomicLong(0);
263     public static AtomicLong tot_wkr_final_transistion_failed =
264       new AtomicLong(0);
265 
266     public static void resetCounters() throws Exception {
267       Class<?> cl = (new Counters()).getClass();
268       Field[] flds = cl.getDeclaredFields();
269       for (Field fld : flds) {
270         ((AtomicLong)fld.get(null)).set(0);
271       }
272     }
273   }
274 }