View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import java.io.IOException;
23  import java.util.LinkedList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.Chore;
33  import org.apache.hadoop.hbase.RemoteExceptionHandler;
34  import org.apache.hadoop.hbase.Stoppable;
35  import org.apache.hadoop.hbase.regionserver.wal.HLog;
36  import org.apache.hadoop.hbase.util.FSUtils;
37  
38  import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
39  
40  /**
41   * This Chore, everytime it runs, will clear the HLogs in the old logs folder
42   * that are deletable for each log cleaner in the chain.
43   */
44  public class LogCleaner extends Chore {
45    static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
46  
47    private final FileSystem fs;
48    private final Path oldLogDir;
49    private List<LogCleanerDelegate> logCleanersChain;
50    private final Configuration conf;
51  
52    /**
53     *
54     * @param p the period of time to sleep between each run
55     * @param s the stopper
56     * @param conf configuration to use
57     * @param fs handle to the FS
58     * @param oldLogDir the path to the archived logs
59     */
60    public LogCleaner(final int p, final Stoppable s,
61                          Configuration conf, FileSystem fs,
62                          Path oldLogDir) {
63      super("LogsCleaner", p, s);
64      this.fs = fs;
65      this.oldLogDir = oldLogDir;
66      this.conf = conf;
67      this.logCleanersChain = new LinkedList<LogCleanerDelegate>();
68  
69      initLogCleanersChain();
70    }
71  
72    /*
73     * Initialize the chain of log cleaners from the configuration. The default
74     * three LogCleanerDelegates in this chain are: TimeToLiveLogCleaner,
75     * ReplicationLogCleaner and SnapshotLogCleaner.
76     */
77    private void initLogCleanersChain() {
78      String[] logCleaners = conf.getStrings(HBASE_MASTER_LOGCLEANER_PLUGINS);
79      if (logCleaners != null) {
80        for (String className : logCleaners) {
81          LogCleanerDelegate logCleaner = newLogCleaner(className, conf);
82          addLogCleaner(logCleaner);
83        }
84      }
85    }
86  
87    /**
88     * A utility method to create new instances of LogCleanerDelegate based
89     * on the class name of the LogCleanerDelegate.
90     * @param className fully qualified class name of the LogCleanerDelegate
91     * @param conf
92     * @return the new instance
93     */
94    public static LogCleanerDelegate newLogCleaner(String className, Configuration conf) {
95      try {
96        Class c = Class.forName(className);
97        LogCleanerDelegate cleaner = (LogCleanerDelegate) c.newInstance();
98        cleaner.setConf(conf);
99        return cleaner;
100     } catch(Exception e) {
101       LOG.warn("Can NOT create LogCleanerDelegate: " + className, e);
102       // skipping if can't instantiate
103       return null;
104     }
105   }
106 
107   /**
108    * Add a LogCleanerDelegate to the log cleaner chain. A log file is deletable
109    * if it is deletable for each LogCleanerDelegate in the chain.
110    * @param logCleaner
111    */
112   public void addLogCleaner(LogCleanerDelegate logCleaner) {
113     if (logCleaner != null && !logCleanersChain.contains(logCleaner)) {
114       logCleanersChain.add(logCleaner);
115       LOG.debug("Add log cleaner in chain: " + logCleaner.getClass().getName());
116     }
117   }
118 
119   @Override
120   protected void chore() {
121     try {
122       FileStatus [] files = FSUtils.listStatus(this.fs, this.oldLogDir, null);
123       if (files == null) return;
124       FILE: for (FileStatus file : files) {
125         Path filePath = file.getPath();
126         if (HLog.validateHLogFilename(filePath.getName())) {
127           for (LogCleanerDelegate logCleaner : logCleanersChain) {
128             if (logCleaner.isStopped()) {
129               LOG.warn("A log cleaner is stopped, won't delete any log.");
130               return;
131             }
132 
133             if (!logCleaner.isLogDeletable(filePath) ) {
134               // this log is not deletable, continue to process next log file
135               continue FILE;
136             }
137           }
138           // delete this log file if it passes all the log cleaners
139           this.fs.delete(filePath, true);
140         } else {
141           LOG.warn("Found a wrongly formated file: "
142               + file.getPath().getName());
143           this.fs.delete(filePath, true);
144         }
145       }
146     } catch (IOException e) {
147       e = RemoteExceptionHandler.checkIOException(e);
148       LOG.warn("Error while cleaning the logs", e);
149     }
150   }
151 
152   @Override
153   public void run() {
154     try {
155       super.run();
156     } finally {
157       for (LogCleanerDelegate lc: this.logCleanersChain) {
158         try {
159           lc.stop("Exiting");
160         } catch (Throwable t) {
161           LOG.warn("Stopping", t);
162         }
163       }
164     }
165   }
166 }