001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.commons.logging.LogFactory;
018    import org.apache.log4j.LogManager;
019    import org.apache.log4j.PropertyConfigurator;
020    import org.apache.oozie.util.Instrumentable;
021    import org.apache.oozie.util.Instrumentation;
022    import org.apache.oozie.util.XLog;
023    import org.apache.oozie.util.XLogStreamer;
024    import org.apache.oozie.util.XConfiguration;
025    import org.apache.oozie.BuildInfo;
026    import org.apache.oozie.ErrorCode;
027    import org.apache.hadoop.conf.Configuration;
028    
029    import java.io.File;
030    import java.io.FileInputStream;
031    import java.io.IOException;
032    import java.io.InputStream;
033    import java.io.Writer;
034    import java.net.URL;
035    import java.util.Properties;
036    import java.util.Map;
037    import java.util.Date;
038    
039    /**
040     * Built-in service that initializes and manages Logging via  Log4j.
041     * <p/>
042     * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
043     * <p/>
044     * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
045     * <p/>
046     * The Log4j configuration files must be a properties file.
047     * <p/>
048     * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
049     * If the file is not found there, it is looked in the classpath.
050     * <p/>
051     * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
052     * <p/>
053     * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
054     * <p/>
055     * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>.
056     * The default value is 10 seconds.
057     */
058    public class XLogService implements Service, Instrumentable {
059        private static final String INSTRUMENTATION_GROUP = "logging";
060    
061        /**
062         * System property that indicates the logs directory.
063         */
064        public static final String OOZIE_LOG_DIR = "oozie.log.dir";
065    
066        /**
067         * System property that indicates the log4j configuration file to load.
068         */
069        public static final String LOG4J_FILE = "oozie.log4j.file";
070    
071        /**
072         * System property that indicates the reload interval of the configuration file.
073         */
074        public static final String LOG4J_RELOAD = "oozie.log4j.reload";
075    
076        /**
077         * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set.
078         */
079        public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
080    
081        /**
082         * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set.
083         */
084        public static final String DEFAULT_RELOAD_INTERVAL = "10";
085    
086        private XLog log;
087        private long interval;
088        private boolean fromClasspath;
089        private String log4jFileName;
090        private boolean logOverWS = true;
091    
092        private static final String STARTUP_MESSAGE = "{E}"
093                + " ******************************************************************************* {E}"
094                + "  STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
095                + "  STARTUP MSG:       revision [{3}]@[{4}]{E}"
096                + "*******************************************************************************";
097    
098        private String oozieLogPath;
099        private String oozieLogName;
100        private int oozieLogRotation = -1;
101    
102        public XLogService() {
103        }
104    
105        /**
106         * Initialize the log service.
107         *
108         * @param services services instance.
109         * @throws ServiceException thrown if the log service could not be initialized.
110         */
111        public void init(Services services) throws ServiceException {
112            String oozieHome = Services.getOozieHome();
113            String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs");
114            System.setProperty(OOZIE_LOG_DIR, oozieLogs);
115            try {
116                LogManager.resetConfiguration();
117                log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES);
118                if (log4jFileName.contains("/")) {
119                    throw new ServiceException(ErrorCode.E0011, log4jFileName);
120                }
121                if (!log4jFileName.endsWith(".properties")) {
122                    throw new ServiceException(ErrorCode.E0012, log4jFileName);
123                }
124                String configPath = ConfigurationService.getConfigurationDirectory();
125                File log4jFile = new File(configPath, log4jFileName);
126                if (log4jFile.exists()) {
127                    fromClasspath = false;
128                }
129                else {
130                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
131                    URL log4jUrl = cl.getResource(log4jFileName);
132                    if (log4jUrl == null) {
133                        throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
134                    }
135                    fromClasspath = true;
136                }
137    
138                if (fromClasspath) {
139                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
140                    URL log4jUrl = cl.getResource(log4jFileName);
141                    PropertyConfigurator.configure(log4jUrl);
142                }
143                else {
144                    interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL));
145                    PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
146                }
147    
148                log = new XLog(LogFactory.getLog(getClass()));
149    
150                log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
151                         BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
152                        .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
153                        BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo()
154                        .getProperty(BuildInfo.BUILD_VC_URL));
155    
156                String from = (fromClasspath) ? "CLASSPATH" : configPath;
157                String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
158                log.info("Log4j configuration file [{0}]", log4jFileName);
159                log.info("Log4j configuration file loaded from [{0}]", from);
160                log.info("Log4j reload interval [{0}]", reload);
161    
162                XLog.Info.reset();
163                XLog.Info.defineParameter(USER);
164                XLog.Info.defineParameter(GROUP);
165                XLogStreamer.Filter.reset();
166                XLogStreamer.Filter.defineParameter(USER);
167                XLogStreamer.Filter.defineParameter(GROUP);
168    
169                // Getting configuration for oozie log via WS
170                ClassLoader cl = Thread.currentThread().getContextClassLoader();
171                InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
172                extractInfoForLogWebService(is);
173            }
174            catch (IOException ex) {
175                throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
176            }
177        }
178    
179        private void extractInfoForLogWebService(InputStream is) throws IOException {
180            Properties props = new Properties();
181            props.load(is);
182    
183            Configuration conf = new XConfiguration();
184            for (Map.Entry entry : props.entrySet()) {
185                conf.set((String) entry.getKey(), (String) entry.getValue());
186            }
187            String logFile = conf.get("log4j.appender.oozie.File");
188            if (logFile == null) {
189                log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' appender");
190            }
191            else {
192                logFile = logFile.trim();
193                int i = logFile.lastIndexOf("/");
194                if (i == -1) {
195                    log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
196                             logFile);
197                    logOverWS = false;
198                }
199                else {
200                    String pattern = conf.get("log4j.appender.oozie.DatePattern");
201                    if (pattern == null) {
202                        log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
203                        logOverWS = false;
204                    }
205                    else {
206                        pattern = pattern.trim();
207                        if (pattern.endsWith("HH")) {
208                            oozieLogRotation = 60 * 60;
209                        }
210                        else {
211                            if (pattern.endsWith("dd")) {
212                                oozieLogRotation = 60 * 60 * 24;
213                            }
214                            else {
215                                log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
216                                         pattern);
217                                logOverWS = false;
218                            }
219                        }
220                        if (oozieLogRotation > 0) {
221                            oozieLogPath = logFile.substring(0, i);
222                            oozieLogName = logFile.substring(i + 1);
223                        }
224                        else {
225                            logOverWS = false;
226                        }
227                    }
228                }
229            }
230        }
231    
232        /**
233         * Destroy the log service.
234         */
235        public void destroy() {
236            LogManager.shutdown();
237            XLog.Info.reset();
238            XLogStreamer.Filter.reset();
239        }
240    
241        /**
242         * Group log info constant.
243         */
244        public static final String USER = "USER";
245    
246        /**
247         * Group log info constant.
248         */
249        public static final String GROUP = "GROUP";
250    
251        /**
252         * Return the public interface for log service.
253         *
254         * @return {@link XLogService}.
255         */
256        public Class<? extends Service> getInterface() {
257            return XLogService.class;
258        }
259    
260        /**
261         * Instruments the log service. <p/> It sets instrumentation variables indicating the config file, reload interval
262         * and if loaded from the classpath.
263         *
264         * @param instr instrumentation to use.
265         */
266        public void instrument(Instrumentation instr) {
267            instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() {
268                public String getValue() {
269                    return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION);
270                }
271            });
272            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
273                public String getValue() {
274                    return log4jFileName;
275                }
276            });
277            instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
278                public Long getValue() {
279                    return interval;
280                }
281            });
282            instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
283                public Boolean getValue() {
284                    return fromClasspath;
285                }
286            });
287            instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
288                public Boolean getValue() {
289                    return logOverWS;
290                }
291            });
292        }
293    
294        /**
295         * Stream the log of a job.
296         *
297         * @param filter log streamer filter.
298         * @param startTime start time for log events to filter.
299         * @param endTime end time for log events to filter.
300         * @param writer writer to stream the log to.
301         * @throws IOException thrown if the log cannot be streamed.
302         */
303        public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException {
304            if (logOverWS) {
305                new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation)
306                        .streamLog(startTime, endTime);
307            }
308            else {
309                writer.write("Log streaming disabled!!");
310            }
311    
312        }
313    
314        String getLog4jProperties() {
315            return log4jFileName;
316        }
317    
318        boolean getFromClasspath() {
319            return fromClasspath;
320        }
321    
322    }