001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.commons.logging.LogFactory;
018    import org.apache.log4j.LogManager;
019    import org.apache.log4j.PropertyConfigurator;
020    import org.apache.oozie.util.Instrumentable;
021    import org.apache.oozie.util.Instrumentation;
022    import org.apache.oozie.util.XLog;
023    import org.apache.oozie.util.XLogStreamer;
024    import org.apache.oozie.util.XConfiguration;
025    import org.apache.oozie.BuildInfo;
026    import org.apache.oozie.ErrorCode;
027    import org.apache.hadoop.conf.Configuration;
028    
029    import java.io.File;
030    import java.io.FileInputStream;
031    import java.io.IOException;
032    import java.io.InputStream;
033    import java.io.Writer;
034    import java.net.URL;
035    import java.util.Properties;
036    import java.util.Map;
037    import java.util.Date;
038    
039    /**
040     * Built-in service that initializes and manages Logging via  Log4j.
041     * <p/>
042     * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
043     * <p/>
044     * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
045     * <p/>
046     * The Log4j configuration files must be a properties file.
047     * <p/>
048     * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
049     * If the file is not found there, it is looked in the classpath.
050     * <p/>
051     * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
052     * <p/>
053     * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
054     * <p/>
055     * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>.
056     * The default value is 10 seconds.
057     */
058    public class XLogService implements Service, Instrumentable {
059        private static final String INSTRUMENTATION_GROUP = "logging";
060    
061        /**
062         * System property that indicates the log4j configuration file to load.
063         */
064        public static final String LOG4J_FILE_ENV = "OOZIE_LOG4J_FILE";
065    
066        /**
067         * System property that indicates the reload interval of the configuration file.
068         */
069        public static final String LOG4J_RELOAD_ENV = "OOZIE_LOG4J_RELOAD";
070    
071        /**
072         * Default value for the log4j configuration file if {@link #LOG4J_FILE_ENV} is not set.
073         */
074        public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
075    
076        /**
077         * Default value for the reload interval if {@link #LOG4J_RELOAD_ENV} is not set.
078         */
079        public static final String DEFAULT_RELOAD_INTERVAL = "10";
080    
081        private XLog log;
082        private long interval;
083        private boolean fromClasspath;
084        private String log4jFileName;
085        private boolean logOverWS = true;
086    
087        private static final String STARTUP_MESSAGE = "{E}"
088                + " ******************************************************************************* {E}"
089                + "  STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
090                + "  STARTUP MSG:       revision [{3}]@[{4}]{E}"
091                + "*******************************************************************************";
092    
093        private String oozieLogPath;
094        private String oozieLogName;
095        private int oozieLogRotation = -1;
096    
097        public XLogService() {
098        }
099    
100        /**
101         * Obtains the value of a system property or if not defined from an environment variable.
102         *
103         * @param envName environment variable name
104         * @param defaultValue default value if not set
105         * @return the value of the environment variable.
106         */
107        private static String getEnvValue(String envName, String defaultValue) {
108            String value = System.getProperty(envName);
109            if (value == null) {
110                value = System.getenv(envName);
111            }
112            return (value != null) ? value : defaultValue;
113        }
114    
115        /**
116         * Initialize the log service.
117         *
118         * @param services services instance.
119         * @throws ServiceException thrown if the log service could not be initialized.
120         */
121        public void init(Services services) throws ServiceException {
122            Services.getOozieHome();
123            try {
124                LogManager.resetConfiguration();
125                log4jFileName = getEnvValue(LOG4J_FILE_ENV, DEFAULT_LOG4J_PROPERTIES);
126                if (log4jFileName.contains("/")) {
127                    throw new ServiceException(ErrorCode.E0011, log4jFileName);
128                }
129                if (!log4jFileName.endsWith(".properties")) {
130                    throw new ServiceException(ErrorCode.E0012, log4jFileName);
131                }
132                String configPath = ConfigurationService.getConfigurationDirectory();
133                File log4jFile = new File(configPath, log4jFileName);
134                if (log4jFile.exists()) {
135                    fromClasspath = false;
136                }
137                else {
138                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
139                    URL log4jUrl = cl.getResource(log4jFileName);
140                    if (log4jUrl == null) {
141                        throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
142                    }
143                    fromClasspath = true;
144                }
145    
146                if (fromClasspath) {
147                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
148                    URL log4jUrl = cl.getResource(log4jFileName);
149                    PropertyConfigurator.configure(log4jUrl);
150                }
151                else {
152                    interval = Long.parseLong(getEnvValue(LOG4J_RELOAD_ENV, DEFAULT_RELOAD_INTERVAL));
153                    PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
154                }
155    
156                log = new XLog(LogFactory.getLog(getClass()));
157    
158                log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
159                         BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
160                        .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
161                        BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo()
162                        .getProperty(BuildInfo.BUILD_VC_URL));
163    
164                String from = (fromClasspath) ? "CLASSPATH" : configPath;
165                String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
166                log.info("Log4j configuration file [{0}]", log4jFileName);
167                log.info("Log4j configuration file loaded from [{0}]", from);
168                log.info("Log4j reload interval [{0}]", reload);
169    
170                XLog.Info.reset();
171                XLog.Info.defineParameter(USER);
172                XLog.Info.defineParameter(GROUP);
173                XLogStreamer.Filter.reset();
174                XLogStreamer.Filter.defineParameter(USER);
175                XLogStreamer.Filter.defineParameter(GROUP);
176    
177                // Getting configuration for oozie log via WS
178                ClassLoader cl = Thread.currentThread().getContextClassLoader();
179                InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
180                extractInfoForLogWebService(is);
181            }
182            catch (IOException ex) {
183                throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
184            }
185        }
186    
187        private void extractInfoForLogWebService(InputStream is) throws IOException {
188            Properties props = new Properties();
189            props.load(is);
190    
191            Configuration conf = new XConfiguration();
192            for (Map.Entry entry : props.entrySet()) {
193                conf.set((String) entry.getKey(), (String) entry.getValue());
194            }
195            String logFile = conf.get("log4j.appender.oozie.File");
196            if (logFile == null) {
197                log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' appender");
198            }
199            else {
200                logFile = logFile.trim();
201                int i = logFile.lastIndexOf("/");
202                if (i == -1) {
203                    log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
204                             logFile);
205                    logOverWS = false;
206                }
207                else {
208                    String pattern = conf.get("log4j.appender.oozie.DatePattern");
209                    if (pattern == null) {
210                        log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
211                        logOverWS = false;
212                    }
213                    else {
214                        pattern = pattern.trim();
215                        if (pattern.endsWith("HH")) {
216                            oozieLogRotation = 60 * 60;
217                        }
218                        else {
219                            if (pattern.endsWith("dd")) {
220                                oozieLogRotation = 60 * 60 * 24;
221                            }
222                            else {
223                                log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
224                                         pattern);
225                                logOverWS = false;
226                            }
227                        }
228                        if (oozieLogRotation > 0) {
229                            oozieLogPath = logFile.substring(0, i);
230                            oozieLogName = logFile.substring(i + 1);
231                        }
232                        else {
233                            logOverWS = false;
234                        }
235                    }
236                }
237            }
238        }
239    
240        /**
241         * Destroy the log service.
242         */
243        public void destroy() {
244            LogManager.shutdown();
245            XLog.Info.reset();
246            XLogStreamer.Filter.reset();
247        }
248    
249        /**
250         * Group log info constant.
251         */
252        public static final String USER = "USER";
253    
254        /**
255         * Group log info constant.
256         */
257        public static final String GROUP = "GROUP";
258    
259        /**
260         * Return the public interface for log service.
261         *
262         * @return {@link XLogService}.
263         */
264        public Class<? extends Service> getInterface() {
265            return XLogService.class;
266        }
267    
268        /**
269         * Instruments the log service. <p/> It sets instrumentation variables indicating the config file, reload interval
270         * and if loaded from the classpath.
271         *
272         * @param instr instrumentation to use.
273         */
274        public void instrument(Instrumentation instr) {
275            instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() {
276                public String getValue() {
277                    return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION);
278                }
279            });
280            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
281                public String getValue() {
282                    return log4jFileName;
283                }
284            });
285            instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
286                public Long getValue() {
287                    return interval;
288                }
289            });
290            instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
291                public Boolean getValue() {
292                    return fromClasspath;
293                }
294            });
295            instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
296                public Boolean getValue() {
297                    return logOverWS;
298                }
299            });
300        }
301    
302        /**
303         * Stream the log of a job.
304         *
305         * @param filter log streamer filter.
306         * @param startTime start time for log events to filter.
307         * @param endTime end time for log events to filter.
308         * @param writer writer to stream the log to.
309         * @throws IOException thrown if the log cannot be streamed.
310         */
311        public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException {
312            if (logOverWS) {
313                new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation)
314                        .streamLog(startTime, endTime);
315            }
316            else {
317                writer.write("Log streaming disabled!!");
318            }
319    
320        }
321    
322        String getLog4jProperties() {
323            return log4jFileName;
324        }
325    
326        boolean getFromClasspath() {
327            return fromClasspath;
328        }
329    
330    }