001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.commons.logging.LogFactory; 018 import org.apache.log4j.LogManager; 019 import org.apache.log4j.PropertyConfigurator; 020 import org.apache.oozie.util.Instrumentable; 021 import org.apache.oozie.util.Instrumentation; 022 import org.apache.oozie.util.XLog; 023 import org.apache.oozie.util.XLogStreamer; 024 import org.apache.oozie.util.XConfiguration; 025 import org.apache.oozie.BuildInfo; 026 import org.apache.oozie.ErrorCode; 027 import org.apache.hadoop.conf.Configuration; 028 029 import java.io.File; 030 import java.io.FileInputStream; 031 import java.io.IOException; 032 import java.io.InputStream; 033 import java.io.Writer; 034 import java.net.URL; 035 import java.util.Properties; 036 import java.util.Map; 037 import java.util.Date; 038 039 /** 040 * Built-in service that initializes and manages Logging via Log4j. 041 * <p/> 042 * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>. 043 * <p/> 044 * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>. 045 * <p/> 046 * The Log4j configuration files must be a properties file. 047 * <p/> 048 * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}. 049 * If the file is not found there, it is looked in the classpath. 050 * <p/> 051 * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled. 052 * <p/> 053 * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled. 054 * <p/> 055 * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. 056 * The default value is 10 seconds. 057 */ 058 public class XLogService implements Service, Instrumentable { 059 private static final String INSTRUMENTATION_GROUP = "logging"; 060 061 /** 062 * System property that indicates the logs directory. 063 */ 064 public static final String OOZIE_LOG_DIR = "oozie.log.dir"; 065 066 /** 067 * System property that indicates the log4j configuration file to load. 068 */ 069 public static final String LOG4J_FILE = "oozie.log4j.file"; 070 071 /** 072 * System property that indicates the reload interval of the configuration file. 073 */ 074 public static final String LOG4J_RELOAD = "oozie.log4j.reload"; 075 076 /** 077 * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set. 078 */ 079 public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties"; 080 081 /** 082 * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set. 083 */ 084 public static final String DEFAULT_RELOAD_INTERVAL = "10"; 085 086 private XLog log; 087 private long interval; 088 private boolean fromClasspath; 089 private String log4jFileName; 090 private boolean logOverWS = true; 091 092 private static final String STARTUP_MESSAGE = "{E}" 093 + " ******************************************************************************* {E}" 094 + " STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}" 095 + " STARTUP MSG: revision [{3}]@[{4}]{E}" 096 + "*******************************************************************************"; 097 098 private String oozieLogPath; 099 private String oozieLogName; 100 private int oozieLogRotation = -1; 101 102 public XLogService() { 103 } 104 105 /** 106 * Initialize the log service. 107 * 108 * @param services services instance. 109 * @throws ServiceException thrown if the log service could not be initialized. 110 */ 111 public void init(Services services) throws ServiceException { 112 String oozieHome = Services.getOozieHome(); 113 String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs"); 114 System.setProperty(OOZIE_LOG_DIR, oozieLogs); 115 try { 116 LogManager.resetConfiguration(); 117 log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES); 118 if (log4jFileName.contains("/")) { 119 throw new ServiceException(ErrorCode.E0011, log4jFileName); 120 } 121 if (!log4jFileName.endsWith(".properties")) { 122 throw new ServiceException(ErrorCode.E0012, log4jFileName); 123 } 124 String configPath = ConfigurationService.getConfigurationDirectory(); 125 File log4jFile = new File(configPath, log4jFileName); 126 if (log4jFile.exists()) { 127 fromClasspath = false; 128 } 129 else { 130 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 131 URL log4jUrl = cl.getResource(log4jFileName); 132 if (log4jUrl == null) { 133 throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath); 134 } 135 fromClasspath = true; 136 } 137 138 if (fromClasspath) { 139 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 140 URL log4jUrl = cl.getResource(log4jFileName); 141 PropertyConfigurator.configure(log4jUrl); 142 } 143 else { 144 interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL)); 145 PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000); 146 } 147 148 log = new XLog(LogFactory.getLog(getClass())); 149 150 log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION), 151 BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo() 152 .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty( 153 BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo() 154 .getProperty(BuildInfo.BUILD_VC_URL)); 155 156 String from = (fromClasspath) ? "CLASSPATH" : configPath; 157 String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec"; 158 log.info("Log4j configuration file [{0}]", log4jFileName); 159 log.info("Log4j configuration file loaded from [{0}]", from); 160 log.info("Log4j reload interval [{0}]", reload); 161 162 XLog.Info.reset(); 163 XLog.Info.defineParameter(USER); 164 XLog.Info.defineParameter(GROUP); 165 XLogStreamer.Filter.reset(); 166 XLogStreamer.Filter.defineParameter(USER); 167 XLogStreamer.Filter.defineParameter(GROUP); 168 169 // Getting configuration for oozie log via WS 170 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 171 InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile); 172 extractInfoForLogWebService(is); 173 } 174 catch (IOException ex) { 175 throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex); 176 } 177 } 178 179 private void extractInfoForLogWebService(InputStream is) throws IOException { 180 Properties props = new Properties(); 181 props.load(is); 182 183 Configuration conf = new XConfiguration(); 184 for (Map.Entry entry : props.entrySet()) { 185 conf.set((String) entry.getKey(), (String) entry.getValue()); 186 } 187 String logFile = conf.get("log4j.appender.oozie.File"); 188 if (logFile == null) { 189 log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' appender"); 190 } 191 else { 192 logFile = logFile.trim(); 193 int i = logFile.lastIndexOf("/"); 194 if (i == -1) { 195 log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender", 196 logFile); 197 logOverWS = false; 198 } 199 else { 200 String pattern = conf.get("log4j.appender.oozie.DatePattern"); 201 if (pattern == null) { 202 log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]"); 203 logOverWS = false; 204 } 205 else { 206 pattern = pattern.trim(); 207 if (pattern.endsWith("HH")) { 208 oozieLogRotation = 60 * 60; 209 } 210 else { 211 if (pattern.endsWith("dd")) { 212 oozieLogRotation = 60 * 60 * 24; 213 } 214 else { 215 log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'", 216 pattern); 217 logOverWS = false; 218 } 219 } 220 if (oozieLogRotation > 0) { 221 oozieLogPath = logFile.substring(0, i); 222 oozieLogName = logFile.substring(i + 1); 223 } 224 else { 225 logOverWS = false; 226 } 227 } 228 } 229 } 230 } 231 232 /** 233 * Destroy the log service. 234 */ 235 public void destroy() { 236 LogManager.shutdown(); 237 XLog.Info.reset(); 238 XLogStreamer.Filter.reset(); 239 } 240 241 /** 242 * Group log info constant. 243 */ 244 public static final String USER = "USER"; 245 246 /** 247 * Group log info constant. 248 */ 249 public static final String GROUP = "GROUP"; 250 251 /** 252 * Return the public interface for log service. 253 * 254 * @return {@link XLogService}. 255 */ 256 public Class<? extends Service> getInterface() { 257 return XLogService.class; 258 } 259 260 /** 261 * Instruments the log service. <p/> It sets instrumentation variables indicating the config file, reload interval 262 * and if loaded from the classpath. 263 * 264 * @param instr instrumentation to use. 265 */ 266 public void instrument(Instrumentation instr) { 267 instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() { 268 public String getValue() { 269 return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION); 270 } 271 }); 272 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 273 public String getValue() { 274 return log4jFileName; 275 } 276 }); 277 instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() { 278 public Long getValue() { 279 return interval; 280 } 281 }); 282 instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() { 283 public Boolean getValue() { 284 return fromClasspath; 285 } 286 }); 287 instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() { 288 public Boolean getValue() { 289 return logOverWS; 290 } 291 }); 292 } 293 294 /** 295 * Stream the log of a job. 296 * 297 * @param filter log streamer filter. 298 * @param startTime start time for log events to filter. 299 * @param endTime end time for log events to filter. 300 * @param writer writer to stream the log to. 301 * @throws IOException thrown if the log cannot be streamed. 302 */ 303 public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException { 304 if (logOverWS) { 305 new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation) 306 .streamLog(startTime, endTime); 307 } 308 else { 309 writer.write("Log streaming disabled!!"); 310 } 311 312 } 313 314 String getLog4jProperties() { 315 return log4jFileName; 316 } 317 318 boolean getFromClasspath() { 319 return fromClasspath; 320 } 321 322 }