001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.commons.logging.LogFactory; 018 import org.apache.log4j.LogManager; 019 import org.apache.log4j.PropertyConfigurator; 020 import org.apache.oozie.util.Instrumentable; 021 import org.apache.oozie.util.Instrumentation; 022 import org.apache.oozie.util.XLog; 023 import org.apache.oozie.util.XLogStreamer; 024 import org.apache.oozie.util.XConfiguration; 025 import org.apache.oozie.BuildInfo; 026 import org.apache.oozie.ErrorCode; 027 import org.apache.hadoop.conf.Configuration; 028 029 import java.io.File; 030 import java.io.FileInputStream; 031 import java.io.IOException; 032 import java.io.InputStream; 033 import java.io.Writer; 034 import java.net.URL; 035 import java.util.Properties; 036 import java.util.Map; 037 import java.util.Date; 038 039 /** 040 * Built-in service that initializes and manages Logging via Log4j. 041 * <p/> 042 * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>. 043 * <p/> 044 * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>. 045 * <p/> 046 * The Log4j configuration files must be a properties file. 047 * <p/> 048 * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}. 049 * If the file is not found there, it is looked in the classpath. 050 * <p/> 051 * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled. 052 * <p/> 053 * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled. 054 * <p/> 055 * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. 056 * The default value is 10 seconds. 057 */ 058 public class XLogService implements Service, Instrumentable { 059 private static final String INSTRUMENTATION_GROUP = "logging"; 060 061 /** 062 * System property that indicates the log4j configuration file to load. 063 */ 064 public static final String LOG4J_FILE_ENV = "OOZIE_LOG4J_FILE"; 065 066 /** 067 * System property that indicates the reload interval of the configuration file. 068 */ 069 public static final String LOG4J_RELOAD_ENV = "OOZIE_LOG4J_RELOAD"; 070 071 /** 072 * Default value for the log4j configuration file if {@link #LOG4J_FILE_ENV} is not set. 073 */ 074 public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties"; 075 076 /** 077 * Default value for the reload interval if {@link #LOG4J_RELOAD_ENV} is not set. 078 */ 079 public static final String DEFAULT_RELOAD_INTERVAL = "10"; 080 081 private XLog log; 082 private long interval; 083 private boolean fromClasspath; 084 private String log4jFileName; 085 private boolean logOverWS = true; 086 087 private static final String STARTUP_MESSAGE = "{E}" 088 + " ******************************************************************************* {E}" 089 + " STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}" 090 + " STARTUP MSG: revision [{3}]@[{4}]{E}" 091 + "*******************************************************************************"; 092 093 private String oozieLogPath; 094 private String oozieLogName; 095 private int oozieLogRotation = -1; 096 097 public XLogService() { 098 } 099 100 /** 101 * Obtains the value of a system property or if not defined from an environment variable. 102 * 103 * @param envName environment variable name 104 * @param defaultValue default value if not set 105 * @return the value of the environment variable. 106 */ 107 private static String getEnvValue(String envName, String defaultValue) { 108 String value = System.getProperty(envName); 109 if (value == null) { 110 value = System.getenv(envName); 111 } 112 return (value != null) ? value : defaultValue; 113 } 114 115 /** 116 * Initialize the log service. 117 * 118 * @param services services instance. 119 * @throws ServiceException thrown if the log service could not be initialized. 120 */ 121 public void init(Services services) throws ServiceException { 122 Services.getOozieHome(); 123 try { 124 LogManager.resetConfiguration(); 125 log4jFileName = getEnvValue(LOG4J_FILE_ENV, DEFAULT_LOG4J_PROPERTIES); 126 if (log4jFileName.contains("/")) { 127 throw new ServiceException(ErrorCode.E0011, log4jFileName); 128 } 129 if (!log4jFileName.endsWith(".properties")) { 130 throw new ServiceException(ErrorCode.E0012, log4jFileName); 131 } 132 String configPath = ConfigurationService.getConfigurationDirectory(); 133 File log4jFile = new File(configPath, log4jFileName); 134 if (log4jFile.exists()) { 135 fromClasspath = false; 136 } 137 else { 138 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 139 URL log4jUrl = cl.getResource(log4jFileName); 140 if (log4jUrl == null) { 141 throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath); 142 } 143 fromClasspath = true; 144 } 145 146 if (fromClasspath) { 147 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 148 URL log4jUrl = cl.getResource(log4jFileName); 149 PropertyConfigurator.configure(log4jUrl); 150 } 151 else { 152 interval = Long.parseLong(getEnvValue(LOG4J_RELOAD_ENV, DEFAULT_RELOAD_INTERVAL)); 153 PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000); 154 } 155 156 log = new XLog(LogFactory.getLog(getClass())); 157 158 log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION), 159 BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo() 160 .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty( 161 BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo() 162 .getProperty(BuildInfo.BUILD_VC_URL)); 163 164 String from = (fromClasspath) ? "CLASSPATH" : configPath; 165 String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec"; 166 log.info("Log4j configuration file [{0}]", log4jFileName); 167 log.info("Log4j configuration file loaded from [{0}]", from); 168 log.info("Log4j reload interval [{0}]", reload); 169 170 XLog.Info.reset(); 171 XLog.Info.defineParameter(USER); 172 XLog.Info.defineParameter(GROUP); 173 XLogStreamer.Filter.reset(); 174 XLogStreamer.Filter.defineParameter(USER); 175 XLogStreamer.Filter.defineParameter(GROUP); 176 177 // Getting configuration for oozie log via WS 178 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 179 InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile); 180 extractInfoForLogWebService(is); 181 } 182 catch (IOException ex) { 183 throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex); 184 } 185 } 186 187 private void extractInfoForLogWebService(InputStream is) throws IOException { 188 Properties props = new Properties(); 189 props.load(is); 190 191 Configuration conf = new XConfiguration(); 192 for (Map.Entry entry : props.entrySet()) { 193 conf.set((String) entry.getKey(), (String) entry.getValue()); 194 } 195 String logFile = conf.get("log4j.appender.oozie.File"); 196 if (logFile == null) { 197 log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' appender"); 198 } 199 else { 200 logFile = logFile.trim(); 201 int i = logFile.lastIndexOf("/"); 202 if (i == -1) { 203 log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender", 204 logFile); 205 logOverWS = false; 206 } 207 else { 208 String pattern = conf.get("log4j.appender.oozie.DatePattern"); 209 if (pattern == null) { 210 log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]"); 211 logOverWS = false; 212 } 213 else { 214 pattern = pattern.trim(); 215 if (pattern.endsWith("HH")) { 216 oozieLogRotation = 60 * 60; 217 } 218 else { 219 if (pattern.endsWith("dd")) { 220 oozieLogRotation = 60 * 60 * 24; 221 } 222 else { 223 log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'", 224 pattern); 225 logOverWS = false; 226 } 227 } 228 if (oozieLogRotation > 0) { 229 oozieLogPath = logFile.substring(0, i); 230 oozieLogName = logFile.substring(i + 1); 231 } 232 else { 233 logOverWS = false; 234 } 235 } 236 } 237 } 238 } 239 240 /** 241 * Destroy the log service. 242 */ 243 public void destroy() { 244 LogManager.shutdown(); 245 XLog.Info.reset(); 246 XLogStreamer.Filter.reset(); 247 } 248 249 /** 250 * Group log info constant. 251 */ 252 public static final String USER = "USER"; 253 254 /** 255 * Group log info constant. 256 */ 257 public static final String GROUP = "GROUP"; 258 259 /** 260 * Return the public interface for log service. 261 * 262 * @return {@link XLogService}. 263 */ 264 public Class<? extends Service> getInterface() { 265 return XLogService.class; 266 } 267 268 /** 269 * Instruments the log service. <p/> It sets instrumentation variables indicating the config file, reload interval 270 * and if loaded from the classpath. 271 * 272 * @param instr instrumentation to use. 273 */ 274 public void instrument(Instrumentation instr) { 275 instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() { 276 public String getValue() { 277 return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION); 278 } 279 }); 280 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 281 public String getValue() { 282 return log4jFileName; 283 } 284 }); 285 instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() { 286 public Long getValue() { 287 return interval; 288 } 289 }); 290 instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() { 291 public Boolean getValue() { 292 return fromClasspath; 293 } 294 }); 295 instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() { 296 public Boolean getValue() { 297 return logOverWS; 298 } 299 }); 300 } 301 302 /** 303 * Stream the log of a job. 304 * 305 * @param filter log streamer filter. 306 * @param startTime start time for log events to filter. 307 * @param endTime end time for log events to filter. 308 * @param writer writer to stream the log to. 309 * @throws IOException thrown if the log cannot be streamed. 310 */ 311 public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException { 312 if (logOverWS) { 313 new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation) 314 .streamLog(startTime, endTime); 315 } 316 else { 317 writer.write("Log streaming disabled!!"); 318 } 319 320 } 321 322 String getLog4jProperties() { 323 return log4jFileName; 324 } 325 326 boolean getFromClasspath() { 327 return fromClasspath; 328 } 329 330 }