001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.oozie.util.Instrumentable; 019 import org.apache.oozie.util.Instrumentation; 020 import org.apache.oozie.util.XLog; 021 import org.apache.oozie.util.XConfiguration; 022 import org.apache.oozie.ErrorCode; 023 024 import java.io.File; 025 import java.io.FileInputStream; 026 import java.io.IOException; 027 import java.io.InputStream; 028 import java.io.StringWriter; 029 import java.util.HashSet; 030 import java.util.Map; 031 import java.util.Set; 032 import java.util.Arrays; 033 034 /** 035 * Built in service that initializes the services configuration. 036 * <p/> 037 * The configuration loading sequence is identical to Hadoop configuration loading sequence. 038 * <p/> 039 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values 040 * are loaded from a site configuration file from the Oozie configuration directory. 041 * <p/> 042 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as 043 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the 044 * initialization of the <code>ConfigurationService</code> fails. 045 * <p/> 046 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory. 047 * <p/> 048 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment 049 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory. 050 * <p/> 051 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values. 052 * <p/> 053 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode 054 * via system properties settings. 055 */ 056 public class ConfigurationService implements Service, Instrumentable { 057 private static final String INSTRUMENTATION_GROUP = "configuration"; 058 059 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService."; 060 061 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties"; 062 063 /** 064 * System property that indicates the name of the site configuration file to load. 065 */ 066 public static final String OOZIE_CONFIG_FILE_ENV = "OOZIE_CONFIG_FILE"; 067 068 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>(); 069 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test."; 070 071 private static final String PASSWORD_PROPERTY_END = ".password"; 072 073 static { 074 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE_ENV); 075 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD_ENV); 076 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); 077 } 078 079 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml"; 080 public static final String SITE_CONFIG_FILE = "oozie-site.xml"; 081 082 private static XLog log = XLog.getLog(ConfigurationService.class); 083 084 private String configDir; 085 private String configFile; 086 087 private LogChangesConfiguration configuration; 088 089 public ConfigurationService() { 090 log = XLog.getLog(ConfigurationService.class); 091 } 092 093 /** 094 * Obtains the value of a system property or if not defined from an environment variable. 095 * 096 * @param envName environment variable name 097 * @param defaultValue default value if not set 098 * @return the value of the environment variable. 099 */ 100 private static String getEnvValue(String envName, String defaultValue) { 101 String value = System.getProperty(envName); 102 if (value == null) { 103 value = System.getenv(envName); 104 String debugValue = (value == null) ? "variable not defined" : "value [" + value + "]"; 105 log.debug("Fetching env var [{0}], {1}", envName, debugValue); 106 } 107 else { 108 log.debug("Fetching env var [{0}], Java system property overriding it, value [{1}]", envName, value); 109 } 110 return (value != null) ? value : defaultValue; 111 } 112 113 /** 114 * Initialize the log service. 115 * 116 * @param services services instance. 117 * @throws ServiceException thrown if the log service could not be initialized. 118 */ 119 public void init(Services services) throws ServiceException { 120 configDir = getConfigurationDirectory(); 121 configFile = getEnvValue(OOZIE_CONFIG_FILE_ENV, SITE_CONFIG_FILE); 122 if (configFile.contains("/")) { 123 throw new ServiceException(ErrorCode.E0022, configFile); 124 } 125 log.info("Oozie home [{0}]", configDir); 126 log.info("Oozie site [{0}]", configFile); 127 configFile = new File(configDir, configFile).toString(); 128 configuration = loadConf(); 129 } 130 131 public static String getConfigurationDirectory() throws ServiceException { 132 String oozieHome = Services.getOozieHome(); 133 String configDir = new File(oozieHome, "conf").toString(); 134 File file = new File(oozieHome); 135 if (!file.exists()) { 136 throw new ServiceException(ErrorCode.E0024, configDir); 137 } 138 return configDir; 139 } 140 141 /** 142 * Destroy the configuration service. 143 */ 144 public void destroy() { 145 configuration = null; 146 } 147 148 /** 149 * Return the public interface for configuration service. 150 * 151 * @return {@link ConfigurationService}. 152 */ 153 public Class<? extends Service> getInterface() { 154 return ConfigurationService.class; 155 } 156 157 /** 158 * Return the services configuration. 159 * 160 * @return the services configuration. 161 */ 162 public Configuration getConf() { 163 if (configuration == null) { 164 throw new IllegalStateException("Not initialized"); 165 } 166 return configuration; 167 } 168 169 /** 170 * Return Oozie configuration directory. 171 * 172 * @return Oozie configuration directory. 173 */ 174 public String getConfDirectory() { 175 return configDir; 176 } 177 178 private InputStream getDefaultConfiguration() throws ServiceException, IOException { 179 ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); 180 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE); 181 if (inputStream == null) { 182 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE); 183 } 184 return inputStream; 185 } 186 187 private LogChangesConfiguration loadConf() throws ServiceException { 188 XConfiguration configuration; 189 try { 190 InputStream inputStream = getDefaultConfiguration(); 191 configuration = new XConfiguration(inputStream); 192 File file = new File(configFile); 193 if (!file.exists()) { 194 log.info("Missing site configuration file [{0}]", configFile); 195 } 196 else { 197 inputStream = new FileInputStream(configFile); 198 XConfiguration siteConfiguration = new XConfiguration(inputStream); 199 XConfiguration.injectDefaults(configuration, siteConfiguration); 200 configuration = siteConfiguration; 201 } 202 } 203 catch (IOException ex) { 204 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex); 205 } 206 207 if (log.isTraceEnabled()) { 208 try { 209 StringWriter writer = new StringWriter(); 210 for (Map.Entry<String, String> entry : configuration) { 211 boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END); 212 String value = (maskValue) ? "**MASKED**" : entry.getValue(); 213 writer.write(" " + entry.getKey() + " = " + value + "\n"); 214 } 215 writer.close(); 216 log.trace("Configuration:\n{0}---", writer.toString()); 217 } 218 catch (IOException ex) { 219 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex); 220 } 221 } 222 223 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS); 224 if (ignoreSysProps != null) { 225 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps)); 226 } 227 228 for (Map.Entry<String, String> entry : configuration) { 229 String sysValue = System.getProperty(entry.getKey()); 230 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) { 231 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue); 232 configuration.set(entry.getKey(), sysValue); 233 } 234 } 235 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { 236 String name = (String) entry.getKey(); 237 if (IGNORE_SYS_PROPS.contains(name)) { 238 log.warn("System property [{0}] in ignore list, ignored", name); 239 } 240 else { 241 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) { 242 if (configuration.get(name) == null) { 243 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name); 244 } 245 } 246 } 247 } 248 249 return new LogChangesConfiguration(configuration); 250 } 251 252 private class LogChangesConfiguration extends XConfiguration { 253 254 public LogChangesConfiguration(Configuration conf) { 255 for (Map.Entry<String, String> entry : conf) { 256 if (get(entry.getKey()) == null) { 257 setValue(entry.getKey(), entry.getValue()); 258 } 259 } 260 } 261 262 public String[] getStrings(String name) { 263 String s = get(name); 264 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0]; 265 } 266 267 public String get(String name, String defaultValue) { 268 String value = get(name); 269 if (value == null) { 270 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END); 271 value = (maskValue) ? "**MASKED**" : defaultValue; 272 log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, value); 273 } 274 return value; 275 } 276 277 public void set(String name, String value) { 278 setValue(name, value); 279 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END); 280 value = (maskValue) ? "**MASKED**" : value; 281 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value); 282 } 283 284 private void setValue(String name, String value) { 285 super.set(name, value); 286 } 287 288 } 289 290 /** 291 * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and 292 * config file used. 293 * 294 * @param instr instrumentation to use. 295 */ 296 public void instrument(Instrumentation instr) { 297 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() { 298 public String getValue() { 299 return configDir; 300 } 301 }); 302 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 303 public String getValue() { 304 return configFile; 305 } 306 }); 307 instr.setConfiguration(configuration); 308 } 309 310 /** 311 * Return a configuration with all sensitive values masked. 312 * 313 * @param conf configuration to mask. 314 * @return masked configuration. 315 */ 316 public static Configuration maskPasswords(Configuration conf) { 317 XConfiguration maskedConf = new XConfiguration(); 318 for (Map.Entry<String, String> entry : conf) { 319 String name = entry.getKey(); 320 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END); 321 String value = (maskValue) ? "**MASKED**" : entry.getValue(); 322 maskedConf.set(name, value); 323 } 324 return maskedConf; 325 } 326 327 }