001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.oozie.util.Instrumentable; 019 import org.apache.oozie.util.Instrumentation; 020 import org.apache.oozie.util.XLog; 021 import org.apache.oozie.util.XConfiguration; 022 import org.apache.oozie.ErrorCode; 023 024 import java.io.File; 025 import java.io.FileInputStream; 026 import java.io.IOException; 027 import java.io.InputStream; 028 import java.io.StringWriter; 029 import java.util.HashSet; 030 import java.util.Map; 031 import java.util.Set; 032 import java.util.Arrays; 033 034 /** 035 * Built in service that initializes the services configuration. 036 * <p/> 037 * The configuration loading sequence is identical to Hadoop configuration loading sequence. 038 * <p/> 039 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values 040 * are loaded from a site configuration file from the Oozie configuration directory. 041 * <p/> 042 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as 043 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the 044 * initialization of the <code>ConfigurationService</code> fails. 045 * <p/> 046 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory. 047 * <p/> 048 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment 049 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory. 050 * <p/> 051 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values. 052 * <p/> 053 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode 054 * via system properties settings. 055 */ 056 public class ConfigurationService implements Service, Instrumentable { 057 private static final String INSTRUMENTATION_GROUP = "configuration"; 058 059 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService."; 060 061 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties"; 062 063 /** 064 * System property that indicates the configuration directory. 065 */ 066 public static final String OOZIE_CONFIG_DIR = "oozie.config.dir"; 067 068 069 /** 070 * System property that indicates the data directory. 071 */ 072 public static final String OOZIE_DATA_DIR = "oozie.data.dir"; 073 074 /** 075 * System property that indicates the name of the site configuration file to load. 076 */ 077 public static final String OOZIE_CONFIG_FILE = "oozie.config.file"; 078 079 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>(); 080 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test."; 081 082 private static final String PASSWORD_PROPERTY_END = ".password"; 083 084 static { 085 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); 086 087 //all this properties are seeded as system properties, no need to log changes 088 IGNORE_SYS_PROPS.add("oozie.http.hostname"); 089 IGNORE_SYS_PROPS.add("oozie.http.port"); 090 091 IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR); 092 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR); 093 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE); 094 IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR); 095 096 IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR); 097 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE); 098 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD); 099 } 100 101 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml"; 102 public static final String SITE_CONFIG_FILE = "oozie-site.xml"; 103 104 private static XLog log = XLog.getLog(ConfigurationService.class); 105 106 private String configDir; 107 private String configFile; 108 109 private LogChangesConfiguration configuration; 110 111 public ConfigurationService() { 112 log = XLog.getLog(ConfigurationService.class); 113 } 114 115 /** 116 * Initialize the log service. 117 * 118 * @param services services instance. 119 * @throws ServiceException thrown if the log service could not be initialized. 120 */ 121 public void init(Services services) throws ServiceException { 122 configDir = getConfigurationDirectory(); 123 configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE); 124 if (configFile.contains("/")) { 125 throw new ServiceException(ErrorCode.E0022, configFile); 126 } 127 log.info("Oozie home dir [{0}]", Services.getOozieHome()); 128 log.info("Oozie conf dir [{0}]", configDir); 129 log.info("Oozie conf file [{0}]", configFile); 130 configFile = new File(configDir, configFile).toString(); 131 configuration = loadConf(); 132 } 133 134 public static String getConfigurationDirectory() throws ServiceException { 135 String oozieHome = Services.getOozieHome(); 136 String configDir = System.getProperty(OOZIE_CONFIG_DIR, oozieHome + "/conf"); 137 File file = new File(configDir); 138 if (!file.exists()) { 139 throw new ServiceException(ErrorCode.E0024, configDir); 140 } 141 return configDir; 142 } 143 144 /** 145 * Destroy the configuration service. 146 */ 147 public void destroy() { 148 configuration = null; 149 } 150 151 /** 152 * Return the public interface for configuration service. 153 * 154 * @return {@link ConfigurationService}. 155 */ 156 public Class<? extends Service> getInterface() { 157 return ConfigurationService.class; 158 } 159 160 /** 161 * Return the services configuration. 162 * 163 * @return the services configuration. 164 */ 165 public Configuration getConf() { 166 if (configuration == null) { 167 throw new IllegalStateException("Not initialized"); 168 } 169 return configuration; 170 } 171 172 /** 173 * Return Oozie configuration directory. 174 * 175 * @return Oozie configuration directory. 176 */ 177 public String getConfigDir() { 178 return configDir; 179 } 180 181 private InputStream getDefaultConfiguration() throws ServiceException, IOException { 182 ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); 183 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE); 184 if (inputStream == null) { 185 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE); 186 } 187 return inputStream; 188 } 189 190 private LogChangesConfiguration loadConf() throws ServiceException { 191 XConfiguration configuration; 192 try { 193 InputStream inputStream = getDefaultConfiguration(); 194 configuration = new XConfiguration(inputStream); 195 File file = new File(configFile); 196 if (!file.exists()) { 197 log.info("Missing site configuration file [{0}]", configFile); 198 } 199 else { 200 inputStream = new FileInputStream(configFile); 201 XConfiguration siteConfiguration = new XConfiguration(inputStream); 202 XConfiguration.injectDefaults(configuration, siteConfiguration); 203 configuration = siteConfiguration; 204 } 205 } 206 catch (IOException ex) { 207 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex); 208 } 209 210 if (log.isTraceEnabled()) { 211 try { 212 StringWriter writer = new StringWriter(); 213 for (Map.Entry<String, String> entry : configuration) { 214 boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END); 215 String value = (maskValue) ? "**MASKED**" : entry.getValue(); 216 writer.write(" " + entry.getKey() + " = " + value + "\n"); 217 } 218 writer.close(); 219 log.trace("Configuration:\n{0}---", writer.toString()); 220 } 221 catch (IOException ex) { 222 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex); 223 } 224 } 225 226 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS); 227 if (ignoreSysProps != null) { 228 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps)); 229 } 230 231 for (Map.Entry<String, String> entry : configuration) { 232 String sysValue = System.getProperty(entry.getKey()); 233 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) { 234 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue); 235 configuration.set(entry.getKey(), sysValue); 236 } 237 } 238 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { 239 String name = (String) entry.getKey(); 240 if (!IGNORE_SYS_PROPS.contains(name)) { 241 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) { 242 if (configuration.get(name) == null) { 243 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name); 244 } 245 } 246 } 247 } 248 249 return new LogChangesConfiguration(configuration); 250 } 251 252 private class LogChangesConfiguration extends XConfiguration { 253 254 public LogChangesConfiguration(Configuration conf) { 255 for (Map.Entry<String, String> entry : conf) { 256 if (get(entry.getKey()) == null) { 257 setValue(entry.getKey(), entry.getValue()); 258 } 259 } 260 } 261 262 public String[] getStrings(String name) { 263 String s = get(name); 264 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0]; 265 } 266 267 public String get(String name, String defaultValue) { 268 String value = get(name); 269 if (value == null) { 270 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END); 271 value = (maskValue) ? "**MASKED**" : defaultValue; 272 log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, value); 273 } 274 return value; 275 } 276 277 public void set(String name, String value) { 278 setValue(name, value); 279 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END); 280 value = (maskValue) ? "**MASKED**" : value; 281 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value); 282 } 283 284 private void setValue(String name, String value) { 285 super.set(name, value); 286 } 287 288 } 289 290 /** 291 * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and 292 * config file used. 293 * 294 * @param instr instrumentation to use. 295 */ 296 public void instrument(Instrumentation instr) { 297 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() { 298 public String getValue() { 299 return configDir; 300 } 301 }); 302 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 303 public String getValue() { 304 return configFile; 305 } 306 }); 307 instr.setConfiguration(configuration); 308 } 309 310 /** 311 * Return a configuration with all sensitive values masked. 312 * 313 * @param conf configuration to mask. 314 * @return masked configuration. 315 */ 316 public static Configuration maskPasswords(Configuration conf) { 317 XConfiguration maskedConf = new XConfiguration(); 318 for (Map.Entry<String, String> entry : conf) { 319 String name = entry.getKey(); 320 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END); 321 String value = (maskValue) ? "**MASKED**" : entry.getValue(); 322 maskedConf.set(name, value); 323 } 324 return maskedConf; 325 } 326 327 }