001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.oozie.util.Instrumentable;
019    import org.apache.oozie.util.Instrumentation;
020    import org.apache.oozie.util.XLog;
021    import org.apache.oozie.util.XConfiguration;
022    import org.apache.oozie.ErrorCode;
023    
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.StringWriter;
029    import java.util.HashSet;
030    import java.util.Map;
031    import java.util.Set;
032    import java.util.Arrays;
033    
034    /**
035     * Built in service that initializes the services configuration.
036     * <p/>
037     * The configuration loading sequence is identical to Hadoop configuration loading sequence.
038     * <p/>
039     * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
040     * are loaded from a site configuration file from the Oozie configuration directory.
041     * <p/>
042     * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
043     * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
044     * initialization of the <code>ConfigurationService</code> fails.
045     * <p/>
046     * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
047     * <p/>
048     * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
049     * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
050     * <p/>
051     * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
052     * <p/>
053     * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
054     * via system properties settings.
055     */
056    public class ConfigurationService implements Service, Instrumentable {
057        private static final String INSTRUMENTATION_GROUP = "configuration";
058    
059        public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
060    
061        public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
062    
063        /**
064         * System property that indicates the name of the site configuration file to load.
065         */
066        public static final String OOZIE_CONFIG_FILE_ENV = "OOZIE_CONFIG_FILE";
067    
068        private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
069        private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
070    
071        private static final String PASSWORD_PROPERTY_END = ".password";
072    
073        static {
074            IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE_ENV);
075            IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD_ENV);
076            IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
077        }
078    
079        public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
080        public static final String SITE_CONFIG_FILE = "oozie-site.xml";
081    
082        private static XLog log = XLog.getLog(ConfigurationService.class);
083    
084        private String configDir;
085        private String configFile;
086    
087        private LogChangesConfiguration configuration;
088    
089        public ConfigurationService() {
090            log = XLog.getLog(ConfigurationService.class);
091        }
092    
093        /**
094         * Obtains the value of a system property or if not defined from an environment variable.
095         *
096         * @param envName environment variable name
097         * @param defaultValue default value if not set
098         * @return the value of the environment variable.
099         */
100        private static String getEnvValue(String envName, String defaultValue) {
101            String value = System.getProperty(envName);
102            if (value == null) {
103                value = System.getenv(envName);
104                String debugValue = (value == null) ? "variable not defined" : "value [" + value + "]";
105                log.debug("Fetching env var [{0}], {1}", envName, debugValue);
106            }
107            else {
108                log.debug("Fetching env var [{0}], Java system property overriding it, value [{1}]", envName, value);
109            }
110            return (value != null) ? value : defaultValue;
111        }
112    
113        /**
114         * Initialize the log service.
115         *
116         * @param services services instance.
117         * @throws ServiceException thrown if the log service could not be initialized.
118         */
119        public void init(Services services) throws ServiceException {
120            configDir = getConfigurationDirectory();
121            configFile = getEnvValue(OOZIE_CONFIG_FILE_ENV, SITE_CONFIG_FILE);
122            if (configFile.contains("/")) {
123                throw new ServiceException(ErrorCode.E0022, configFile);
124            }
125            log.info("Oozie home [{0}]", configDir);
126            log.info("Oozie site [{0}]", configFile);
127            configFile = new File(configDir, configFile).toString();
128            configuration = loadConf();
129        }
130    
131        public static String getConfigurationDirectory() throws ServiceException {
132            String oozieHome = Services.getOozieHome();
133            String configDir = new File(oozieHome, "conf").toString();
134            File file = new File(oozieHome);
135            if (!file.exists()) {
136                throw new ServiceException(ErrorCode.E0024, configDir);
137            }
138            return configDir;
139        }
140    
141        /**
142         * Destroy the configuration service.
143         */
144        public void destroy() {
145            configuration = null;
146        }
147    
148        /**
149         * Return the public interface for configuration service.
150         *
151         * @return {@link ConfigurationService}.
152         */
153        public Class<? extends Service> getInterface() {
154            return ConfigurationService.class;
155        }
156    
157        /**
158         * Return the services configuration.
159         *
160         * @return the services configuration.
161         */
162        public Configuration getConf() {
163            if (configuration == null) {
164                throw new IllegalStateException("Not initialized");
165            }
166            return configuration;
167        }
168    
169        /**
170         * Return Oozie configuration directory.
171         *
172         * @return Oozie configuration directory.
173         */
174        public String getConfDirectory() {
175            return configDir;
176        }
177    
178        private InputStream getDefaultConfiguration() throws ServiceException, IOException {
179            ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
180            InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
181            if (inputStream == null) {
182                throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
183            }
184            return inputStream;
185        }
186    
187        private LogChangesConfiguration loadConf() throws ServiceException {
188            XConfiguration configuration;
189            try {
190                InputStream inputStream = getDefaultConfiguration();
191                configuration = new XConfiguration(inputStream);
192                File file = new File(configFile);
193                if (!file.exists()) {
194                    log.info("Missing site configuration file [{0}]", configFile);
195                }
196                else {
197                    inputStream = new FileInputStream(configFile);
198                    XConfiguration siteConfiguration = new XConfiguration(inputStream);
199                    XConfiguration.injectDefaults(configuration, siteConfiguration);
200                    configuration = siteConfiguration;
201                }
202            }
203            catch (IOException ex) {
204                throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
205            }
206    
207            if (log.isTraceEnabled()) {
208                try {
209                    StringWriter writer = new StringWriter();
210                    for (Map.Entry<String, String> entry : configuration) {
211                        boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END);
212                        String value = (maskValue) ? "**MASKED**" : entry.getValue();
213                        writer.write(" " + entry.getKey() + " = " + value + "\n");
214                    }
215                    writer.close();
216                    log.trace("Configuration:\n{0}---", writer.toString());
217                }
218                catch (IOException ex) {
219                    throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
220                }
221            }
222    
223            String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
224            if (ignoreSysProps != null) {
225                IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
226            }
227    
228            for (Map.Entry<String, String> entry : configuration) {
229                String sysValue = System.getProperty(entry.getKey());
230                if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
231                    log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
232                    configuration.set(entry.getKey(), sysValue);
233                }
234            }
235            for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
236                String name = (String) entry.getKey();
237                if (IGNORE_SYS_PROPS.contains(name)) {
238                    log.warn("System property [{0}] in ignore list, ignored", name);
239                }
240                else {
241                    if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
242                        if (configuration.get(name) == null) {
243                            log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
244                        }
245                    }
246                }
247            }
248    
249            return new LogChangesConfiguration(configuration);
250        }
251    
252        private class LogChangesConfiguration extends XConfiguration {
253    
254            public LogChangesConfiguration(Configuration conf) {
255                for (Map.Entry<String, String> entry : conf) {
256                    if (get(entry.getKey()) == null) {
257                        setValue(entry.getKey(), entry.getValue());
258                    }
259                }
260            }
261    
262            public String[] getStrings(String name) {
263                String s = get(name);
264                return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
265            }
266    
267            public String get(String name, String defaultValue) {
268                String value = get(name);
269                if (value == null) {
270                    boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
271                    value = (maskValue) ? "**MASKED**" : defaultValue;
272                    log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, value);
273                }
274                return value;
275            }
276    
277            public void set(String name, String value) {
278                setValue(name, value);
279                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
280                value = (maskValue) ? "**MASKED**" : value;
281                log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
282            }
283    
284            private void setValue(String name, String value) {
285                super.set(name, value);
286            }
287    
288        }
289    
290        /**
291         * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
292         * config file used.
293         *
294         * @param instr instrumentation to use.
295         */
296        public void instrument(Instrumentation instr) {
297            instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
298                public String getValue() {
299                    return configDir;
300                }
301            });
302            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
303                public String getValue() {
304                    return configFile;
305                }
306            });
307            instr.setConfiguration(configuration);
308        }
309    
310        /**
311         * Return a configuration with all sensitive values masked.
312         *
313         * @param conf configuration to mask.
314         * @return masked configuration.
315         */
316        public static Configuration maskPasswords(Configuration conf) {
317            XConfiguration maskedConf = new XConfiguration();
318            for (Map.Entry<String, String> entry : conf) {
319                String name = entry.getKey();
320                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
321                String value = (maskValue) ? "**MASKED**" : entry.getValue();
322                maskedConf.set(name, value);
323            }
324            return maskedConf;
325        }
326    
327    }