001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.oozie.util.Instrumentable;
019    import org.apache.oozie.util.Instrumentation;
020    import org.apache.oozie.util.XLog;
021    import org.apache.oozie.util.XConfiguration;
022    import org.apache.oozie.ErrorCode;
023    
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.StringWriter;
029    import java.util.HashSet;
030    import java.util.Map;
031    import java.util.Set;
032    import java.util.Arrays;
033    
034    /**
035     * Built in service that initializes the services configuration.
036     * <p/>
037     * The configuration loading sequence is identical to Hadoop configuration loading sequence.
038     * <p/>
039     * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
040     * are loaded from a site configuration file from the Oozie configuration directory.
041     * <p/>
042     * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
043     * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
044     * initialization of the <code>ConfigurationService</code> fails.
045     * <p/>
046     * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
047     * <p/>
048     * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
049     * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
050     * <p/>
051     * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
052     * <p/>
053     * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
054     * via system properties settings.
055     */
056    public class ConfigurationService implements Service, Instrumentable {
057        private static final String INSTRUMENTATION_GROUP = "configuration";
058    
059        public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
060    
061        public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
062    
063        /**
064         * System property that indicates the configuration directory.
065         */
066        public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
067    
068    
069        /**
070         * System property that indicates the data directory.
071         */
072        public static final String OOZIE_DATA_DIR = "oozie.data.dir";
073    
074        /**
075         * System property that indicates the name of the site configuration file to load.
076         */
077        public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
078    
079        private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
080        private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
081    
082        private static final String PASSWORD_PROPERTY_END = ".password";
083    
084        static {
085            IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
086    
087            //all this properties are seeded as system properties, no need to log changes
088            IGNORE_SYS_PROPS.add("oozie.http.hostname");
089            IGNORE_SYS_PROPS.add("oozie.http.port");
090    
091            IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
092            IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
093            IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
094            IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
095    
096            IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
097            IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
098            IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
099        }
100    
101        public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
102        public static final String SITE_CONFIG_FILE = "oozie-site.xml";
103    
104        private static XLog log = XLog.getLog(ConfigurationService.class);
105    
106        private String configDir;
107        private String configFile;
108    
109        private LogChangesConfiguration configuration;
110    
111        public ConfigurationService() {
112            log = XLog.getLog(ConfigurationService.class);
113        }
114    
115        /**
116         * Initialize the log service.
117         *
118         * @param services services instance.
119         * @throws ServiceException thrown if the log service could not be initialized.
120         */
121        public void init(Services services) throws ServiceException {
122            configDir = getConfigurationDirectory();
123            configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
124            if (configFile.contains("/")) {
125                throw new ServiceException(ErrorCode.E0022, configFile);
126            }
127            log.info("Oozie home dir  [{0}]", Services.getOozieHome());
128            log.info("Oozie conf dir  [{0}]", configDir);
129            log.info("Oozie conf file [{0}]", configFile);
130            configFile = new File(configDir, configFile).toString();
131            configuration = loadConf();
132        }
133    
134        public static String getConfigurationDirectory() throws ServiceException {
135            String oozieHome = Services.getOozieHome();
136            String configDir = System.getProperty(OOZIE_CONFIG_DIR, oozieHome + "/conf");
137            File file = new File(configDir);
138            if (!file.exists()) {
139                throw new ServiceException(ErrorCode.E0024, configDir);
140            }
141            return configDir;
142        }
143    
144        /**
145         * Destroy the configuration service.
146         */
147        public void destroy() {
148            configuration = null;
149        }
150    
151        /**
152         * Return the public interface for configuration service.
153         *
154         * @return {@link ConfigurationService}.
155         */
156        public Class<? extends Service> getInterface() {
157            return ConfigurationService.class;
158        }
159    
160        /**
161         * Return the services configuration.
162         *
163         * @return the services configuration.
164         */
165        public Configuration getConf() {
166            if (configuration == null) {
167                throw new IllegalStateException("Not initialized");
168            }
169            return configuration;
170        }
171    
172        /**
173         * Return Oozie configuration directory.
174         *
175         * @return Oozie configuration directory.
176         */
177        public String getConfigDir() {
178            return configDir;
179        }
180    
181        private InputStream getDefaultConfiguration() throws ServiceException, IOException {
182            ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
183            InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
184            if (inputStream == null) {
185                throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
186            }
187            return inputStream;
188        }
189    
190        private LogChangesConfiguration loadConf() throws ServiceException {
191            XConfiguration configuration;
192            try {
193                InputStream inputStream = getDefaultConfiguration();
194                configuration = new XConfiguration(inputStream);
195                File file = new File(configFile);
196                if (!file.exists()) {
197                    log.info("Missing site configuration file [{0}]", configFile);
198                }
199                else {
200                    inputStream = new FileInputStream(configFile);
201                    XConfiguration siteConfiguration = new XConfiguration(inputStream);
202                    XConfiguration.injectDefaults(configuration, siteConfiguration);
203                    configuration = siteConfiguration;
204                }
205            }
206            catch (IOException ex) {
207                throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
208            }
209    
210            if (log.isTraceEnabled()) {
211                try {
212                    StringWriter writer = new StringWriter();
213                    for (Map.Entry<String, String> entry : configuration) {
214                        boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END);
215                        String value = (maskValue) ? "**MASKED**" : entry.getValue();
216                        writer.write(" " + entry.getKey() + " = " + value + "\n");
217                    }
218                    writer.close();
219                    log.trace("Configuration:\n{0}---", writer.toString());
220                }
221                catch (IOException ex) {
222                    throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
223                }
224            }
225    
226            String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
227            if (ignoreSysProps != null) {
228                IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
229            }
230    
231            for (Map.Entry<String, String> entry : configuration) {
232                String sysValue = System.getProperty(entry.getKey());
233                if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
234                    log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
235                    configuration.set(entry.getKey(), sysValue);
236                }
237            }
238            for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
239                String name = (String) entry.getKey();
240                if (!IGNORE_SYS_PROPS.contains(name)) {
241                    if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
242                        if (configuration.get(name) == null) {
243                            log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
244                        }
245                    }
246                }
247            }
248    
249            return new LogChangesConfiguration(configuration);
250        }
251    
252        private class LogChangesConfiguration extends XConfiguration {
253    
254            public LogChangesConfiguration(Configuration conf) {
255                for (Map.Entry<String, String> entry : conf) {
256                    if (get(entry.getKey()) == null) {
257                        setValue(entry.getKey(), entry.getValue());
258                    }
259                }
260            }
261    
262            public String[] getStrings(String name) {
263                String s = get(name);
264                return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
265            }
266    
267            public String get(String name, String defaultValue) {
268                String value = get(name);
269                if (value == null) {
270                    boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
271                    value = (maskValue) ? "**MASKED**" : defaultValue;
272                    log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, value);
273                }
274                return value;
275            }
276    
277            public void set(String name, String value) {
278                setValue(name, value);
279                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
280                value = (maskValue) ? "**MASKED**" : value;
281                log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
282            }
283    
284            private void setValue(String name, String value) {
285                super.set(name, value);
286            }
287    
288        }
289    
290        /**
291         * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
292         * config file used.
293         *
294         * @param instr instrumentation to use.
295         */
296        public void instrument(Instrumentation instr) {
297            instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
298                public String getValue() {
299                    return configDir;
300                }
301            });
302            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
303                public String getValue() {
304                    return configFile;
305                }
306            });
307            instr.setConfiguration(configuration);
308        }
309    
310        /**
311         * Return a configuration with all sensitive values masked.
312         *
313         * @param conf configuration to mask.
314         * @return masked configuration.
315         */
316        public static Configuration maskPasswords(Configuration conf) {
317            XConfiguration maskedConf = new XConfiguration();
318            for (Map.Entry<String, String> entry : conf) {
319                String name = entry.getKey();
320                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
321                String value = (maskValue) ? "**MASKED**" : entry.getValue();
322                maskedConf.set(name, value);
323            }
324            return maskedConf;
325        }
326    
327    }