001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.mapred.JobClient;
018    import org.apache.hadoop.mapred.JobConf;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.ErrorCode;
022    import org.apache.oozie.util.ParamChecker;
023    import org.apache.oozie.util.XConfiguration;
024    import org.apache.oozie.util.XLog;
025    
026    import java.io.IOException;
027    import java.net.URI;
028    import java.net.URISyntaxException;
029    import java.util.Set;
030    import java.util.HashSet;
031    
032    /**
033     * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The
034     * default accessor used is the base accessor which just injects the UGI into the configuration instance used to
035     * create/obtain JobClient and ileSystem instances. <p/> The HadoopAccess class to use can be configured in the
036     * <code>oozie-site.xml</code> using the <code>oozie.service.HadoopAccessorService.accessor.class</code> property.
037     */
038    public class HadoopAccessorService implements Service {
039    
040        public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService.";
041        public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX + "jobTracker.whitelist";
042        public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist";
043    
044        private Set<String> jobTrackerWhitelist = new HashSet<String>();
045        private Set<String> nameNodeWhitelist = new HashSet<String>();
046    
047        public void init(Services services) throws ServiceException {
048            for (String name : services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)) {
049                String tmp = name.toLowerCase().trim();
050                if (tmp.length() == 0) {
051                    continue;
052                }
053                jobTrackerWhitelist.add(tmp);
054            }
055            XLog.getLog(getClass()).info(
056                    "JOB_TRACKER_WHITELIST :" + services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)
057                            + ", Total entries :" + jobTrackerWhitelist.size());
058            for (String name : services.getConf().getStringCollection(NAME_NODE_WHITELIST)) {
059                String tmp = name.toLowerCase().trim();
060                if (tmp.length() == 0) {
061                    continue;
062                }
063                nameNodeWhitelist.add(tmp);
064            }
065            XLog.getLog(getClass()).info(
066                    "NAME_NODE_WHITELIST :" + services.getConf().getStringCollection(NAME_NODE_WHITELIST)
067                            + ", Total entries :" + nameNodeWhitelist.size());
068            init(services.getConf());
069        }
070    
071        public void init(Configuration serviceConf) throws ServiceException {
072        }
073    
074        public void destroy() {
075        }
076    
077        public Class<? extends Service> getInterface() {
078            return HadoopAccessorService.class;
079        }
080    
081        /**
082         * Return a JobClient created with the provided user/group.
083         * 
084         * @param conf JobConf with all necessary information to create the
085         *        JobClient.
086         * @return JobClient created with the provided user/group.
087         * @throws HadoopAccessorException if the client could not be created.
088         */
089        public JobClient createJobClient(String user, String group, JobConf conf) throws HadoopAccessorException {
090            validateJobTracker(conf.get("mapred.job.tracker"));
091            conf = createConfiguration(user, group, conf);
092            try {
093                return new JobClient(conf);
094            }
095            catch (IOException e) {
096                throw new HadoopAccessorException(ErrorCode.E0902, e);
097            }
098        }
099    
100        /**
101         * Return a FileSystem created with the provided user/group.
102         * 
103         * @param conf Configuration with all necessary information to create the
104         *        FileSystem.
105         * @return FileSystem created with the provided user/group.
106         * @throws HadoopAccessorException if the filesystem could not be created.
107         */
108        public FileSystem createFileSystem(String user, String group, Configuration conf) throws HadoopAccessorException {
109            try {
110                validateNameNode(new URI(conf.get("fs.default.name")).getAuthority());
111                conf = createConfiguration(user, group, conf);
112                return FileSystem.get(conf);
113            }
114            catch (IOException e) {
115                throw new HadoopAccessorException(ErrorCode.E0902, e);
116            }
117            catch (URISyntaxException e) {
118                throw new HadoopAccessorException(ErrorCode.E0902, e);
119            }
120        }
121    
122        /**
123         * Return a FileSystem created with the provided user/group for the
124         * specified URI.
125         * 
126         * @param uri file system URI.
127         * @param conf Configuration with all necessary information to create the
128         *        FileSystem.
129         * @return FileSystem created with the provided user/group.
130         * @throws HadoopAccessorException if the filesystem could not be created.
131         */
132        public FileSystem createFileSystem(String user, String group, URI uri, Configuration conf)
133                throws HadoopAccessorException {
134            validateNameNode(uri.getAuthority());
135            conf = createConfiguration(user, group, conf);
136            try {
137                return FileSystem.get(uri, conf);
138            }
139            catch (IOException e) {
140                throw new HadoopAccessorException(ErrorCode.E0902, e);
141            }
142        }
143    
144        /**
145         * Validate Job tracker
146         * @param jobTrackerUri
147         * @throws HadoopAccessorException
148         */
149        protected void validateJobTracker(String jobTrackerUri) throws HadoopAccessorException {
150            validate(jobTrackerUri, jobTrackerWhitelist, ErrorCode.E0900);
151        }
152    
153        /**
154         * Validate Namenode list
155         * @param nameNodeUri
156         * @throws HadoopAccessorException
157         */
158        protected void validateNameNode(String nameNodeUri) throws HadoopAccessorException {
159            validate(nameNodeUri, nameNodeWhitelist, ErrorCode.E0901);
160        }
161    
162        private void validate(String uri, Set<String> whitelist, ErrorCode error) throws HadoopAccessorException {
163            if (uri != null) {
164                uri = uri.toLowerCase().trim();
165                if (whitelist.size() > 0 && !whitelist.contains(uri)) {
166                    throw new HadoopAccessorException(error, uri);
167                }
168            }
169        }
170    
171        @SuppressWarnings("unchecked")
172        private <C extends Configuration> C createConfiguration(String user, String group, C conf) {
173            ParamChecker.notEmpty(user, "user");
174            ParamChecker.notEmpty(group, "group");
175            C fsConf = (C) ((conf instanceof JobConf) ? new JobConf() : new Configuration());
176            XConfiguration.copy(conf, fsConf);
177            fsConf.set("user.name", user);
178            fsConf.set("hadoop.job.ugi", user + "," + group);
179            return fsConf;
180        }
181    
182    }