001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.mapred.JobClient;
018    import org.apache.hadoop.mapred.JobConf;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.fs.Path;
021    import org.apache.hadoop.conf.Configuration;
022    import org.apache.hadoop.security.UserGroupInformation;
023    import org.apache.hadoop.filecache.DistributedCache;
024    import org.apache.oozie.ErrorCode;
025    import org.apache.oozie.util.ParamChecker;
026    import org.apache.oozie.util.XConfiguration;
027    import org.apache.oozie.util.XLog;
028    
029    import java.io.IOException;
030    import java.net.URI;
031    import java.net.URISyntaxException;
032    import java.security.PrivilegedExceptionAction;
033    import java.util.Set;
034    import java.util.HashSet;
035    
036    /**
037     * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The
038     * default accessor used is the base accessor which just injects the UGI into the configuration instance used to
039     * create/obtain JobClient and ileSystem instances. <p/> The HadoopAccess class to use can be configured in the
040     * <code>oozie-site.xml</code> using the <code>oozie.service.HadoopAccessorService.accessor.class</code> property.
041     */
042    public class HadoopAccessorService implements Service {
043    
044        public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService.";
045        public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX + "jobTracker.whitelist";
046        public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist";
047    
048        private Set<String> jobTrackerWhitelist = new HashSet<String>();
049        private Set<String> nameNodeWhitelist = new HashSet<String>();
050    
051        public void init(Services services) throws ServiceException {
052            for (String name : services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)) {
053                String tmp = name.toLowerCase().trim();
054                if (tmp.length() == 0) {
055                    continue;
056                }
057                jobTrackerWhitelist.add(tmp);
058            }
059            XLog.getLog(getClass()).info(
060                    "JOB_TRACKER_WHITELIST :" + services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)
061                            + ", Total entries :" + jobTrackerWhitelist.size());
062            for (String name : services.getConf().getStringCollection(NAME_NODE_WHITELIST)) {
063                String tmp = name.toLowerCase().trim();
064                if (tmp.length() == 0) {
065                    continue;
066                }
067                nameNodeWhitelist.add(tmp);
068            }
069            XLog.getLog(getClass()).info(
070                    "NAME_NODE_WHITELIST :" + services.getConf().getStringCollection(NAME_NODE_WHITELIST)
071                            + ", Total entries :" + nameNodeWhitelist.size());
072            init(services.getConf());
073        }
074    
075        public void init(Configuration serviceConf) throws ServiceException {
076        }
077    
078        public void destroy() {
079        }
080    
081        public Class<? extends Service> getInterface() {
082            return HadoopAccessorService.class;
083        }
084    
085        /**
086         * Return a JobClient created with the provided user/group.
087         * 
088         * @param conf JobConf with all necessary information to create the
089         *        JobClient.
090         * @return JobClient created with the provided user/group.
091         * @throws HadoopAccessorException if the client could not be created.
092         */
093        public JobClient createJobClient(String user, String group, JobConf conf) throws HadoopAccessorException {
094            validateJobTracker(conf.get("mapred.job.tracker"));
095            conf = createConfiguration(user, group, conf);
096            try {
097                return new JobClient(conf);
098            }
099            catch (IOException e) {
100                throw new HadoopAccessorException(ErrorCode.E0902, e);
101            }
102        }
103    
104        /**
105         * Return a FileSystem created with the provided user/group.
106         * 
107         * @param conf Configuration with all necessary information to create the
108         *        FileSystem.
109         * @return FileSystem created with the provided user/group.
110         * @throws HadoopAccessorException if the filesystem could not be created.
111         */
112        public FileSystem createFileSystem(String user, String group, Configuration conf) throws HadoopAccessorException {
113            try {
114                validateNameNode(new URI(conf.get("fs.default.name")).getAuthority());
115                conf = createConfiguration(user, group, conf);
116                return FileSystem.get(conf);
117            }
118            catch (IOException e) {
119                throw new HadoopAccessorException(ErrorCode.E0902, e);
120            }
121            catch (URISyntaxException e) {
122                throw new HadoopAccessorException(ErrorCode.E0902, e);
123            }
124        }
125    
126        /**
127         * Return a FileSystem created with the provided user/group for the
128         * specified URI.
129         * 
130         * @param uri file system URI.
131         * @param conf Configuration with all necessary information to create the
132         *        FileSystem.
133         * @return FileSystem created with the provided user/group.
134         * @throws HadoopAccessorException if the filesystem could not be created.
135         */
136        public FileSystem createFileSystem(String user, String group, URI uri, Configuration conf)
137                throws HadoopAccessorException {
138            validateNameNode(uri.getAuthority());
139            conf = createConfiguration(user, group, conf);
140            try {
141                return FileSystem.get(uri, conf);
142            }
143            catch (IOException e) {
144                throw new HadoopAccessorException(ErrorCode.E0902, e);
145            }
146        }
147    
148        /**
149         * Validate Job tracker
150         * @param jobTrackerUri
151         * @throws HadoopAccessorException
152         */
153        protected void validateJobTracker(String jobTrackerUri) throws HadoopAccessorException {
154            validate(jobTrackerUri, jobTrackerWhitelist, ErrorCode.E0900);
155        }
156    
157        /**
158         * Validate Namenode list
159         * @param nameNodeUri
160         * @throws HadoopAccessorException
161         */
162        protected void validateNameNode(String nameNodeUri) throws HadoopAccessorException {
163            validate(nameNodeUri, nameNodeWhitelist, ErrorCode.E0901);
164        }
165    
166        private void validate(String uri, Set<String> whitelist, ErrorCode error) throws HadoopAccessorException {
167            if (uri != null) {
168                uri = uri.toLowerCase().trim();
169                if (whitelist.size() > 0 && !whitelist.contains(uri)) {
170                    throw new HadoopAccessorException(error, uri);
171                }
172            }
173        }
174    
175        @SuppressWarnings("unchecked")
176        private <C extends Configuration> C createConfiguration(String user, String group, C conf) {
177            ParamChecker.notEmpty(user, "user");
178            ParamChecker.notEmpty(group, "group");
179            C fsConf = (C) ((conf instanceof JobConf) ? new JobConf() : new Configuration());
180            XConfiguration.copy(conf, fsConf);
181            fsConf.set("user.name", user);
182            fsConf.set("hadoop.job.ugi", user + "," + group);
183            return fsConf;
184        }
185    
186        /**
187         * Add a file to the ClassPath via the DistributedCache.
188         */
189        public void addFileToClassPath(String user, String group, final Path file, final Configuration conf)
190                throws IOException {
191            Configuration defaultConf = createConfiguration(user, group, conf);
192            DistributedCache.addFileToClassPath(file, defaultConf);
193            DistributedCache.addFileToClassPath(file, conf);
194        }
195    
196    }