001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.mapred.JobClient;
018    import org.apache.hadoop.mapred.JobConf;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.fs.Path;
021    import org.apache.hadoop.conf.Configuration;
022    import org.apache.hadoop.security.UserGroupInformation;
023    import org.apache.hadoop.security.token.Token;
024    import org.apache.hadoop.filecache.DistributedCache;
025    import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
026    import org.apache.hadoop.io.Text;
027    import org.apache.oozie.util.XLog;
028    import org.apache.oozie.util.XConfiguration;
029    import org.apache.oozie.util.ParamChecker;
030    import org.apache.oozie.ErrorCode;
031    import org.apache.oozie.service.HadoopAccessorService;
032    import org.apache.oozie.service.HadoopAccessorException;
033    import org.apache.oozie.service.Service;
034    import org.apache.oozie.service.ServiceException;
035    
036    import java.io.IOException;
037    import java.net.URI;
038    import java.net.URISyntaxException;
039    import java.security.PrivilegedExceptionAction;
040    import java.util.concurrent.ConcurrentMap;
041    import java.util.concurrent.ConcurrentHashMap;
042    
043    /**
044     * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The
045     * default accessor used is the base accessor which just injects the UGI into the configuration instance used to
046     * create/obtain JobClient and ileSystem instances. <p/> The HadoopAccess class to use can be configured in the
047     * <code>oozie-site.xml</code> using the <code>oozie.service.HadoopAccessorService.accessor.class</code> property.
048     */
049    public class KerberosHadoopAccessorService extends HadoopAccessorService {
050    
051        public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService.";
052    
053        public static final String KERBEROS_AUTH_ENABLED = CONF_PREFIX + "kerberos.enabled";
054        public static final String KERBEROS_KEYTAB = CONF_PREFIX + "keytab.file";
055        public static final String KERBEROS_PRINCIPAL = CONF_PREFIX + "kerberos.principal";
056    
057        private ConcurrentMap<String, UserGroupInformation> userUgiMap;
058    
059        public void init(Configuration serviceConf) throws ServiceException {
060            boolean kerberosAuthOn = serviceConf.getBoolean(KERBEROS_AUTH_ENABLED, true);
061            XLog.getLog(getClass()).info("Oozie Kerberos Authentication [{0}]", (kerberosAuthOn) ? "enabled" : "disabled");
062            if (kerberosAuthOn) {
063                try {
064                    String keytabFile = serviceConf.get(KERBEROS_KEYTAB,
065                                                        System.getProperty("user.home") + "/oozie.keytab").trim();
066                    if (keytabFile.length() == 0) {
067                        throw new ServiceException(ErrorCode.E0026, KERBEROS_KEYTAB);
068                    }
069                    String principal = serviceConf.get(KERBEROS_PRINCIPAL, "oozie/localhost@LOCALHOST");
070                    if (principal.length() == 0) {
071                        throw new ServiceException(ErrorCode.E0026, KERBEROS_PRINCIPAL);
072                    }
073                    Configuration conf = new Configuration();
074                    conf.set("hadoop.security.authentication", "kerberos");
075                    UserGroupInformation.setConfiguration(conf);
076                    UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
077                    XLog.getLog(getClass()).info("Got Kerberos ticket, keytab [{0}], Oozie principal principal [{1}]",
078                                                 keytabFile, principal);
079                }
080                catch (ServiceException ex) {
081                    throw ex;
082                }
083                catch (Exception ex) {
084                    throw new ServiceException(ErrorCode.E0100, getClass().getName(), ex.getMessage(), ex);
085                }
086            }
087            else {
088                Configuration conf = new Configuration();
089                conf.set("hadoop.security.authentication", "simple");
090                UserGroupInformation.setConfiguration(conf);
091            }
092            userUgiMap = new ConcurrentHashMap<String, UserGroupInformation>();
093        }
094    
095        public void destroy() {
096            userUgiMap = null;
097            super.destroy();
098        }
099    
100        private UserGroupInformation getUGI(String user) throws IOException {
101            UserGroupInformation ugi = userUgiMap.get(user);
102            if (ugi == null) {
103                // taking care of a race condition, the latest UGI will be discarded
104                ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
105                userUgiMap.putIfAbsent(user, ugi);
106            }
107            return ugi;
108        }
109    
110        /**
111         * Return a JobClient created with the provided user/group.
112         *
113         * @param conf JobConf with all necessary information to create the JobClient.
114         * @return JobClient created with the provided user/group.
115         * @throws HadoopAccessorException if the client could not be created.
116         */
117        public JobClient createJobClient(String user, String group, final JobConf conf) throws HadoopAccessorException {
118            ParamChecker.notEmpty(user, "user");
119            ParamChecker.notEmpty(group, "group");
120            validateJobTracker(conf.get("mapred.job.tracker"));
121            try {
122                UserGroupInformation ugi = getUGI(user);
123                JobClient jobClient = ugi.doAs(new PrivilegedExceptionAction<JobClient>() {
124                    public JobClient run() throws Exception {
125                        return new JobClient(conf);
126                    }
127                });
128                Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token"));
129                conf.getCredentials().addToken(new Text("mr token"), mrdt);
130                return jobClient;
131            }
132            catch (InterruptedException ex) {
133                throw new HadoopAccessorException(ErrorCode.E0902, ex);
134            }
135            catch (IOException ex) {
136                throw new HadoopAccessorException(ErrorCode.E0902, ex);
137            }
138        }
139    
140        /**
141         * Return a FileSystem created with the provided user/group.
142         *
143         * @param conf Configuration with all necessary information to create the FileSystem.
144         * @return FileSystem created with the provided user/group.
145         * @throws HadoopAccessorException if the filesystem could not be created.
146         */
147        public FileSystem createFileSystem(String user, String group, final Configuration conf)
148                throws HadoopAccessorException {
149            ParamChecker.notEmpty(user, "user");
150            ParamChecker.notEmpty(group, "group");
151            try {
152                validateNameNode(new URI(conf.get("fs.default.name")).getAuthority());
153                UserGroupInformation ugi = getUGI(user);
154                return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
155                    public FileSystem run() throws Exception {
156                        Configuration defaultConf = new Configuration();
157                        XConfiguration.copy(conf, defaultConf);
158                        return FileSystem.get(defaultConf);
159                    }
160                });
161            }
162            catch (InterruptedException ex) {
163                throw new HadoopAccessorException(ErrorCode.E0902, ex);
164            }
165            catch (IOException ex) {
166                throw new HadoopAccessorException(ErrorCode.E0902, ex);
167            }
168            catch (URISyntaxException ex) {
169                throw new HadoopAccessorException(ErrorCode.E0902, ex);
170            }
171        }
172    
173        /**
174         * Return a FileSystem created with the provided user/group for the specified URI.
175         *
176         * @param uri file system URI.
177         * @param conf Configuration with all necessary information to create the FileSystem.
178         * @return FileSystem created with the provided user/group.
179         * @throws HadoopAccessorException if the filesystem could not be created.
180         */
181        public FileSystem createFileSystem(String user, String group, final URI uri, final Configuration conf)
182                throws HadoopAccessorException {
183            ParamChecker.notEmpty(user, "user");
184            ParamChecker.notEmpty(group, "group");
185            validateNameNode(uri.getAuthority());
186            try {
187                UserGroupInformation ugi = getUGI(user);
188                return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
189                    public FileSystem run() throws Exception {
190                        Configuration defaultConf = new Configuration();
191                        XConfiguration.copy(conf, defaultConf);
192                        return FileSystem.get(uri, defaultConf);
193                    }
194                });
195            }
196            catch (InterruptedException ex) {
197                throw new HadoopAccessorException(ErrorCode.E0902, ex);
198            }
199            catch (IOException ex) {
200                throw new HadoopAccessorException(ErrorCode.E0902, ex);
201            }
202        }
203    
204    
205        public void addFileToClassPath(String user, String group, final Path file, final Configuration conf)
206                throws IOException {
207            ParamChecker.notEmpty(user, "user");
208            ParamChecker.notEmpty(group, "group");
209            try {
210                UserGroupInformation ugi = getUGI(user);
211                ugi.doAs(new PrivilegedExceptionAction<Void>() {
212                    public Void run() throws Exception {
213                        Configuration defaultConf = new Configuration();
214                        XConfiguration.copy(conf, defaultConf);
215                        //Doing this NOP add first to have the FS created and cached
216                        DistributedCache.addFileToClassPath(file, defaultConf);
217    
218                        DistributedCache.addFileToClassPath(file, conf);
219                        return null;
220                    }
221                });
222    
223            }
224            catch (InterruptedException ex) {
225                throw new IOException(ex);
226            }
227    
228        }
229    
230    }