001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.hadoop.mapred.JobClient; 018 import org.apache.hadoop.mapred.JobConf; 019 import org.apache.hadoop.fs.FileSystem; 020 import org.apache.hadoop.fs.Path; 021 import org.apache.hadoop.conf.Configuration; 022 import org.apache.hadoop.security.UserGroupInformation; 023 import org.apache.hadoop.filecache.DistributedCache; 024 import org.apache.oozie.ErrorCode; 025 import org.apache.oozie.util.ParamChecker; 026 import org.apache.oozie.util.XConfiguration; 027 import org.apache.oozie.util.XLog; 028 029 import java.io.IOException; 030 import java.net.URI; 031 import java.net.URISyntaxException; 032 import java.security.PrivilegedExceptionAction; 033 import java.util.Set; 034 import java.util.HashSet; 035 036 /** 037 * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The 038 * default accessor used is the base accessor which just injects the UGI into the configuration instance used to 039 * create/obtain JobClient and ileSystem instances. <p/> The HadoopAccess class to use can be configured in the 040 * <code>oozie-site.xml</code> using the <code>oozie.service.HadoopAccessorService.accessor.class</code> property. 041 */ 042 public class HadoopAccessorService implements Service { 043 044 public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService."; 045 public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX + "jobTracker.whitelist"; 046 public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist"; 047 048 private Set<String> jobTrackerWhitelist = new HashSet<String>(); 049 private Set<String> nameNodeWhitelist = new HashSet<String>(); 050 051 public void init(Services services) throws ServiceException { 052 for (String name : services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)) { 053 String tmp = name.toLowerCase().trim(); 054 if (tmp.length() == 0) { 055 continue; 056 } 057 jobTrackerWhitelist.add(tmp); 058 } 059 XLog.getLog(getClass()).info( 060 "JOB_TRACKER_WHITELIST :" + services.getConf().getStringCollection(JOB_TRACKER_WHITELIST) 061 + ", Total entries :" + jobTrackerWhitelist.size()); 062 for (String name : services.getConf().getStringCollection(NAME_NODE_WHITELIST)) { 063 String tmp = name.toLowerCase().trim(); 064 if (tmp.length() == 0) { 065 continue; 066 } 067 nameNodeWhitelist.add(tmp); 068 } 069 XLog.getLog(getClass()).info( 070 "NAME_NODE_WHITELIST :" + services.getConf().getStringCollection(NAME_NODE_WHITELIST) 071 + ", Total entries :" + nameNodeWhitelist.size()); 072 init(services.getConf()); 073 } 074 075 public void init(Configuration serviceConf) throws ServiceException { 076 } 077 078 public void destroy() { 079 } 080 081 public Class<? extends Service> getInterface() { 082 return HadoopAccessorService.class; 083 } 084 085 /** 086 * Return a JobClient created with the provided user/group. 087 * 088 * @param conf JobConf with all necessary information to create the 089 * JobClient. 090 * @return JobClient created with the provided user/group. 091 * @throws HadoopAccessorException if the client could not be created. 092 */ 093 public JobClient createJobClient(String user, String group, JobConf conf) throws HadoopAccessorException { 094 validateJobTracker(conf.get("mapred.job.tracker")); 095 conf = createConfiguration(user, group, conf); 096 try { 097 return new JobClient(conf); 098 } 099 catch (IOException e) { 100 throw new HadoopAccessorException(ErrorCode.E0902, e); 101 } 102 } 103 104 /** 105 * Return a FileSystem created with the provided user/group. 106 * 107 * @param conf Configuration with all necessary information to create the 108 * FileSystem. 109 * @return FileSystem created with the provided user/group. 110 * @throws HadoopAccessorException if the filesystem could not be created. 111 */ 112 public FileSystem createFileSystem(String user, String group, Configuration conf) throws HadoopAccessorException { 113 try { 114 validateNameNode(new URI(conf.get("fs.default.name")).getAuthority()); 115 conf = createConfiguration(user, group, conf); 116 return FileSystem.get(conf); 117 } 118 catch (IOException e) { 119 throw new HadoopAccessorException(ErrorCode.E0902, e); 120 } 121 catch (URISyntaxException e) { 122 throw new HadoopAccessorException(ErrorCode.E0902, e); 123 } 124 } 125 126 /** 127 * Return a FileSystem created with the provided user/group for the 128 * specified URI. 129 * 130 * @param uri file system URI. 131 * @param conf Configuration with all necessary information to create the 132 * FileSystem. 133 * @return FileSystem created with the provided user/group. 134 * @throws HadoopAccessorException if the filesystem could not be created. 135 */ 136 public FileSystem createFileSystem(String user, String group, URI uri, Configuration conf) 137 throws HadoopAccessorException { 138 validateNameNode(uri.getAuthority()); 139 conf = createConfiguration(user, group, conf); 140 try { 141 return FileSystem.get(uri, conf); 142 } 143 catch (IOException e) { 144 throw new HadoopAccessorException(ErrorCode.E0902, e); 145 } 146 } 147 148 /** 149 * Validate Job tracker 150 * @param jobTrackerUri 151 * @throws HadoopAccessorException 152 */ 153 protected void validateJobTracker(String jobTrackerUri) throws HadoopAccessorException { 154 validate(jobTrackerUri, jobTrackerWhitelist, ErrorCode.E0900); 155 } 156 157 /** 158 * Validate Namenode list 159 * @param nameNodeUri 160 * @throws HadoopAccessorException 161 */ 162 protected void validateNameNode(String nameNodeUri) throws HadoopAccessorException { 163 validate(nameNodeUri, nameNodeWhitelist, ErrorCode.E0901); 164 } 165 166 private void validate(String uri, Set<String> whitelist, ErrorCode error) throws HadoopAccessorException { 167 if (uri != null) { 168 uri = uri.toLowerCase().trim(); 169 if (whitelist.size() > 0 && !whitelist.contains(uri)) { 170 throw new HadoopAccessorException(error, uri); 171 } 172 } 173 } 174 175 @SuppressWarnings("unchecked") 176 private <C extends Configuration> C createConfiguration(String user, String group, C conf) { 177 ParamChecker.notEmpty(user, "user"); 178 ParamChecker.notEmpty(group, "group"); 179 C fsConf = (C) ((conf instanceof JobConf) ? new JobConf() : new Configuration()); 180 XConfiguration.copy(conf, fsConf); 181 fsConf.set("user.name", user); 182 fsConf.set("hadoop.job.ugi", user + "," + group); 183 return fsConf; 184 } 185 186 /** 187 * Add a file to the ClassPath via the DistributedCache. 188 */ 189 public void addFileToClassPath(String user, String group, final Path file, final Configuration conf) 190 throws IOException { 191 Configuration defaultConf = createConfiguration(user, group, conf); 192 DistributedCache.addFileToClassPath(file, defaultConf); 193 DistributedCache.addFileToClassPath(file, conf); 194 } 195 196 }