001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.hadoop.mapred.JobClient; 018 import org.apache.hadoop.mapred.JobConf; 019 import org.apache.hadoop.fs.FileSystem; 020 import org.apache.hadoop.conf.Configuration; 021 import org.apache.oozie.ErrorCode; 022 import org.apache.oozie.util.ParamChecker; 023 import org.apache.oozie.util.XConfiguration; 024 import org.apache.oozie.util.XLog; 025 026 import java.io.IOException; 027 import java.net.URI; 028 import java.net.URISyntaxException; 029 import java.util.Set; 030 import java.util.HashSet; 031 032 /** 033 * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The 034 * default accessor used is the base accessor which just injects the UGI into the configuration instance used to 035 * create/obtain JobClient and ileSystem instances. <p/> The HadoopAccess class to use can be configured in the 036 * <code>oozie-site.xml</code> using the <code>oozie.service.HadoopAccessorService.accessor.class</code> property. 037 */ 038 public class HadoopAccessorService implements Service { 039 040 public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService."; 041 public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX + "jobTracker.whitelist"; 042 public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist"; 043 044 private Set<String> jobTrackerWhitelist = new HashSet<String>(); 045 private Set<String> nameNodeWhitelist = new HashSet<String>(); 046 047 public void init(Services services) throws ServiceException { 048 for (String name : services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)) { 049 String tmp = name.toLowerCase().trim(); 050 if (tmp.length() == 0) { 051 continue; 052 } 053 jobTrackerWhitelist.add(tmp); 054 } 055 XLog.getLog(getClass()).info( 056 "JOB_TRACKER_WHITELIST :" + services.getConf().getStringCollection(JOB_TRACKER_WHITELIST) 057 + ", Total entries :" + jobTrackerWhitelist.size()); 058 for (String name : services.getConf().getStringCollection(NAME_NODE_WHITELIST)) { 059 String tmp = name.toLowerCase().trim(); 060 if (tmp.length() == 0) { 061 continue; 062 } 063 nameNodeWhitelist.add(tmp); 064 } 065 XLog.getLog(getClass()).info( 066 "NAME_NODE_WHITELIST :" + services.getConf().getStringCollection(NAME_NODE_WHITELIST) 067 + ", Total entries :" + nameNodeWhitelist.size()); 068 init(services.getConf()); 069 } 070 071 public void init(Configuration serviceConf) throws ServiceException { 072 } 073 074 public void destroy() { 075 } 076 077 public Class<? extends Service> getInterface() { 078 return HadoopAccessorService.class; 079 } 080 081 /** 082 * Return a JobClient created with the provided user/group. 083 * 084 * @param conf JobConf with all necessary information to create the 085 * JobClient. 086 * @return JobClient created with the provided user/group. 087 * @throws HadoopAccessorException if the client could not be created. 088 */ 089 public JobClient createJobClient(String user, String group, JobConf conf) throws HadoopAccessorException { 090 validateJobTracker(conf.get("mapred.job.tracker")); 091 conf = createConfiguration(user, group, conf); 092 try { 093 return new JobClient(conf); 094 } 095 catch (IOException e) { 096 throw new HadoopAccessorException(ErrorCode.E0902, e); 097 } 098 } 099 100 /** 101 * Return a FileSystem created with the provided user/group. 102 * 103 * @param conf Configuration with all necessary information to create the 104 * FileSystem. 105 * @return FileSystem created with the provided user/group. 106 * @throws HadoopAccessorException if the filesystem could not be created. 107 */ 108 public FileSystem createFileSystem(String user, String group, Configuration conf) throws HadoopAccessorException { 109 try { 110 validateNameNode(new URI(conf.get("fs.default.name")).getAuthority()); 111 conf = createConfiguration(user, group, conf); 112 return FileSystem.get(conf); 113 } 114 catch (IOException e) { 115 throw new HadoopAccessorException(ErrorCode.E0902, e); 116 } 117 catch (URISyntaxException e) { 118 throw new HadoopAccessorException(ErrorCode.E0902, e); 119 } 120 } 121 122 /** 123 * Return a FileSystem created with the provided user/group for the 124 * specified URI. 125 * 126 * @param uri file system URI. 127 * @param conf Configuration with all necessary information to create the 128 * FileSystem. 129 * @return FileSystem created with the provided user/group. 130 * @throws HadoopAccessorException if the filesystem could not be created. 131 */ 132 public FileSystem createFileSystem(String user, String group, URI uri, Configuration conf) 133 throws HadoopAccessorException { 134 validateNameNode(uri.getAuthority()); 135 conf = createConfiguration(user, group, conf); 136 try { 137 return FileSystem.get(uri, conf); 138 } 139 catch (IOException e) { 140 throw new HadoopAccessorException(ErrorCode.E0902, e); 141 } 142 } 143 144 /** 145 * Validate Job tracker 146 * @param jobTrackerUri 147 * @throws HadoopAccessorException 148 */ 149 protected void validateJobTracker(String jobTrackerUri) throws HadoopAccessorException { 150 validate(jobTrackerUri, jobTrackerWhitelist, ErrorCode.E0900); 151 } 152 153 /** 154 * Validate Namenode list 155 * @param nameNodeUri 156 * @throws HadoopAccessorException 157 */ 158 protected void validateNameNode(String nameNodeUri) throws HadoopAccessorException { 159 validate(nameNodeUri, nameNodeWhitelist, ErrorCode.E0901); 160 } 161 162 private void validate(String uri, Set<String> whitelist, ErrorCode error) throws HadoopAccessorException { 163 if (uri != null) { 164 uri = uri.toLowerCase().trim(); 165 if (whitelist.size() > 0 && !whitelist.contains(uri)) { 166 throw new HadoopAccessorException(error, uri); 167 } 168 } 169 } 170 171 @SuppressWarnings("unchecked") 172 private <C extends Configuration> C createConfiguration(String user, String group, C conf) { 173 ParamChecker.notEmpty(user, "user"); 174 ParamChecker.notEmpty(group, "group"); 175 C fsConf = (C) ((conf instanceof JobConf) ? new JobConf() : new Configuration()); 176 XConfiguration.copy(conf, fsConf); 177 fsConf.set("user.name", user); 178 fsConf.set("hadoop.job.ugi", user + "," + group); 179 return fsConf; 180 } 181 182 }