001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import java.io.IOException; 018 import java.net.URISyntaxException; 019 import java.util.List; 020 021 import org.apache.hadoop.conf.Configuration; 022 import org.apache.hadoop.fs.FileStatus; 023 import org.apache.hadoop.fs.FileSystem; 024 import org.apache.hadoop.fs.Path; 025 import org.apache.hadoop.fs.permission.FsPermission; 026 import org.apache.oozie.action.ActionExecutor; 027 import org.apache.oozie.action.ActionExecutorException; 028 import org.apache.oozie.client.WorkflowAction; 029 import org.apache.oozie.service.HadoopAccessorException; 030 import org.apache.oozie.service.HadoopAccessorService; 031 import org.apache.oozie.service.Services; 032 import org.apache.oozie.util.XmlUtils; 033 import org.jdom.Element; 034 035 /** 036 * File system action executor. <p/> This executes the file system mkdir, move and delete commands 037 */ 038 public class FsActionExecutor extends ActionExecutor { 039 040 public FsActionExecutor() { 041 super("fs"); 042 } 043 044 Path getPath(Element element, String attribute) { 045 String str = element.getAttributeValue(attribute).trim(); 046 return new Path(str); 047 } 048 049 void validatePath(Path path, boolean withScheme) throws ActionExecutorException { 050 String scheme = path.toUri().getScheme(); 051 if (withScheme) { 052 if (scheme == null) { 053 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS001", 054 "Missing scheme in path [{0}]", path); 055 } 056 else { 057 if (!scheme.equals("hdfs")) { 058 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS002", 059 "Scheme [{0}] not support in path [{1}]", scheme, path); 060 } 061 } 062 } 063 else { 064 if (scheme != null) { 065 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS003", 066 "Scheme [{0}] not allowed in path [{1}]", scheme, path); 067 } 068 } 069 } 070 071 @SuppressWarnings("unchecked") 072 void doOperations(Context context, Element element) throws ActionExecutorException { 073 try { 074 FileSystem fs = context.getAppFileSystem(); 075 boolean recovery = fs.exists(getRecoveryPath(context)); 076 if (!recovery) { 077 fs.mkdirs(getRecoveryPath(context)); 078 } 079 for (Element commandElement : (List<Element>) element.getChildren()) { 080 String command = commandElement.getName(); 081 if (command.equals("mkdir")) { 082 Path path = getPath(commandElement, "path"); 083 mkdir(context, path); 084 } 085 else { 086 if (command.equals("delete")) { 087 Path path = getPath(commandElement, "path"); 088 delete(context, path); 089 } 090 else { 091 if (command.equals("move")) { 092 Path source = getPath(commandElement, "source"); 093 Path target = getPath(commandElement, "target"); 094 move(context, source, target, recovery); 095 } 096 else { 097 if (command.equals("chmod")) { 098 Path path = getPath(commandElement, "path"); 099 String str = commandElement.getAttributeValue("dir-files"); 100 boolean dirFiles = (str == null) || Boolean.parseBoolean(str); 101 String permissionsMask = commandElement.getAttributeValue("permissions").trim(); 102 chmod(context, path, permissionsMask, dirFiles); 103 } 104 } 105 } 106 } 107 } 108 } 109 catch (Exception ex) { 110 throw convertException(ex); 111 } 112 } 113 114 /** 115 * @param path 116 * @param context 117 * @return FileSystem 118 * @throws HadoopAccessorException 119 */ 120 private FileSystem getFileSystemFor(Path path, Context context) throws HadoopAccessorException { 121 String user = context.getWorkflow().getUser(); 122 String group = context.getWorkflow().getGroup(); 123 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(), 124 context.getProtoActionConf()); 125 } 126 127 /** 128 * @param path 129 * @param user 130 * @param group 131 * @return FileSystem 132 * @throws HadoopAccessorException 133 */ 134 private FileSystem getFileSystemFor(Path path, String user, String group) throws HadoopAccessorException { 135 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(), 136 new Configuration()); 137 } 138 139 void mkdir(Context context, Path path) throws ActionExecutorException { 140 try { 141 validatePath(path, true); 142 FileSystem fs = getFileSystemFor(path, context); 143 144 if (!fs.exists(path)) { 145 if (!fs.mkdirs(path)) { 146 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS004", 147 "mkdir, path [{0}] could not create directory", path); 148 } 149 } 150 } 151 catch (Exception ex) { 152 throw convertException(ex); 153 } 154 } 155 156 /** 157 * Delete path 158 * 159 * @param context 160 * @param path 161 * @throws ActionExecutorException 162 */ 163 public void delete(Context context, Path path) throws ActionExecutorException { 164 try { 165 validatePath(path, true); 166 FileSystem fs = getFileSystemFor(path, context); 167 168 if (fs.exists(path)) { 169 if (!fs.delete(path, true)) { 170 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005", 171 "delete, path [{0}] could not delete path", path); 172 } 173 } 174 } 175 catch (Exception ex) { 176 throw convertException(ex); 177 } 178 } 179 180 /** 181 * Delete path 182 * 183 * @param user 184 * @param group 185 * @param path 186 * @throws ActionExecutorException 187 */ 188 public void delete(String user, String group, Path path) throws ActionExecutorException { 189 try { 190 validatePath(path, true); 191 FileSystem fs = getFileSystemFor(path, user, group); 192 193 if (fs.exists(path)) { 194 if (!fs.delete(path, true)) { 195 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005", 196 "delete, path [{0}] could not delete path", path); 197 } 198 } 199 } 200 catch (Exception ex) { 201 throw convertException(ex); 202 } 203 } 204 205 /** 206 * Move source to target 207 * 208 * @param context 209 * @param source 210 * @param target 211 * @param recovery 212 * @throws ActionExecutorException 213 */ 214 public void move(Context context, Path source, Path target, boolean recovery) throws ActionExecutorException { 215 try { 216 validatePath(source, true); 217 validatePath(target, false); 218 FileSystem fs = getFileSystemFor(source, context); 219 220 if (!fs.exists(source) && !recovery) { 221 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS006", 222 "move, source path [{0}] does not exist", source); 223 } 224 225 Path path = new Path(source, target); 226 if (fs.exists(path) && !recovery) { 227 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS007", 228 "move, target path [{0}] already exists", target); 229 } 230 231 if (!fs.rename(source, target) && !recovery) { 232 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS008", 233 "move, could not move [{0}] to [{1}]", source, target); 234 } 235 } 236 catch (Exception ex) { 237 throw convertException(ex); 238 } 239 } 240 241 void chmod(Context context, Path path, String permissions, boolean dirFiles) throws ActionExecutorException { 242 try { 243 validatePath(path, true); 244 FileSystem fs = getFileSystemFor(path, context); 245 246 if (!fs.exists(path)) { 247 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009", 248 "chmod, path [{0}] does not exist", path); 249 } 250 251 FileStatus pathStatus = fs.getFileStatus(path); 252 253 Path[] paths; 254 if (dirFiles && pathStatus.isDir()) { 255 FileStatus[] filesStatus = fs.listStatus(path); 256 paths = new Path[filesStatus.length]; 257 for (int i = 0; i < filesStatus.length; i++) { 258 paths[i] = filesStatus[i].getPath(); 259 } 260 } 261 else { 262 paths = new Path[]{path}; 263 } 264 265 FsPermission newFsPermission = createShortPermission(permissions, path); 266 fs.setPermission(path, newFsPermission); 267 for (Path p : paths) { 268 fs.setPermission(p, newFsPermission); 269 } 270 } 271 catch (Exception ex) { 272 throw convertException(ex); 273 } 274 } 275 276 FsPermission createShortPermission(String permissions, Path path) throws ActionExecutorException { 277 if (permissions.length() == 3) { 278 char user = permissions.charAt(0); 279 char group = permissions.charAt(1); 280 char other = permissions.charAt(2); 281 int useri = user - '0'; 282 int groupi = group - '0'; 283 int otheri = other - '0'; 284 int mask = useri * 100 + groupi * 10 + otheri; 285 short omask = Short.parseShort(Integer.toString(mask), 8); 286 return new FsPermission(omask); 287 } 288 else { 289 if (permissions.length() == 10) { 290 return FsPermission.valueOf(permissions); 291 } 292 else { 293 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS010", 294 "chmod, path [{0}] invalid permissions mask [{1}]", path, permissions); 295 } 296 } 297 } 298 299 @Override 300 public void check(Context context, WorkflowAction action) throws ActionExecutorException { 301 } 302 303 @Override 304 public void kill(Context context, WorkflowAction action) throws ActionExecutorException { 305 } 306 307 @Override 308 public void start(Context context, WorkflowAction action) throws ActionExecutorException { 309 try { 310 context.setStartData("-", "-", "-"); 311 Element actionXml = XmlUtils.parseXml(action.getConf()); 312 doOperations(context, actionXml); 313 context.setExecutionData("OK", null); 314 } 315 catch (Exception ex) { 316 throw convertException(ex); 317 } 318 } 319 320 @Override 321 public void end(Context context, WorkflowAction action) throws ActionExecutorException { 322 String externalStatus = action.getExternalStatus(); 323 WorkflowAction.Status status = externalStatus.equals("OK") ? WorkflowAction.Status.OK : 324 WorkflowAction.Status.ERROR; 325 context.setEndData(status, getActionSignal(status)); 326 if (!context.getProtoActionConf().getBoolean("oozie.action.keep.action.dir", false)) { 327 try { 328 FileSystem fs = context.getAppFileSystem(); 329 fs.delete(context.getActionDir(), true); 330 } 331 catch (Exception ex) { 332 throw convertException(ex); 333 } 334 } 335 } 336 337 @Override 338 public boolean isCompleted(String externalStatus) { 339 return true; 340 } 341 342 /** 343 * @param context 344 * @return 345 * @throws HadoopAccessorException 346 * @throws IOException 347 * @throws URISyntaxException 348 */ 349 public Path getRecoveryPath(Context context) throws HadoopAccessorException, IOException, URISyntaxException { 350 return new Path(context.getActionDir(), "fs-" + context.getRecoveryId()); 351 } 352 353 }