001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action; 016 017 import org.apache.hadoop.fs.FileSystem; 018 import org.apache.hadoop.fs.Path; 019 import org.apache.hadoop.conf.Configuration; 020 import org.apache.oozie.client.WorkflowAction; 021 import org.apache.oozie.client.WorkflowJob; 022 import org.apache.oozie.util.ELEvaluator; 023 import org.apache.oozie.util.ParamChecker; 024 import org.apache.oozie.util.XLog; 025 import org.apache.oozie.service.HadoopAccessorException; 026 import org.apache.oozie.service.Services; 027 028 import java.io.IOException; 029 import java.net.URISyntaxException; 030 import java.util.HashMap; 031 import java.util.Map; 032 import java.util.Properties; 033 import java.util.LinkedHashMap; 034 035 /** 036 * Base action executor class. <p/> All the action executors should extend this class. 037 */ 038 public abstract class ActionExecutor { 039 040 /** 041 * Configuration prefix for action executor (sub-classes) properties. 042 */ 043 public static final String CONF_PREFIX = "oozie.action."; 044 045 /** 046 * Error code used by {@link #convertException} when there is not register error information for an exception. 047 */ 048 public static final String ERROR_OTHER = "OTHER"; 049 050 private static class ErrorInfo { 051 ActionExecutorException.ErrorType errorType; 052 String errorCode; 053 054 private ErrorInfo(ActionExecutorException.ErrorType errorType, String errorCode) { 055 this.errorType = errorType; 056 this.errorCode = errorCode; 057 } 058 } 059 060 private static boolean initMode = false; 061 private static Map<String, Map<Class, ErrorInfo>> ERROR_INFOS = new HashMap<String, Map<Class, ErrorInfo>>(); 062 063 /** 064 * Context information passed to the ActionExecutor methods. 065 */ 066 public interface Context { 067 068 /** 069 * Create the callback URL for the action. 070 * 071 * @param externalStatusVar variable for the caller to inject the external status. 072 * @return the callback URL. 073 */ 074 public String getCallbackUrl(String externalStatusVar); 075 076 /** 077 * Return a proto configuration for actions with auth properties already set. 078 * 079 * @return a proto configuration for actions with auth properties already set. 080 */ 081 public Configuration getProtoActionConf(); 082 083 /** 084 * Return the workflow job. 085 * 086 * @return the workflow job. 087 */ 088 public WorkflowJob getWorkflow(); 089 090 /** 091 * Return an ELEvaluator with the context injected. 092 * 093 * @return configured ELEvaluator. 094 */ 095 public ELEvaluator getELEvaluator(); 096 097 /** 098 * Set a workflow action variable. <p/> Convenience method that prefixes the variable name with the action name 099 * plus a '.'. 100 * 101 * @param name variable name. 102 * @param value variable value, <code>null</code> removes the variable. 103 */ 104 public void setVar(String name, String value); 105 106 /** 107 * Get a workflow action variable. <p/> Convenience method that prefixes the variable name with the action name 108 * plus a '.'. 109 * 110 * @param name variable name. 111 * @return the variable value, <code>null</code> if not set. 112 */ 113 public String getVar(String name); 114 115 /** 116 * Set the action tracking information for an successfully started action. 117 * 118 * @param externalId the action external ID. 119 * @param trackerUri the action tracker URI. 120 * @param consoleUrl the action console URL. 121 */ 122 void setStartData(String externalId, String trackerUri, String consoleUrl); 123 124 /** 125 * Set the action execution completion information for an action. The action status is set to {@link 126 * org.apache.oozie.client.WorkflowAction.Status#DONE} 127 * 128 * @param externalStatus the action external end status. 129 * @param actionData the action data on completion, <code>null</code> if none. 130 */ 131 void setExecutionData(String externalStatus, Properties actionData); 132 133 /** 134 * Set the action end completion information for a completed action. 135 * 136 * @param status the action end status, it can be {@link org.apache.oozie.client.WorkflowAction.Status#OK} or 137 * {@link org.apache.oozie.client.WorkflowAction.Status#ERROR}. 138 * @param signalValue the action external end status. 139 */ 140 void setEndData(WorkflowAction.Status status, String signalValue); 141 142 /** 143 * Return if the executor invocation is a retry or not. 144 * 145 * @return if the executor invocation is a retry or not. 146 */ 147 boolean isRetry(); 148 149 /** 150 * Sets the external status for the action in context. 151 * 152 * @param externalStatus the external status. 153 */ 154 void setExternalStatus(String externalStatus); 155 156 /** 157 * Get the Action Recovery ID. 158 * 159 * @return recovery ID. 160 */ 161 String getRecoveryId(); 162 163 /* 164 * @return the path that will be used to store action specific data 165 * @throws IOException @throws URISyntaxException @throws HadoopAccessorException 166 */ 167 public Path getActionDir() throws HadoopAccessorException, IOException, URISyntaxException; 168 169 /** 170 * @return filesystem handle for the application deployment fs. 171 * @throws IOException 172 * @throws URISyntaxException 173 * @throws HadoopAccessorException 174 */ 175 public FileSystem getAppFileSystem() throws HadoopAccessorException, IOException, URISyntaxException; 176 177 public void setErrorInfo(String str, String exMsg); 178 } 179 180 /** 181 * Define the default maximum number of retry attempts for transient errors (total attempts = 1 + MAX_RETRIES). 182 */ 183 public static final int MAX_RETRIES = 3; 184 185 /** 186 * Define the default inteval in seconds between retries. 187 */ 188 public static final long RETRY_INTERVAL = 60; 189 190 private String type; 191 private int maxRetries; 192 private long retryInterval; 193 194 /** 195 * Create an action executor with default retry parameters. 196 * 197 * @param type action executor type. 198 */ 199 protected ActionExecutor(String type) { 200 this(type, MAX_RETRIES, RETRY_INTERVAL); 201 } 202 203 /** 204 * Create an action executor. 205 * 206 * @param type action executor type. 207 * @param retryAttempts retry attempts. 208 * @param retryInterval retry interval, in seconds. 209 */ 210 protected ActionExecutor(String type, int retryAttempts, long retryInterval) { 211 this.type = ParamChecker.notEmpty(type, "type"); 212 this.maxRetries = retryAttempts; 213 this.retryInterval = retryInterval; 214 } 215 216 /** 217 * Clear all init settings for all action types. 218 */ 219 public static void resetInitInfo() { 220 if (!initMode) { 221 throw new IllegalStateException("Error, action type info locked"); 222 } 223 ERROR_INFOS.clear(); 224 } 225 226 /** 227 * Enable action type initialization. 228 */ 229 public static void enableInit() { 230 initMode = true; 231 } 232 233 /** 234 * Disable action type initialization. 235 */ 236 public static void disableInit() { 237 initMode = false; 238 } 239 240 /** 241 * Invoked once at system initialization time. <p/> It can be used to register error information for the expected 242 * exceptions. Exceptions should be register from subclasses to superclasses to ensure proper detection, same thing 243 * that it is done in a normal catch. <p/> This method should invoke the {@link #registerError} method to register 244 * all its possible errors. <p/> Subclasses overriding must invoke super. 245 */ 246 public void initActionType() { 247 ERROR_INFOS.put(getType(), new LinkedHashMap<Class, ErrorInfo>()); 248 } 249 250 /** 251 * Return the system ID, this ID is defined in Oozie configuration. 252 * 253 * @return the system ID. 254 */ 255 public String getOozieSystemId() { 256 return Services.get().getSystemId(); 257 } 258 259 /** 260 * Return the runtime directory of the Oozie instance. <p/> The directory is created under TMP and it is always a 261 * new directory per system initialization. 262 * 263 * @return the runtime directory of the Oozie instance. 264 */ 265 public String getOozieRuntimeDir() { 266 return Services.get().getRuntimeDir(); 267 } 268 269 /** 270 * Return Oozie configuration. <p/> This is useful for actions that need access to configuration properties. 271 * 272 * @return Oozie configuration. 273 */ 274 public Configuration getOozieConf() { 275 return Services.get().getConf(); 276 } 277 278 /** 279 * Register error handling information for an exception. 280 * 281 * @param exClass excpetion class name (to work in case of a particular exception not being in the classpath, needed 282 * to be able to handle multiple version of Hadoop or other JARs used by executors with the same codebase). 283 * @param errorType error type for the exception. 284 * @param errorCode error code for the exception. 285 */ 286 protected void registerError(String exClass, ActionExecutorException.ErrorType errorType, String errorCode) { 287 if (!initMode) { 288 throw new IllegalStateException("Error, action type info locked"); 289 } 290 try { 291 Class klass = Thread.currentThread().getContextClassLoader().loadClass(exClass); 292 Map<Class, ErrorInfo> executorErrorInfo = ERROR_INFOS.get(getType()); 293 executorErrorInfo.put(klass, new ErrorInfo(errorType, errorCode)); 294 } 295 catch (ClassNotFoundException ex) { 296 XLog.getLog(getClass()).warn( 297 "Exception [{0}] no in classpath, ActionExecutor [{1}] will handled it as ERROR", exClass, 298 getType()); 299 } 300 } 301 302 /** 303 * Return the action executor type. 304 * 305 * @return the action executor type. 306 */ 307 public String getType() { 308 return type; 309 } 310 311 /** 312 * Return the maximum number of retries for the action executor. 313 * 314 * @return the maximum number of retries for the action executor. 315 */ 316 public int getMaxRetries() { 317 return maxRetries; 318 } 319 320 /** 321 * Set the maximum number of retries for the action executor. 322 * 323 * @param maxRetries the maximum number of retries. 324 */ 325 public void setMaxRetries(int maxRetries) { 326 this.maxRetries = maxRetries; 327 } 328 329 /** 330 * Return the retry interval for the action executor in seconds. 331 * 332 * @return the retry interval for the action executor in seconds. 333 */ 334 public long getRetryInterval() { 335 return retryInterval; 336 } 337 338 /** 339 * Sets the retry interval for the action executor. 340 * 341 * @param retryInterval retry interval in seconds. 342 */ 343 public void setRetryInterval(long retryInterval) { 344 this.retryInterval = retryInterval; 345 } 346 347 /** 348 * Utility method to handle exceptions in the {@link #start}, {@link #end}, {@link #kill} and {@link #check} methods 349 * <p/> It uses the error registry to convert exceptions to {@link ActionExecutorException}s. 350 * 351 * @param ex exception to convert. 352 * @return ActionExecutorException converted exception. 353 */ 354 @SuppressWarnings({"ThrowableInstanceNeverThrown"}) 355 protected ActionExecutorException convertException(Exception ex) { 356 if (ex instanceof ActionExecutorException) { 357 return (ActionExecutorException) ex; 358 } 359 for (Map.Entry<Class, ErrorInfo> errorInfo : ERROR_INFOS.get(getType()).entrySet()) { 360 if (errorInfo.getKey().isInstance(ex)) { 361 return new ActionExecutorException(errorInfo.getValue().errorType, errorInfo.getValue().errorCode, 362 "{0}", ex.getMessage(), ex); 363 } 364 } 365 String errorCode = ex.getClass().getName(); 366 errorCode = errorCode.substring(errorCode.lastIndexOf(".") + 1); 367 return new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, errorCode, "{0}", ex.getMessage(), 368 ex); 369 } 370 371 /** 372 * Convenience method that return the signal for an Action based on the action status. 373 * 374 * @param status action status. 375 * @return the action signal. 376 */ 377 protected String getActionSignal(WorkflowAction.Status status) { 378 switch (status) { 379 case OK: 380 return "OK"; 381 case ERROR: 382 case KILLED: 383 return "ERROR"; 384 default: 385 throw new IllegalArgumentException("Action status for signal can only be OK or ERROR"); 386 } 387 } 388 389 /** 390 * Return the path that will be used to store action specific data 391 * 392 * @param jobId Worfklow ID 393 * @param action Action 394 * @param key An Identifier 395 * @param temp temp directory flag 396 * @return A string that has the path 397 */ 398 protected String getActionDirPath(String jobId, WorkflowAction action, String key, boolean temp) { 399 String name = jobId + "/" + action.getName() + "--" + key; 400 if (temp) { 401 name += ".temp"; 402 } 403 return getOozieSystemId() + "/" + name; 404 } 405 406 /** 407 * Return the path that will be used to store action specific data. 408 * 409 * @param jobId Workflow ID 410 * @param action Action 411 * @param key An identifier 412 * @param temp Temp directory flag 413 * @return Path to the directory 414 */ 415 public Path getActionDir(String jobId, WorkflowAction action, String key, boolean temp) { 416 return new Path(getActionDirPath(jobId, action, key, temp)); 417 } 418 419 /** 420 * Start an action. <p/> The {@link Context#setStartData} method must be called within this method. <p/> If the 421 * action has completed, the {@link Context#setExecutionData} method must be called within this method. 422 * 423 * @param context executor context. 424 * @param action the action to start. 425 * @throws ActionExecutorException thrown if the action could not start. 426 */ 427 public abstract void start(Context context, WorkflowAction action) throws ActionExecutorException; 428 429 /** 430 * End an action after it has executed. <p/> The {@link Context#setEndData} method must be called within this 431 * method. 432 * 433 * @param context executor context. 434 * @param action the action to end. 435 * @throws ActionExecutorException thrown if the action could not end. 436 */ 437 public abstract void end(Context context, WorkflowAction action) throws ActionExecutorException; 438 439 /** 440 * Check if an action has completed. This method must be implemented by Async Action Executors. <p/> If the action 441 * has completed, the {@link Context#setExecutionData} method must be called within this method. <p/> If the action 442 * has not completed, the {@link Context#setExternalStatus} method must be called within this method. 443 * 444 * @param context executor context. 445 * @param action the action to end. 446 * @throws ActionExecutorException thrown if the action could not be checked. 447 */ 448 public abstract void check(Context context, WorkflowAction action) throws ActionExecutorException; 449 450 /** 451 * Kill an action. <p/> The {@link Context#setEndData} method must be called within this method. 452 * 453 * @param context executor context. 454 * @param action the action to kill. 455 * @throws ActionExecutorException thrown if the action could not be killed. 456 */ 457 public abstract void kill(Context context, WorkflowAction action) throws ActionExecutorException; 458 459 /** 460 * Return if the external status indicates that the action has completed. 461 * 462 * @param externalStatus external status to check. 463 * @return if the external status indicates that the action has completed. 464 */ 465 public abstract boolean isCompleted(String externalStatus); 466 467 }