001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie; 016 017 import org.apache.oozie.util.XLogStreamer; 018 import org.apache.oozie.service.XLogService; 019 import org.apache.oozie.service.DagXLogInfoService; 020 import org.apache.hadoop.conf.Configuration; 021 import org.apache.oozie.client.CoordinatorJob; 022 import org.apache.oozie.client.WorkflowJob; 023 import org.apache.oozie.client.OozieClient; 024 import org.apache.oozie.command.wf.CompletedActionCommand; 025 import org.apache.oozie.command.CommandException; 026 import org.apache.oozie.command.Command; 027 import org.apache.oozie.command.wf.JobCommand; 028 import org.apache.oozie.command.wf.JobsCommand; 029 import org.apache.oozie.command.wf.KillCommand; 030 import org.apache.oozie.command.wf.ReRunCommand; 031 import org.apache.oozie.command.wf.ResumeCommand; 032 import org.apache.oozie.command.wf.SubmitCommand; 033 import org.apache.oozie.command.wf.SubmitHttpCommand; 034 import org.apache.oozie.command.wf.SubmitPigCommand; 035 import org.apache.oozie.command.wf.SubmitMRCommand; 036 import org.apache.oozie.command.wf.StartCommand; 037 import org.apache.oozie.command.wf.SuspendCommand; 038 import org.apache.oozie.command.wf.DefinitionCommand; 039 import org.apache.oozie.command.wf.ExternalIdCommand; 040 import org.apache.oozie.command.wf.WorkflowActionInfoCommand; 041 import org.apache.oozie.service.Services; 042 import org.apache.oozie.service.CallableQueueService; 043 import org.apache.oozie.util.ParamChecker; 044 import org.apache.oozie.util.XConfiguration; 045 import org.apache.oozie.util.XLog; 046 047 import java.io.Writer; 048 import java.util.Date; 049 import java.util.List; 050 import java.util.Properties; 051 import java.util.Set; 052 import java.util.HashSet; 053 import java.util.StringTokenizer; 054 import java.util.Map; 055 import java.util.HashMap; 056 import java.util.ArrayList; 057 import java.io.IOException; 058 059 /** 060 * The DagEngine bean provides all the DAG engine functionality for WS calls. 061 */ 062 public class DagEngine extends BaseEngine { 063 064 private static final int HIGH_PRIORITY = 2; 065 066 /** 067 * Create a system Dag engine, with no user and no group. 068 */ 069 public DagEngine() { 070 } 071 072 /** 073 * Create a Dag engine to perform operations on behave of a user. 074 * 075 * @param user user name. 076 * @param authToken the authentication token. 077 */ 078 public DagEngine(String user, String authToken) { 079 this.user = ParamChecker.notEmpty(user, "user"); 080 this.authToken = ParamChecker.notEmpty(authToken, "authToken"); 081 } 082 083 /** 084 * Submit a workflow job. <p/> It validates configuration properties. 085 * 086 * @param conf job configuration. 087 * @param startJob indicates if the job should be started or not. 088 * @return the job Id. 089 * @throws DagEngineException thrown if the job could not be created. 090 */ 091 @Override 092 public String submitJob(Configuration conf, boolean startJob) throws DagEngineException { 093 validateSubmitConfiguration(conf); 094 SubmitCommand submit = new SubmitCommand(conf, getAuthToken()); 095 try { 096 String jobId = submit.call(); 097 if (startJob) { 098 start(jobId); 099 } 100 return jobId; 101 } 102 catch (CommandException ex) { 103 throw new DagEngineException(ex); 104 } 105 } 106 107 /** 108 * Submit a pig/mapreduce job through HTTP. 109 * <p/> 110 * It validates configuration properties. 111 * 112 * @param conf job configuration. 113 * @param jobType job type - can be "pig" or "mapreduce". 114 * @return the job Id. 115 * @throws DagEngineException thrown if the job could not be created. 116 */ 117 public String submitHttpJob(Configuration conf, String jobType) throws DagEngineException { 118 validateSubmitConfiguration(conf); 119 120 SubmitHttpCommand submit = null; 121 if (jobType.equals("pig")) { 122 submit = new SubmitPigCommand(conf, getAuthToken()); 123 } 124 else if (jobType.equals("mapreduce")) { 125 submit = new SubmitMRCommand(conf, getAuthToken()); 126 } 127 128 try { 129 String jobId = submit.call(); 130 start(jobId); 131 return jobId; 132 } 133 catch (CommandException ex) { 134 throw new DagEngineException(ex); 135 } 136 } 137 138 public static void main(String[] args) throws Exception { 139 // Configuration conf = new XConfiguration(IOUtils.getResourceAsReader( 140 // "org/apache/oozie/coord/conf.xml", -1)); 141 142 Configuration conf = new XConfiguration(); 143 144 // String appXml = 145 // IOUtils.getResourceAsString("org/apache/oozie/coord/test1.xml", -1); 146 conf.set(OozieClient.APP_PATH, "file:///Users/danielwo/oozie/workflows/examples/seed/workflows/map-reduce"); 147 conf.set(OozieClient.USER_NAME, "danielwo"); 148 conf.set(OozieClient.GROUP_NAME, "other"); 149 150 conf.set("inputDir", " blah "); 151 152 // System.out.println("appXml :"+ appXml + "\n conf :"+ conf); 153 new Services().init(); 154 try { 155 DagEngine de = new DagEngine("me", "TESTING_WF"); 156 String jobId = de.submitJob(conf, true); 157 System.out.println("WF Job Id " + jobId); 158 159 Thread.sleep(20000); 160 } 161 finally { 162 Services.get().destroy(); 163 } 164 } 165 166 private void validateSubmitConfiguration(Configuration conf) throws DagEngineException { 167 if (conf.get(OozieClient.APP_PATH) == null) { 168 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH); 169 } 170 } 171 172 /** 173 * Start a job. 174 * 175 * @param jobId job Id. 176 * @throws DagEngineException thrown if the job could not be started. 177 */ 178 @Override 179 public void start(String jobId) throws DagEngineException { 180 // Changing to synchronous call from asynchronous queuing to prevent the 181 // loss of command if the queue is full or the queue is lost in case of 182 // failure. 183 try { 184 new StartCommand(jobId).call(); 185 } 186 catch (CommandException e) { 187 throw new DagEngineException(e); 188 } 189 } 190 191 /** 192 * Resume a job. 193 * 194 * @param jobId job Id. 195 * @throws DagEngineException thrown if the job could not be resumed. 196 */ 197 @Override 198 public void resume(String jobId) throws DagEngineException { 199 // Changing to synchronous call from asynchronous queuing to prevent the 200 // loss of command if the queue is full or the queue is lost in case of 201 // failure. 202 try { 203 new ResumeCommand(jobId).call(); 204 } 205 catch (CommandException e) { 206 throw new DagEngineException(e); 207 } 208 } 209 210 /** 211 * Suspend a job. 212 * 213 * @param jobId job Id. 214 * @throws DagEngineException thrown if the job could not be suspended. 215 */ 216 @Override 217 public void suspend(String jobId) throws DagEngineException { 218 // Changing to synchronous call from asynchronous queuing to prevent the 219 // loss of command if the queue is full or the queue is lost in case of 220 // failure. 221 try { 222 new SuspendCommand(jobId).call(); 223 } 224 catch (CommandException e) { 225 throw new DagEngineException(e); 226 } 227 } 228 229 /** 230 * Kill a job. 231 * 232 * @param jobId job Id. 233 * @throws DagEngineException thrown if the job could not be killed. 234 */ 235 @Override 236 public void kill(String jobId) throws DagEngineException { 237 // Changing to synchronous call from asynchronous queuing to prevent the 238 // loss of command if the queue is full or the queue is lost in case of 239 // failure. 240 try { 241 new KillCommand(jobId).call(); 242 XLog.getLog(getClass()).info("User " + user + " killed the WF job " + jobId); 243 } 244 catch (CommandException e) { 245 throw new DagEngineException(e); 246 } 247 } 248 249 /* (non-Javadoc) 250 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String) 251 */ 252 @Override 253 public void change(String jobId, String changeValue) throws DagEngineException { 254 // This code should not be reached. 255 throw new DagEngineException(ErrorCode.E1017); 256 } 257 258 /** 259 * Rerun a job. 260 * 261 * @param jobId job Id to rerun. 262 * @param conf configuration information for the rerun. 263 * @throws DagEngineException thrown if the job could not be rerun. 264 */ 265 @Override 266 public void reRun(String jobId, Configuration conf) throws DagEngineException { 267 try { 268 validateReRunConfiguration(conf); 269 new ReRunCommand(jobId, conf, getAuthToken()).call(); 270 start(jobId); 271 } 272 catch (CommandException ex) { 273 throw new DagEngineException(ex); 274 } 275 } 276 277 private void validateReRunConfiguration(Configuration conf) throws DagEngineException { 278 if (conf.get(OozieClient.APP_PATH) == null) { 279 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH); 280 } 281 if (conf.get(OozieClient.RERUN_SKIP_NODES) == null) { 282 throw new DagEngineException(ErrorCode.E0401, OozieClient.RERUN_SKIP_NODES); 283 } 284 } 285 286 /** 287 * Process an action callback. 288 * 289 * @param actionId the action Id. 290 * @param externalStatus the action external status. 291 * @param actionData the action output data, <code>null</code> if none. 292 * @throws DagEngineException thrown if the callback could not be processed. 293 */ 294 public void processCallback(String actionId, String externalStatus, Properties actionData) 295 throws DagEngineException { 296 XLog.Info.get().clearParameter(XLogService.GROUP); 297 XLog.Info.get().clearParameter(XLogService.USER); 298 Command<Void, ?> command = new CompletedActionCommand(actionId, externalStatus, actionData, HIGH_PRIORITY); 299 if (!Services.get().get(CallableQueueService.class).queue(command)) { 300 XLog.getLog(this.getClass()).warn(XLog.OPS, "queue is full or system is in SAFEMODE, ignoring callback"); 301 } 302 } 303 304 /** 305 * Return the info about a job. 306 * 307 * @param jobId job Id. 308 * @return the workflow job info. 309 * @throws DagEngineException thrown if the job info could not be obtained. 310 */ 311 @Override 312 public WorkflowJob getJob(String jobId) throws DagEngineException { 313 try { 314 return new JobCommand(jobId).call(); 315 } 316 catch (CommandException ex) { 317 throw new DagEngineException(ex); 318 } 319 } 320 321 /** 322 * Return the info about a job with actions subset. 323 * 324 * @param jobId job Id 325 * @param start starting from this index in the list of actions belonging to the job 326 * @param length number of actions to be returned 327 * @return the workflow job info. 328 * @throws DagEngineException thrown if the job info could not be obtained. 329 */ 330 @Override 331 public WorkflowJob getJob(String jobId, int start, int length) throws DagEngineException { 332 try { 333 return new JobCommand(jobId, start, length).call(); 334 } 335 catch (CommandException ex) { 336 throw new DagEngineException(ex); 337 } 338 } 339 340 /** 341 * Return the a job definition. 342 * 343 * @param jobId job Id. 344 * @return the job definition. 345 * @throws DagEngineException thrown if the job definition could no be obtained. 346 */ 347 @Override 348 public String getDefinition(String jobId) throws DagEngineException { 349 try { 350 return new DefinitionCommand(jobId).call(); 351 } 352 catch (CommandException ex) { 353 throw new DagEngineException(ex); 354 } 355 } 356 357 /** 358 * Stream the log of a job. 359 * 360 * @param jobId job Id. 361 * @param writer writer to stream the log to. 362 * @throws IOException thrown if the log cannot be streamed. 363 * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId. 364 */ 365 @Override 366 public void streamLog(String jobId, Writer writer) throws IOException, DagEngineException { 367 XLogStreamer.Filter filter = new XLogStreamer.Filter(); 368 filter.setParameter(DagXLogInfoService.JOB, jobId); 369 WorkflowJob job = getJob(jobId); 370 Date lastTime = job.getEndTime(); 371 if (lastTime == null) { 372 lastTime = job.getLastModifiedTime(); 373 } 374 Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), lastTime, writer); 375 } 376 377 private static final Set<String> FILTER_NAMES = new HashSet<String>(); 378 379 static { 380 FILTER_NAMES.add(OozieClient.FILTER_USER); 381 FILTER_NAMES.add(OozieClient.FILTER_NAME); 382 FILTER_NAMES.add(OozieClient.FILTER_GROUP); 383 FILTER_NAMES.add(OozieClient.FILTER_STATUS); 384 } 385 386 /** 387 * Validate a jobs filter. 388 * 389 * @param filter filter to validate. 390 * @return the parsed filter. 391 * @throws DagEngineException thrown if the filter is invalid. 392 */ 393 protected Map<String, List<String>> parseFilter(String filter) throws DagEngineException { 394 Map<String, List<String>> map = new HashMap<String, List<String>>(); 395 if (filter != null) { 396 StringTokenizer st = new StringTokenizer(filter, ";"); 397 while (st.hasMoreTokens()) { 398 String token = st.nextToken(); 399 if (token.contains("=")) { 400 String[] pair = token.split("="); 401 if (pair.length != 2) { 402 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 403 } 404 if (!FILTER_NAMES.contains(pair[0])) { 405 throw new DagEngineException(ErrorCode.E0420, filter, XLog 406 .format("invalid name [{0}]", pair[0])); 407 } 408 if (pair[0].equals("status")) { 409 try { 410 WorkflowJob.Status.valueOf(pair[1]); 411 } 412 catch (IllegalArgumentException ex) { 413 throw new DagEngineException(ErrorCode.E0420, filter, XLog.format("invalid status [{0}]", 414 pair[1])); 415 } 416 } 417 List<String> list = map.get(pair[0]); 418 if (list == null) { 419 list = new ArrayList<String>(); 420 map.put(pair[0], list); 421 } 422 list.add(pair[1]); 423 } 424 else { 425 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 426 } 427 } 428 } 429 return map; 430 } 431 432 /** 433 * Return the info about a set of jobs. 434 * 435 * @param filterStr job filter. Refer to the {@link org.apache.oozie.client.OozieClient} for the filter syntax. 436 * @param start offset, base 1. 437 * @param len number of jobs to return. 438 * @return job info for all matching jobs, the jobs don't contain node action information. 439 * @throws DagEngineException thrown if the jobs info could not be obtained. 440 */ 441 @SuppressWarnings("unchecked") 442 public WorkflowsInfo getJobs(String filterStr, int start, int len) throws DagEngineException { 443 Map<String, List<String>> filter = parseFilter(filterStr); 444 try { 445 return new JobsCommand(filter, start, len).call(); 446 } 447 catch (CommandException dce) { 448 throw new DagEngineException(dce); 449 } 450 } 451 452 /** 453 * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes. 454 * 455 * @param externalId external ID provided at job submission time. 456 * @return the associated workflow job ID if any, <code>null</code> if none. 457 * @throws DagEngineException thrown if the lookup could not be done. 458 */ 459 @Override 460 public String getJobIdForExternalId(String externalId) throws DagEngineException { 461 try { 462 return new ExternalIdCommand(externalId).call(); 463 } 464 catch (CommandException dce) { 465 throw new DagEngineException(dce); 466 } 467 } 468 469 @Override 470 public CoordinatorJob getCoordJob(String jobId) throws BaseEngineException { 471 throw new BaseEngineException(new XException(ErrorCode.E0301)); 472 } 473 474 @Override 475 public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BaseEngineException { 476 throw new BaseEngineException(new XException(ErrorCode.E0301)); 477 } 478 479 public WorkflowActionBean getWorkflowAction(String actionId) throws BaseEngineException { 480 try { 481 return new WorkflowActionInfoCommand(actionId).call(); 482 } 483 catch (CommandException ex) { 484 throw new BaseEngineException(ex); 485 } 486 } 487 488 @Override 489 public String dryrunSubmit(Configuration conf, boolean startJob) throws BaseEngineException { 490 return null; 491 } 492 }