001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie; 016 017 import org.apache.oozie.util.XLogStreamer; 018 import org.apache.oozie.service.XLogService; 019 import org.apache.oozie.service.DagXLogInfoService; 020 import org.apache.hadoop.conf.Configuration; 021 import org.apache.oozie.client.CoordinatorJob; 022 import org.apache.oozie.client.WorkflowJob; 023 import org.apache.oozie.client.OozieClient; 024 import org.apache.oozie.command.wf.CompletedActionCommand; 025 import org.apache.oozie.command.CommandException; 026 import org.apache.oozie.command.Command; 027 import org.apache.oozie.command.wf.JobCommand; 028 import org.apache.oozie.command.wf.JobsCommand; 029 import org.apache.oozie.command.wf.KillCommand; 030 import org.apache.oozie.command.wf.ReRunCommand; 031 import org.apache.oozie.command.wf.ResumeCommand; 032 import org.apache.oozie.command.wf.SubmitCommand; 033 import org.apache.oozie.command.wf.SubmitHttpCommand; 034 import org.apache.oozie.command.wf.SubmitPigCommand; 035 import org.apache.oozie.command.wf.SubmitMRCommand; 036 import org.apache.oozie.command.wf.StartCommand; 037 import org.apache.oozie.command.wf.SuspendCommand; 038 import org.apache.oozie.command.wf.DefinitionCommand; 039 import org.apache.oozie.command.wf.ExternalIdCommand; 040 import org.apache.oozie.command.wf.WorkflowActionInfoCommand; 041 import org.apache.oozie.service.Services; 042 import org.apache.oozie.service.CallableQueueService; 043 import org.apache.oozie.util.ParamChecker; 044 import org.apache.oozie.util.XConfiguration; 045 import org.apache.oozie.util.XLog; 046 047 import java.io.Writer; 048 import java.util.List; 049 import java.util.Properties; 050 import java.util.Set; 051 import java.util.HashSet; 052 import java.util.StringTokenizer; 053 import java.util.Map; 054 import java.util.HashMap; 055 import java.util.ArrayList; 056 import java.io.IOException; 057 058 /** 059 * The DagEngine bean provides all the DAG engine functionality for WS calls. 060 */ 061 public class DagEngine extends BaseEngine { 062 063 private static final int HIGH_PRIORITY = 2; 064 065 /** 066 * Create a system Dag engine, with no user and no group. 067 */ 068 public DagEngine() { 069 } 070 071 /** 072 * Create a Dag engine to perform operations on behave of a user. 073 * 074 * @param user user name. 075 * @param authToken the authentication token. 076 */ 077 public DagEngine(String user, String authToken) { 078 this.user = ParamChecker.notEmpty(user, "user"); 079 this.authToken = ParamChecker.notEmpty(authToken, "authToken"); 080 } 081 082 /** 083 * Submit a workflow job. <p/> It validates configuration properties. 084 * 085 * @param conf job configuration. 086 * @param startJob indicates if the job should be started or not. 087 * @return the job Id. 088 * @throws DagEngineException thrown if the job could not be created. 089 */ 090 @Override 091 public String submitJob(Configuration conf, boolean startJob) throws DagEngineException { 092 validateSubmitConfiguration(conf); 093 SubmitCommand submit = new SubmitCommand(conf, getAuthToken()); 094 try { 095 String jobId = submit.call(); 096 if (startJob) { 097 start(jobId); 098 } 099 return jobId; 100 } 101 catch (CommandException ex) { 102 throw new DagEngineException(ex); 103 } 104 } 105 106 /** 107 * Submit a pig/mapreduce job through HTTP. 108 * <p/> 109 * It validates configuration properties. 110 * 111 * @param conf job configuration. 112 * @param jobType job type - can be "pig" or "mapreduce". 113 * @return the job Id. 114 * @throws DagEngineException thrown if the job could not be created. 115 */ 116 public String submitHttpJob(Configuration conf, String jobType) throws DagEngineException { 117 validateSubmitConfiguration(conf); 118 119 SubmitHttpCommand submit = null; 120 if (jobType.equals("pig")) { 121 submit = new SubmitPigCommand(conf, getAuthToken()); 122 } 123 else if (jobType.equals("mapreduce")) { 124 submit = new SubmitMRCommand(conf, getAuthToken()); 125 } 126 127 try { 128 String jobId = submit.call(); 129 start(jobId); 130 return jobId; 131 } 132 catch (CommandException ex) { 133 throw new DagEngineException(ex); 134 } 135 } 136 137 public static void main(String[] args) throws Exception { 138 // Configuration conf = new XConfiguration(IOUtils.getResourceAsReader( 139 // "org/apache/oozie/coord/conf.xml", -1)); 140 141 Configuration conf = new XConfiguration(); 142 143 // String appXml = 144 // IOUtils.getResourceAsString("org/apache/oozie/coord/test1.xml", -1); 145 conf.set(OozieClient.APP_PATH, "file:///Users/danielwo/oozie/workflows/examples/seed/workflows/map-reduce"); 146 conf.set(OozieClient.USER_NAME, "danielwo"); 147 conf.set(OozieClient.GROUP_NAME, "other"); 148 149 conf.set("inputDir", " blah "); 150 151 // System.out.println("appXml :"+ appXml + "\n conf :"+ conf); 152 new Services().init(); 153 try { 154 DagEngine de = new DagEngine("me", "TESTING_WF"); 155 String jobId = de.submitJob(conf, true); 156 System.out.println("WF Job Id " + jobId); 157 158 Thread.sleep(20000); 159 } 160 finally { 161 Services.get().destroy(); 162 } 163 } 164 165 private void validateSubmitConfiguration(Configuration conf) throws DagEngineException { 166 if (conf.get(OozieClient.APP_PATH) == null) { 167 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH); 168 } 169 } 170 171 /** 172 * Start a job. 173 * 174 * @param jobId job Id. 175 * @throws DagEngineException thrown if the job could not be started. 176 */ 177 @Override 178 public void start(String jobId) throws DagEngineException { 179 // Changing to synchronous call from asynchronous queuing to prevent the 180 // loss of command if the queue is full or the queue is lost in case of 181 // failure. 182 try { 183 new StartCommand(jobId).call(); 184 } 185 catch (CommandException e) { 186 throw new DagEngineException(e); 187 } 188 } 189 190 /** 191 * Resume a job. 192 * 193 * @param jobId job Id. 194 * @throws DagEngineException thrown if the job could not be resumed. 195 */ 196 @Override 197 public void resume(String jobId) throws DagEngineException { 198 // Changing to synchronous call from asynchronous queuing to prevent the 199 // loss of command if the queue is full or the queue is lost in case of 200 // failure. 201 try { 202 new ResumeCommand(jobId).call(); 203 } 204 catch (CommandException e) { 205 throw new DagEngineException(e); 206 } 207 } 208 209 /** 210 * Suspend a job. 211 * 212 * @param jobId job Id. 213 * @throws DagEngineException thrown if the job could not be suspended. 214 */ 215 @Override 216 public void suspend(String jobId) throws DagEngineException { 217 // Changing to synchronous call from asynchronous queuing to prevent the 218 // loss of command if the queue is full or the queue is lost in case of 219 // failure. 220 try { 221 new SuspendCommand(jobId).call(); 222 } 223 catch (CommandException e) { 224 throw new DagEngineException(e); 225 } 226 } 227 228 /** 229 * Kill a job. 230 * 231 * @param jobId job Id. 232 * @throws DagEngineException thrown if the job could not be killed. 233 */ 234 @Override 235 public void kill(String jobId) throws DagEngineException { 236 // Changing to synchronous call from asynchronous queuing to prevent the 237 // loss of command if the queue is full or the queue is lost in case of 238 // failure. 239 try { 240 new KillCommand(jobId).call(); 241 XLog.getLog(getClass()).info("User " + user + " killed the WF job " + jobId); 242 } 243 catch (CommandException e) { 244 throw new DagEngineException(e); 245 } 246 } 247 248 /* (non-Javadoc) 249 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String) 250 */ 251 @Override 252 public void change(String jobId, String changeValue) throws DagEngineException { 253 // This code should not be reached. 254 throw new DagEngineException(ErrorCode.E1017); 255 } 256 257 /** 258 * Rerun a job. 259 * 260 * @param jobId job Id to rerun. 261 * @param conf configuration information for the rerun. 262 * @throws DagEngineException thrown if the job could not be rerun. 263 */ 264 @Override 265 public void reRun(String jobId, Configuration conf) throws DagEngineException { 266 try { 267 validateReRunConfiguration(conf); 268 new ReRunCommand(jobId, conf, getAuthToken()).call(); 269 start(jobId); 270 } 271 catch (CommandException ex) { 272 throw new DagEngineException(ex); 273 } 274 } 275 276 private void validateReRunConfiguration(Configuration conf) throws DagEngineException { 277 if (conf.get(OozieClient.APP_PATH) == null) { 278 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH); 279 } 280 if (conf.get(OozieClient.RERUN_SKIP_NODES) == null) { 281 throw new DagEngineException(ErrorCode.E0401, OozieClient.RERUN_SKIP_NODES); 282 } 283 } 284 285 /** 286 * Process an action callback. 287 * 288 * @param actionId the action Id. 289 * @param externalStatus the action external status. 290 * @param actionData the action output data, <code>null</code> if none. 291 * @throws DagEngineException thrown if the callback could not be processed. 292 */ 293 public void processCallback(String actionId, String externalStatus, Properties actionData) 294 throws DagEngineException { 295 XLog.Info.get().clearParameter(XLogService.GROUP); 296 XLog.Info.get().clearParameter(XLogService.USER); 297 Command<Void, ?> command = new CompletedActionCommand(actionId, externalStatus, actionData, HIGH_PRIORITY); 298 if (!Services.get().get(CallableQueueService.class).queue(command)) { 299 XLog.getLog(this.getClass()).warn(XLog.OPS, "queue is full or system is in SAFEMODE, ignoring callback"); 300 } 301 } 302 303 /** 304 * Return the info about a job. 305 * 306 * @param jobId job Id. 307 * @return the workflow job info. 308 * @throws DagEngineException thrown if the job info could not be obtained. 309 */ 310 @Override 311 public WorkflowJob getJob(String jobId) throws DagEngineException { 312 try { 313 return new JobCommand(jobId).call(); 314 } 315 catch (CommandException ex) { 316 throw new DagEngineException(ex); 317 } 318 } 319 320 /** 321 * Return the info about a job with actions subset. 322 * 323 * @param jobId job Id 324 * @param start starting from this index in the list of actions belonging to the job 325 * @param length number of actions to be returned 326 * @return the workflow job info. 327 * @throws DagEngineException thrown if the job info could not be obtained. 328 */ 329 @Override 330 public WorkflowJob getJob(String jobId, int start, int length) throws DagEngineException { 331 try { 332 return new JobCommand(jobId, start, length).call(); 333 } 334 catch (CommandException ex) { 335 throw new DagEngineException(ex); 336 } 337 } 338 339 /** 340 * Return the a job definition. 341 * 342 * @param jobId job Id. 343 * @return the job definition. 344 * @throws DagEngineException thrown if the job definition could no be obtained. 345 */ 346 @Override 347 public String getDefinition(String jobId) throws DagEngineException { 348 try { 349 return new DefinitionCommand(jobId).call(); 350 } 351 catch (CommandException ex) { 352 throw new DagEngineException(ex); 353 } 354 } 355 356 /** 357 * Stream the log of a job. 358 * 359 * @param jobId job Id. 360 * @param writer writer to stream the log to. 361 * @throws IOException thrown if the log cannot be streamed. 362 * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId. 363 */ 364 @Override 365 public void streamLog(String jobId, Writer writer) throws IOException, DagEngineException { 366 XLogStreamer.Filter filter = new XLogStreamer.Filter(); 367 filter.setParameter(DagXLogInfoService.JOB, jobId); 368 WorkflowJob job = getJob(jobId); 369 Services.get().get(XLogService.class).streamLog(filter, job.getStartTime(), job.getEndTime(), writer); 370 } 371 372 private static final Set<String> FILTER_NAMES = new HashSet<String>(); 373 374 static { 375 FILTER_NAMES.add(OozieClient.FILTER_USER); 376 FILTER_NAMES.add(OozieClient.FILTER_NAME); 377 FILTER_NAMES.add(OozieClient.FILTER_GROUP); 378 FILTER_NAMES.add(OozieClient.FILTER_STATUS); 379 } 380 381 /** 382 * Validate a jobs filter. 383 * 384 * @param filter filter to validate. 385 * @return the parsed filter. 386 * @throws DagEngineException thrown if the filter is invalid. 387 */ 388 protected Map<String, List<String>> parseFilter(String filter) throws DagEngineException { 389 Map<String, List<String>> map = new HashMap<String, List<String>>(); 390 if (filter != null) { 391 StringTokenizer st = new StringTokenizer(filter, ";"); 392 while (st.hasMoreTokens()) { 393 String token = st.nextToken(); 394 if (token.contains("=")) { 395 String[] pair = token.split("="); 396 if (pair.length != 2) { 397 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 398 } 399 if (!FILTER_NAMES.contains(pair[0])) { 400 throw new DagEngineException(ErrorCode.E0420, filter, XLog 401 .format("invalid name [{0}]", pair[0])); 402 } 403 if (pair[0].equals("status")) { 404 try { 405 WorkflowJob.Status.valueOf(pair[1]); 406 } 407 catch (IllegalArgumentException ex) { 408 throw new DagEngineException(ErrorCode.E0420, filter, XLog.format("invalid status [{0}]", 409 pair[1])); 410 } 411 } 412 List<String> list = map.get(pair[0]); 413 if (list == null) { 414 list = new ArrayList<String>(); 415 map.put(pair[0], list); 416 } 417 list.add(pair[1]); 418 } 419 else { 420 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 421 } 422 } 423 } 424 return map; 425 } 426 427 /** 428 * Return the info about a set of jobs. 429 * 430 * @param filterStr job filter. Refer to the {@link org.apache.oozie.client.OozieClient} for the filter syntax. 431 * @param start offset, base 1. 432 * @param len number of jobs to return. 433 * @return job info for all matching jobs, the jobs don't contain node action information. 434 * @throws DagEngineException thrown if the jobs info could not be obtained. 435 */ 436 @SuppressWarnings("unchecked") 437 public WorkflowsInfo getJobs(String filterStr, int start, int len) throws DagEngineException { 438 Map<String, List<String>> filter = parseFilter(filterStr); 439 try { 440 return new JobsCommand(filter, start, len).call(); 441 } 442 catch (CommandException dce) { 443 throw new DagEngineException(dce); 444 } 445 } 446 447 /** 448 * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes. 449 * 450 * @param externalId external ID provided at job submission time. 451 * @return the associated workflow job ID if any, <code>null</code> if none. 452 * @throws DagEngineException thrown if the lookup could not be done. 453 */ 454 @Override 455 public String getJobIdForExternalId(String externalId) throws DagEngineException { 456 try { 457 return new ExternalIdCommand(externalId).call(); 458 } 459 catch (CommandException dce) { 460 throw new DagEngineException(dce); 461 } 462 } 463 464 @Override 465 public CoordinatorJob getCoordJob(String jobId) throws BaseEngineException { 466 throw new BaseEngineException(new XException(ErrorCode.E0301)); 467 } 468 469 @Override 470 public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BaseEngineException { 471 throw new BaseEngineException(new XException(ErrorCode.E0301)); 472 } 473 474 public WorkflowActionBean getWorkflowAction(String actionId) throws BaseEngineException { 475 try { 476 return new WorkflowActionInfoCommand(actionId).call(); 477 } 478 catch (CommandException ex) { 479 throw new BaseEngineException(ex); 480 } 481 } 482 483 @Override 484 public String dryrunSubmit(Configuration conf, boolean startJob) throws BaseEngineException { 485 return null; 486 } 487 }