001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.coord; 016 017 import java.io.StringReader; 018 import java.util.Date; 019 import java.util.List; 020 021 import org.apache.hadoop.conf.Configuration; 022 import org.apache.oozie.CoordinatorActionBean; 023 import org.apache.oozie.ErrorCode; 024 import org.apache.oozie.client.CoordinatorAction; 025 import org.apache.oozie.command.CommandException; 026 import org.apache.oozie.coord.CoordELEvaluator; 027 import org.apache.oozie.coord.CoordELFunctions; 028 import org.apache.oozie.coord.CoordUtils; 029 import org.apache.oozie.coord.CoordinatorJobException; 030 import org.apache.oozie.coord.SyncCoordAction; 031 import org.apache.oozie.coord.TimeUnit; 032 import org.apache.oozie.service.Services; 033 import org.apache.oozie.service.UUIDService; 034 import org.apache.oozie.util.DateUtils; 035 import org.apache.oozie.util.ELEvaluator; 036 import org.apache.oozie.util.XConfiguration; 037 import org.apache.oozie.util.XmlUtils; 038 import org.jdom.Element; 039 040 public class CoordCommandUtils { 041 public static int CURRENT = 0; 042 public static int LATEST = 1; 043 public static int FUTURE = 2; 044 public static int UNEXPECTED = -1; 045 public static final String RESOLVED_UNRESOLVED_SEPARATOR = ";"; 046 047 /** 048 * parse a function like coord:latest(n)/future() and return the 'n'. 049 * <p/> 050 * @param function 051 * @param event 052 * @param appInst 053 * @param conf 054 * @param restArg 055 * @return int instanceNumber 056 * @throws Exception 057 */ 058 public static int getInstanceNumber(String function, Element event, SyncCoordAction appInst, Configuration conf, 059 StringBuilder restArg) throws Exception { 060 ELEvaluator eval = CoordELEvaluator 061 .createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf); 062 String newFunc = CoordELFunctions.evalAndWrap(eval, function); 063 int funcType = getFuncType(newFunc); 064 if (funcType == CURRENT || funcType == LATEST) { 065 return parseOneArg(newFunc); 066 } 067 else { 068 return parseMoreArgs(newFunc, restArg); 069 } 070 } 071 072 private static int parseOneArg(String funcName) throws Exception { 073 int firstPos = funcName.indexOf("("); 074 int lastPos = funcName.lastIndexOf(")"); 075 if (firstPos >= 0 && lastPos > firstPos) { 076 String tmp = funcName.substring(firstPos + 1, lastPos).trim(); 077 if (tmp.length() > 0) { 078 return Integer.parseInt(tmp); 079 } 080 } 081 throw new RuntimeException("Unformatted function :" + funcName); 082 } 083 084 private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception { 085 int firstPos = funcName.indexOf("("); 086 int secondPos = funcName.lastIndexOf(","); 087 int lastPos = funcName.lastIndexOf(")"); 088 if (firstPos >= 0 && secondPos > firstPos) { 089 String tmp = funcName.substring(firstPos + 1, secondPos).trim(); 090 if (tmp.length() > 0) { 091 restArg.append(funcName.substring(secondPos + 1, lastPos).trim()); 092 return Integer.parseInt(tmp); 093 } 094 } 095 throw new RuntimeException("Unformatted function :" + funcName); 096 } 097 098 /** 099 * @param EL function name 100 * @return type of EL function 101 */ 102 public static int getFuncType(String function) { 103 if (function.indexOf("current") >= 0) { 104 return CURRENT; 105 } 106 else if (function.indexOf("latest") >= 0) { 107 return LATEST; 108 } 109 else if (function.indexOf("future") >= 0) { 110 return FUTURE; 111 } 112 return UNEXPECTED; 113 // throw new RuntimeException("Unexpected instance name "+ function); 114 } 115 116 /** 117 * @param startInst: EL function name 118 * @param endInst: EL function name 119 * @throws CommandException if both are not the same function 120 */ 121 public static void checkIfBothSameType(String startInst, String endInst) throws CommandException { 122 if (getFuncType(startInst) != getFuncType(endInst)) { 123 throw new CommandException(ErrorCode.E1010, 124 " start-instance and end-instance both should be either latest or current or future\n" 125 + " start " + startInst + " and end " + endInst); 126 } 127 } 128 129 /** 130 * Resolve list of <instance> </instance> tags. 131 * 132 * @param event 133 * @param instances 134 * @param actionInst 135 * @param conf 136 * @param eval: ELEvalautor 137 * @throws Exception 138 */ 139 public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst, 140 Configuration conf, ELEvaluator eval) throws Exception { 141 for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) { 142 if (instances.length() > 0) { 143 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 144 } 145 instances.append(materializeInstance(event, eInstance.getTextTrim(), actionInst, conf, eval)); 146 } 147 event.removeChildren("instance", event.getNamespace()); 148 } 149 150 /** 151 * Resolve <start-instance> <end-insatnce> tag. Don't resolve any 152 * latest()/future() 153 * 154 * @param event 155 * @param instances 156 * @param appInst 157 * @param conf 158 * @param eval: ELEvalautor 159 * @throws Exception 160 */ 161 public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst, 162 Configuration conf, ELEvaluator eval) throws Exception { 163 Element eStartInst = event.getChild("start-instance", event.getNamespace()); 164 Element eEndInst = event.getChild("end-instance", event.getNamespace()); 165 if (eStartInst != null && eEndInst != null) { 166 String strStart = eStartInst.getTextTrim(); 167 String strEnd = eEndInst.getTextTrim(); 168 checkIfBothSameType(strStart, strEnd); 169 StringBuilder restArg = new StringBuilder(); // To store rest 170 // arguments for 171 // future 172 // function 173 int startIndex = getInstanceNumber(strStart, event, appInst, conf, restArg); 174 restArg.delete(0, restArg.length()); 175 int endIndex = getInstanceNumber(strEnd, event, appInst, conf, restArg); 176 if (startIndex > endIndex) { 177 throw new CommandException(ErrorCode.E1010, 178 " start-instance should be equal or earlier than the end-instance \n" 179 + XmlUtils.prettyPrint(event)); 180 } 181 int funcType = getFuncType(strStart); 182 if (funcType == CURRENT) { 183 // Everything could be resolved NOW. no latest() ELs 184 for (int i = endIndex; i >= startIndex; i--) { 185 String matInstance = materializeInstance(event, "${coord:current(" + i + ")}", appInst, conf, eval); 186 if (matInstance == null || matInstance.length() == 0) { 187 // Earlier than dataset's initial instance 188 break; 189 } 190 if (instances.length() > 0) { 191 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 192 } 193 instances.append(matInstance); 194 } 195 } 196 else { // latest(n)/future() EL is present 197 for (; startIndex <= endIndex; startIndex++) { 198 if (instances.length() > 0) { 199 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 200 } 201 if (funcType == LATEST) { 202 instances.append("${coord:latest(" + startIndex + ")}"); 203 } 204 else { // For future 205 instances.append("${coord:future(" + startIndex + ",'" + restArg + "')}"); 206 } 207 } 208 } 209 // Remove start-instance and end-instances 210 event.removeChild("start-instance", event.getNamespace()); 211 event.removeChild("end-instance", event.getNamespace()); 212 } 213 } 214 215 /** 216 * Materialize one instance like current(-2) 217 * 218 * @param event : <data-in> 219 * @param expr : instance like current(-1) 220 * @param appInst : application specific info 221 * @param conf 222 * @param evalInst :ELEvaluator 223 * @return materialized date string 224 * @throws Exception 225 */ 226 public static String materializeInstance(Element event, String expr, SyncCoordAction appInst, Configuration conf, 227 ELEvaluator evalInst) throws Exception { 228 if (event == null) { 229 return null; 230 } 231 // ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, 232 // appInst, conf); 233 return CoordELFunctions.evalAndWrap(evalInst, expr); 234 } 235 236 /** 237 * Create two new tags with <uris> and <unresolved-instances>. 238 * 239 * @param event 240 * @param instances 241 * @param dependencyList 242 * @throws Exception 243 */ 244 public static void separateResolvedAndUnresolved(Element event, StringBuilder instances, StringBuffer dependencyList) 245 throws Exception { 246 StringBuilder unresolvedInstances = new StringBuilder(); 247 StringBuilder urisWithDoneFlag = new StringBuilder(); 248 String uris = createEarlyURIs(event, instances.toString(), unresolvedInstances, urisWithDoneFlag); 249 if (uris.length() > 0) { 250 Element uriInstance = new Element("uris", event.getNamespace()); 251 uriInstance.addContent(uris); 252 event.getContent().add(1, uriInstance); 253 if (dependencyList.length() > 0) { 254 dependencyList.append(CoordELFunctions.INSTANCE_SEPARATOR); 255 } 256 dependencyList.append(urisWithDoneFlag); 257 } 258 if (unresolvedInstances.length() > 0) { 259 Element elemInstance = new Element("unresolved-instances", event.getNamespace()); 260 elemInstance.addContent(unresolvedInstances.toString()); 261 event.getContent().add(1, elemInstance); 262 } 263 } 264 265 /** 266 * The function create a list of URIs separated by "," using the instances 267 * time stamp and URI-template 268 * 269 * @param event : <data-in> event 270 * @param instances : List of time stamp separated by "," 271 * @param unresolvedInstances : list of instance with latest function 272 * @param urisWithDoneFlag : list of URIs with the done flag appended 273 * @return : list of URIs separated by ";" as a string. 274 * @throws Exception 275 */ 276 public static String createEarlyURIs(Element event, String instances, StringBuilder unresolvedInstances, 277 StringBuilder urisWithDoneFlag) throws Exception { 278 if (instances == null || instances.length() == 0) { 279 return ""; 280 } 281 String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR); 282 StringBuilder uris = new StringBuilder(); 283 284 Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag", 285 event.getNamespace()); 286 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement); 287 288 for (int i = 0; i < instanceList.length; i++) { 289 if(instanceList[i].trim().length() == 0) { 290 continue; 291 } 292 int funcType = getFuncType(instanceList[i]); 293 if (funcType == LATEST || funcType == FUTURE) { 294 if (unresolvedInstances.length() > 0) { 295 unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR); 296 } 297 unresolvedInstances.append(instanceList[i]); 298 continue; 299 } 300 ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]); 301 if (uris.length() > 0) { 302 uris.append(CoordELFunctions.INSTANCE_SEPARATOR); 303 urisWithDoneFlag.append(CoordELFunctions.INSTANCE_SEPARATOR); 304 } 305 306 String uriPath = CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace()) 307 .getChild("uri-template", event.getNamespace()).getTextTrim()); 308 uris.append(uriPath); 309 if (doneFlag.length() > 0) { 310 uriPath += "/" + doneFlag; 311 } 312 urisWithDoneFlag.append(uriPath); 313 } 314 return uris.toString(); 315 } 316 317 /** 318 * @param eSla 319 * @param nominalTime 320 * @param conf 321 * @return boolean to determine whether the SLA element is present or not 322 * @throws CoordinatorJobException 323 */ 324 public static boolean materializeSLA(Element eSla, Date nominalTime, Configuration conf) 325 throws CoordinatorJobException { 326 if (eSla == null) { 327 // eAppXml.getNamespace("sla")); 328 return false; 329 } 330 try { 331 ELEvaluator evalSla = CoordELEvaluator.createSLAEvaluator(nominalTime, conf); 332 List<Element> elemList = eSla.getChildren(); 333 for (Element elem : elemList) { 334 String updated; 335 try { 336 updated = CoordELFunctions.evalAndWrap(evalSla, elem.getText().trim()); 337 } 338 catch (Exception e) { 339 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 340 } 341 elem.removeContent(); 342 elem.addContent(updated); 343 } 344 } 345 catch (Exception e) { 346 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 347 } 348 return true; 349 } 350 351 /** 352 * Materialize one instance for specific nominal time. It includes: 1. 353 * Materialize data events (i.e. <data-in> and <data-out>) 2. Materialize 354 * data properties (i.e dataIn(<DS>) and dataOut(<DS>) 3. remove 'start' and 355 * 'end' tag 4. Add 'instance_number' and 'nominal-time' tag 356 * 357 * @param jobId coordinator job id 358 * @param dryrun true if it is dryrun 359 * @param eAction frequency unexploded-job 360 * @param nominalTime materialization time 361 * @param instanceCount instance numbers 362 * @param conf job configuration 363 * @param actionBean CoordinatorActionBean to materialize 364 * @return one materialized action for specific nominal time 365 * @throws Exception 366 */ 367 @SuppressWarnings("unchecked") 368 public static String materializeOneInstance(String jobId, boolean dryrun, Element eAction, Date nominalTime, 369 int instanceCount, Configuration conf, CoordinatorActionBean actionBean) throws Exception { 370 String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, instanceCount + ""); 371 SyncCoordAction appInst = new SyncCoordAction(); 372 appInst.setActionId(actionId); 373 appInst.setName(eAction.getAttributeValue("name")); 374 appInst.setNominalTime(nominalTime); 375 int frequency = Integer.parseInt(eAction.getAttributeValue("frequency")); 376 appInst.setFrequency(frequency); 377 appInst.setTimeUnit(TimeUnit.valueOf(eAction.getAttributeValue("freq_timeunit"))); 378 appInst.setTimeZone(DateUtils.getTimeZone(eAction.getAttributeValue("timezone"))); 379 appInst.setEndOfDuration(TimeUnit.valueOf(eAction.getAttributeValue("end_of_duration"))); 380 381 StringBuffer dependencyList = new StringBuffer(); 382 383 Element inputList = eAction.getChild("input-events", eAction.getNamespace()); 384 List<Element> dataInList = null; 385 if (inputList != null) { 386 dataInList = inputList.getChildren("data-in", eAction.getNamespace()); 387 materializeDataEvents(dataInList, appInst, conf, dependencyList); 388 } 389 390 Element outputList = eAction.getChild("output-events", eAction.getNamespace()); 391 List<Element> dataOutList = null; 392 if (outputList != null) { 393 dataOutList = outputList.getChildren("data-out", eAction.getNamespace()); 394 StringBuffer tmp = new StringBuffer(); 395 // no dependency checks 396 materializeDataEvents(dataOutList, appInst, conf, tmp); 397 } 398 399 eAction.removeAttribute("start"); 400 eAction.removeAttribute("end"); 401 eAction.setAttribute("instance-number", Integer.toString(instanceCount)); 402 eAction.setAttribute("action-nominal-time", DateUtils.formatDateUTC(nominalTime)); 403 404 boolean isSla = CoordCommandUtils.materializeSLA(eAction.getChild("action", eAction.getNamespace()).getChild( 405 "info", eAction.getNamespace("sla")), nominalTime, conf); 406 407 // Setting up action bean 408 actionBean.setCreatedConf(XmlUtils.prettyPrint(conf).toString()); 409 actionBean.setRunConf(XmlUtils.prettyPrint(conf).toString()); 410 actionBean.setCreatedTime(new Date()); 411 actionBean.setJobId(jobId); 412 actionBean.setId(actionId); 413 actionBean.setLastModifiedTime(new Date()); 414 actionBean.setStatus(CoordinatorAction.Status.WAITING); 415 actionBean.setActionNumber(instanceCount); 416 actionBean.setMissingDependencies(dependencyList.toString()); 417 actionBean.setNominalTime(nominalTime); 418 if (isSla == true) { 419 actionBean.setSlaXml(XmlUtils.prettyPrint( 420 eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"))) 421 .toString()); 422 } 423 424 // actionBean.setTrackerUri(trackerUri);//TOOD: 425 // actionBean.setConsoleUrl(consoleUrl); //TODO: 426 // actionBean.setType(type);//TODO: 427 // actionBean.setErrorInfo(errorCode, errorMessage); //TODO: 428 // actionBean.setExternalStatus(externalStatus);//TODO 429 if (!dryrun) { 430 return XmlUtils.prettyPrint(eAction).toString(); 431 } 432 else { 433 String action = XmlUtils.prettyPrint(eAction).toString(); 434 CoordActionInputCheckCommand coordActionInput = new CoordActionInputCheckCommand(actionBean.getId()); 435 StringBuilder actionXml = new StringBuilder(action); 436 StringBuilder existList = new StringBuilder(); 437 StringBuilder nonExistList = new StringBuilder(); 438 StringBuilder nonResolvedList = new StringBuilder(); 439 getResolvedList(actionBean.getMissingDependencies(), nonExistList, nonResolvedList); 440 Date actualTime = new Date(); 441 Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf())); 442 coordActionInput.checkInput(actionXml, existList, nonExistList, actionConf, actualTime); 443 return actionXml.toString(); 444 } 445 } 446 447 /** 448 * Materialize all <input-events>/<data-in> or <output-events>/<data-out> 449 * tags Create uris for resolved instances. Create unresolved instance for 450 * latest()/future(). 451 * 452 * @param events 453 * @param appInst 454 * @param conf 455 * @throws Exception 456 */ 457 public static void materializeDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf, 458 StringBuffer dependencyList) throws Exception { 459 460 if (events == null) { 461 return; 462 } 463 StringBuffer unresolvedList = new StringBuffer(); 464 for (Element event : events) { 465 StringBuilder instances = new StringBuilder(); 466 ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf); 467 // Handle list of instance tag 468 resolveInstances(event, instances, appInst, conf, eval); 469 // Handle start-instance and end-instance 470 resolveInstanceRange(event, instances, appInst, conf, eval); 471 // Separate out the unresolved instances 472 separateResolvedAndUnresolved(event, instances, dependencyList); 473 String tmpUnresolved = event.getChildTextTrim("unresolved-instances", event.getNamespace()); 474 if (tmpUnresolved != null) { 475 if (unresolvedList.length() > 0) { 476 unresolvedList.append(CoordELFunctions.INSTANCE_SEPARATOR); 477 } 478 unresolvedList.append(tmpUnresolved); 479 } 480 } 481 if (unresolvedList.length() > 0) { 482 dependencyList.append(RESOLVED_UNRESOLVED_SEPARATOR); 483 dependencyList.append(unresolvedList); 484 } 485 return; 486 } 487 488 /** 489 * Get resolved string from missDepList 490 * 491 * @param missDepList 492 * @param resolved 493 * @param unresolved 494 * @return resolved string 495 */ 496 public static String getResolvedList(String missDepList, StringBuilder resolved, StringBuilder unresolved) { 497 if (missDepList != null) { 498 int index = missDepList.indexOf(RESOLVED_UNRESOLVED_SEPARATOR); 499 if (index < 0) { 500 resolved.append(missDepList); 501 } 502 else { 503 resolved.append(missDepList.substring(0, index)); 504 unresolved.append(missDepList.substring(index + 1)); 505 } 506 } 507 return resolved.toString(); 508 } 509 510 }