001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.coord; 016 017 import java.io.IOException; 018 import java.util.Calendar; 019 import java.util.Date; 020 import java.util.TimeZone; 021 022 import org.apache.hadoop.conf.Configuration; 023 import org.apache.hadoop.fs.Path; 024 025 import org.apache.oozie.client.OozieClient; 026 import org.apache.oozie.util.DateUtils; 027 import org.apache.oozie.util.ELEvaluator; 028 import org.apache.oozie.util.ParamChecker; 029 import org.apache.oozie.util.XLog; 030 import org.apache.oozie.service.HadoopAccessorException; 031 import org.apache.oozie.service.Services; 032 import org.apache.oozie.service.HadoopAccessorService; 033 034 /** 035 * This class implements the EL function related to coordinator 036 */ 037 038 public class CoordELFunctions { 039 final private static String DATASET = "oozie.coord.el.dataset.bean"; 040 final private static String COORD_ACTION = "oozie.coord.el.app.bean"; 041 final public static String CONFIGURATION = "oozie.coord.el.conf"; 042 // INSTANCE_SEPARATOR is used to separate multiple directories into one tag. 043 final public static String INSTANCE_SEPARATOR = "#"; 044 final public static String DIR_SEPARATOR = ","; 045 // TODO: in next release, support flexibility 046 private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS"; 047 048 /** 049 * Used in defining the frequency in 'day' unit. <p/> domain: <code> val > 0</code> and should be integer. 050 * 051 * @param val frequency in number of days. 052 * @return number of days and also set the frequency timeunit to "day" 053 */ 054 public static int ph1_coord_days(int val) { 055 val = ParamChecker.checkGTZero(val, "n"); 056 ELEvaluator eval = ELEvaluator.getCurrent(); 057 eval.setVariable("timeunit", TimeUnit.DAY); 058 eval.setVariable("endOfDuration", TimeUnit.NONE); 059 return val; 060 } 061 062 /** 063 * Used in defining the frequency in 'month' unit. <p/> domain: <code> val > 0</code> and should be integer. 064 * 065 * @param val frequency in number of months. 066 * @return number of months and also set the frequency timeunit to "month" 067 */ 068 public static int ph1_coord_months(int val) { 069 val = ParamChecker.checkGTZero(val, "n"); 070 ELEvaluator eval = ELEvaluator.getCurrent(); 071 eval.setVariable("timeunit", TimeUnit.MONTH); 072 eval.setVariable("endOfDuration", TimeUnit.NONE); 073 return val; 074 } 075 076 /** 077 * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val > 0</code> and should 078 * be integer. 079 * 080 * @param val frequency in number of hours. 081 * @return number of minutes and also set the frequency timeunit to "minute" 082 */ 083 public static int ph1_coord_hours(int val) { 084 val = ParamChecker.checkGTZero(val, "n"); 085 ELEvaluator eval = ELEvaluator.getCurrent(); 086 eval.setVariable("timeunit", TimeUnit.MINUTE); 087 eval.setVariable("endOfDuration", TimeUnit.NONE); 088 return val * 60; 089 } 090 091 /** 092 * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val > 0</code> and should be integer. 093 * 094 * @param val frequency in number of minutes. 095 * @return number of minutes and also set the frequency timeunit to "minute" 096 */ 097 public static int ph1_coord_minutes(int val) { 098 val = ParamChecker.checkGTZero(val, "n"); 099 ELEvaluator eval = ELEvaluator.getCurrent(); 100 eval.setVariable("timeunit", TimeUnit.MINUTE); 101 eval.setVariable("endOfDuration", TimeUnit.NONE); 102 return val; 103 } 104 105 /** 106 * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will 107 * start at 00:00 hour of each day. <p/> domain: <code> val > 0</code> and should be integer. 108 * 109 * @param val frequency in number of days. 110 * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day" 111 */ 112 public static int ph1_coord_endOfDays(int val) { 113 val = ParamChecker.checkGTZero(val, "n"); 114 ELEvaluator eval = ELEvaluator.getCurrent(); 115 eval.setVariable("timeunit", TimeUnit.DAY); 116 eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY); 117 return val; 118 } 119 120 /** 121 * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will 122 * start at first day of each month at 00:00 hour. <p/> domain: <code> val > 0</code> and should be integer. 123 * 124 * @param val: frequency in number of months. 125 * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month" 126 */ 127 public static int ph1_coord_endOfMonths(int val) { 128 val = ParamChecker.checkGTZero(val, "n"); 129 ELEvaluator eval = ELEvaluator.getCurrent(); 130 eval.setVariable("timeunit", TimeUnit.MONTH); 131 eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH); 132 return val; 133 } 134 135 /** 136 * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/> 137 * 1. Timezone of both dataset and job <p/> 2. Action creation Time 138 * 139 * @return difference in minutes (DataSet TZ Offset - Application TZ offset) 140 */ 141 public static int ph2_coord_tzOffset() { 142 Date actionCreationTime = getActionCreationtime(); 143 TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ"); 144 TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ"); 145 // Apply the TZ into Calendar object 146 Calendar dsTime = Calendar.getInstance(dsTZ); 147 dsTime.setTime(actionCreationTime); 148 Calendar jobTime = Calendar.getInstance(jobTZ); 149 jobTime.setTime(actionCreationTime); 150 return (dsTime.get(Calendar.ZONE_OFFSET) - jobTime.get(Calendar.ZONE_OFFSET)) / (1000 * 60); 151 } 152 153 public static int ph3_coord_tzOffset() { 154 return ph2_coord_tzOffset(); 155 } 156 157 /** 158 * Returns the a date string while given a base date in 'strBaseDate', 159 * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH). 160 * 161 * @param strBaseDate -- base date 162 * @param offset -- any number 163 * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH 164 * @return date string 165 * @throws Exception 166 */ 167 public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { 168 Calendar baseCalDate = DateUtils.getCalendar(strBaseDate); 169 StringBuilder buffer = new StringBuilder(); 170 baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset); 171 buffer.append(DateUtils.formatDateUTC(baseCalDate)); 172 return buffer.toString(); 173 } 174 175 public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { 176 return ph2_coord_dateOffset(strBaseDate, offset, unit); 177 } 178 179 /** 180 * Determine the date-time in UTC of n-th future available dataset instance 181 * from nominal Time but not beyond the instance specified as 'instance. 182 * <p/> 183 * It depends on: 184 * <p/> 185 * 1. Data set frequency 186 * <p/> 187 * 2. Data set Time unit (day, month, minute) 188 * <p/> 189 * 3. Data set Time zone/DST 190 * <p/> 191 * 4. End Day/Month flag 192 * <p/> 193 * 5. Data set initial instance 194 * <p/> 195 * 6. Action Creation Time 196 * <p/> 197 * 7. Existence of dataset's directory 198 * 199 * @param n :instance count 200 * <p/> 201 * domain: n >= 0, n is integer 202 * @param instance: How many future instance it should check? value should 203 * be >=0 204 * @return date-time in UTC of the n-th instance 205 * <p/> 206 * @throws Exception 207 */ 208 public static String ph3_coord_future(int n, int instance) throws Exception { 209 ParamChecker.checkGEZero(n, "future:n"); 210 ParamChecker.checkGTZero(instance, "future:instance"); 211 if (isSyncDataSet()) {// For Sync Dataset 212 return coord_future_sync(n, instance); 213 } 214 else { 215 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 216 } 217 } 218 219 private static String coord_future_sync(int n, int instance) throws Exception { 220 ELEvaluator eval = ELEvaluator.getCurrent(); 221 String retVal = ""; 222 int datasetFrequency = (int) getDSFrequency();// in minutes 223 TimeUnit dsTimeUnit = getDSTimeUnit(); 224 int[] instCount = new int[1]; 225 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount); 226 if (nominalInstanceCal != null) { 227 Calendar initInstance = getInitialInstanceCal(); 228 nominalInstanceCal = (Calendar) initInstance.clone(); 229 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 230 231 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 232 if (ds == null) { 233 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 234 } 235 String uriTemplate = ds.getUriTemplate(); 236 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION); 237 if (conf == null) { 238 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION); 239 } 240 int available = 0, checkedInstance = 0; 241 boolean resolved = false; 242 String user = ParamChecker 243 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME); 244 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME), 245 OozieClient.GROUP_NAME); 246 String doneFlag = ds.getDoneFlag(); 247 while (instance >= checkedInstance) { 248 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal); 249 String uriPath = uriEval.evaluate(uriTemplate, String.class); 250 String pathWithDoneFlag = uriPath; 251 if (doneFlag.length() > 0) { 252 pathWithDoneFlag += "/" + doneFlag; 253 } 254 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) { 255 XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag); 256 if (available == n) { 257 XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag); 258 resolved = true; 259 retVal = DateUtils.formatDateUTC(nominalInstanceCal); 260 eval.setVariable("resolved_path", uriPath); 261 break; 262 } 263 available++; 264 } 265 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), 266 // -datasetFrequency); 267 nominalInstanceCal = (Calendar) initInstance.clone(); 268 instCount[0]++; 269 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 270 checkedInstance++; 271 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 272 } 273 if (!resolved) { 274 // return unchanged future function with variable 'is_resolved' 275 // to 'false' 276 eval.setVariable("is_resolved", Boolean.FALSE); 277 retVal = "${coord:future(" + n + ", " + instance + ")}"; 278 } 279 else { 280 eval.setVariable("is_resolved", Boolean.TRUE); 281 } 282 } 283 else {// No feasible nominal time 284 eval.setVariable("is_resolved", Boolean.TRUE); 285 retVal = ""; 286 } 287 return retVal; 288 } 289 290 /** 291 * Return nominal time or Action Creation Time. 292 * <p/> 293 * 294 * @return coordinator action creation or materialization date time 295 * @throws Exception if unable to format the Date object to String 296 */ 297 public static String ph2_coord_nominalTime() throws Exception { 298 ELEvaluator eval = ELEvaluator.getCurrent(); 299 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 300 "Coordinator Action"); 301 return DateUtils.formatDateUTC(action.getNominalTime()); 302 } 303 304 public static String ph3_coord_nominalTime() throws Exception { 305 return ph2_coord_nominalTime(); 306 } 307 308 /** 309 * Convert from standard date-time formatting to a desired format. 310 * <p/> 311 * @param dateTimeStr - A timestamp in standard (ISO8601) format. 312 * @param format - A string representing the desired format. 313 * @return coordinator action creation or materialization date time 314 * @throws Exception if unable to format the Date object to String 315 */ 316 public static String ph2_coord_formatTime(String dateTimeStr, String format) 317 throws Exception { 318 Date dateTime = DateUtils.parseDateUTC(dateTimeStr); 319 return DateUtils.formatDateCustom(dateTime, format); 320 } 321 322 public static String ph3_coord_formatTime(String dateTimeStr, String format) 323 throws Exception { 324 return ph2_coord_formatTime(dateTimeStr, format); 325 } 326 327 /** 328 * Return Action Id. <p/> 329 * 330 * @return coordinator action Id 331 */ 332 public static String ph2_coord_actionId() throws Exception { 333 ELEvaluator eval = ELEvaluator.getCurrent(); 334 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 335 "Coordinator Action"); 336 return action.getActionId(); 337 } 338 339 public static String ph3_coord_actionId() throws Exception { 340 return ph2_coord_actionId(); 341 } 342 343 /** 344 * Return Job Name. <p/> 345 * 346 * @return coordinator name 347 */ 348 public static String ph2_coord_name() throws Exception { 349 ELEvaluator eval = ELEvaluator.getCurrent(); 350 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 351 "Coordinator Action"); 352 return action.getName(); 353 } 354 355 public static String ph3_coord_name() throws Exception { 356 return ph2_coord_name(); 357 } 358 359 /** 360 * Return Action Start time. <p/> 361 * 362 * @return coordinator action start time 363 * @throws Exception if unable to format the Date object to String 364 */ 365 public static String ph2_coord_actualTime() throws Exception { 366 ELEvaluator eval = ELEvaluator.getCurrent(); 367 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 368 if (coordAction == null) { 369 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 370 } 371 return DateUtils.formatDateUTC(coordAction.getActualTime()); 372 } 373 374 public static String ph3_coord_actualTime() throws Exception { 375 return ph2_coord_actualTime(); 376 } 377 378 /** 379 * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level 380 * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of 381 * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something 382 * unresolved, this function will echo back the original function <p/> otherwise it sends the uris. 383 * 384 * @param dataInName : Datain name 385 * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest) 386 * , echo back <p/> the function without resolving the function. 387 */ 388 public static String ph3_coord_dataIn(String dataInName) { 389 String uris = ""; 390 ELEvaluator eval = ELEvaluator.getCurrent(); 391 uris = (String) eval.getVariable(".datain." + dataInName); 392 Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved"); 393 if (unresolved != null && unresolved.booleanValue() == true) { 394 return "${coord:dataIn('" + dataInName + "')}"; 395 } 396 return uris; 397 } 398 399 /** 400 * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level 401 * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris. 402 * 403 * @param dataOutName : Dataout name 404 * @return the list of URI's separated by INSTANCE_SEPARATOR 405 */ 406 public static String ph3_coord_dataOut(String dataOutName) { 407 String uris = ""; 408 ELEvaluator eval = ELEvaluator.getCurrent(); 409 uris = (String) eval.getVariable(".dataout." + dataOutName); 410 return uris; 411 } 412 413 /** 414 * Determine the date-time in UTC of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency <p/> 2. 415 * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data 416 * set initial instance <p/> 6. Action Creation Time 417 * 418 * @param n instance count domain: n is integer 419 * @return date-time in UTC of the n-th instance returns 'null' means n-th instance is earlier than Initial-Instance 420 * of DS 421 * @throws Exception 422 */ 423 public static String ph2_coord_current(int n) throws Exception { 424 if (isSyncDataSet()) { // For Sync Dataset 425 return coord_current_sync(n); 426 } 427 else { 428 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 429 } 430 } 431 432 /** 433 * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency 434 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. 435 * Data set initial instance <p/> 6. Action Creation Time 436 * 437 * @param n instance count <p/> domain: n is integer 438 * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS 439 * @throws Exception 440 */ 441 public static int ph2_coord_hoursInDay(int n) throws Exception { 442 int datasetFrequency = (int) getDSFrequency(); 443 // /Calendar nominalInstanceCal = 444 // getCurrentInstance(getActionCreationtime()); 445 Calendar nominalInstanceCal = getEffectiveNominalTime(); 446 if (nominalInstanceCal == null) { 447 return -1; 448 } 449 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n); 450 /* 451 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) 452 * { return -1; } 453 */ 454 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ 455 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 456 return DateUtils.hoursInDay(nominalInstanceCal); 457 } 458 459 public static int ph3_coord_hoursInDay(int n) throws Exception { 460 return ph2_coord_hoursInDay(n); 461 } 462 463 /** 464 * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency . 465 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. 466 * Data set initial instance <p/> 6. Action Creation Time 467 * 468 * @param n instance count. domain: n is integer 469 * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS 470 * @throws Exception 471 */ 472 public static int ph2_coord_daysInMonth(int n) throws Exception { 473 int datasetFrequency = (int) getDSFrequency();// in minutes 474 // Calendar nominalInstanceCal = 475 // getCurrentInstance(getActionCreationtime()); 476 Calendar nominalInstanceCal = getEffectiveNominalTime(); 477 if (nominalInstanceCal == null) { 478 return -1; 479 } 480 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n); 481 /* 482 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) 483 * { return -1; } 484 */ 485 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ 486 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 487 return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH); 488 } 489 490 public static int ph3_coord_daysInMonth(int n) throws Exception { 491 return ph2_coord_daysInMonth(n); 492 } 493 494 /** 495 * Determine the date-time in UTC of n-th latest available dataset instance. <p/> It depends on: <p/> 1. Data set 496 * frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month 497 * flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of dataset's directory 498 * 499 * @param n :instance count <p/> domain: n > 0, n is integer 500 * @return date-time in UTC of the n-th instance <p/> returns 'null' means n-th instance is earlier than 501 * Initial-Instance of DS 502 * @throws Exception 503 */ 504 public static String ph3_coord_latest(int n) throws Exception { 505 if (n > 0) { 506 throw new IllegalArgumentException("paramter should be <= 0 but it is " + n); 507 } 508 if (isSyncDataSet()) {// For Sync Dataset 509 return coord_latest_sync(n); 510 } 511 else { 512 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 513 } 514 } 515 516 /** 517 * Configure an evaluator with data set and application specific information. <p/> Helper method of associating 518 * dataset and application object 519 * 520 * @param evaluator : to set variables 521 * @param ds : Data Set object 522 * @param coordAction : Application instance 523 */ 524 public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) { 525 evaluator.setVariable(COORD_ACTION, coordAction); 526 evaluator.setVariable(DATASET, ds); 527 } 528 529 /** 530 * Helper method to wrap around with "${..}". <p/> 531 * 532 * 533 * @param eval :EL evaluator 534 * @param expr : expression to evaluate 535 * @return Resolved expression or echo back the same expression 536 * @throws Exception 537 */ 538 public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception { 539 try { 540 eval.setVariable(".wrap", null); 541 String result = eval.evaluate(expr, String.class); 542 if (eval.getVariable(".wrap") != null) { 543 return "${" + result + "}"; 544 } 545 else { 546 return result; 547 } 548 } 549 catch (Exception e) { 550 throw new Exception("Unable to evaluate :" + expr + ":\n", e); 551 } 552 } 553 554 // Set of echo functions 555 556 public static String ph1_coord_current_echo(String n) { 557 return echoUnResolved("current", n); 558 } 559 560 public static String ph2_coord_current_echo(String n) { 561 return echoUnResolved("current", n); 562 } 563 564 public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) { 565 return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit); 566 } 567 568 public static String ph1_coord_formatTime_echo(String dateTime, String format) { 569 // Quote the dateTime value since it would contain a ':'. 570 return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format); 571 } 572 573 public static String ph1_coord_latest_echo(String n) { 574 return echoUnResolved("latest", n); 575 } 576 577 public static String ph2_coord_latest_echo(String n) { 578 return ph1_coord_latest_echo(n); 579 } 580 581 public static String ph1_coord_future_echo(String n, String instance) { 582 return echoUnResolved("future", n + ", " + instance + ""); 583 } 584 585 public static String ph2_coord_future_echo(String n, String instance) { 586 return ph1_coord_future_echo(n, instance); 587 } 588 589 public static String ph1_coord_dataIn_echo(String n) { 590 ELEvaluator eval = ELEvaluator.getCurrent(); 591 String val = (String) eval.getVariable("oozie.dataname." + n); 592 if (val == null || val.equals("data-in") == false) { 593 XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid"); 594 throw new RuntimeException("data_in_name " + n + " is not valid"); 595 } 596 return echoUnResolved("dataIn", "'" + n + "'"); 597 } 598 599 public static String ph1_coord_dataOut_echo(String n) { 600 ELEvaluator eval = ELEvaluator.getCurrent(); 601 String val = (String) eval.getVariable("oozie.dataname." + n); 602 if (val == null || val.equals("data-out") == false) { 603 XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid"); 604 throw new RuntimeException("data_out_name " + n + " is not valid"); 605 } 606 return echoUnResolved("dataOut", "'" + n + "'"); 607 } 608 609 public static String ph1_coord_nominalTime_echo() { 610 return echoUnResolved("nominalTime", ""); 611 } 612 613 public static String ph1_coord_nominalTime_echo_wrap() { 614 // return "${coord:nominalTime()}"; // no resolution 615 return echoUnResolved("nominalTime", ""); 616 } 617 618 public static String ph1_coord_nominalTime_echo_fixed() { 619 return "2009-03-06T010:00"; // Dummy resolution 620 } 621 622 public static String ph1_coord_actionId_echo() { 623 return echoUnResolved("actionId", ""); 624 } 625 626 public static String ph1_coord_name_echo() { 627 return echoUnResolved("name", ""); 628 } 629 630 // The following echo functions are not used in any phases yet 631 // They are here for future purpose. 632 public static String coord_minutes_echo(String n) { 633 return echoUnResolved("minutes", n); 634 } 635 636 public static String coord_hours_echo(String n) { 637 return echoUnResolved("hours", n); 638 } 639 640 public static String coord_days_echo(String n) { 641 return echoUnResolved("days", n); 642 } 643 644 public static String coord_endOfDay_echo(String n) { 645 return echoUnResolved("endOfDay", n); 646 } 647 648 public static String coord_months_echo(String n) { 649 return echoUnResolved("months", n); 650 } 651 652 public static String coord_endOfMonth_echo(String n) { 653 return echoUnResolved("endOfMonth", n); 654 } 655 656 public static String coord_actualTime_echo() { 657 return echoUnResolved("actualTime", ""); 658 } 659 660 // This echo function will always return "24" for validation only. 661 // This evaluation ****should not**** replace the original XML 662 // Create a temporary string and validate the function 663 // This is **required** for evaluating an expression like 664 // coord:HoursInDay(0) + 3 665 // actual evaluation will happen in phase 2 or phase 3. 666 public static String ph1_coord_hoursInDay_echo(String n) { 667 return "24"; 668 // return echoUnResolved("hoursInDay", n); 669 } 670 671 // This echo function will always return "30" for validation only. 672 // This evaluation ****should not**** replace the original XML 673 // Create a temporary string and validate the function 674 // This is **required** for evaluating an expression like 675 // coord:daysInMonth(0) + 3 676 // actual evaluation will happen in phase 2 or phase 3. 677 public static String ph1_coord_daysInMonth_echo(String n) { 678 // return echoUnResolved("daysInMonth", n); 679 return "30"; 680 } 681 682 // This echo function will always return "3" for validation only. 683 // This evaluation ****should not**** replace the original XML 684 // Create a temporary string and validate the function 685 // This is **required** for evaluating an expression like coord:tzOffset + 2 686 // actual evaluation will happen in phase 2 or phase 3. 687 public static String ph1_coord_tzOffset_echo() { 688 // return echoUnResolved("tzOffset", ""); 689 return "3"; 690 } 691 692 // Local methods 693 /** 694 * @param n 695 * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the 696 * Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset. 697 * @throws Exception 698 */ 699 private static String coord_current_sync(int n) throws Exception { 700 int datasetFrequency = getDSFrequency();// in minutes 701 TimeUnit dsTimeUnit = getDSTimeUnit(); 702 int[] instCount = new int[1];// used as pass by ref 703 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount); 704 if (nominalInstanceCal == null) { 705 return ""; 706 } 707 nominalInstanceCal = getInitialInstanceCal(); 708 int absInstanceCount = instCount[0] + n; 709 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount); 710 711 if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) { 712 return ""; 713 } 714 String str = DateUtils.formatDateUTC(nominalInstanceCal); 715 return str; 716 } 717 718 /** 719 * @param offset 720 * @return n-th available latest instance Date-Time for SYNC data-set 721 * @throws Exception 722 */ 723 private static String coord_latest_sync(int offset) throws Exception { 724 if (offset > 0) { 725 throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0" 726 + offset); 727 } 728 ELEvaluator eval = ELEvaluator.getCurrent(); 729 String retVal = ""; 730 int datasetFrequency = (int) getDSFrequency();// in minutes 731 TimeUnit dsTimeUnit = getDSTimeUnit(); 732 int[] instCount = new int[1]; 733 Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount); 734 if (nominalInstanceCal != null) { 735 Calendar initInstance = getInitialInstanceCal(); 736 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 737 if (ds == null) { 738 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 739 } 740 String uriTemplate = ds.getUriTemplate(); 741 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION); 742 if (conf == null) { 743 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION); 744 } 745 int available = 0; 746 boolean resolved = false; 747 String user = ParamChecker 748 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME); 749 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME), 750 OozieClient.GROUP_NAME); 751 String doneFlag = ds.getDoneFlag(); 752 while (nominalInstanceCal.compareTo(initInstance) >= 0) { 753 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal); 754 String uriPath = uriEval.evaluate(uriTemplate, String.class); 755 String pathWithDoneFlag = uriPath; 756 if (doneFlag.length() > 0) { 757 pathWithDoneFlag += "/" + doneFlag; 758 } 759 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) { 760 XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag); 761 if (available == offset) { 762 XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag); 763 resolved = true; 764 retVal = DateUtils.formatDateUTC(nominalInstanceCal); 765 eval.setVariable("resolved_path", uriPath); 766 break; 767 } 768 769 available--; 770 } 771 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), 772 // -datasetFrequency); 773 nominalInstanceCal = (Calendar) initInstance.clone(); 774 instCount[0]--; 775 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 776 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 777 } 778 if (!resolved) { 779 // return unchanged latest function with variable 'is_resolved' 780 // to 'false' 781 eval.setVariable("is_resolved", Boolean.FALSE); 782 retVal = "${coord:latest(" + offset + ")}"; 783 } 784 else { 785 eval.setVariable("is_resolved", Boolean.TRUE); 786 } 787 } 788 else {// No feasible nominal time 789 eval.setVariable("is_resolved", Boolean.FALSE); 790 } 791 return retVal; 792 } 793 794 // TODO : Not an efficient way. In a loop environment, we could do something 795 // outside the loop 796 /** 797 * Check whether a URI path exists 798 * 799 * @param sPath 800 * @param conf 801 * @return 802 * @throws IOException 803 */ 804 805 private static boolean isPathAvailable(String sPath, String user, String group, Configuration conf) 806 throws IOException, HadoopAccessorException { 807 // sPath += "/" + END_OF_OPERATION_INDICATOR_FILE; 808 Path path = new Path(sPath); 809 return Services.get().get(HadoopAccessorService.class). 810 createFileSystem(user, group, path.toUri(), conf).exists(path); 811 } 812 813 /** 814 * @param tm 815 * @return a new Evaluator to be used for URI-template evaluation 816 */ 817 private static ELEvaluator getUriEvaluator(Calendar tm) { 818 ELEvaluator retEval = new ELEvaluator(); 819 retEval.setVariable("YEAR", tm.get(Calendar.YEAR)); 820 retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm 821 .get(Calendar.MONTH) + 1)); 822 retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm 823 .get(Calendar.DAY_OF_MONTH)); 824 retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm 825 .get(Calendar.HOUR_OF_DAY)); 826 retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm 827 .get(Calendar.MINUTE)); 828 return retEval; 829 } 830 831 /** 832 * @return whether a data set is SYNCH or ASYNC 833 */ 834 private static boolean isSyncDataSet() { 835 ELEvaluator eval = ELEvaluator.getCurrent(); 836 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 837 if (ds == null) { 838 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 839 } 840 return ds.getType().equalsIgnoreCase("SYNC"); 841 } 842 843 /** 844 * Check whether a function should be resolved. 845 * 846 * @param functionName 847 * @param n 848 * @return null if the functionName needs to be resolved otherwise return the calling function unresolved. 849 */ 850 private static String checkIfResolved(String functionName, String n) { 851 ELEvaluator eval = ELEvaluator.getCurrent(); 852 String replace = (String) eval.getVariable("resolve_" + functionName); 853 if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't 854 // resolve 855 // return "${coord:" + functionName + "(" + n +")}"; //Unresolved 856 eval.setVariable(".wrap", "true"); 857 return "coord:" + functionName + "(" + n + ")"; // Unresolved 858 } 859 return null; // Resolved it 860 } 861 862 private static String echoUnResolved(String functionName, String n) { 863 return echoUnResolvedPre(functionName, n, "coord:"); 864 } 865 866 private static String echoUnResolvedPre(String functionName, String n, String prefix) { 867 ELEvaluator eval = ELEvaluator.getCurrent(); 868 eval.setVariable(".wrap", "true"); 869 return prefix + functionName + "(" + n + ")"; // Unresolved 870 } 871 872 /** 873 * @return the initial instance of a DataSet in DATE 874 */ 875 private static Date getInitialInstance() { 876 return getInitialInstanceCal().getTime(); 877 // return ds.getInitInstance(); 878 } 879 880 /** 881 * @return the initial instance of a DataSet in Calendar 882 */ 883 private static Calendar getInitialInstanceCal() { 884 ELEvaluator eval = ELEvaluator.getCurrent(); 885 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 886 if (ds == null) { 887 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 888 } 889 Calendar effInitTS = Calendar.getInstance(); 890 effInitTS.setTime(ds.getInitInstance()); 891 effInitTS.setTimeZone(ds.getTimeZone()); 892 // To adjust EOD/EOM 893 DateUtils.moveToEnd(effInitTS, getDSEndOfFlag()); 894 return effInitTS; 895 // return ds.getInitInstance(); 896 } 897 898 /** 899 * @return Nominal or action creation Time when all the dependencies of an application instance are met. 900 */ 901 private static Date getActionCreationtime() { 902 ELEvaluator eval = ELEvaluator.getCurrent(); 903 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 904 if (coordAction == null) { 905 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 906 } 907 return coordAction.getNominalTime(); 908 } 909 910 /** 911 * @return Actual Time when all the dependencies of an application instance are met. 912 */ 913 private static Date getActualTime() { 914 ELEvaluator eval = ELEvaluator.getCurrent(); 915 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 916 if (coordAction == null) { 917 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 918 } 919 return coordAction.getActualTime(); 920 } 921 922 /** 923 * @return TimeZone for the application or job. 924 */ 925 private static TimeZone getJobTZ() { 926 ELEvaluator eval = ELEvaluator.getCurrent(); 927 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 928 if (coordAction == null) { 929 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 930 } 931 return coordAction.getTimeZone(); 932 } 933 934 /** 935 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time) 936 * 937 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of 938 * the dataset. 939 */ 940 private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) { 941 Date datasetInitialInstance = getInitialInstance(); 942 TimeUnit dsTimeUnit = getDSTimeUnit(); 943 TimeZone dsTZ = getDatasetTZ(); 944 // Convert Date to Calendar for corresponding TZ 945 Calendar current = Calendar.getInstance(); 946 current.setTime(datasetInitialInstance); 947 current.setTimeZone(dsTZ); 948 949 Calendar calEffectiveTime = Calendar.getInstance(); 950 calEffectiveTime.setTime(effectiveTime); 951 calEffectiveTime.setTimeZone(dsTZ); 952 instanceCount[0] = 0; 953 if (current.compareTo(calEffectiveTime) > 0) { 954 // Nominal Time < initial Instance 955 // TODO: getClass() call doesn't work from static method. 956 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+ 957 // current.getTime()); 958 return null; 959 } 960 Calendar origCurrent = (Calendar) current.clone(); 961 while (current.compareTo(calEffectiveTime) <= 0) { 962 current = (Calendar) origCurrent.clone(); 963 instanceCount[0]++; 964 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency()); 965 } 966 instanceCount[0]--; 967 968 current = (Calendar) origCurrent.clone(); 969 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency()); 970 return current; 971 } 972 973 private static Calendar getEffectiveNominalTime() { 974 Date datasetInitialInstance = getInitialInstance(); 975 TimeZone dsTZ = getDatasetTZ(); 976 // Convert Date to Calendar for corresponding TZ 977 Calendar current = Calendar.getInstance(); 978 current.setTime(datasetInitialInstance); 979 current.setTimeZone(dsTZ); 980 981 Calendar calEffectiveTime = Calendar.getInstance(); 982 calEffectiveTime.setTime(getActionCreationtime()); 983 calEffectiveTime.setTimeZone(dsTZ); 984 if (current.compareTo(calEffectiveTime) > 0) { 985 // Nominal Time < initial Instance 986 // TODO: getClass() call doesn't work from static method. 987 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+ 988 // current.getTime()); 989 return null; 990 } 991 return calEffectiveTime; 992 } 993 994 /** 995 * @return dataset frequency in minutes 996 */ 997 private static int getDSFrequency() { 998 ELEvaluator eval = ELEvaluator.getCurrent(); 999 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1000 if (ds == null) { 1001 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1002 } 1003 return ds.getFrequency(); 1004 } 1005 1006 /** 1007 * @return dataset TimeUnit 1008 */ 1009 private static TimeUnit getDSTimeUnit() { 1010 ELEvaluator eval = ELEvaluator.getCurrent(); 1011 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1012 if (ds == null) { 1013 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1014 } 1015 return ds.getTimeUnit(); 1016 } 1017 1018 /** 1019 * @return dataset TimeZone 1020 */ 1021 private static TimeZone getDatasetTZ() { 1022 ELEvaluator eval = ELEvaluator.getCurrent(); 1023 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1024 if (ds == null) { 1025 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1026 } 1027 return ds.getTimeZone(); 1028 } 1029 1030 /** 1031 * @return dataset TimeUnit 1032 */ 1033 private static TimeUnit getDSEndOfFlag() { 1034 ELEvaluator eval = ELEvaluator.getCurrent(); 1035 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1036 if (ds == null) { 1037 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1038 } 1039 return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration(); 1040 } 1041 1042 /** 1043 * Return the user that submitted the coordinator job. 1044 * 1045 * @return the user that submitted the coordinator job. 1046 */ 1047 public static String coord_user() { 1048 ELEvaluator eval = ELEvaluator.getCurrent(); 1049 return (String) eval.getVariable(OozieClient.USER_NAME); 1050 } 1051 }