001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.coord; 016 017 import java.io.IOException; 018 import java.util.Calendar; 019 import java.util.Date; 020 import java.util.TimeZone; 021 022 import org.apache.hadoop.conf.Configuration; 023 import org.apache.hadoop.fs.Path; 024 025 import org.apache.oozie.client.OozieClient; 026 import org.apache.oozie.util.DateUtils; 027 import org.apache.oozie.util.ELEvaluator; 028 import org.apache.oozie.util.ParamChecker; 029 import org.apache.oozie.util.XLog; 030 import org.apache.oozie.service.HadoopAccessorException; 031 import org.apache.oozie.service.Services; 032 import org.apache.oozie.service.HadoopAccessorService; 033 034 /** 035 * This class implements the EL function related to coordinator 036 */ 037 038 public class CoordELFunctions { 039 final private static String DATASET = "oozie.coord.el.dataset.bean"; 040 final private static String COORD_ACTION = "oozie.coord.el.app.bean"; 041 final public static String CONFIGURATION = "oozie.coord.el.conf"; 042 // INSTANCE_SEPARATOR is used to separate multiple directories into one tag. 043 final public static String INSTANCE_SEPARATOR = "#"; 044 final public static String DIR_SEPARATOR = ","; 045 // TODO: in next release, support flexibility 046 private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS"; 047 048 /** 049 * Used in defining the frequency in 'day' unit. <p/> domain: <code> val > 0</code> and should be integer. 050 * 051 * @param val frequency in number of days. 052 * @return number of days and also set the frequency timeunit to "day" 053 */ 054 public static int ph1_coord_days(int val) { 055 val = ParamChecker.checkGTZero(val, "n"); 056 ELEvaluator eval = ELEvaluator.getCurrent(); 057 eval.setVariable("timeunit", TimeUnit.DAY); 058 eval.setVariable("endOfDuration", TimeUnit.NONE); 059 return val; 060 } 061 062 /** 063 * Used in defining the frequency in 'month' unit. <p/> domain: <code> val > 0</code> and should be integer. 064 * 065 * @param val frequency in number of months. 066 * @return number of months and also set the frequency timeunit to "month" 067 */ 068 public static int ph1_coord_months(int val) { 069 val = ParamChecker.checkGTZero(val, "n"); 070 ELEvaluator eval = ELEvaluator.getCurrent(); 071 eval.setVariable("timeunit", TimeUnit.MONTH); 072 eval.setVariable("endOfDuration", TimeUnit.NONE); 073 return val; 074 } 075 076 /** 077 * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val > 0</code> and should 078 * be integer. 079 * 080 * @param val frequency in number of hours. 081 * @return number of minutes and also set the frequency timeunit to "minute" 082 */ 083 public static int ph1_coord_hours(int val) { 084 val = ParamChecker.checkGTZero(val, "n"); 085 ELEvaluator eval = ELEvaluator.getCurrent(); 086 eval.setVariable("timeunit", TimeUnit.MINUTE); 087 eval.setVariable("endOfDuration", TimeUnit.NONE); 088 return val * 60; 089 } 090 091 /** 092 * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val > 0</code> and should be integer. 093 * 094 * @param val frequency in number of minutes. 095 * @return number of minutes and also set the frequency timeunit to "minute" 096 */ 097 public static int ph1_coord_minutes(int val) { 098 val = ParamChecker.checkGTZero(val, "n"); 099 ELEvaluator eval = ELEvaluator.getCurrent(); 100 eval.setVariable("timeunit", TimeUnit.MINUTE); 101 eval.setVariable("endOfDuration", TimeUnit.NONE); 102 return val; 103 } 104 105 /** 106 * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will 107 * start at 00:00 hour of each day. <p/> domain: <code> val > 0</code> and should be integer. 108 * 109 * @param val frequency in number of days. 110 * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day" 111 */ 112 public static int ph1_coord_endOfDays(int val) { 113 val = ParamChecker.checkGTZero(val, "n"); 114 ELEvaluator eval = ELEvaluator.getCurrent(); 115 eval.setVariable("timeunit", TimeUnit.DAY); 116 eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY); 117 return val; 118 } 119 120 /** 121 * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will 122 * start at first day of each month at 00:00 hour. <p/> domain: <code> val > 0</code> and should be integer. 123 * 124 * @param val: frequency in number of months. 125 * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month" 126 */ 127 public static int ph1_coord_endOfMonths(int val) { 128 val = ParamChecker.checkGTZero(val, "n"); 129 ELEvaluator eval = ELEvaluator.getCurrent(); 130 eval.setVariable("timeunit", TimeUnit.MONTH); 131 eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH); 132 return val; 133 } 134 135 /** 136 * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/> 137 * 1. Timezone of both dataset and job <p/> 2. Action creation Time 138 * 139 * @return difference in minutes (DataSet TZ Offset - Application TZ offset) 140 */ 141 public static int ph2_coord_tzOffset() { 142 Date actionCreationTime = getActionCreationtime(); 143 TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ"); 144 TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ"); 145 // Apply the TZ into Calendar object 146 Calendar dsTime = Calendar.getInstance(dsTZ); 147 dsTime.setTime(actionCreationTime); 148 Calendar jobTime = Calendar.getInstance(jobTZ); 149 jobTime.setTime(actionCreationTime); 150 return (dsTime.get(Calendar.ZONE_OFFSET) - jobTime.get(Calendar.ZONE_OFFSET)) / (1000 * 60); 151 } 152 153 public static int ph3_coord_tzOffset() { 154 return ph2_coord_tzOffset(); 155 } 156 157 /** 158 * Returns the a date string while given a base date in 'strBaseDate', 159 * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH). 160 * 161 * @param strBaseDate -- base date 162 * @param offset -- any number 163 * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH 164 * @return date string 165 * @throws Exception 166 */ 167 public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { 168 Calendar baseCalDate = DateUtils.getCalendar(strBaseDate); 169 StringBuilder buffer = new StringBuilder(); 170 baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset); 171 buffer.append(DateUtils.formatDateUTC(baseCalDate)); 172 return buffer.toString(); 173 } 174 175 public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { 176 return ph2_coord_dateOffset(strBaseDate, offset, unit); 177 } 178 179 /** 180 * Determine the date-time in UTC of n-th future available dataset instance 181 * from nominal Time but not beyond the instance specified as 'instance. 182 * <p/> 183 * It depends on: 184 * <p/> 185 * 1. Data set frequency 186 * <p/> 187 * 2. Data set Time unit (day, month, minute) 188 * <p/> 189 * 3. Data set Time zone/DST 190 * <p/> 191 * 4. End Day/Month flag 192 * <p/> 193 * 5. Data set initial instance 194 * <p/> 195 * 6. Action Creation Time 196 * <p/> 197 * 7. Existence of dataset's directory 198 * 199 * @param n :instance count 200 * <p/> 201 * domain: n >= 0, n is integer 202 * @param instance: How many future instance it should check? value should 203 * be >=0 204 * @return date-time in UTC of the n-th instance 205 * <p/> 206 * @throws Exception 207 */ 208 public static String ph3_coord_future(int n, int instance) throws Exception { 209 ParamChecker.checkGEZero(n, "future:n"); 210 ParamChecker.checkGTZero(instance, "future:instance"); 211 if (isSyncDataSet()) {// For Sync Dataset 212 return coord_future_sync(n, instance); 213 } 214 else { 215 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 216 } 217 } 218 219 private static String coord_future_sync(int n, int instance) throws Exception { 220 ELEvaluator eval = ELEvaluator.getCurrent(); 221 String retVal = ""; 222 int datasetFrequency = (int) getDSFrequency();// in minutes 223 TimeUnit dsTimeUnit = getDSTimeUnit(); 224 int[] instCount = new int[1]; 225 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount); 226 if (nominalInstanceCal != null) { 227 Calendar initInstance = getInitialInstanceCal(); 228 nominalInstanceCal = (Calendar) initInstance.clone(); 229 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 230 231 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 232 if (ds == null) { 233 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 234 } 235 String uriTemplate = ds.getUriTemplate(); 236 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION); 237 if (conf == null) { 238 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION); 239 } 240 int available = 0, checkedInstance = 0; 241 boolean resolved = false; 242 String user = ParamChecker 243 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME); 244 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME), 245 OozieClient.GROUP_NAME); 246 String doneFlag = ds.getDoneFlag(); 247 while (instance >= checkedInstance) { 248 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal); 249 String uriPath = uriEval.evaluate(uriTemplate, String.class); 250 String pathWithDoneFlag = uriPath; 251 if (doneFlag.length() > 0) { 252 pathWithDoneFlag += "/" + doneFlag; 253 } 254 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) { 255 XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag); 256 if (available == n) { 257 XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag); 258 resolved = true; 259 retVal = DateUtils.formatDateUTC(nominalInstanceCal); 260 eval.setVariable("resolved_path", uriPath); 261 break; 262 } 263 available++; 264 } 265 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), 266 // -datasetFrequency); 267 nominalInstanceCal = (Calendar) initInstance.clone(); 268 instCount[0]++; 269 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 270 checkedInstance++; 271 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 272 } 273 if (!resolved) { 274 // return unchanged future function with variable 'is_resolved' 275 // to 'false' 276 eval.setVariable("is_resolved", Boolean.FALSE); 277 retVal = "${coord:future(" + n + ", " + instance + ")}"; 278 } 279 else { 280 eval.setVariable("is_resolved", Boolean.TRUE); 281 } 282 } 283 else {// No feasible nominal time 284 eval.setVariable("is_resolved", Boolean.TRUE); 285 retVal = ""; 286 } 287 return retVal; 288 } 289 290 /** 291 * Return nominal time or Action Creation Time. 292 * <p/> 293 * 294 * @return coordinator action creation or materialization date time 295 * @throws Exception if unable to format the Date object to String 296 */ 297 public static String ph2_coord_nominalTime() throws Exception { 298 ELEvaluator eval = ELEvaluator.getCurrent(); 299 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 300 "Coordinator Action"); 301 return DateUtils.formatDateUTC(action.getNominalTime()); 302 } 303 304 public static String ph3_coord_nominalTime() throws Exception { 305 return ph2_coord_nominalTime(); 306 } 307 308 /** 309 * Return Action Id. <p/> 310 * 311 * @return coordinator action Id 312 */ 313 public static String ph2_coord_actionId() throws Exception { 314 ELEvaluator eval = ELEvaluator.getCurrent(); 315 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 316 "Coordinator Action"); 317 return action.getActionId(); 318 } 319 320 public static String ph3_coord_actionId() throws Exception { 321 return ph2_coord_actionId(); 322 } 323 324 /** 325 * Return Job Name. <p/> 326 * 327 * @return coordinator name 328 */ 329 public static String ph2_coord_name() throws Exception { 330 ELEvaluator eval = ELEvaluator.getCurrent(); 331 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 332 "Coordinator Action"); 333 return action.getName(); 334 } 335 336 public static String ph3_coord_name() throws Exception { 337 return ph2_coord_name(); 338 } 339 340 /** 341 * Return Action Start time. <p/> 342 * 343 * @return coordinator action start time 344 * @throws Exception if unable to format the Date object to String 345 */ 346 public static String ph2_coord_actualTime() throws Exception { 347 ELEvaluator eval = ELEvaluator.getCurrent(); 348 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 349 if (coordAction == null) { 350 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 351 } 352 return DateUtils.formatDateUTC(coordAction.getActualTime()); 353 } 354 355 public static String ph3_coord_actualTime() throws Exception { 356 return ph2_coord_actualTime(); 357 } 358 359 /** 360 * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level 361 * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of 362 * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something 363 * unresolved, this function will echo back the original function <p/> otherwise it sends the uris. 364 * 365 * @param dataInName : Datain name 366 * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest) 367 * , echo back <p/> the function without resolving the function. 368 */ 369 public static String ph3_coord_dataIn(String dataInName) { 370 String uris = ""; 371 ELEvaluator eval = ELEvaluator.getCurrent(); 372 uris = (String) eval.getVariable(".datain." + dataInName); 373 Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved"); 374 if (unresolved != null && unresolved.booleanValue() == true) { 375 return "${coord:dataIn('" + dataInName + "')}"; 376 } 377 return uris; 378 } 379 380 /** 381 * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level 382 * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris. 383 * 384 * @param dataOutName : Dataout name 385 * @return the list of URI's separated by INSTANCE_SEPARATOR 386 */ 387 public static String ph3_coord_dataOut(String dataOutName) { 388 String uris = ""; 389 ELEvaluator eval = ELEvaluator.getCurrent(); 390 uris = (String) eval.getVariable(".dataout." + dataOutName); 391 return uris; 392 } 393 394 /** 395 * Determine the date-time in UTC of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency <p/> 2. 396 * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data 397 * set initial instance <p/> 6. Action Creation Time 398 * 399 * @param n instance count domain: n is integer 400 * @return date-time in UTC of the n-th instance returns 'null' means n-th instance is earlier than Initial-Instance 401 * of DS 402 * @throws Exception 403 */ 404 public static String ph2_coord_current(int n) throws Exception { 405 if (isSyncDataSet()) { // For Sync Dataset 406 return coord_current_sync(n); 407 } 408 else { 409 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 410 } 411 } 412 413 /** 414 * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency 415 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. 416 * Data set initial instance <p/> 6. Action Creation Time 417 * 418 * @param n instance count <p/> domain: n is integer 419 * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS 420 * @throws Exception 421 */ 422 public static int ph2_coord_hoursInDay(int n) throws Exception { 423 int datasetFrequency = (int) getDSFrequency(); 424 // /Calendar nominalInstanceCal = 425 // getCurrentInstance(getActionCreationtime()); 426 Calendar nominalInstanceCal = getEffectiveNominalTime(); 427 if (nominalInstanceCal == null) { 428 return -1; 429 } 430 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n); 431 /* 432 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) 433 * { return -1; } 434 */ 435 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ 436 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 437 return DateUtils.hoursInDay(nominalInstanceCal); 438 } 439 440 public static int ph3_coord_hoursInDay(int n) throws Exception { 441 return ph2_coord_hoursInDay(n); 442 } 443 444 /** 445 * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency . 446 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. 447 * Data set initial instance <p/> 6. Action Creation Time 448 * 449 * @param n instance count. domain: n is integer 450 * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS 451 * @throws Exception 452 */ 453 public static int ph2_coord_daysInMonth(int n) throws Exception { 454 int datasetFrequency = (int) getDSFrequency();// in minutes 455 // Calendar nominalInstanceCal = 456 // getCurrentInstance(getActionCreationtime()); 457 Calendar nominalInstanceCal = getEffectiveNominalTime(); 458 if (nominalInstanceCal == null) { 459 return -1; 460 } 461 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n); 462 /* 463 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) 464 * { return -1; } 465 */ 466 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ 467 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 468 return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH); 469 } 470 471 public static int ph3_coord_daysInMonth(int n) throws Exception { 472 return ph2_coord_daysInMonth(n); 473 } 474 475 /** 476 * Determine the date-time in UTC of n-th latest available dataset instance. <p/> It depends on: <p/> 1. Data set 477 * frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month 478 * flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of dataset's directory 479 * 480 * @param n :instance count <p/> domain: n > 0, n is integer 481 * @return date-time in UTC of the n-th instance <p/> returns 'null' means n-th instance is earlier than 482 * Initial-Instance of DS 483 * @throws Exception 484 */ 485 public static String ph3_coord_latest(int n) throws Exception { 486 if (n > 0) { 487 throw new IllegalArgumentException("paramter should be <= 0 but it is " + n); 488 } 489 if (isSyncDataSet()) {// For Sync Dataset 490 return coord_latest_sync(n); 491 } 492 else { 493 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 494 } 495 } 496 497 /** 498 * Configure an evaluator with data set and application specific information. <p/> Helper method of associating 499 * dataset and application object 500 * 501 * @param evaluator : to set variables 502 * @param ds : Data Set object 503 * @param coordAction : Application instance 504 */ 505 public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) { 506 evaluator.setVariable(COORD_ACTION, coordAction); 507 evaluator.setVariable(DATASET, ds); 508 } 509 510 /** 511 * Helper method to wrap around with "${..}". <p/> 512 * 513 * 514 * @param eval :EL evaluator 515 * @param expr : expression to evaluate 516 * @return Resolved expression or echo back the same expression 517 * @throws Exception 518 */ 519 public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception { 520 try { 521 eval.setVariable(".wrap", null); 522 String result = eval.evaluate(expr, String.class); 523 if (eval.getVariable(".wrap") != null) { 524 return "${" + result + "}"; 525 } 526 else { 527 return result; 528 } 529 } 530 catch (Exception e) { 531 throw new Exception("Unable to evaluate :" + expr + ":\n", e); 532 } 533 } 534 535 // Set of echo functions 536 537 public static String ph1_coord_current_echo(String n) { 538 return echoUnResolved("current", n); 539 } 540 541 public static String ph2_coord_current_echo(String n) { 542 return echoUnResolved("current", n); 543 } 544 545 public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) { 546 return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit); 547 } 548 549 public static String ph1_coord_latest_echo(String n) { 550 return echoUnResolved("latest", n); 551 } 552 553 public static String ph2_coord_latest_echo(String n) { 554 return ph1_coord_latest_echo(n); 555 } 556 557 public static String ph1_coord_future_echo(String n, String instance) { 558 return echoUnResolved("future", n + ", " + instance + ""); 559 } 560 561 public static String ph2_coord_future_echo(String n, String instance) { 562 return ph1_coord_future_echo(n, instance); 563 } 564 565 public static String ph1_coord_dataIn_echo(String n) { 566 ELEvaluator eval = ELEvaluator.getCurrent(); 567 String val = (String) eval.getVariable("oozie.dataname." + n); 568 if (val == null || val.equals("data-in") == false) { 569 XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid"); 570 throw new RuntimeException("data_in_name " + n + " is not valid"); 571 } 572 return echoUnResolved("dataIn", "'" + n + "'"); 573 } 574 575 public static String ph1_coord_dataOut_echo(String n) { 576 ELEvaluator eval = ELEvaluator.getCurrent(); 577 String val = (String) eval.getVariable("oozie.dataname." + n); 578 if (val == null || val.equals("data-out") == false) { 579 XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid"); 580 throw new RuntimeException("data_out_name " + n + " is not valid"); 581 } 582 return echoUnResolved("dataOut", "'" + n + "'"); 583 } 584 585 public static String ph1_coord_nominalTime_echo() { 586 return echoUnResolved("nominalTime", ""); 587 } 588 589 public static String ph1_coord_nominalTime_echo_wrap() { 590 // return "${coord:nominalTime()}"; // no resolution 591 return echoUnResolved("nominalTime", ""); 592 } 593 594 public static String ph1_coord_nominalTime_echo_fixed() { 595 return "2009-03-06T010:00"; // Dummy resolution 596 } 597 598 public static String ph1_coord_actionId_echo() { 599 return echoUnResolved("actionId", ""); 600 } 601 602 public static String ph1_coord_name_echo() { 603 return echoUnResolved("name", ""); 604 } 605 606 // The following echo functions are not used in any phases yet 607 // They are here for future purpose. 608 public static String coord_minutes_echo(String n) { 609 return echoUnResolved("minutes", n); 610 } 611 612 public static String coord_hours_echo(String n) { 613 return echoUnResolved("hours", n); 614 } 615 616 public static String coord_days_echo(String n) { 617 return echoUnResolved("days", n); 618 } 619 620 public static String coord_endOfDay_echo(String n) { 621 return echoUnResolved("endOfDay", n); 622 } 623 624 public static String coord_months_echo(String n) { 625 return echoUnResolved("months", n); 626 } 627 628 public static String coord_endOfMonth_echo(String n) { 629 return echoUnResolved("endOfMonth", n); 630 } 631 632 public static String coord_actualTime_echo() { 633 return echoUnResolved("actualTime", ""); 634 } 635 636 // This echo function will always return "24" for validation only. 637 // This evaluation ****should not**** replace the original XML 638 // Create a temporary string and validate the function 639 // This is **required** for evaluating an expression like 640 // coord:HoursInDay(0) + 3 641 // actual evaluation will happen in phase 2 or phase 3. 642 public static String ph1_coord_hoursInDay_echo(String n) { 643 return "24"; 644 // return echoUnResolved("hoursInDay", n); 645 } 646 647 // This echo function will always return "30" for validation only. 648 // This evaluation ****should not**** replace the original XML 649 // Create a temporary string and validate the function 650 // This is **required** for evaluating an expression like 651 // coord:daysInMonth(0) + 3 652 // actual evaluation will happen in phase 2 or phase 3. 653 public static String ph1_coord_daysInMonth_echo(String n) { 654 // return echoUnResolved("daysInMonth", n); 655 return "30"; 656 } 657 658 // This echo function will always return "3" for validation only. 659 // This evaluation ****should not**** replace the original XML 660 // Create a temporary string and validate the function 661 // This is **required** for evaluating an expression like coord:tzOffset + 2 662 // actual evaluation will happen in phase 2 or phase 3. 663 public static String ph1_coord_tzOffset_echo() { 664 // return echoUnResolved("tzOffset", ""); 665 return "3"; 666 } 667 668 // Local methods 669 /** 670 * @param n 671 * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the 672 * Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset. 673 * @throws Exception 674 */ 675 private static String coord_current_sync(int n) throws Exception { 676 int datasetFrequency = getDSFrequency();// in minutes 677 TimeUnit dsTimeUnit = getDSTimeUnit(); 678 int[] instCount = new int[1];// used as pass by ref 679 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount); 680 if (nominalInstanceCal == null) { 681 return ""; 682 } 683 nominalInstanceCal = getInitialInstanceCal(); 684 int absInstanceCount = instCount[0] + n; 685 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount); 686 687 if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) { 688 return ""; 689 } 690 String str = DateUtils.formatDateUTC(nominalInstanceCal); 691 return str; 692 } 693 694 /** 695 * @param offset 696 * @return n-th available latest instance Date-Time for SYNC data-set 697 * @throws Exception 698 */ 699 private static String coord_latest_sync(int offset) throws Exception { 700 if (offset > 0) { 701 throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0" 702 + offset); 703 } 704 ELEvaluator eval = ELEvaluator.getCurrent(); 705 String retVal = ""; 706 int datasetFrequency = (int) getDSFrequency();// in minutes 707 TimeUnit dsTimeUnit = getDSTimeUnit(); 708 int[] instCount = new int[1]; 709 Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount); 710 if (nominalInstanceCal != null) { 711 Calendar initInstance = getInitialInstanceCal(); 712 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 713 if (ds == null) { 714 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 715 } 716 String uriTemplate = ds.getUriTemplate(); 717 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION); 718 if (conf == null) { 719 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION); 720 } 721 int available = 0; 722 boolean resolved = false; 723 String user = ParamChecker 724 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME); 725 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME), 726 OozieClient.GROUP_NAME); 727 String doneFlag = ds.getDoneFlag(); 728 while (nominalInstanceCal.compareTo(initInstance) >= 0) { 729 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal); 730 String uriPath = uriEval.evaluate(uriTemplate, String.class); 731 String pathWithDoneFlag = uriPath; 732 if (doneFlag.length() > 0) { 733 pathWithDoneFlag += "/" + doneFlag; 734 } 735 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) { 736 XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag); 737 if (available == offset) { 738 XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag); 739 resolved = true; 740 retVal = DateUtils.formatDateUTC(nominalInstanceCal); 741 eval.setVariable("resolved_path", uriPath); 742 break; 743 } 744 745 available--; 746 } 747 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), 748 // -datasetFrequency); 749 nominalInstanceCal = (Calendar) initInstance.clone(); 750 instCount[0]--; 751 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 752 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 753 } 754 if (!resolved) { 755 // return unchanged latest function with variable 'is_resolved' 756 // to 'false' 757 eval.setVariable("is_resolved", Boolean.FALSE); 758 retVal = "${coord:latest(" + offset + ")}"; 759 } 760 else { 761 eval.setVariable("is_resolved", Boolean.TRUE); 762 } 763 } 764 else {// No feasible nominal time 765 eval.setVariable("is_resolved", Boolean.FALSE); 766 } 767 return retVal; 768 } 769 770 // TODO : Not an efficient way. In a loop environment, we could do something 771 // outside the loop 772 /** 773 * Check whether a URI path exists 774 * 775 * @param sPath 776 * @param conf 777 * @return 778 * @throws IOException 779 */ 780 781 private static boolean isPathAvailable(String sPath, String user, String group, Configuration conf) 782 throws IOException, HadoopAccessorException { 783 // sPath += "/" + END_OF_OPERATION_INDICATOR_FILE; 784 Path path = new Path(sPath); 785 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(), 786 new Configuration()).exists(path); 787 } 788 789 /** 790 * @param tm 791 * @return a new Evaluator to be used for URI-template evaluation 792 */ 793 private static ELEvaluator getUriEvaluator(Calendar tm) { 794 ELEvaluator retEval = new ELEvaluator(); 795 retEval.setVariable("YEAR", tm.get(Calendar.YEAR)); 796 retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm 797 .get(Calendar.MONTH) + 1)); 798 retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm 799 .get(Calendar.DAY_OF_MONTH)); 800 retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm 801 .get(Calendar.HOUR_OF_DAY)); 802 retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm 803 .get(Calendar.MINUTE)); 804 return retEval; 805 } 806 807 /** 808 * @return whether a data set is SYNCH or ASYNC 809 */ 810 private static boolean isSyncDataSet() { 811 ELEvaluator eval = ELEvaluator.getCurrent(); 812 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 813 if (ds == null) { 814 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 815 } 816 return ds.getType().equalsIgnoreCase("SYNC"); 817 } 818 819 /** 820 * Check whether a function should be resolved. 821 * 822 * @param functionName 823 * @param n 824 * @return null if the functionName needs to be resolved otherwise return the calling function unresolved. 825 */ 826 private static String checkIfResolved(String functionName, String n) { 827 ELEvaluator eval = ELEvaluator.getCurrent(); 828 String replace = (String) eval.getVariable("resolve_" + functionName); 829 if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't 830 // resolve 831 // return "${coord:" + functionName + "(" + n +")}"; //Unresolved 832 eval.setVariable(".wrap", "true"); 833 return "coord:" + functionName + "(" + n + ")"; // Unresolved 834 } 835 return null; // Resolved it 836 } 837 838 private static String echoUnResolved(String functionName, String n) { 839 return echoUnResolvedPre(functionName, n, "coord:"); 840 } 841 842 private static String echoUnResolvedPre(String functionName, String n, String prefix) { 843 ELEvaluator eval = ELEvaluator.getCurrent(); 844 eval.setVariable(".wrap", "true"); 845 return prefix + functionName + "(" + n + ")"; // Unresolved 846 } 847 848 /** 849 * @return the initial instance of a DataSet in DATE 850 */ 851 private static Date getInitialInstance() { 852 return getInitialInstanceCal().getTime(); 853 // return ds.getInitInstance(); 854 } 855 856 /** 857 * @return the initial instance of a DataSet in Calendar 858 */ 859 private static Calendar getInitialInstanceCal() { 860 ELEvaluator eval = ELEvaluator.getCurrent(); 861 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 862 if (ds == null) { 863 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 864 } 865 Calendar effInitTS = Calendar.getInstance(); 866 effInitTS.setTime(ds.getInitInstance()); 867 effInitTS.setTimeZone(ds.getTimeZone()); 868 // To adjust EOD/EOM 869 DateUtils.moveToEnd(effInitTS, getDSEndOfFlag()); 870 return effInitTS; 871 // return ds.getInitInstance(); 872 } 873 874 /** 875 * @return Nominal or action creation Time when all the dependencies of an application instance are met. 876 */ 877 private static Date getActionCreationtime() { 878 ELEvaluator eval = ELEvaluator.getCurrent(); 879 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 880 if (coordAction == null) { 881 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 882 } 883 return coordAction.getNominalTime(); 884 } 885 886 /** 887 * @return Actual Time when all the dependencies of an application instance are met. 888 */ 889 private static Date getActualTime() { 890 ELEvaluator eval = ELEvaluator.getCurrent(); 891 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 892 if (coordAction == null) { 893 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 894 } 895 return coordAction.getActualTime(); 896 } 897 898 /** 899 * @return TimeZone for the application or job. 900 */ 901 private static TimeZone getJobTZ() { 902 ELEvaluator eval = ELEvaluator.getCurrent(); 903 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 904 if (coordAction == null) { 905 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 906 } 907 return coordAction.getTimeZone(); 908 } 909 910 /** 911 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time) 912 * 913 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of 914 * the dataset. 915 */ 916 private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) { 917 Date datasetInitialInstance = getInitialInstance(); 918 TimeUnit dsTimeUnit = getDSTimeUnit(); 919 TimeZone dsTZ = getDatasetTZ(); 920 // Convert Date to Calendar for corresponding TZ 921 Calendar current = Calendar.getInstance(); 922 current.setTime(datasetInitialInstance); 923 current.setTimeZone(dsTZ); 924 925 Calendar calEffectiveTime = Calendar.getInstance(); 926 calEffectiveTime.setTime(effectiveTime); 927 calEffectiveTime.setTimeZone(dsTZ); 928 instanceCount[0] = 0; 929 if (current.compareTo(calEffectiveTime) > 0) { 930 // Nominal Time < initial Instance 931 // TODO: getClass() call doesn't work from static method. 932 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+ 933 // current.getTime()); 934 return null; 935 } 936 Calendar origCurrent = (Calendar) current.clone(); 937 while (current.compareTo(calEffectiveTime) <= 0) { 938 current = (Calendar) origCurrent.clone(); 939 instanceCount[0]++; 940 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency()); 941 } 942 instanceCount[0]--; 943 944 current = (Calendar) origCurrent.clone(); 945 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency()); 946 return current; 947 } 948 949 private static Calendar getEffectiveNominalTime() { 950 Date datasetInitialInstance = getInitialInstance(); 951 TimeZone dsTZ = getDatasetTZ(); 952 // Convert Date to Calendar for corresponding TZ 953 Calendar current = Calendar.getInstance(); 954 current.setTime(datasetInitialInstance); 955 current.setTimeZone(dsTZ); 956 957 Calendar calEffectiveTime = Calendar.getInstance(); 958 calEffectiveTime.setTime(getActionCreationtime()); 959 calEffectiveTime.setTimeZone(dsTZ); 960 if (current.compareTo(calEffectiveTime) > 0) { 961 // Nominal Time < initial Instance 962 // TODO: getClass() call doesn't work from static method. 963 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+ 964 // current.getTime()); 965 return null; 966 } 967 return calEffectiveTime; 968 } 969 970 /** 971 * @return dataset frequency in minutes 972 */ 973 private static int getDSFrequency() { 974 ELEvaluator eval = ELEvaluator.getCurrent(); 975 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 976 if (ds == null) { 977 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 978 } 979 return ds.getFrequency(); 980 } 981 982 /** 983 * @return dataset TimeUnit 984 */ 985 private static TimeUnit getDSTimeUnit() { 986 ELEvaluator eval = ELEvaluator.getCurrent(); 987 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 988 if (ds == null) { 989 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 990 } 991 return ds.getTimeUnit(); 992 } 993 994 /** 995 * @return dataset TimeZone 996 */ 997 private static TimeZone getDatasetTZ() { 998 ELEvaluator eval = ELEvaluator.getCurrent(); 999 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1000 if (ds == null) { 1001 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1002 } 1003 return ds.getTimeZone(); 1004 } 1005 1006 /** 1007 * @return dataset TimeUnit 1008 */ 1009 private static TimeUnit getDSEndOfFlag() { 1010 ELEvaluator eval = ELEvaluator.getCurrent(); 1011 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1012 if (ds == null) { 1013 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1014 } 1015 return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration(); 1016 } 1017 1018 }