001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.coord;
016    
017    import java.io.IOException;
018    import java.util.Calendar;
019    import java.util.Date;
020    import java.util.TimeZone;
021    
022    import org.apache.hadoop.conf.Configuration;
023    import org.apache.hadoop.fs.Path;
024    
025    import org.apache.oozie.client.OozieClient;
026    import org.apache.oozie.util.DateUtils;
027    import org.apache.oozie.util.ELEvaluator;
028    import org.apache.oozie.util.ParamChecker;
029    import org.apache.oozie.util.XLog;
030    import org.apache.oozie.service.HadoopAccessorException;
031    import org.apache.oozie.service.Services;
032    import org.apache.oozie.service.HadoopAccessorService;
033    
034    /**
035     * This class implements the EL function related to coordinator
036     */
037    
038    public class CoordELFunctions {
039        final private static String DATASET = "oozie.coord.el.dataset.bean";
040        final private static String COORD_ACTION = "oozie.coord.el.app.bean";
041        final public static String CONFIGURATION = "oozie.coord.el.conf";
042        // INSTANCE_SEPARATOR is used to separate multiple directories into one tag.
043        final public static String INSTANCE_SEPARATOR = "#";
044        final public static String DIR_SEPARATOR = ",";
045        // TODO: in next release, support flexibility
046        private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS";
047    
048        /**
049         * Used in defining the frequency in 'day' unit. <p/> domain: <code> val &gt; 0</code> and should be integer.
050         *
051         * @param val frequency in number of days.
052         * @return number of days and also set the frequency timeunit to "day"
053         */
054        public static int ph1_coord_days(int val) {
055            val = ParamChecker.checkGTZero(val, "n");
056            ELEvaluator eval = ELEvaluator.getCurrent();
057            eval.setVariable("timeunit", TimeUnit.DAY);
058            eval.setVariable("endOfDuration", TimeUnit.NONE);
059            return val;
060        }
061    
062        /**
063         * Used in defining the frequency in 'month' unit. <p/> domain: <code> val &gt; 0</code> and should be integer.
064         *
065         * @param val frequency in number of months.
066         * @return number of months and also set the frequency timeunit to "month"
067         */
068        public static int ph1_coord_months(int val) {
069            val = ParamChecker.checkGTZero(val, "n");
070            ELEvaluator eval = ELEvaluator.getCurrent();
071            eval.setVariable("timeunit", TimeUnit.MONTH);
072            eval.setVariable("endOfDuration", TimeUnit.NONE);
073            return val;
074        }
075    
076        /**
077         * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val &gt; 0</code> and should
078         * be integer.
079         *
080         * @param val frequency in number of hours.
081         * @return number of minutes and also set the frequency timeunit to "minute"
082         */
083        public static int ph1_coord_hours(int val) {
084            val = ParamChecker.checkGTZero(val, "n");
085            ELEvaluator eval = ELEvaluator.getCurrent();
086            eval.setVariable("timeunit", TimeUnit.MINUTE);
087            eval.setVariable("endOfDuration", TimeUnit.NONE);
088            return val * 60;
089        }
090    
091        /**
092         * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val &gt; 0</code> and should be integer.
093         *
094         * @param val frequency in number of minutes.
095         * @return number of minutes and also set the frequency timeunit to "minute"
096         */
097        public static int ph1_coord_minutes(int val) {
098            val = ParamChecker.checkGTZero(val, "n");
099            ELEvaluator eval = ELEvaluator.getCurrent();
100            eval.setVariable("timeunit", TimeUnit.MINUTE);
101            eval.setVariable("endOfDuration", TimeUnit.NONE);
102            return val;
103        }
104    
105        /**
106         * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will
107         * start at 00:00 hour of each day. <p/> domain: <code> val &gt; 0</code> and should be integer.
108         *
109         * @param val frequency in number of days.
110         * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day"
111         */
112        public static int ph1_coord_endOfDays(int val) {
113            val = ParamChecker.checkGTZero(val, "n");
114            ELEvaluator eval = ELEvaluator.getCurrent();
115            eval.setVariable("timeunit", TimeUnit.DAY);
116            eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY);
117            return val;
118        }
119    
120        /**
121         * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will
122         * start at first day of each month at 00:00 hour. <p/> domain: <code> val &gt; 0</code> and should be integer.
123         *
124         * @param val: frequency in number of months.
125         * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month"
126         */
127        public static int ph1_coord_endOfMonths(int val) {
128            val = ParamChecker.checkGTZero(val, "n");
129            ELEvaluator eval = ELEvaluator.getCurrent();
130            eval.setVariable("timeunit", TimeUnit.MONTH);
131            eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH);
132            return val;
133        }
134    
135        /**
136         * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/>
137         * 1. Timezone of both dataset and job <p/> 2. Action creation Time
138         *
139         * @return difference in minutes (DataSet TZ Offset - Application TZ offset)
140         */
141        public static int ph2_coord_tzOffset() {
142            Date actionCreationTime = getActionCreationtime();
143            TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ");
144            TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ");
145            // Apply the TZ into Calendar object
146            Calendar dsTime = Calendar.getInstance(dsTZ);
147            dsTime.setTime(actionCreationTime);
148            Calendar jobTime = Calendar.getInstance(jobTZ);
149            jobTime.setTime(actionCreationTime);
150            return (dsTime.get(Calendar.ZONE_OFFSET) - jobTime.get(Calendar.ZONE_OFFSET)) / (1000 * 60);
151        }
152    
153        public static int ph3_coord_tzOffset() {
154            return ph2_coord_tzOffset();
155        }
156    
157        /**
158         * Returns the a date string while given a base date in 'strBaseDate',
159         * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH).
160         * 
161         * @param strBaseDate -- base date
162         * @param offset -- any number
163         * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH
164         * @return date string
165         * @throws Exception
166         */
167        public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
168            Calendar baseCalDate = DateUtils.getCalendar(strBaseDate);
169            StringBuilder buffer = new StringBuilder();
170            baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset);
171            buffer.append(DateUtils.formatDateUTC(baseCalDate));
172            return buffer.toString();
173        }
174    
175        public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
176            return ph2_coord_dateOffset(strBaseDate, offset, unit);
177        }
178    
179        /**
180         * Determine the date-time in UTC of n-th future available dataset instance
181         * from nominal Time but not beyond the instance specified as 'instance.
182         * <p/>
183         * It depends on:
184         * <p/>
185         * 1. Data set frequency
186         * <p/>
187         * 2. Data set Time unit (day, month, minute)
188         * <p/>
189         * 3. Data set Time zone/DST
190         * <p/>
191         * 4. End Day/Month flag
192         * <p/>
193         * 5. Data set initial instance
194         * <p/>
195         * 6. Action Creation Time
196         * <p/>
197         * 7. Existence of dataset's directory
198         * 
199         * @param n :instance count
200         *        <p/>
201         *        domain: n >= 0, n is integer
202         * @param instance: How many future instance it should check? value should
203         *        be >=0
204         * @return date-time in UTC of the n-th instance
205         *         <p/>
206         * @throws Exception
207         */
208        public static String ph3_coord_future(int n, int instance) throws Exception {
209            ParamChecker.checkGEZero(n, "future:n");
210            ParamChecker.checkGTZero(instance, "future:instance");
211            if (isSyncDataSet()) {// For Sync Dataset
212                return coord_future_sync(n, instance);
213            }
214            else {
215                throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
216            }
217        }
218    
219        private static String coord_future_sync(int n, int instance) throws Exception {
220            ELEvaluator eval = ELEvaluator.getCurrent();
221            String retVal = "";
222            int datasetFrequency = (int) getDSFrequency();// in minutes
223            TimeUnit dsTimeUnit = getDSTimeUnit();
224            int[] instCount = new int[1];
225            Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
226            if (nominalInstanceCal != null) {
227                Calendar initInstance = getInitialInstanceCal();
228                nominalInstanceCal = (Calendar) initInstance.clone();
229                nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
230    
231                SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
232                if (ds == null) {
233                    throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
234                }
235                String uriTemplate = ds.getUriTemplate();
236                Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
237                if (conf == null) {
238                    throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
239                }
240                int available = 0, checkedInstance = 0;
241                boolean resolved = false;
242                String user = ParamChecker
243                        .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
244                String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME),
245                        OozieClient.GROUP_NAME);
246                String doneFlag = ds.getDoneFlag();
247                while (instance >= checkedInstance) {
248                    ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
249                    String uriPath = uriEval.evaluate(uriTemplate, String.class);
250                    String pathWithDoneFlag = uriPath;
251                    if (doneFlag.length() > 0) {
252                        pathWithDoneFlag += "/" + doneFlag;
253                    }
254                    if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
255                        XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag);
256                        if (available == n) {
257                            XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag);
258                            resolved = true;
259                            retVal = DateUtils.formatDateUTC(nominalInstanceCal);
260                            eval.setVariable("resolved_path", uriPath);
261                            break;
262                        }
263                        available++;
264                    }
265                    // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
266                    // -datasetFrequency);
267                    nominalInstanceCal = (Calendar) initInstance.clone();
268                    instCount[0]++;
269                    nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
270                    checkedInstance++;
271                    // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
272                }
273                if (!resolved) {
274                    // return unchanged future function with variable 'is_resolved'
275                    // to 'false'
276                    eval.setVariable("is_resolved", Boolean.FALSE);
277                    retVal = "${coord:future(" + n + ", " + instance + ")}";
278                }
279                else {
280                    eval.setVariable("is_resolved", Boolean.TRUE);
281                }
282            }
283            else {// No feasible nominal time
284                eval.setVariable("is_resolved", Boolean.TRUE);
285                retVal = "";
286            }
287            return retVal;
288        }
289    
290        /**
291         * Return nominal time or Action Creation Time.
292         * <p/>
293         * 
294         * @return coordinator action creation or materialization date time
295         * @throws Exception if unable to format the Date object to String
296         */
297        public static String ph2_coord_nominalTime() throws Exception {
298            ELEvaluator eval = ELEvaluator.getCurrent();
299            SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
300                    "Coordinator Action");
301            return DateUtils.formatDateUTC(action.getNominalTime());
302        }
303    
304        public static String ph3_coord_nominalTime() throws Exception {
305            return ph2_coord_nominalTime();
306        }
307    
308        /**
309         * Return Action Id. <p/>
310         * 
311         * @return coordinator action Id
312         */
313        public static String ph2_coord_actionId() throws Exception {
314            ELEvaluator eval = ELEvaluator.getCurrent();
315            SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
316                    "Coordinator Action");
317            return action.getActionId();
318        }
319    
320        public static String ph3_coord_actionId() throws Exception {
321            return ph2_coord_actionId();
322        }
323    
324        /**
325         * Return Job Name. <p/>
326         * 
327         * @return coordinator name
328         */
329        public static String ph2_coord_name() throws Exception {
330            ELEvaluator eval = ELEvaluator.getCurrent();
331            SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
332                    "Coordinator Action");
333            return action.getName();
334        }
335    
336        public static String ph3_coord_name() throws Exception {
337            return ph2_coord_name();
338        }
339    
340        /**
341         * Return Action Start time. <p/>
342         * 
343         * @return coordinator action start time
344         * @throws Exception if unable to format the Date object to String
345         */
346        public static String ph2_coord_actualTime() throws Exception {
347            ELEvaluator eval = ELEvaluator.getCurrent();
348            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
349            if (coordAction == null) {
350                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
351            }
352            return DateUtils.formatDateUTC(coordAction.getActualTime());
353        }
354    
355        public static String ph3_coord_actualTime() throws Exception {
356            return ph2_coord_actualTime();
357        }
358    
359        /**
360         * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level
361         * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
362         * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
363         * unresolved, this function will echo back the original function <p/> otherwise it sends the uris.
364         *
365         * @param dataInName : Datain name
366         * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest)
367         *         , echo back <p/> the function without resolving the function.
368         */
369        public static String ph3_coord_dataIn(String dataInName) {
370            String uris = "";
371            ELEvaluator eval = ELEvaluator.getCurrent();
372            uris = (String) eval.getVariable(".datain." + dataInName);
373            Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
374            if (unresolved != null && unresolved.booleanValue() == true) {
375                return "${coord:dataIn('" + dataInName + "')}";
376            }
377            return uris;
378        }
379    
380        /**
381         * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level
382         * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris.
383         *
384         * @param dataOutName : Dataout name
385         * @return the list of URI's separated by INSTANCE_SEPARATOR
386         */
387        public static String ph3_coord_dataOut(String dataOutName) {
388            String uris = "";
389            ELEvaluator eval = ELEvaluator.getCurrent();
390            uris = (String) eval.getVariable(".dataout." + dataOutName);
391            return uris;
392        }
393    
394        /**
395         * Determine the date-time in UTC of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency <p/> 2.
396         * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data
397         * set initial instance <p/> 6. Action Creation Time
398         *
399         * @param n instance count domain: n is integer
400         * @return date-time in UTC of the n-th instance returns 'null' means n-th instance is earlier than Initial-Instance
401         *         of DS
402         * @throws Exception
403         */
404        public static String ph2_coord_current(int n) throws Exception {
405            if (isSyncDataSet()) { // For Sync Dataset
406                return coord_current_sync(n);
407            }
408            else {
409                throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
410            }
411        }
412    
413        /**
414         * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency
415         * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
416         * Data set initial instance <p/> 6. Action Creation Time
417         *
418         * @param n instance count <p/> domain: n is integer
419         * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
420         * @throws Exception
421         */
422        public static int ph2_coord_hoursInDay(int n) throws Exception {
423            int datasetFrequency = (int) getDSFrequency();
424            // /Calendar nominalInstanceCal =
425            // getCurrentInstance(getActionCreationtime());
426            Calendar nominalInstanceCal = getEffectiveNominalTime();
427            if (nominalInstanceCal == null) {
428                return -1;
429            }
430            nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
431            /*
432             * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
433             * { return -1; }
434             */
435            nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
436            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
437            return DateUtils.hoursInDay(nominalInstanceCal);
438        }
439    
440        public static int ph3_coord_hoursInDay(int n) throws Exception {
441            return ph2_coord_hoursInDay(n);
442        }
443    
444        /**
445         * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency .
446         * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
447         * Data set initial instance <p/> 6. Action Creation Time
448         *
449         * @param n instance count. domain: n is integer
450         * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
451         * @throws Exception
452         */
453        public static int ph2_coord_daysInMonth(int n) throws Exception {
454            int datasetFrequency = (int) getDSFrequency();// in minutes
455            // Calendar nominalInstanceCal =
456            // getCurrentInstance(getActionCreationtime());
457            Calendar nominalInstanceCal = getEffectiveNominalTime();
458            if (nominalInstanceCal == null) {
459                return -1;
460            }
461            nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
462            /*
463             * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
464             * { return -1; }
465             */
466            nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
467            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
468            return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH);
469        }
470    
471        public static int ph3_coord_daysInMonth(int n) throws Exception {
472            return ph2_coord_daysInMonth(n);
473        }
474    
475        /**
476         * Determine the date-time in UTC of n-th latest available dataset instance. <p/> It depends on: <p/> 1. Data set
477         * frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month
478         * flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of dataset's directory
479         *
480         * @param n :instance count <p/> domain: n > 0, n is integer
481         * @return date-time in UTC of the n-th instance <p/> returns 'null' means n-th instance is earlier than
482         *         Initial-Instance of DS
483         * @throws Exception
484         */
485        public static String ph3_coord_latest(int n) throws Exception {
486            if (n > 0) {
487                throw new IllegalArgumentException("paramter should be <= 0 but it is " + n);
488            }
489            if (isSyncDataSet()) {// For Sync Dataset
490                return coord_latest_sync(n);
491            }
492            else {
493                throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
494            }
495        }
496    
497        /**
498         * Configure an evaluator with data set and application specific information. <p/> Helper method of associating
499         * dataset and application object
500         * 
501         * @param evaluator : to set variables
502         * @param ds : Data Set object
503         * @param coordAction : Application instance
504         */
505        public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) {
506            evaluator.setVariable(COORD_ACTION, coordAction);
507            evaluator.setVariable(DATASET, ds);
508        }
509    
510        /**
511         * Helper method to wrap around with "${..}". <p/>
512         *
513         * 
514         * @param eval :EL evaluator
515         * @param expr : expression to evaluate
516         * @return Resolved expression or echo back the same expression
517         * @throws Exception
518         */
519        public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception {
520            try {
521                eval.setVariable(".wrap", null);
522                String result = eval.evaluate(expr, String.class);
523                if (eval.getVariable(".wrap") != null) {
524                    return "${" + result + "}";
525                }
526                else {
527                    return result;
528                }
529            }
530            catch (Exception e) {
531                throw new Exception("Unable to evaluate :" + expr + ":\n", e);
532            }
533        }
534    
535        // Set of echo functions
536    
537        public static String ph1_coord_current_echo(String n) {
538            return echoUnResolved("current", n);
539        }
540    
541        public static String ph2_coord_current_echo(String n) {
542            return echoUnResolved("current", n);
543        }
544    
545        public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) {
546            return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit);
547        }
548    
549        public static String ph1_coord_latest_echo(String n) {
550            return echoUnResolved("latest", n);
551        }
552    
553        public static String ph2_coord_latest_echo(String n) {
554            return ph1_coord_latest_echo(n);
555        }
556    
557        public static String ph1_coord_future_echo(String n, String instance) {
558            return echoUnResolved("future", n + ", " + instance + "");
559        }
560    
561        public static String ph2_coord_future_echo(String n, String instance) {
562            return ph1_coord_future_echo(n, instance);
563        }
564    
565        public static String ph1_coord_dataIn_echo(String n) {
566            ELEvaluator eval = ELEvaluator.getCurrent();
567            String val = (String) eval.getVariable("oozie.dataname." + n);
568            if (val == null || val.equals("data-in") == false) {
569                XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid");
570                throw new RuntimeException("data_in_name " + n + " is not valid");
571            }
572            return echoUnResolved("dataIn", "'" + n + "'");
573        }
574    
575        public static String ph1_coord_dataOut_echo(String n) {
576            ELEvaluator eval = ELEvaluator.getCurrent();
577            String val = (String) eval.getVariable("oozie.dataname." + n);
578            if (val == null || val.equals("data-out") == false) {
579                XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid");
580                throw new RuntimeException("data_out_name " + n + " is not valid");
581            }
582            return echoUnResolved("dataOut", "'" + n + "'");
583        }
584    
585        public static String ph1_coord_nominalTime_echo() {
586            return echoUnResolved("nominalTime", "");
587        }
588    
589        public static String ph1_coord_nominalTime_echo_wrap() {
590            // return "${coord:nominalTime()}"; // no resolution
591            return echoUnResolved("nominalTime", "");
592        }
593    
594        public static String ph1_coord_nominalTime_echo_fixed() {
595            return "2009-03-06T010:00"; // Dummy resolution
596        }
597    
598        public static String ph1_coord_actionId_echo() {
599            return echoUnResolved("actionId", "");
600        }
601    
602        public static String ph1_coord_name_echo() {
603            return echoUnResolved("name", "");
604        }
605    
606        // The following echo functions are not used in any phases yet
607        // They are here for future purpose.
608        public static String coord_minutes_echo(String n) {
609            return echoUnResolved("minutes", n);
610        }
611    
612        public static String coord_hours_echo(String n) {
613            return echoUnResolved("hours", n);
614        }
615    
616        public static String coord_days_echo(String n) {
617            return echoUnResolved("days", n);
618        }
619    
620        public static String coord_endOfDay_echo(String n) {
621            return echoUnResolved("endOfDay", n);
622        }
623    
624        public static String coord_months_echo(String n) {
625            return echoUnResolved("months", n);
626        }
627    
628        public static String coord_endOfMonth_echo(String n) {
629            return echoUnResolved("endOfMonth", n);
630        }
631    
632        public static String coord_actualTime_echo() {
633            return echoUnResolved("actualTime", "");
634        }
635    
636        // This echo function will always return "24" for validation only.
637        // This evaluation ****should not**** replace the original XML
638        // Create a temporary string and validate the function
639        // This is **required** for evaluating an expression like
640        // coord:HoursInDay(0) + 3
641        // actual evaluation will happen in phase 2 or phase 3.
642        public static String ph1_coord_hoursInDay_echo(String n) {
643            return "24";
644            // return echoUnResolved("hoursInDay", n);
645        }
646    
647        // This echo function will always return "30" for validation only.
648        // This evaluation ****should not**** replace the original XML
649        // Create a temporary string and validate the function
650        // This is **required** for evaluating an expression like
651        // coord:daysInMonth(0) + 3
652        // actual evaluation will happen in phase 2 or phase 3.
653        public static String ph1_coord_daysInMonth_echo(String n) {
654            // return echoUnResolved("daysInMonth", n);
655            return "30";
656        }
657    
658        // This echo function will always return "3" for validation only.
659        // This evaluation ****should not**** replace the original XML
660        // Create a temporary string and validate the function
661        // This is **required** for evaluating an expression like coord:tzOffset + 2
662        // actual evaluation will happen in phase 2 or phase 3.
663        public static String ph1_coord_tzOffset_echo() {
664            // return echoUnResolved("tzOffset", "");
665            return "3";
666        }
667    
668        // Local methods
669        /**
670         * @param n
671         * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the
672         *         Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset.
673         * @throws Exception
674         */
675        private static String coord_current_sync(int n) throws Exception {
676            int datasetFrequency = getDSFrequency();// in minutes
677            TimeUnit dsTimeUnit = getDSTimeUnit();
678            int[] instCount = new int[1];// used as pass by ref
679            Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
680            if (nominalInstanceCal == null) {
681                return "";
682            }
683            nominalInstanceCal = getInitialInstanceCal();
684            int absInstanceCount = instCount[0] + n;
685            nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount);
686    
687            if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
688                return "";
689            }
690            String str = DateUtils.formatDateUTC(nominalInstanceCal);
691            return str;
692        }
693    
694        /**
695         * @param offset
696         * @return n-th available latest instance Date-Time for SYNC data-set
697         * @throws Exception
698         */
699        private static String coord_latest_sync(int offset) throws Exception {
700            if (offset > 0) {
701                throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0"
702                        + offset);
703            }
704            ELEvaluator eval = ELEvaluator.getCurrent();
705            String retVal = "";
706            int datasetFrequency = (int) getDSFrequency();// in minutes
707            TimeUnit dsTimeUnit = getDSTimeUnit();
708            int[] instCount = new int[1];
709            Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
710            if (nominalInstanceCal != null) {
711                Calendar initInstance = getInitialInstanceCal();
712                SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
713                if (ds == null) {
714                    throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
715                }
716                String uriTemplate = ds.getUriTemplate();
717                Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
718                if (conf == null) {
719                    throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
720                }
721                int available = 0;
722                boolean resolved = false;
723                String user = ParamChecker
724                        .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
725                String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME),
726                                                     OozieClient.GROUP_NAME);
727                String doneFlag = ds.getDoneFlag();
728                while (nominalInstanceCal.compareTo(initInstance) >= 0) {
729                    ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
730                    String uriPath = uriEval.evaluate(uriTemplate, String.class);
731                    String pathWithDoneFlag = uriPath;
732                    if (doneFlag.length() > 0) {
733                        pathWithDoneFlag += "/" + doneFlag;
734                    }
735                    if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
736                        XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag);
737                        if (available == offset) {
738                            XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
739                            resolved = true;
740                            retVal = DateUtils.formatDateUTC(nominalInstanceCal);
741                            eval.setVariable("resolved_path", uriPath);
742                            break;
743                        }
744    
745                        available--;
746                    }
747                    // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
748                    // -datasetFrequency);
749                    nominalInstanceCal = (Calendar) initInstance.clone();
750                    instCount[0]--;
751                    nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
752                    // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
753                }
754                if (!resolved) {
755                    // return unchanged latest function with variable 'is_resolved'
756                    // to 'false'
757                    eval.setVariable("is_resolved", Boolean.FALSE);
758                    retVal = "${coord:latest(" + offset + ")}";
759                }
760                else {
761                    eval.setVariable("is_resolved", Boolean.TRUE);
762                }
763            }
764            else {// No feasible nominal time
765                eval.setVariable("is_resolved", Boolean.FALSE);
766            }
767            return retVal;
768        }
769    
770        // TODO : Not an efficient way. In a loop environment, we could do something
771        // outside the loop
772        /**
773         * Check whether a URI path exists
774         *
775         * @param sPath
776         * @param conf
777         * @return
778         * @throws IOException
779         */
780    
781        private static boolean isPathAvailable(String sPath, String user, String group, Configuration conf)
782                throws IOException, HadoopAccessorException {
783            // sPath += "/" + END_OF_OPERATION_INDICATOR_FILE;
784            Path path = new Path(sPath);
785            return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
786                        new Configuration()).exists(path);
787        }
788    
789        /**
790         * @param tm
791         * @return a new Evaluator to be used for URI-template evaluation
792         */
793        private static ELEvaluator getUriEvaluator(Calendar tm) {
794            ELEvaluator retEval = new ELEvaluator();
795            retEval.setVariable("YEAR", tm.get(Calendar.YEAR));
796            retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm
797                    .get(Calendar.MONTH) + 1));
798            retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm
799                    .get(Calendar.DAY_OF_MONTH));
800            retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm
801                    .get(Calendar.HOUR_OF_DAY));
802            retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm
803                    .get(Calendar.MINUTE));
804            return retEval;
805        }
806    
807        /**
808         * @return whether a data set is SYNCH or ASYNC
809         */
810        private static boolean isSyncDataSet() {
811            ELEvaluator eval = ELEvaluator.getCurrent();
812            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
813            if (ds == null) {
814                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
815            }
816            return ds.getType().equalsIgnoreCase("SYNC");
817        }
818    
819        /**
820         * Check whether a function should be resolved.
821         *
822         * @param functionName
823         * @param n
824         * @return null if the functionName needs to be resolved otherwise return the calling function unresolved.
825         */
826        private static String checkIfResolved(String functionName, String n) {
827            ELEvaluator eval = ELEvaluator.getCurrent();
828            String replace = (String) eval.getVariable("resolve_" + functionName);
829            if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't
830                // resolve
831                // return "${coord:" + functionName + "(" + n +")}"; //Unresolved
832                eval.setVariable(".wrap", "true");
833                return "coord:" + functionName + "(" + n + ")"; // Unresolved
834            }
835            return null; // Resolved it
836        }
837    
838        private static String echoUnResolved(String functionName, String n) {
839            return echoUnResolvedPre(functionName, n, "coord:");
840        }
841    
842        private static String echoUnResolvedPre(String functionName, String n, String prefix) {
843            ELEvaluator eval = ELEvaluator.getCurrent();
844            eval.setVariable(".wrap", "true");
845            return prefix + functionName + "(" + n + ")"; // Unresolved
846        }
847    
848        /**
849         * @return the initial instance of a DataSet in DATE
850         */
851        private static Date getInitialInstance() {
852            return getInitialInstanceCal().getTime();
853            // return ds.getInitInstance();
854        }
855    
856        /**
857         * @return the initial instance of a DataSet in Calendar
858         */
859        private static Calendar getInitialInstanceCal() {
860            ELEvaluator eval = ELEvaluator.getCurrent();
861            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
862            if (ds == null) {
863                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
864            }
865            Calendar effInitTS = Calendar.getInstance();
866            effInitTS.setTime(ds.getInitInstance());
867            effInitTS.setTimeZone(ds.getTimeZone());
868            // To adjust EOD/EOM
869            DateUtils.moveToEnd(effInitTS, getDSEndOfFlag());
870            return effInitTS;
871            // return ds.getInitInstance();
872        }
873    
874        /**
875         * @return Nominal or action creation Time when all the dependencies of an application instance are met.
876         */
877        private static Date getActionCreationtime() {
878            ELEvaluator eval = ELEvaluator.getCurrent();
879            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
880            if (coordAction == null) {
881                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
882            }
883            return coordAction.getNominalTime();
884        }
885    
886        /**
887         * @return Actual Time when all the dependencies of an application instance are met.
888         */
889        private static Date getActualTime() {
890            ELEvaluator eval = ELEvaluator.getCurrent();
891            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
892            if (coordAction == null) {
893                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
894            }
895            return coordAction.getActualTime();
896        }
897    
898        /**
899         * @return TimeZone for the application or job.
900         */
901        private static TimeZone getJobTZ() {
902            ELEvaluator eval = ELEvaluator.getCurrent();
903            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
904            if (coordAction == null) {
905                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
906            }
907            return coordAction.getTimeZone();
908        }
909    
910        /**
911         * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time)
912         *
913         * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
914         *         the dataset.
915         */
916        private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) {
917            Date datasetInitialInstance = getInitialInstance();
918            TimeUnit dsTimeUnit = getDSTimeUnit();
919            TimeZone dsTZ = getDatasetTZ();
920            // Convert Date to Calendar for corresponding TZ
921            Calendar current = Calendar.getInstance();
922            current.setTime(datasetInitialInstance);
923            current.setTimeZone(dsTZ);
924    
925            Calendar calEffectiveTime = Calendar.getInstance();
926            calEffectiveTime.setTime(effectiveTime);
927            calEffectiveTime.setTimeZone(dsTZ);
928            instanceCount[0] = 0;
929            if (current.compareTo(calEffectiveTime) > 0) {
930                // Nominal Time < initial Instance
931                // TODO: getClass() call doesn't work from static method.
932                // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
933                // current.getTime());
934                return null;
935            }
936            Calendar origCurrent = (Calendar) current.clone();
937            while (current.compareTo(calEffectiveTime) <= 0) {
938                current = (Calendar) origCurrent.clone();
939                instanceCount[0]++;
940                current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency());
941            }
942            instanceCount[0]--;
943    
944            current = (Calendar) origCurrent.clone();
945            current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency());
946            return current;
947        }
948    
949        private static Calendar getEffectiveNominalTime() {
950            Date datasetInitialInstance = getInitialInstance();
951            TimeZone dsTZ = getDatasetTZ();
952            // Convert Date to Calendar for corresponding TZ
953            Calendar current = Calendar.getInstance();
954            current.setTime(datasetInitialInstance);
955            current.setTimeZone(dsTZ);
956    
957            Calendar calEffectiveTime = Calendar.getInstance();
958            calEffectiveTime.setTime(getActionCreationtime());
959            calEffectiveTime.setTimeZone(dsTZ);
960            if (current.compareTo(calEffectiveTime) > 0) {
961                // Nominal Time < initial Instance
962                // TODO: getClass() call doesn't work from static method.
963                // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
964                // current.getTime());
965                return null;
966            }
967            return calEffectiveTime;
968        }
969    
970        /**
971         * @return dataset frequency in minutes
972         */
973        private static int getDSFrequency() {
974            ELEvaluator eval = ELEvaluator.getCurrent();
975            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
976            if (ds == null) {
977                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
978            }
979            return ds.getFrequency();
980        }
981    
982        /**
983         * @return dataset TimeUnit
984         */
985        private static TimeUnit getDSTimeUnit() {
986            ELEvaluator eval = ELEvaluator.getCurrent();
987            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
988            if (ds == null) {
989                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
990            }
991            return ds.getTimeUnit();
992        }
993    
994        /**
995         * @return dataset TimeZone
996         */
997        private static TimeZone getDatasetTZ() {
998            ELEvaluator eval = ELEvaluator.getCurrent();
999            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1000            if (ds == null) {
1001                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1002            }
1003            return ds.getTimeZone();
1004        }
1005    
1006        /**
1007         * @return dataset TimeUnit
1008         */
1009        private static TimeUnit getDSEndOfFlag() {
1010            ELEvaluator eval = ELEvaluator.getCurrent();
1011            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1012            if (ds == null) {
1013                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1014            }
1015            return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration();
1016        }
1017    
1018    }