001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.coord;
016    
017    import java.util.Calendar;
018    import java.util.Date;
019    import java.util.HashMap;
020    import java.util.Iterator;
021    import java.util.List;
022    import java.util.Map;
023    import java.util.TimeZone;
024    
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.oozie.service.ELService;
027    import org.apache.oozie.service.Services;
028    import org.apache.oozie.util.DateUtils;
029    import org.apache.oozie.util.ELEvaluator;
030    import org.apache.oozie.util.XLog;
031    import org.apache.oozie.util.XmlUtils;
032    import org.jdom.Element;
033    
034    /**
035     * This class provide different evaluators required at different stages
036     */
037    public class CoordELEvaluator {
038        public static final Integer MINUTE = 1;
039        public static final Integer HOUR = 60 * MINUTE;
040    
041        /**
042         * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
043         * 1)
044         *
045         * @param conf : Configuration containing property variables
046         * @return configured ELEvaluator
047         */
048        public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
049            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
050            setConfigToEval(eval, conf);
051            return eval;
052        }
053    
054        /**
055         * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
056         *
057         * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
058         * @param appInst : Application Instance related information such as Action creation Time
059         * @param conf :Configuration to substitute any variables
060         * @return configured ELEvaluator
061         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
062         */
063        public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
064                throws Exception {
065            return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
066        }
067    
068        public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
069                                                             Configuration conf) throws Exception {
070            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
071            setConfigToEval(eval, conf);
072            SyncCoordDataset ds = getDSObject(event);
073            CoordELFunctions.configureEvaluator(eval, ds, appInst);
074            return eval;
075        }
076    
077        public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
078                                                               HashMap<String, String> dataNameList) throws Exception {
079            ELEvaluator eval = createELEvaluatorForGroup(conf, group);
080            for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
081                String key = it.next();
082                String value = dataNameList.get(key);
083                eval.setVariable("oozie.dataname." + key, value);
084            }
085            return eval;
086        }
087    
088        /**
089         * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
090         * should be resolved when all other data dependencies are met.
091         *
092         * @param actualTime : Action start time
093         * @param nominalTime : Action creation time
094         * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
095         * @param conf :Configuration to substitute any variables
096         * @return configured ELEvaluator
097         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
098         */
099        public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
100                throws Exception {
101            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
102            setConfigToEval(eval, conf);
103            SyncCoordDataset ds = getDSObject(dEvent);
104            SyncCoordAction appInst = new SyncCoordAction();// TODO:
105            appInst.setNominalTime(nominalTime);
106            appInst.setActualTime(actualTime);// TODO:
107            CoordELFunctions.configureEvaluator(eval, ds, appInst);
108            eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
109            return eval;
110        }
111    
112        /**
113         * Create a SLA evaluator to be used during Materialization
114         *
115         * @param nominalTime
116         * @param conf
117         * @return
118         * @throws Exception
119         */
120        public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
121            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
122            setConfigToEval(eval, conf);
123            SyncCoordAction appInst = new SyncCoordAction();// TODO:
124            appInst.setNominalTime(nominalTime);
125            CoordELFunctions.configureEvaluator(eval, null, appInst);
126            return eval;
127        }
128    
129        /**
130         * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
131         *
132         * @param eJob : XML element for the application instance
133         * @param conf :Configuration to substitute any variables
134         * @return configured ELEvaluator
135         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
136         */
137        public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
138            ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
139            setConfigToEval(e, conf);
140            SyncCoordAction appInst = new SyncCoordAction();
141            String strNominalTime = eJob.getAttributeValue("action-nominal-time");
142            if (strNominalTime != null) {
143                appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime));
144                appInst.setActionId(actionId);
145                appInst.setName(eJob.getAttributeValue("name"));
146            }
147            CoordELFunctions.configureEvaluator(e, null, appInst);
148            Element events = eJob.getChild("input-events", eJob.getNamespace());
149            if (events != null) {
150                for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
151                    if (data.getChild("uris", data.getNamespace()) != null) {
152                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
153                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
154                        e.setVariable(".datain." + data.getAttributeValue("name"), uris);
155                    }
156                    else {
157                    }
158                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
159                        e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
160                        // check
161                        // null
162                    }
163                }
164            }
165            events = eJob.getChild("output-events", eJob.getNamespace());
166            if (events != null) {
167                for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
168                    if (data.getChild("uris", data.getNamespace()) != null) {
169                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
170                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
171                        e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
172                    }
173                    else {
174                    }// TODO
175                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
176                        e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
177                        // check
178                        // null
179                    }
180                }
181            }
182            return e;
183        }
184    
185        /**
186         * Create a new Evaluator to resolve URI temple with time specific constant
187         *
188         * @param strDate : Date-time
189         * @return configured ELEvaluator
190         * @throws Exception If there is any date-time string in wrong format, the exception is thrown
191         */
192        public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
193            ELEvaluator eval = new ELEvaluator();
194            Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC
195            // always???
196            date.setTime(DateUtils.parseDateUTC(strDate));
197            eval.setVariable("YEAR", date.get(Calendar.YEAR));
198            eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
199            eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
200            eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
201            eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
202            return eval;
203        }
204    
205        /**
206         * Create Dataset object using the Dataset XML information
207         *
208         * @param eData
209         * @return
210         * @throws Exception
211         */
212        private static SyncCoordDataset getDSObject(Element eData) throws Exception {
213            SyncCoordDataset ds = new SyncCoordDataset();
214            Element eDataset = eData.getChild("dataset", eData.getNamespace());
215            // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
216            Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance"));
217            ds.setInitInstance(initInstance);
218            if (eDataset.getAttributeValue("frequency") != null) {
219                int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
220                ds.setFrequency(frequency);
221                ds.setType("SYNC");
222                if (eDataset.getAttributeValue("freq_timeunit") == null) {
223                    throw new RuntimeException("No freq_timeunit defined in data set definition\n"
224                            + XmlUtils.prettyPrint(eDataset));
225                }
226                ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
227                if (eDataset.getAttributeValue("timezone") == null) {
228                    throw new RuntimeException("No timezone defined in data set definition\n"
229                            + XmlUtils.prettyPrint(eDataset));
230                }
231                ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
232                if (eDataset.getAttributeValue("end_of_duration") == null) {
233                    throw new RuntimeException("No end_of_duration defined in data set definition\n"
234                            + XmlUtils.prettyPrint(eDataset));
235                }
236                ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
237    
238                Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
239                String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
240                ds.setDoneFlag(doneFlag);
241            }
242            else {
243                ds.setType("ASYNC");
244            }
245            String name = eDataset.getAttributeValue("name");
246            ds.setName(name);
247            // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
248            // eData.getNamespace()));
249            String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
250            ds.setUriTemplate(uriTemplate);
251            // ds.setTimeUnit(TimeUnit.MINUTES);
252            return ds;
253        }
254    
255        /**
256         * Set all job configurations properties into evaluator.
257         *
258         * @param eval : Evaluator to set variables
259         * @param conf : configurations to set Evaluator
260         */
261        private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
262            for (Map.Entry<String, String> entry : conf) {
263                eval.setVariable(entry.getKey(), entry.getValue());
264            }
265        }
266    
267        /**
268         * make any one digit number to two digit string pre-appending a"0"
269         *
270         * @param num : number to make sting
271         * @return :String of length at least two digit.
272         */
273        private static String make2Digits(int num) {
274            String ret = "" + num;
275            if (num <= 9) {
276                ret = "0" + ret;
277            }
278            return ret;
279        }
280    }