001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.coord;
016    
017    import java.util.Calendar;
018    import java.util.Date;
019    import java.util.HashMap;
020    import java.util.Iterator;
021    import java.util.List;
022    import java.util.Map;
023    import java.util.TimeZone;
024    
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.oozie.service.ELService;
027    import org.apache.oozie.service.Services;
028    import org.apache.oozie.util.DateUtils;
029    import org.apache.oozie.util.ELEvaluator;
030    import org.apache.oozie.util.XLog;
031    import org.apache.oozie.util.XmlUtils;
032    import org.jdom.Element;
033    
034    /**
035     * This class provide different evaluators required at different stages
036     */
037    public class CoordELEvaluator {
038        public static final Integer MINUTE = 1;
039        public static final Integer HOUR = 60 * MINUTE;
040    
041        /**
042         * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
043         * 1)
044         *
045         * @param conf : Configuration containing property variables
046         * @return configured ELEvaluator
047         */
048        public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
049            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
050            setConfigToEval(eval, conf);
051            return eval;
052        }
053    
054        /**
055         * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
056         *
057         * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
058         * @param appInst : Application Instance related information such as Action creation Time
059         * @param conf :Configuration to substitute any variables
060         * @return configured ELEvaluator
061         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
062         */
063        public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
064                throws Exception {
065            return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
066        }
067    
068        public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
069                                                             Configuration conf) throws Exception {
070            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
071            setConfigToEval(eval, conf);
072            SyncCoordDataset ds = getDSObject(event);
073            CoordELFunctions.configureEvaluator(eval, ds, appInst);
074            return eval;
075        }
076    
077        public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
078                                                               HashMap<String, String> dataNameList) throws Exception {
079            ELEvaluator eval = createELEvaluatorForGroup(conf, group);
080            for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
081                String key = it.next();
082                String value = dataNameList.get(key);
083                eval.setVariable("oozie.dataname." + key, value);
084            }
085            return eval;
086        }
087    
088        /**
089         * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
090         * should be resolved when all other data dependencies are met.
091         *
092         * @param actualTime : Action start time
093         * @param nominalTime : Action creation time
094         * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
095         * @param conf :Configuration to substitute any variables
096         * @return configured ELEvaluator
097         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
098         */
099        public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
100                throws Exception {
101            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
102            setConfigToEval(eval, conf);
103            SyncCoordDataset ds = getDSObject(dEvent);
104            SyncCoordAction appInst = new SyncCoordAction();// TODO:
105            appInst.setNominalTime(nominalTime);
106            appInst.setActualTime(actualTime);// TODO:
107            CoordELFunctions.configureEvaluator(eval, ds, appInst);
108            // Configuration tmpConf = new Configuration();
109            Configuration tmpConf = CoordUtils.getHadoopConf(conf);
110            // TODO:Set hadoop properties
111            eval.setVariable(CoordELFunctions.CONFIGURATION, tmpConf);
112            return eval;
113        }
114    
115        /**
116         * Create a SLA evaluator to be used during Materialization
117         *
118         * @param nominalTime
119         * @param conf
120         * @return
121         * @throws Exception
122         */
123        public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
124            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
125            setConfigToEval(eval, conf);
126            SyncCoordAction appInst = new SyncCoordAction();// TODO:
127            appInst.setNominalTime(nominalTime);
128            CoordELFunctions.configureEvaluator(eval, null, appInst);
129            return eval;
130        }
131    
132        /**
133         * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
134         *
135         * @param eJob : XML element for the application instance
136         * @param conf :Configuration to substitute any variables
137         * @return configured ELEvaluator
138         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
139         */
140        public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
141            ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
142            setConfigToEval(e, conf);
143            SyncCoordAction appInst = new SyncCoordAction();
144            String strNominalTime = eJob.getAttributeValue("action-nominal-time");
145            if (strNominalTime != null) {
146                appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime));
147                appInst.setActionId(actionId);
148                appInst.setName(eJob.getAttributeValue("name"));
149            }
150            CoordELFunctions.configureEvaluator(e, null, appInst);
151            Element events = eJob.getChild("input-events", eJob.getNamespace());
152            if (events != null) {
153                for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
154                    if (data.getChild("uris", data.getNamespace()) != null) {
155                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
156                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
157                        e.setVariable(".datain." + data.getAttributeValue("name"), uris);
158                    }
159                    else {
160                    }
161                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
162                        e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
163                        // check
164                        // null
165                    }
166                }
167            }
168            events = eJob.getChild("output-events", eJob.getNamespace());
169            if (events != null) {
170                for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
171                    if (data.getChild("uris", data.getNamespace()) != null) {
172                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
173                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
174                        e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
175                    }
176                    else {
177                    }// TODO
178                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
179                        e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
180                        // check
181                        // null
182                    }
183                }
184            }
185            return e;
186        }
187    
188        /**
189         * Create a new Evaluator to resolve URI temple with time specific constant
190         *
191         * @param strDate : Date-time
192         * @return configured ELEvaluator
193         * @throws Exception If there is any date-time string in wrong format, the exception is thrown
194         */
195        public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
196            ELEvaluator eval = new ELEvaluator();
197            Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC
198            // always???
199            date.setTime(DateUtils.parseDateUTC(strDate));
200            eval.setVariable("YEAR", date.get(Calendar.YEAR));
201            eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
202            eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
203            eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
204            eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
205            return eval;
206        }
207    
208        /**
209         * Create Dataset object using the Dataset XML information
210         *
211         * @param eData
212         * @return
213         * @throws Exception
214         */
215        private static SyncCoordDataset getDSObject(Element eData) throws Exception {
216            SyncCoordDataset ds = new SyncCoordDataset();
217            Element eDataset = eData.getChild("dataset", eData.getNamespace());
218            // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
219            Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance"));
220            ds.setInitInstance(initInstance);
221            if (eDataset.getAttributeValue("frequency") != null) {
222                int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
223                ds.setFrequency(frequency);
224                ds.setType("SYNC");
225                if (eDataset.getAttributeValue("freq_timeunit") == null) {
226                    throw new RuntimeException("No freq_timeunit defined in data set definition\n"
227                            + XmlUtils.prettyPrint(eDataset));
228                }
229                ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
230                if (eDataset.getAttributeValue("timezone") == null) {
231                    throw new RuntimeException("No timezone defined in data set definition\n"
232                            + XmlUtils.prettyPrint(eDataset));
233                }
234                ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
235                if (eDataset.getAttributeValue("end_of_duration") == null) {
236                    throw new RuntimeException("No end_of_duration defined in data set definition\n"
237                            + XmlUtils.prettyPrint(eDataset));
238                }
239                ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
240    
241                Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
242                String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
243                ds.setDoneFlag(doneFlag);
244            }
245            else {
246                ds.setType("ASYNC");
247            }
248            String name = eDataset.getAttributeValue("name");
249            ds.setName(name);
250            // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
251            // eData.getNamespace()));
252            String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
253            ds.setUriTemplate(uriTemplate);
254            // ds.setTimeUnit(TimeUnit.MINUTES);
255            return ds;
256        }
257    
258        /**
259         * Set all job configurations properties into evaluator.
260         *
261         * @param eval : Evaluator to set variables
262         * @param conf : configurations to set Evaluator
263         */
264        private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
265            for (Map.Entry<String, String> entry : conf) {
266                eval.setVariable(entry.getKey(), entry.getValue());
267            }
268        }
269    
270        /**
271         * make any one digit number to two digit string pre-appending a"0"
272         *
273         * @param num : number to make sting
274         * @return :String of length at least two digit.
275         */
276        private static String make2Digits(int num) {
277            String ret = "" + num;
278            if (num <= 9) {
279                ret = "0" + ret;
280            }
281            return ret;
282        }
283    }