001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.coord;
016    
017    import java.io.StringReader;
018    import java.util.Date;
019    import java.util.List;
020    
021    import org.apache.hadoop.conf.Configuration;
022    import org.apache.oozie.CoordinatorActionBean;
023    import org.apache.oozie.ErrorCode;
024    import org.apache.oozie.client.CoordinatorAction;
025    import org.apache.oozie.command.CommandException;
026    import org.apache.oozie.coord.CoordELEvaluator;
027    import org.apache.oozie.coord.CoordELFunctions;
028    import org.apache.oozie.coord.CoordUtils;
029    import org.apache.oozie.coord.CoordinatorJobException;
030    import org.apache.oozie.coord.SyncCoordAction;
031    import org.apache.oozie.coord.TimeUnit;
032    import org.apache.oozie.service.Services;
033    import org.apache.oozie.service.UUIDService;
034    import org.apache.oozie.util.DateUtils;
035    import org.apache.oozie.util.ELEvaluator;
036    import org.apache.oozie.util.XConfiguration;
037    import org.apache.oozie.util.XmlUtils;
038    import org.jdom.Element;
039    
040    public class CoordCommandUtils {
041        public static int CURRENT = 0;
042        public static int LATEST = 1;
043        public static int FUTURE = 2;
044        public static int UNEXPECTED = -1;
045        public static final String RESOLVED_UNRESOLVED_SEPARATOR = ";";
046    
047        /**
048         * parse a function like coord:latest(n)/future() and return the 'n'.
049         * <p/>
050         * @param function
051         * @param event
052         * @param appInst
053         * @param conf
054         * @param restArg
055         * @return int instanceNumber
056         * @throws Exception
057         */
058        public static int getInstanceNumber(String function, Element event, SyncCoordAction appInst, Configuration conf,
059                StringBuilder restArg) throws Exception {
060            ELEvaluator eval = CoordELEvaluator
061                    .createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf);
062            String newFunc = CoordELFunctions.evalAndWrap(eval, function);
063            int funcType = getFuncType(newFunc);
064            if (funcType == CURRENT || funcType == LATEST) {
065                return parseOneArg(newFunc);
066            }
067            else {
068                return parseMoreArgs(newFunc, restArg);
069            }
070        }
071    
072        private static int parseOneArg(String funcName) throws Exception {
073            int firstPos = funcName.indexOf("(");
074            int lastPos = funcName.lastIndexOf(")");
075            if (firstPos >= 0 && lastPos > firstPos) {
076                String tmp = funcName.substring(firstPos + 1, lastPos).trim();
077                if (tmp.length() > 0) {
078                    return Integer.parseInt(tmp);
079                }
080            }
081            throw new RuntimeException("Unformatted function :" + funcName);
082        }
083    
084        private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception {
085            int firstPos = funcName.indexOf("(");
086            int secondPos = funcName.lastIndexOf(",");
087            int lastPos = funcName.lastIndexOf(")");
088            if (firstPos >= 0 && secondPos > firstPos) {
089                String tmp = funcName.substring(firstPos + 1, secondPos).trim();
090                if (tmp.length() > 0) {
091                    restArg.append(funcName.substring(secondPos + 1, lastPos).trim());
092                    return Integer.parseInt(tmp);
093                }
094            }
095            throw new RuntimeException("Unformatted function :" + funcName);
096        }
097    
098        /**
099         * @param EL function name
100         * @return type of EL function
101         */
102        public static int getFuncType(String function) {
103            if (function.indexOf("current") >= 0) {
104                return CURRENT;
105            }
106            else if (function.indexOf("latest") >= 0) {
107                return LATEST;
108            }
109            else if (function.indexOf("future") >= 0) {
110                return FUTURE;
111            }
112            return UNEXPECTED;
113            // throw new RuntimeException("Unexpected instance name "+ function);
114        }
115    
116        /**
117         * @param startInst: EL function name
118         * @param endInst: EL function name
119         * @throws CommandException if both are not the same function
120         */
121        public static void checkIfBothSameType(String startInst, String endInst) throws CommandException {
122            if (getFuncType(startInst) != getFuncType(endInst)) {
123                throw new CommandException(ErrorCode.E1010,
124                        " start-instance and end-instance both should be either latest or current or future\n"
125                                + " start " + startInst + " and end " + endInst);
126            }
127        }
128    
129        /**
130         * Resolve list of <instance> </instance> tags.
131         *
132         * @param event
133         * @param instances
134         * @param actionInst
135         * @param conf
136         * @param eval: ELEvalautor
137         * @throws Exception
138         */
139        public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst,
140                Configuration conf, ELEvaluator eval) throws Exception {
141            for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) {
142                if (instances.length() > 0) {
143                    instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
144                }
145                instances.append(materializeInstance(event, eInstance.getTextTrim(), actionInst, conf, eval));
146            }
147            event.removeChildren("instance", event.getNamespace());
148        }
149    
150        /**
151         * Resolve <start-instance> <end-insatnce> tag. Don't resolve any
152         * latest()/future()
153         *
154         * @param event
155         * @param instances
156         * @param appInst
157         * @param conf
158         * @param eval: ELEvalautor
159         * @throws Exception
160         */
161        public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst,
162                Configuration conf, ELEvaluator eval) throws Exception {
163            Element eStartInst = event.getChild("start-instance", event.getNamespace());
164            Element eEndInst = event.getChild("end-instance", event.getNamespace());
165            if (eStartInst != null && eEndInst != null) {
166                String strStart = eStartInst.getTextTrim();
167                String strEnd = eEndInst.getTextTrim();
168                checkIfBothSameType(strStart, strEnd);
169                StringBuilder restArg = new StringBuilder(); // To store rest
170                                                             // arguments for
171                                                             // future
172                                                             // function
173                int startIndex = getInstanceNumber(strStart, event, appInst, conf, restArg);
174                restArg.delete(0, restArg.length());
175                int endIndex = getInstanceNumber(strEnd, event, appInst, conf, restArg);
176                if (startIndex > endIndex) {
177                    throw new CommandException(ErrorCode.E1010,
178                            " start-instance should be equal or earlier than the end-instance \n"
179                                    + XmlUtils.prettyPrint(event));
180                }
181                int funcType = getFuncType(strStart);
182                if (funcType == CURRENT) {
183                    // Everything could be resolved NOW. no latest() ELs
184                    for (int i = endIndex; i >= startIndex; i--) {
185                        String matInstance = materializeInstance(event, "${coord:current(" + i + ")}", appInst, conf, eval);
186                        if (matInstance == null || matInstance.length() == 0) {
187                            // Earlier than dataset's initial instance
188                            break;
189                        }
190                        if (instances.length() > 0) {
191                            instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
192                        }
193                        instances.append(matInstance);
194                    }
195                }
196                else { // latest(n)/future() EL is present
197                    for (; startIndex <= endIndex; startIndex++) {
198                        if (instances.length() > 0) {
199                            instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
200                        }
201                        if (funcType == LATEST) {
202                            instances.append("${coord:latest(" + startIndex + ")}");
203                        }
204                        else { // For future
205                            instances.append("${coord:future(" + startIndex + ",'" + restArg + "')}");
206                        }
207                    }
208                }
209                // Remove start-instance and end-instances
210                event.removeChild("start-instance", event.getNamespace());
211                event.removeChild("end-instance", event.getNamespace());
212            }
213        }
214    
215        /**
216         * Materialize one instance like current(-2)
217         *
218         * @param event : <data-in>
219         * @param expr : instance like current(-1)
220         * @param appInst : application specific info
221         * @param conf
222         * @param evalInst :ELEvaluator
223         * @return materialized date string
224         * @throws Exception
225         */
226        public static String materializeInstance(Element event, String expr, SyncCoordAction appInst, Configuration conf,
227                ELEvaluator evalInst) throws Exception {
228            if (event == null) {
229                return null;
230            }
231            // ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event,
232            // appInst, conf);
233            return CoordELFunctions.evalAndWrap(evalInst, expr);
234        }
235    
236        /**
237         * Create two new tags with <uris> and <unresolved-instances>.
238         *
239         * @param event
240         * @param instances
241         * @param dependencyList
242         * @throws Exception
243         */
244        public static void separateResolvedAndUnresolved(Element event, StringBuilder instances, StringBuffer dependencyList)
245                throws Exception {
246            StringBuilder unresolvedInstances = new StringBuilder();
247            StringBuilder urisWithDoneFlag = new StringBuilder();
248            String uris = createEarlyURIs(event, instances.toString(), unresolvedInstances, urisWithDoneFlag);
249            if (uris.length() > 0) {
250                Element uriInstance = new Element("uris", event.getNamespace());
251                uriInstance.addContent(uris);
252                event.getContent().add(1, uriInstance);
253                if (dependencyList.length() > 0) {
254                    dependencyList.append(CoordELFunctions.INSTANCE_SEPARATOR);
255                }
256                dependencyList.append(urisWithDoneFlag);
257            }
258            if (unresolvedInstances.length() > 0) {
259                Element elemInstance = new Element("unresolved-instances", event.getNamespace());
260                elemInstance.addContent(unresolvedInstances.toString());
261                event.getContent().add(1, elemInstance);
262            }
263        }
264    
265        /**
266         * The function create a list of URIs separated by "," using the instances
267         * time stamp and URI-template
268         *
269         * @param event : <data-in> event
270         * @param instances : List of time stamp separated by ","
271         * @param unresolvedInstances : list of instance with latest function
272         * @param urisWithDoneFlag : list of URIs with the done flag appended
273         * @return : list of URIs separated by ";" as a string.
274         * @throws Exception
275         */
276        public static String createEarlyURIs(Element event, String instances, StringBuilder unresolvedInstances,
277                StringBuilder urisWithDoneFlag) throws Exception {
278            if (instances == null || instances.length() == 0) {
279                return "";
280            }
281            String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR);
282            StringBuilder uris = new StringBuilder();
283    
284            Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag",
285                    event.getNamespace());
286            String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
287    
288            for (int i = 0; i < instanceList.length; i++) {
289                if(instanceList[i].trim().length() == 0) {
290                    continue;
291                }
292                int funcType = getFuncType(instanceList[i]);
293                if (funcType == LATEST || funcType == FUTURE) {
294                    if (unresolvedInstances.length() > 0) {
295                        unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR);
296                    }
297                    unresolvedInstances.append(instanceList[i]);
298                    continue;
299                }
300                ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]);
301                if (uris.length() > 0) {
302                    uris.append(CoordELFunctions.INSTANCE_SEPARATOR);
303                    urisWithDoneFlag.append(CoordELFunctions.INSTANCE_SEPARATOR);
304                }
305    
306                String uriPath = CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace())
307                        .getChild("uri-template", event.getNamespace()).getTextTrim());
308                uris.append(uriPath);
309                if (doneFlag.length() > 0) {
310                    uriPath += "/" + doneFlag;
311                }
312                urisWithDoneFlag.append(uriPath);
313            }
314            return uris.toString();
315        }
316    
317        /**
318         * @param eSla
319         * @param nominalTime
320         * @param conf
321         * @return boolean to determine whether the SLA element is present or not
322         * @throws CoordinatorJobException
323         */
324        public static boolean materializeSLA(Element eSla, Date nominalTime, Configuration conf)
325                throws CoordinatorJobException {
326            if (eSla == null) {
327                // eAppXml.getNamespace("sla"));
328                return false;
329            }
330            try {
331                ELEvaluator evalSla = CoordELEvaluator.createSLAEvaluator(nominalTime, conf);
332                List<Element> elemList = eSla.getChildren();
333                for (Element elem : elemList) {
334                    String updated;
335                    try {
336                        updated = CoordELFunctions.evalAndWrap(evalSla, elem.getText().trim());
337                    }
338                    catch (Exception e) {
339                        throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
340                    }
341                    elem.removeContent();
342                    elem.addContent(updated);
343                }
344            }
345            catch (Exception e) {
346                throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
347            }
348            return true;
349        }
350    
351        /**
352         * Materialize one instance for specific nominal time. It includes: 1.
353         * Materialize data events (i.e. <data-in> and <data-out>) 2. Materialize
354         * data properties (i.e dataIn(<DS>) and dataOut(<DS>) 3. remove 'start' and
355         * 'end' tag 4. Add 'instance_number' and 'nominal-time' tag
356         *
357         * @param jobId coordinator job id
358         * @param dryrun true if it is dryrun
359         * @param eAction frequency unexploded-job
360         * @param nominalTime materialization time
361         * @param instanceCount instance numbers
362         * @param conf job configuration
363         * @param actionBean CoordinatorActionBean to materialize
364         * @return one materialized action for specific nominal time
365         * @throws Exception
366         */
367        @SuppressWarnings("unchecked")
368        public static String materializeOneInstance(String jobId, boolean dryrun, Element eAction, Date nominalTime,
369                int instanceCount, Configuration conf, CoordinatorActionBean actionBean) throws Exception {
370            String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, instanceCount + "");
371            SyncCoordAction appInst = new SyncCoordAction();
372            appInst.setActionId(actionId);
373            appInst.setName(eAction.getAttributeValue("name"));
374            appInst.setNominalTime(nominalTime);
375            int frequency = Integer.parseInt(eAction.getAttributeValue("frequency"));
376            appInst.setFrequency(frequency);
377            appInst.setTimeUnit(TimeUnit.valueOf(eAction.getAttributeValue("freq_timeunit")));
378            appInst.setTimeZone(DateUtils.getTimeZone(eAction.getAttributeValue("timezone")));
379            appInst.setEndOfDuration(TimeUnit.valueOf(eAction.getAttributeValue("end_of_duration")));
380    
381            StringBuffer dependencyList = new StringBuffer();
382    
383            Element inputList = eAction.getChild("input-events", eAction.getNamespace());
384            List<Element> dataInList = null;
385            if (inputList != null) {
386                dataInList = inputList.getChildren("data-in", eAction.getNamespace());
387                materializeDataEvents(dataInList, appInst, conf, dependencyList);
388            }
389    
390            Element outputList = eAction.getChild("output-events", eAction.getNamespace());
391            List<Element> dataOutList = null;
392            if (outputList != null) {
393                dataOutList = outputList.getChildren("data-out", eAction.getNamespace());
394                StringBuffer tmp = new StringBuffer();
395                // no dependency checks
396                materializeDataEvents(dataOutList, appInst, conf, tmp);
397            }
398    
399            eAction.removeAttribute("start");
400            eAction.removeAttribute("end");
401            eAction.setAttribute("instance-number", Integer.toString(instanceCount));
402            eAction.setAttribute("action-nominal-time", DateUtils.formatDateUTC(nominalTime));
403    
404            boolean isSla = CoordCommandUtils.materializeSLA(eAction.getChild("action", eAction.getNamespace()).getChild(
405                    "info", eAction.getNamespace("sla")), nominalTime, conf);
406    
407            // Setting up action bean
408            actionBean.setCreatedConf(XmlUtils.prettyPrint(conf).toString());
409            actionBean.setRunConf(XmlUtils.prettyPrint(conf).toString());
410            actionBean.setCreatedTime(new Date());
411            actionBean.setJobId(jobId);
412            actionBean.setId(actionId);
413            actionBean.setLastModifiedTime(new Date());
414            actionBean.setStatus(CoordinatorAction.Status.WAITING);
415            actionBean.setActionNumber(instanceCount);
416            actionBean.setMissingDependencies(dependencyList.toString());
417            actionBean.setNominalTime(nominalTime);
418            if (isSla == true) {
419                actionBean.setSlaXml(XmlUtils.prettyPrint(
420                        eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla")))
421                        .toString());
422            }
423    
424            // actionBean.setTrackerUri(trackerUri);//TOOD:
425            // actionBean.setConsoleUrl(consoleUrl); //TODO:
426            // actionBean.setType(type);//TODO:
427            // actionBean.setErrorInfo(errorCode, errorMessage); //TODO:
428            // actionBean.setExternalStatus(externalStatus);//TODO
429            if (!dryrun) {
430                return XmlUtils.prettyPrint(eAction).toString();
431            }
432            else {
433                String action = XmlUtils.prettyPrint(eAction).toString();
434                CoordActionInputCheckCommand coordActionInput = new CoordActionInputCheckCommand(actionBean.getId());
435                StringBuilder actionXml = new StringBuilder(action);
436                StringBuilder existList = new StringBuilder();
437                StringBuilder nonExistList = new StringBuilder();
438                StringBuilder nonResolvedList = new StringBuilder();
439                getResolvedList(actionBean.getMissingDependencies(), nonExistList, nonResolvedList);
440                Date actualTime = new Date();
441                Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
442                coordActionInput.checkInput(actionXml, existList, nonExistList, actionConf, actualTime);
443                return actionXml.toString();
444            }
445        }
446    
447        /**
448         * Materialize all <input-events>/<data-in> or <output-events>/<data-out>
449         * tags Create uris for resolved instances. Create unresolved instance for
450         * latest()/future().
451         *
452         * @param events
453         * @param appInst
454         * @param conf
455         * @throws Exception
456         */
457        public static void materializeDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf,
458                StringBuffer dependencyList) throws Exception {
459    
460            if (events == null) {
461                return;
462            }
463            StringBuffer unresolvedList = new StringBuffer();
464            for (Element event : events) {
465                StringBuilder instances = new StringBuilder();
466                ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
467                // Handle list of instance tag
468                resolveInstances(event, instances, appInst, conf, eval);
469                // Handle start-instance and end-instance
470                resolveInstanceRange(event, instances, appInst, conf, eval);
471                // Separate out the unresolved instances
472                separateResolvedAndUnresolved(event, instances, dependencyList);
473                String tmpUnresolved = event.getChildTextTrim("unresolved-instances", event.getNamespace());
474                if (tmpUnresolved != null) {
475                    if (unresolvedList.length() > 0) {
476                        unresolvedList.append(CoordELFunctions.INSTANCE_SEPARATOR);
477                    }
478                    unresolvedList.append(tmpUnresolved);
479                }
480            }
481            if (unresolvedList.length() > 0) {
482                dependencyList.append(RESOLVED_UNRESOLVED_SEPARATOR);
483                dependencyList.append(unresolvedList);
484            }
485            return;
486        }
487    
488        /**
489         * Get resolved string from missDepList
490         *
491         * @param missDepList
492         * @param resolved
493         * @param unresolved
494         * @return resolved string
495         */
496        public static String getResolvedList(String missDepList, StringBuilder resolved, StringBuilder unresolved) {
497            if (missDepList != null) {
498                int index = missDepList.indexOf(RESOLVED_UNRESOLVED_SEPARATOR);
499                if (index < 0) {
500                    resolved.append(missDepList);
501                }
502                else {
503                    resolved.append(missDepList.substring(0, index));
504                    unresolved.append(missDepList.substring(index + 1));
505                }
506            }
507            return resolved.toString();
508        }
509    
510    }