001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.coord;
016    
017    import org.apache.hadoop.conf.Configuration;
018    
019    import org.apache.oozie.client.CoordinatorAction;
020    import org.apache.oozie.client.OozieClient;
021    import org.apache.oozie.CoordinatorActionBean;
022    import org.apache.oozie.DagEngine;
023    import org.apache.oozie.DagEngineException;
024    import org.apache.oozie.ErrorCode;
025    import org.apache.oozie.command.CommandException;
026    import org.apache.oozie.service.DagEngineService;
027    import org.apache.oozie.store.StoreException;
028    import org.apache.oozie.store.CoordinatorStore;
029    import org.apache.oozie.service.Services;
030    import org.apache.oozie.util.JobUtils;
031    import org.apache.oozie.util.ParamChecker;
032    import org.apache.oozie.util.XLog;
033    import org.apache.oozie.util.XmlUtils;
034    import org.apache.oozie.util.XConfiguration;
035    import org.apache.oozie.util.db.SLADbOperations;
036    import org.apache.oozie.client.SLAEvent.SlaAppType;
037    import org.apache.oozie.client.SLAEvent.Status;
038    
039    import org.jdom.Element;
040    import org.jdom.JDOMException;
041    
042    import java.io.IOException;
043    import java.io.StringReader;
044    
045    public class CoordActionStartCommand extends CoordinatorCommand<Void> {
046    
047        public static final String EL_ERROR = "EL_ERROR";
048        public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR";
049        public static final String COULD_NOT_START = "COULD_NOT_START";
050        public static final String START_DATA_MISSING = "START_DATA_MISSING";
051        public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING";
052    
053        private final XLog log = XLog.getLog(getClass());
054        private String actionId = null;
055        private String user = null;
056        private String authToken = null;
057        private CoordinatorActionBean coordAction = null;
058    
059        public CoordActionStartCommand(String id, String user, String token) {
060            super("coord_action_start", "coord_action_start", 1, XLog.OPS);
061            this.actionId = ParamChecker.notEmpty(id, "id");
062            this.user = ParamChecker.notEmpty(user, "user");
063            this.authToken = ParamChecker.notEmpty(token, "token");
064        }
065    
066        /**
067         * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
068         * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
069         * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
070         * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
071         * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
072         * Merge Action createdConf with actionXml to create new runConf with replaced variables
073         *
074         * @param action CoordinatorActionBean
075         * @return Configuration
076         * @throws CommandException
077         */
078        private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
079            String createdConf = action.getCreatedConf();
080            String actionXml = action.getActionXml();
081            Element workflowProperties = null;
082            try {
083                workflowProperties = XmlUtils.parseXml(actionXml);
084            }
085            catch (JDOMException e1) {
086                log.warn("Configuration parse error in:" + actionXml);
087                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
088            }
089            // generate the 'runConf' for this action
090            // Step 1: runConf = createdConf
091            Configuration runConf = null;
092            try {
093                runConf = new XConfiguration(new StringReader(createdConf));
094            }
095            catch (IOException e1) {
096                log.warn("Configuration parse error in:" + createdConf);
097                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
098            }
099            // Step 2: Merge local properties into runConf
100            // extract 'property' tags under 'configuration' block in the
101            // coordinator.xml (saved in actionxml column)
102            // convert Element to XConfiguration
103            Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace())
104                    .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
105                                                                                      workflowProperties.getNamespace());
106            if (configElement != null) {
107                String strConfig = XmlUtils.prettyPrint(configElement).toString();
108                Configuration localConf;
109                try {
110                    localConf = new XConfiguration(new StringReader(strConfig));
111                }
112                catch (IOException e1) {
113                    log.warn("Configuration parse error in:" + strConfig);
114                    throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
115                }
116    
117                // copy configuration properties in coordinator.xml to the runConf
118                XConfiguration.copy(localConf, runConf);
119            }
120    
121            // Step 3: Extract value of 'app-path' in actionxml, and save it as a
122            // new property called 'oozie.wf.application.path'
123            // WF Engine requires the path to the workflow.xml to be saved under
124            // this property name
125            String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow",
126                                                                                                               workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue();
127            runConf.set("oozie.wf.application.path", appPath);
128            return runConf;
129        }
130    
131        protected Void call(CoordinatorStore store) throws StoreException, CommandException {
132            boolean makeFail = true;
133            String errCode = "";
134            String errMsg = "";
135            ParamChecker.notEmpty(user, "user");
136            ParamChecker.notEmpty(authToken, "authToken");
137    
138            // CoordinatorActionBean coordAction = store.getCoordinatorAction(id, true);
139            log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
140            if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
141                // log.debug("getting.. job id: " + coordAction.getJobId());
142                // create merged runConf to pass to WF Engine
143                Configuration runConf = mergeConfig(coordAction);
144                coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
145                // log.debug("%%% merged runconf=" +
146                // XmlUtils.prettyPrint(runConf).toString());
147                DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
148                try {
149                    boolean startJob = true;
150                    Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
151                    SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), store, Status.STARTED,
152                                                    SlaAppType.COORDINATOR_ACTION);
153    
154                    // Normalize workflow appPath here;
155                    JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
156                    
157                    String wfId = dagEngine.submitJob(conf, startJob);
158                    coordAction.setStatus(CoordinatorAction.Status.RUNNING);
159                    coordAction.setExternalId(wfId);
160                    store.updateCoordinatorAction(coordAction);
161                    makeFail = false;
162                }
163                catch (StoreException se) {
164                    makeFail = false;
165                    throw se;
166                }
167                catch (DagEngineException dee) {
168                    errMsg = dee.getMessage();
169                    errCode = "E1005";
170                    log.warn("can not create DagEngine for submitting jobs", dee);
171                }
172                catch (CommandException ce) {
173                    errMsg = ce.getMessage();
174                    errCode = ce.getErrorCode().toString();
175                    log.warn("command exception occured ", ce);
176                }
177                catch (java.io.IOException ioe) {
178                    errMsg = ioe.getMessage();
179                    errCode = "E1005";
180                    log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
181                }
182                catch (Exception ex) {
183                    errMsg = ex.getMessage();
184                    errCode = "E1005";
185                    log.warn("can not create DagEngine for submitting jobs", ex);
186                }
187                finally {
188                    if (makeFail == true) { // No DB exception occurs
189                        log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
190                        coordAction.setStatus(CoordinatorAction.Status.FAILED);
191                        if (errMsg.length() > 254) { // Because table column size is 255
192                            errMsg = errMsg.substring(0, 255);
193                        }
194                        coordAction.setErrorMessage(errMsg);
195                        coordAction.setErrorCode(errCode);
196                        store.updateCoordinatorAction(coordAction);
197                        queueCallable(new CoordActionReadyCommand(coordAction.getJobId()));
198                    }
199                }
200            }
201            return null;
202        }
203    
204        @Override
205        protected Void execute(CoordinatorStore store) throws StoreException, CommandException {
206            log.info("STARTED CoordActionStartCommand  actionId=" + actionId);
207            try {
208                coordAction = store.getEntityManager().find(CoordinatorActionBean.class, actionId);
209                setLogInfo(coordAction);
210                if (lock(coordAction.getJobId())) {
211                    call(store);
212                }
213                else {
214                    queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL);
215                    log.warn("CoordActionStartCommand lock was not acquired - failed jobId=" + coordAction.getJobId()
216                            + ", actionId=" + actionId + ". Requeing the same.");
217                }
218            }
219            catch (InterruptedException e) {
220                queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL);
221                log.warn("CoordActionStartCommand lock acquiring failed with exception " + e.getMessage() + " for jobId="
222                        + coordAction.getJobId() + ", actionId=" + actionId + " Requeing the same.");
223            }
224            finally {
225                log.info("ENDED CoordActionStartCommand  actionId=" + actionId);
226            }
227            return null;
228        }
229    }