001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.coord;
016    
017    import org.apache.hadoop.conf.Configuration;
018    
019    import org.apache.oozie.client.CoordinatorAction;
020    import org.apache.oozie.CoordinatorActionBean;
021    import org.apache.oozie.DagEngine;
022    import org.apache.oozie.DagEngineException;
023    import org.apache.oozie.ErrorCode;
024    import org.apache.oozie.command.CommandException;
025    import org.apache.oozie.service.DagEngineService;
026    import org.apache.oozie.store.StoreException;
027    import org.apache.oozie.store.CoordinatorStore;
028    import org.apache.oozie.service.Services;
029    import org.apache.oozie.util.ParamChecker;
030    import org.apache.oozie.util.XLog;
031    import org.apache.oozie.util.XmlUtils;
032    import org.apache.oozie.util.XConfiguration;
033    import org.apache.oozie.util.db.SLADbOperations;
034    import org.apache.oozie.client.SLAEvent.SlaAppType;
035    import org.apache.oozie.client.SLAEvent.Status;
036    
037    import org.jdom.Element;
038    import org.jdom.JDOMException;
039    
040    import java.io.IOException;
041    import java.io.StringReader;
042    
043    public class CoordActionStartCommand extends CoordinatorCommand<Void> {
044    
045        public static final String EL_ERROR = "EL_ERROR";
046        public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR";
047        public static final String COULD_NOT_START = "COULD_NOT_START";
048        public static final String START_DATA_MISSING = "START_DATA_MISSING";
049        public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING";
050    
051        private final XLog log = XLog.getLog(getClass());
052        private String actionId = null;
053        private String user = null;
054        private String authToken = null;
055        private CoordinatorActionBean coordAction = null;
056    
057        public CoordActionStartCommand(String id, String user, String token) {
058            super("coord_action_start", "coord_action_start", 1, XLog.OPS);
059            this.actionId = ParamChecker.notEmpty(id, "id");
060            this.user = ParamChecker.notEmpty(user, "user");
061            this.authToken = ParamChecker.notEmpty(token, "token");
062        }
063    
064        /**
065         * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
066         * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
067         * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
068         * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
069         * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
070         * Merge Action createdConf with actionXml to create new runConf with replaced variables
071         *
072         * @param action CoordinatorActionBean
073         * @return Configuration
074         * @throws CommandException
075         */
076        private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
077            String createdConf = action.getCreatedConf();
078            String actionXml = action.getActionXml();
079            Element workflowProperties = null;
080            try {
081                workflowProperties = XmlUtils.parseXml(actionXml);
082            }
083            catch (JDOMException e1) {
084                log.warn("Configuration parse error in:" + actionXml);
085                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
086            }
087            // generate the 'runConf' for this action
088            // Step 1: runConf = createdConf
089            Configuration runConf = null;
090            try {
091                runConf = new XConfiguration(new StringReader(createdConf));
092            }
093            catch (IOException e1) {
094                log.warn("Configuration parse error in:" + createdConf);
095                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
096            }
097            // Step 2: Merge local properties into runConf
098            // extract 'property' tags under 'configuration' block in the
099            // coordinator.xml (saved in actionxml column)
100            // convert Element to XConfiguration
101            Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace())
102                    .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
103                                                                                      workflowProperties.getNamespace());
104            if (configElement != null) {
105                String strConfig = XmlUtils.prettyPrint(configElement).toString();
106                Configuration localConf;
107                try {
108                    localConf = new XConfiguration(new StringReader(strConfig));
109                }
110                catch (IOException e1) {
111                    log.warn("Configuration parse error in:" + strConfig);
112                    throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
113                }
114    
115                // copy configuration properties in coordinator.xml to the runConf
116                XConfiguration.copy(localConf, runConf);
117            }
118    
119            // Step 3: Extract value of 'app-path' in actionxml, and save it as a
120            // new property called 'oozie.wf.application.path'
121            // WF Engine requires the path to the workflow.xml to be saved under
122            // this property name
123            String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow",
124                                                                                                               workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue();
125            runConf.set("oozie.wf.application.path", appPath);
126            return runConf;
127        }
128    
129        protected Void call(CoordinatorStore store) throws StoreException, CommandException {
130            boolean makeFail = true;
131            String errCode = "";
132            String errMsg = "";
133            ParamChecker.notEmpty(user, "user");
134            ParamChecker.notEmpty(authToken, "authToken");
135    
136            // CoordinatorActionBean coordAction = store.getCoordinatorAction(id, true);
137            log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
138            if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
139                // log.debug("getting.. job id: " + coordAction.getJobId());
140                // create merged runConf to pass to WF Engine
141                Configuration runConf = mergeConfig(coordAction);
142                coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
143                // log.debug("%%% merged runconf=" +
144                // XmlUtils.prettyPrint(runConf).toString());
145                DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
146                try {
147                    boolean startJob = true;
148                    Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
149                    SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), store, Status.STARTED,
150                                                    SlaAppType.COORDINATOR_ACTION);
151                    String wfId = dagEngine.submitJob(conf, startJob);
152                    coordAction.setStatus(CoordinatorAction.Status.RUNNING);
153                    coordAction.setExternalId(wfId);
154                    store.updateCoordinatorAction(coordAction);
155                    makeFail = false;
156                }
157                catch (StoreException se) {
158                    makeFail = false;
159                    throw se;
160                }
161                catch (DagEngineException dee) {
162                    errMsg = dee.getMessage();
163                    errCode = "E1005";
164                    log.warn("can not create DagEngine for submitting jobs", dee);
165                }
166                catch (CommandException ce) {
167                    errMsg = ce.getMessage();
168                    errCode = ce.getErrorCode().toString();
169                    log.warn("command exception occured ", ce);
170                }
171                catch (java.io.IOException ioe) {
172                    errMsg = ioe.getMessage();
173                    errCode = "E1005";
174                    log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
175                }
176                catch (Exception ex) {
177                    errMsg = ex.getMessage();
178                    errCode = "E1005";
179                    log.warn("can not create DagEngine for submitting jobs", ex);
180                }
181                finally {
182                    if (makeFail == true) { // No DB exception occurs
183                        log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
184                        coordAction.setStatus(CoordinatorAction.Status.FAILED);
185                        if (errMsg.length() > 254) { // Because table column size is 255
186                            errMsg = errMsg.substring(0, 255);
187                        }
188                        coordAction.setErrorMessage(errMsg);
189                        coordAction.setErrorCode(errCode);
190                        store.updateCoordinatorAction(coordAction);
191                        queueCallable(new CoordActionReadyCommand(coordAction.getJobId()));
192                    }
193                }
194            }
195            return null;
196        }
197    
198        @Override
199        protected Void execute(CoordinatorStore store) throws StoreException, CommandException {
200            log.info("STARTED CoordActionStartCommand  actionId=" + actionId);
201            try {
202                coordAction = store.getEntityManager().find(CoordinatorActionBean.class, actionId);
203                setLogInfo(coordAction);
204                if (lock(coordAction.getJobId())) {
205                    call(store);
206                }
207                else {
208                    queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL);
209                    log.warn("CoordActionStartCommand lock was not acquired - failed jobId=" + coordAction.getJobId()
210                            + ", actionId=" + actionId + ". Requeing the same.");
211                }
212            }
213            catch (InterruptedException e) {
214                queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL);
215                log.warn("CoordActionStartCommand lock acquiring failed with exception " + e.getMessage() + " for jobId="
216                        + coordAction.getJobId() + ", actionId=" + actionId + " Requeing the same.");
217            }
218            finally {
219                log.info("ENDED CoordActionStartCommand  actionId=" + actionId);
220            }
221            return null;
222        }
223    }