001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.coord; 016 017 import org.apache.hadoop.conf.Configuration; 018 019 import org.apache.oozie.client.CoordinatorAction; 020 import org.apache.oozie.CoordinatorActionBean; 021 import org.apache.oozie.DagEngine; 022 import org.apache.oozie.DagEngineException; 023 import org.apache.oozie.ErrorCode; 024 import org.apache.oozie.command.CommandException; 025 import org.apache.oozie.service.DagEngineService; 026 import org.apache.oozie.store.StoreException; 027 import org.apache.oozie.store.CoordinatorStore; 028 import org.apache.oozie.service.Services; 029 import org.apache.oozie.util.ParamChecker; 030 import org.apache.oozie.util.XLog; 031 import org.apache.oozie.util.XmlUtils; 032 import org.apache.oozie.util.XConfiguration; 033 import org.apache.oozie.util.db.SLADbOperations; 034 import org.apache.oozie.client.SLAEvent.SlaAppType; 035 import org.apache.oozie.client.SLAEvent.Status; 036 037 import org.jdom.Element; 038 import org.jdom.JDOMException; 039 040 import java.io.IOException; 041 import java.io.StringReader; 042 043 public class CoordActionStartCommand extends CoordinatorCommand<Void> { 044 045 public static final String EL_ERROR = "EL_ERROR"; 046 public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR"; 047 public static final String COULD_NOT_START = "COULD_NOT_START"; 048 public static final String START_DATA_MISSING = "START_DATA_MISSING"; 049 public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING"; 050 051 private final XLog log = XLog.getLog(getClass()); 052 private String actionId = null; 053 private String user = null; 054 private String authToken = null; 055 private CoordinatorActionBean coordAction = null; 056 057 public CoordActionStartCommand(String id, String user, String token) { 058 super("coord_action_start", "coord_action_start", 1, XLog.OPS); 059 this.actionId = ParamChecker.notEmpty(id, "id"); 060 this.user = ParamChecker.notEmpty(user, "user"); 061 this.authToken = ParamChecker.notEmpty(token, "token"); 062 } 063 064 /** 065 * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from 066 * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract 067 * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf 068 * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf 069 * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table. 070 * Merge Action createdConf with actionXml to create new runConf with replaced variables 071 * 072 * @param action CoordinatorActionBean 073 * @return Configuration 074 * @throws CommandException 075 */ 076 private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException { 077 String createdConf = action.getCreatedConf(); 078 String actionXml = action.getActionXml(); 079 Element workflowProperties = null; 080 try { 081 workflowProperties = XmlUtils.parseXml(actionXml); 082 } 083 catch (JDOMException e1) { 084 log.warn("Configuration parse error in:" + actionXml); 085 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 086 } 087 // generate the 'runConf' for this action 088 // Step 1: runConf = createdConf 089 Configuration runConf = null; 090 try { 091 runConf = new XConfiguration(new StringReader(createdConf)); 092 } 093 catch (IOException e1) { 094 log.warn("Configuration parse error in:" + createdConf); 095 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 096 } 097 // Step 2: Merge local properties into runConf 098 // extract 'property' tags under 'configuration' block in the 099 // coordinator.xml (saved in actionxml column) 100 // convert Element to XConfiguration 101 Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace()) 102 .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration", 103 workflowProperties.getNamespace()); 104 if (configElement != null) { 105 String strConfig = XmlUtils.prettyPrint(configElement).toString(); 106 Configuration localConf; 107 try { 108 localConf = new XConfiguration(new StringReader(strConfig)); 109 } 110 catch (IOException e1) { 111 log.warn("Configuration parse error in:" + strConfig); 112 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 113 } 114 115 // copy configuration properties in coordinator.xml to the runConf 116 XConfiguration.copy(localConf, runConf); 117 } 118 119 // Step 3: Extract value of 'app-path' in actionxml, and save it as a 120 // new property called 'oozie.wf.application.path' 121 // WF Engine requires the path to the workflow.xml to be saved under 122 // this property name 123 String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow", 124 workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue(); 125 runConf.set("oozie.wf.application.path", appPath); 126 return runConf; 127 } 128 129 protected Void call(CoordinatorStore store) throws StoreException, CommandException { 130 boolean makeFail = true; 131 String errCode = ""; 132 String errMsg = ""; 133 ParamChecker.notEmpty(user, "user"); 134 ParamChecker.notEmpty(authToken, "authToken"); 135 136 // CoordinatorActionBean coordAction = store.getCoordinatorAction(id, true); 137 log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus()); 138 if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) { 139 // log.debug("getting.. job id: " + coordAction.getJobId()); 140 // create merged runConf to pass to WF Engine 141 Configuration runConf = mergeConfig(coordAction); 142 coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString()); 143 // log.debug("%%% merged runconf=" + 144 // XmlUtils.prettyPrint(runConf).toString()); 145 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken); 146 try { 147 boolean startJob = true; 148 Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf())); 149 SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), store, Status.STARTED, 150 SlaAppType.COORDINATOR_ACTION); 151 String wfId = dagEngine.submitJob(conf, startJob); 152 coordAction.setStatus(CoordinatorAction.Status.RUNNING); 153 coordAction.setExternalId(wfId); 154 store.updateCoordinatorAction(coordAction); 155 makeFail = false; 156 } 157 catch (StoreException se) { 158 makeFail = false; 159 throw se; 160 } 161 catch (DagEngineException dee) { 162 errMsg = dee.getMessage(); 163 errCode = "E1005"; 164 log.warn("can not create DagEngine for submitting jobs", dee); 165 } 166 catch (CommandException ce) { 167 errMsg = ce.getMessage(); 168 errCode = ce.getErrorCode().toString(); 169 log.warn("command exception occured ", ce); 170 } 171 catch (java.io.IOException ioe) { 172 errMsg = ioe.getMessage(); 173 errCode = "E1005"; 174 log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe); 175 } 176 catch (Exception ex) { 177 errMsg = ex.getMessage(); 178 errCode = "E1005"; 179 log.warn("can not create DagEngine for submitting jobs", ex); 180 } 181 finally { 182 if (makeFail == true) { // No DB exception occurs 183 log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg); 184 coordAction.setStatus(CoordinatorAction.Status.FAILED); 185 if (errMsg.length() > 254) { // Because table column size is 255 186 errMsg = errMsg.substring(0, 255); 187 } 188 coordAction.setErrorMessage(errMsg); 189 coordAction.setErrorCode(errCode); 190 store.updateCoordinatorAction(coordAction); 191 queueCallable(new CoordActionReadyCommand(coordAction.getJobId())); 192 } 193 } 194 } 195 return null; 196 } 197 198 @Override 199 protected Void execute(CoordinatorStore store) throws StoreException, CommandException { 200 log.info("STARTED CoordActionStartCommand actionId=" + actionId); 201 try { 202 coordAction = store.getEntityManager().find(CoordinatorActionBean.class, actionId); 203 setLogInfo(coordAction); 204 if (lock(coordAction.getJobId())) { 205 call(store); 206 } 207 else { 208 queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL); 209 log.warn("CoordActionStartCommand lock was not acquired - failed jobId=" + coordAction.getJobId() 210 + ", actionId=" + actionId + ". Requeing the same."); 211 } 212 } 213 catch (InterruptedException e) { 214 queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL); 215 log.warn("CoordActionStartCommand lock acquiring failed with exception " + e.getMessage() + " for jobId=" 216 + coordAction.getJobId() + ", actionId=" + actionId + " Requeing the same."); 217 } 218 finally { 219 log.info("ENDED CoordActionStartCommand actionId=" + actionId); 220 } 221 return null; 222 } 223 }