001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.coord; 016 017 import org.apache.hadoop.conf.Configuration; 018 019 import org.apache.oozie.client.CoordinatorAction; 020 import org.apache.oozie.client.OozieClient; 021 import org.apache.oozie.CoordinatorActionBean; 022 import org.apache.oozie.DagEngine; 023 import org.apache.oozie.DagEngineException; 024 import org.apache.oozie.ErrorCode; 025 import org.apache.oozie.command.CommandException; 026 import org.apache.oozie.service.DagEngineService; 027 import org.apache.oozie.store.StoreException; 028 import org.apache.oozie.store.CoordinatorStore; 029 import org.apache.oozie.service.Services; 030 import org.apache.oozie.util.JobUtils; 031 import org.apache.oozie.util.ParamChecker; 032 import org.apache.oozie.util.XLog; 033 import org.apache.oozie.util.XmlUtils; 034 import org.apache.oozie.util.XConfiguration; 035 import org.apache.oozie.util.db.SLADbOperations; 036 import org.apache.oozie.client.SLAEvent.SlaAppType; 037 import org.apache.oozie.client.SLAEvent.Status; 038 039 import org.jdom.Element; 040 import org.jdom.JDOMException; 041 042 import java.io.IOException; 043 import java.io.StringReader; 044 045 public class CoordActionStartCommand extends CoordinatorCommand<Void> { 046 047 public static final String EL_ERROR = "EL_ERROR"; 048 public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR"; 049 public static final String COULD_NOT_START = "COULD_NOT_START"; 050 public static final String START_DATA_MISSING = "START_DATA_MISSING"; 051 public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING"; 052 053 private final XLog log = XLog.getLog(getClass()); 054 private String actionId = null; 055 private String user = null; 056 private String authToken = null; 057 private CoordinatorActionBean coordAction = null; 058 059 public CoordActionStartCommand(String id, String user, String token) { 060 super("coord_action_start", "coord_action_start", 1, XLog.OPS); 061 this.actionId = ParamChecker.notEmpty(id, "id"); 062 this.user = ParamChecker.notEmpty(user, "user"); 063 this.authToken = ParamChecker.notEmpty(token, "token"); 064 } 065 066 /** 067 * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from 068 * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract 069 * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf 070 * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf 071 * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table. 072 * Merge Action createdConf with actionXml to create new runConf with replaced variables 073 * 074 * @param action CoordinatorActionBean 075 * @return Configuration 076 * @throws CommandException 077 */ 078 private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException { 079 String createdConf = action.getCreatedConf(); 080 String actionXml = action.getActionXml(); 081 Element workflowProperties = null; 082 try { 083 workflowProperties = XmlUtils.parseXml(actionXml); 084 } 085 catch (JDOMException e1) { 086 log.warn("Configuration parse error in:" + actionXml); 087 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 088 } 089 // generate the 'runConf' for this action 090 // Step 1: runConf = createdConf 091 Configuration runConf = null; 092 try { 093 runConf = new XConfiguration(new StringReader(createdConf)); 094 } 095 catch (IOException e1) { 096 log.warn("Configuration parse error in:" + createdConf); 097 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 098 } 099 // Step 2: Merge local properties into runConf 100 // extract 'property' tags under 'configuration' block in the 101 // coordinator.xml (saved in actionxml column) 102 // convert Element to XConfiguration 103 Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace()) 104 .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration", 105 workflowProperties.getNamespace()); 106 if (configElement != null) { 107 String strConfig = XmlUtils.prettyPrint(configElement).toString(); 108 Configuration localConf; 109 try { 110 localConf = new XConfiguration(new StringReader(strConfig)); 111 } 112 catch (IOException e1) { 113 log.warn("Configuration parse error in:" + strConfig); 114 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 115 } 116 117 // copy configuration properties in coordinator.xml to the runConf 118 XConfiguration.copy(localConf, runConf); 119 } 120 121 // Step 3: Extract value of 'app-path' in actionxml, and save it as a 122 // new property called 'oozie.wf.application.path' 123 // WF Engine requires the path to the workflow.xml to be saved under 124 // this property name 125 String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow", 126 workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue(); 127 runConf.set("oozie.wf.application.path", appPath); 128 return runConf; 129 } 130 131 protected Void call(CoordinatorStore store) throws StoreException, CommandException { 132 boolean makeFail = true; 133 String errCode = ""; 134 String errMsg = ""; 135 ParamChecker.notEmpty(user, "user"); 136 ParamChecker.notEmpty(authToken, "authToken"); 137 138 // CoordinatorActionBean coordAction = store.getCoordinatorAction(id, true); 139 log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus()); 140 if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) { 141 // log.debug("getting.. job id: " + coordAction.getJobId()); 142 // create merged runConf to pass to WF Engine 143 Configuration runConf = mergeConfig(coordAction); 144 coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString()); 145 // log.debug("%%% merged runconf=" + 146 // XmlUtils.prettyPrint(runConf).toString()); 147 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken); 148 try { 149 boolean startJob = true; 150 Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf())); 151 SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), store, Status.STARTED, 152 SlaAppType.COORDINATOR_ACTION); 153 154 // Normalize workflow appPath here; 155 JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf); 156 157 String wfId = dagEngine.submitJob(conf, startJob); 158 coordAction.setStatus(CoordinatorAction.Status.RUNNING); 159 coordAction.setExternalId(wfId); 160 store.updateCoordinatorAction(coordAction); 161 makeFail = false; 162 } 163 catch (StoreException se) { 164 makeFail = false; 165 throw se; 166 } 167 catch (DagEngineException dee) { 168 errMsg = dee.getMessage(); 169 errCode = "E1005"; 170 log.warn("can not create DagEngine for submitting jobs", dee); 171 } 172 catch (CommandException ce) { 173 errMsg = ce.getMessage(); 174 errCode = ce.getErrorCode().toString(); 175 log.warn("command exception occured ", ce); 176 } 177 catch (java.io.IOException ioe) { 178 errMsg = ioe.getMessage(); 179 errCode = "E1005"; 180 log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe); 181 } 182 catch (Exception ex) { 183 errMsg = ex.getMessage(); 184 errCode = "E1005"; 185 log.warn("can not create DagEngine for submitting jobs", ex); 186 } 187 finally { 188 if (makeFail == true) { // No DB exception occurs 189 log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg); 190 coordAction.setStatus(CoordinatorAction.Status.FAILED); 191 if (errMsg.length() > 254) { // Because table column size is 255 192 errMsg = errMsg.substring(0, 255); 193 } 194 coordAction.setErrorMessage(errMsg); 195 coordAction.setErrorCode(errCode); 196 store.updateCoordinatorAction(coordAction); 197 queueCallable(new CoordActionReadyCommand(coordAction.getJobId())); 198 } 199 } 200 } 201 return null; 202 } 203 204 @Override 205 protected Void execute(CoordinatorStore store) throws StoreException, CommandException { 206 log.info("STARTED CoordActionStartCommand actionId=" + actionId); 207 try { 208 coordAction = store.getEntityManager().find(CoordinatorActionBean.class, actionId); 209 setLogInfo(coordAction); 210 if (lock(coordAction.getJobId())) { 211 call(store); 212 } 213 else { 214 queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL); 215 log.warn("CoordActionStartCommand lock was not acquired - failed jobId=" + coordAction.getJobId() 216 + ", actionId=" + actionId + ". Requeing the same."); 217 } 218 } 219 catch (InterruptedException e) { 220 queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL); 221 log.warn("CoordActionStartCommand lock acquiring failed with exception " + e.getMessage() + " for jobId=" 222 + coordAction.getJobId() + ", actionId=" + actionId + " Requeing the same."); 223 } 224 finally { 225 log.info("ENDED CoordActionStartCommand actionId=" + actionId); 226 } 227 return null; 228 } 229 }