001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.oozie; 016 017 import org.apache.oozie.client.OozieClientException; 018 import org.apache.oozie.action.ActionExecutor; 019 import org.apache.oozie.action.ActionExecutorException; 020 import org.apache.oozie.DagEngine; 021 import org.apache.oozie.LocalOozieClient; 022 import org.apache.oozie.WorkflowJobBean; 023 import org.apache.oozie.service.DagEngineService; 024 import org.apache.oozie.service.WorkflowAppService; 025 import org.apache.oozie.client.WorkflowAction; 026 import org.apache.oozie.client.OozieClient; 027 import org.apache.oozie.client.WorkflowJob; 028 import org.apache.oozie.command.CommandException; 029 import org.apache.oozie.util.JobUtils; 030 import org.apache.oozie.util.PropertiesUtils; 031 import org.apache.oozie.util.XmlUtils; 032 import org.apache.oozie.util.XConfiguration; 033 import org.apache.oozie.util.XLog; 034 import org.apache.oozie.service.Services; 035 import org.apache.hadoop.conf.Configuration; 036 import org.jdom.Element; 037 import org.jdom.Namespace; 038 039 import java.io.StringReader; 040 import java.io.IOException; 041 import java.util.Set; 042 import java.util.HashSet; 043 044 public class SubWorkflowActionExecutor extends ActionExecutor { 045 public static final String ACTION_TYPE = "sub-workflow"; 046 public static final String LOCAL = "local"; 047 048 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>(); 049 050 static { 051 String[] badUserProps = {PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES, 052 PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB, 053 PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN, 054 PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS}; 055 056 String[] badDefaultProps = {PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI, 057 WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME}; 058 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES); 059 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES); 060 } 061 062 protected SubWorkflowActionExecutor() { 063 super(ACTION_TYPE); 064 } 065 066 public void initActionType() { 067 super.initActionType(); 068 } 069 070 protected OozieClient getWorkflowClient(Context context, String oozieUri) { 071 OozieClient oozieClient; 072 if (oozieUri.equals(LOCAL)) { 073 WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow(); 074 String user = workflow.getUser(); 075 String group = workflow.getGroup(); 076 String authToken = workflow.getAuthToken(); 077 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken); 078 oozieClient = new LocalOozieClient(dagEngine); 079 } 080 else { 081 // TODO we need to add authToken to the WC for the remote case 082 oozieClient = new OozieClient(oozieUri); 083 } 084 return oozieClient; 085 } 086 087 protected void injectInline(Element eConf, Configuration subWorkflowConf) throws IOException, 088 ActionExecutorException { 089 if (eConf != null) { 090 String strConf = XmlUtils.prettyPrint(eConf).toString(); 091 Configuration conf = new XConfiguration(new StringReader(strConf)); 092 try { 093 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_DEFAULT_PROPERTIES); 094 } 095 catch (CommandException ex) { 096 throw convertException(ex); 097 } 098 XConfiguration.copy(conf, subWorkflowConf); 099 } 100 } 101 102 @SuppressWarnings("unchecked") 103 protected void injectCallback(Context context, Configuration conf) { 104 String callback = context.getCallbackUrl("$status"); 105 if (conf.get(OozieClient.WORKFLOW_NOTIFICATION_URL) != null) { 106 XLog.getLog(getClass()) 107 .warn("Sub-Workflow configuration has a custom job end notification URI, overriding"); 108 } 109 conf.set(OozieClient.WORKFLOW_NOTIFICATION_URL, callback); 110 } 111 112 protected void injectRecovery(String externalId, Configuration conf) { 113 conf.set(OozieClient.EXTERNAL_ID, externalId); 114 } 115 116 protected String checkIfRunning(OozieClient oozieClient, String extId) throws OozieClientException { 117 String jobId = oozieClient.getJobId(extId); 118 if (jobId.equals("")) { 119 return null; 120 } 121 return jobId; 122 } 123 124 public void start(Context context, WorkflowAction action) throws ActionExecutorException { 125 try { 126 Element eConf = XmlUtils.parseXml(action.getConf()); 127 Namespace ns = eConf.getNamespace(); 128 Element e = eConf.getChild("oozie", ns); 129 String oozieUri = (e == null) ? LOCAL : e.getTextTrim(); 130 OozieClient oozieClient = getWorkflowClient(context, oozieUri); 131 String subWorkflowId = null; 132 String extId = context.getRecoveryId(); 133 String runningJobId = null; 134 if (extId != null) { 135 runningJobId = checkIfRunning(oozieClient, extId); 136 } 137 if (runningJobId == null) { 138 String appPath = eConf.getChild("app-path", ns).getTextTrim(); 139 140 XConfiguration subWorkflowConf = new XConfiguration(); 141 if (eConf.getChild(("propagate-configuration"), ns) != null) { 142 Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf())); 143 XConfiguration.copy(parentConf, subWorkflowConf); 144 } 145 146 // the proto has the necessary credentials 147 Configuration protoActionConf = context.getProtoActionConf(); 148 XConfiguration.copy(protoActionConf, subWorkflowConf); 149 subWorkflowConf.set(OozieClient.APP_PATH, appPath); 150 injectInline(eConf.getChild("configuration", ns), subWorkflowConf); 151 injectCallback(context, subWorkflowConf); 152 injectRecovery(extId, subWorkflowConf); 153 154 //TODO: this has to be refactored later to be done in a single place for REST calls and this 155 JobUtils.normalizeAppPath(context.getWorkflow().getUser(), context.getWorkflow().getGroup(), 156 subWorkflowConf); 157 158 subWorkflowId = oozieClient.run(subWorkflowConf.toProperties()); 159 } 160 else { 161 subWorkflowId = runningJobId; 162 } 163 WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId); 164 String consoleUrl = workflow.getConsoleUrl(); 165 context.setStartData(subWorkflowId, oozieUri, consoleUrl); 166 if (runningJobId != null) { 167 check(context, action); 168 } 169 } 170 catch (Exception ex) { 171 throw convertException(ex); 172 } 173 } 174 175 public void end(Context context, WorkflowAction action) throws ActionExecutorException { 176 try { 177 String externalStatus = action.getExternalStatus(); 178 WorkflowAction.Status status = externalStatus.equals("SUCCEEDED") ? WorkflowAction.Status.OK 179 : WorkflowAction.Status.ERROR; 180 context.setEndData(status, getActionSignal(status)); 181 } 182 catch (Exception ex) { 183 throw convertException(ex); 184 } 185 } 186 187 public void check(Context context, WorkflowAction action) throws ActionExecutorException { 188 try { 189 String subWorkflowId = action.getExternalId(); 190 String oozieUri = action.getTrackerUri(); 191 OozieClient oozieClient = getWorkflowClient(context, oozieUri); 192 WorkflowJob subWorkflow = oozieClient.getJobInfo(subWorkflowId); 193 WorkflowJob.Status status = subWorkflow.getStatus(); 194 switch (status) { 195 case FAILED: 196 case KILLED: 197 case SUCCEEDED: 198 context.setExecutionData(status.toString(), null); 199 break; 200 default: 201 context.setExternalStatus(status.toString()); 202 break; 203 } 204 } 205 catch (Exception ex) { 206 throw convertException(ex); 207 } 208 } 209 210 public void kill(Context context, WorkflowAction action) throws ActionExecutorException { 211 try { 212 String subWorkflowId = action.getExternalId(); 213 String oozieUri = action.getTrackerUri(); 214 OozieClient oozieClient = getWorkflowClient(context, oozieUri); 215 oozieClient.kill(subWorkflowId); 216 context.setEndData(WorkflowAction.Status.KILLED, getActionSignal(WorkflowAction.Status.KILLED)); 217 } 218 catch (Exception ex) { 219 throw convertException(ex); 220 } 221 } 222 223 private static Set<String> FINAL_STATUS = new HashSet<String>(); 224 225 static { 226 FINAL_STATUS.add("SUCCEEDED"); 227 FINAL_STATUS.add("KILLED"); 228 FINAL_STATUS.add("FAILED"); 229 } 230 231 public boolean isCompleted(String externalStatus) { 232 return FINAL_STATUS.contains(externalStatus); 233 } 234 }