001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.oozie;
016    
017    import org.apache.oozie.client.OozieClientException;
018    import org.apache.oozie.action.ActionExecutor;
019    import org.apache.oozie.action.ActionExecutorException;
020    import org.apache.oozie.DagEngine;
021    import org.apache.oozie.LocalOozieClient;
022    import org.apache.oozie.WorkflowJobBean;
023    import org.apache.oozie.service.DagEngineService;
024    import org.apache.oozie.service.WorkflowAppService;
025    import org.apache.oozie.client.WorkflowAction;
026    import org.apache.oozie.client.OozieClient;
027    import org.apache.oozie.client.WorkflowJob;
028    import org.apache.oozie.command.CommandException;
029    import org.apache.oozie.util.JobUtils;
030    import org.apache.oozie.util.PropertiesUtils;
031    import org.apache.oozie.util.XmlUtils;
032    import org.apache.oozie.util.XConfiguration;
033    import org.apache.oozie.util.XLog;
034    import org.apache.oozie.service.Services;
035    import org.apache.hadoop.conf.Configuration;
036    import org.jdom.Element;
037    import org.jdom.Namespace;
038    
039    import java.io.StringReader;
040    import java.io.IOException;
041    import java.util.Set;
042    import java.util.HashSet;
043    
044    public class SubWorkflowActionExecutor extends ActionExecutor {
045        public static final String ACTION_TYPE = "sub-workflow";
046        public static final String LOCAL = "local";
047    
048        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
049    
050        static {
051            String[] badUserProps = {PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
052                    PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
053                    PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
054                    PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS};
055    
056            String[] badDefaultProps = {PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
057                    WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME};
058            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
059            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
060        }
061    
062        protected SubWorkflowActionExecutor() {
063            super(ACTION_TYPE);
064        }
065    
066        public void initActionType() {
067            super.initActionType();
068        }
069    
070        protected OozieClient getWorkflowClient(Context context, String oozieUri) {
071            OozieClient oozieClient;
072            if (oozieUri.equals(LOCAL)) {
073                WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
074                String user = workflow.getUser();
075                String group = workflow.getGroup();
076                String authToken = workflow.getAuthToken();
077                DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
078                oozieClient = new LocalOozieClient(dagEngine);
079            }
080            else {
081                // TODO we need to add authToken to the WC for the remote case
082                oozieClient = new OozieClient(oozieUri);
083            }
084            return oozieClient;
085        }
086    
087        protected void injectInline(Element eConf, Configuration subWorkflowConf) throws IOException,
088                ActionExecutorException {
089            if (eConf != null) {
090                String strConf = XmlUtils.prettyPrint(eConf).toString();
091                Configuration conf = new XConfiguration(new StringReader(strConf));
092                try {
093                    PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_DEFAULT_PROPERTIES);
094                }
095                catch (CommandException ex) {
096                    throw convertException(ex);
097                }
098                XConfiguration.copy(conf, subWorkflowConf);
099            }
100        }
101    
102        @SuppressWarnings("unchecked")
103        protected void injectCallback(Context context, Configuration conf) {
104            String callback = context.getCallbackUrl("$status");
105            if (conf.get(OozieClient.WORKFLOW_NOTIFICATION_URL) != null) {
106                XLog.getLog(getClass())
107                        .warn("Sub-Workflow configuration has a custom job end notification URI, overriding");
108            }
109            conf.set(OozieClient.WORKFLOW_NOTIFICATION_URL, callback);
110        }
111    
112        protected void injectRecovery(String externalId, Configuration conf) {
113            conf.set(OozieClient.EXTERNAL_ID, externalId);
114        }
115    
116        protected String checkIfRunning(OozieClient oozieClient, String extId) throws OozieClientException {
117            String jobId = oozieClient.getJobId(extId);
118            if (jobId.equals("")) {
119                return null;
120            }
121            return jobId;
122        }
123    
124        public void start(Context context, WorkflowAction action) throws ActionExecutorException {
125            try {
126                Element eConf = XmlUtils.parseXml(action.getConf());
127                Namespace ns = eConf.getNamespace();
128                Element e = eConf.getChild("oozie", ns);
129                String oozieUri = (e == null) ? LOCAL : e.getTextTrim();
130                OozieClient oozieClient = getWorkflowClient(context, oozieUri);
131                String subWorkflowId = null;
132                String extId = context.getRecoveryId();
133                String runningJobId = null;
134                if (extId != null) {
135                    runningJobId = checkIfRunning(oozieClient, extId);
136                }
137                if (runningJobId == null) {
138                    String appPath = eConf.getChild("app-path", ns).getTextTrim();
139    
140                    XConfiguration subWorkflowConf = new XConfiguration();
141                    if (eConf.getChild(("propagate-configuration"), ns) != null) {
142                        Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
143                        XConfiguration.copy(parentConf, subWorkflowConf);
144                    }
145    
146                    // the proto has the necessary credentials
147                    Configuration protoActionConf = context.getProtoActionConf();
148                    XConfiguration.copy(protoActionConf, subWorkflowConf);
149                    subWorkflowConf.set(OozieClient.APP_PATH, appPath);
150                    injectInline(eConf.getChild("configuration", ns), subWorkflowConf);
151                    injectCallback(context, subWorkflowConf);
152                    injectRecovery(extId, subWorkflowConf);
153    
154                    //TODO: this has to be refactored later to be done in a single place for REST calls and this
155                    JobUtils.normalizeAppPath(context.getWorkflow().getUser(), context.getWorkflow().getGroup(),
156                                              subWorkflowConf);
157    
158                    subWorkflowId = oozieClient.run(subWorkflowConf.toProperties());
159                }
160                else {
161                    subWorkflowId = runningJobId;
162                }
163                WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId);
164                String consoleUrl = workflow.getConsoleUrl();
165                context.setStartData(subWorkflowId, oozieUri, consoleUrl);
166                if (runningJobId != null) {
167                    check(context, action);
168                }
169            }
170            catch (Exception ex) {
171                throw convertException(ex);
172            }
173        }
174    
175        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
176            try {
177                String externalStatus = action.getExternalStatus();
178                WorkflowAction.Status status = externalStatus.equals("SUCCEEDED") ? WorkflowAction.Status.OK
179                                               : WorkflowAction.Status.ERROR;
180                context.setEndData(status, getActionSignal(status));
181            }
182            catch (Exception ex) {
183                throw convertException(ex);
184            }
185        }
186    
187        public void check(Context context, WorkflowAction action) throws ActionExecutorException {
188            try {
189                String subWorkflowId = action.getExternalId();
190                String oozieUri = action.getTrackerUri();
191                OozieClient oozieClient = getWorkflowClient(context, oozieUri);
192                WorkflowJob subWorkflow = oozieClient.getJobInfo(subWorkflowId);
193                WorkflowJob.Status status = subWorkflow.getStatus();
194                switch (status) {
195                    case FAILED:
196                    case KILLED:
197                    case SUCCEEDED:
198                        context.setExecutionData(status.toString(), null);
199                        break;
200                    default:
201                        context.setExternalStatus(status.toString());
202                        break;
203                }
204            }
205            catch (Exception ex) {
206                throw convertException(ex);
207            }
208        }
209    
210        public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
211            try {
212                String subWorkflowId = action.getExternalId();
213                String oozieUri = action.getTrackerUri();
214                OozieClient oozieClient = getWorkflowClient(context, oozieUri);
215                oozieClient.kill(subWorkflowId);
216                context.setEndData(WorkflowAction.Status.KILLED, getActionSignal(WorkflowAction.Status.KILLED));
217            }
218            catch (Exception ex) {
219                throw convertException(ex);
220            }
221        }
222    
223        private static Set<String> FINAL_STATUS = new HashSet<String>();
224    
225        static {
226            FINAL_STATUS.add("SUCCEEDED");
227            FINAL_STATUS.add("KILLED");
228            FINAL_STATUS.add("FAILED");
229        }
230    
231        public boolean isCompleted(String externalStatus) {
232            return FINAL_STATUS.contains(externalStatus);
233        }
234    }