001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie;
016    
017    import org.apache.oozie.util.XLogStreamer;
018    import org.apache.oozie.service.XLogService;
019    import org.apache.oozie.service.DagXLogInfoService;
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.client.CoordinatorJob;
022    import org.apache.oozie.client.WorkflowJob;
023    import org.apache.oozie.client.OozieClient;
024    import org.apache.oozie.command.wf.CompletedActionCommand;
025    import org.apache.oozie.command.CommandException;
026    import org.apache.oozie.command.Command;
027    import org.apache.oozie.command.wf.JobCommand;
028    import org.apache.oozie.command.wf.JobsCommand;
029    import org.apache.oozie.command.wf.KillCommand;
030    import org.apache.oozie.command.wf.ReRunCommand;
031    import org.apache.oozie.command.wf.ResumeCommand;
032    import org.apache.oozie.command.wf.SubmitCommand;
033    import org.apache.oozie.command.wf.SubmitHttpCommand;
034    import org.apache.oozie.command.wf.SubmitPigCommand;
035    import org.apache.oozie.command.wf.SubmitMRCommand;
036    import org.apache.oozie.command.wf.StartCommand;
037    import org.apache.oozie.command.wf.SuspendCommand;
038    import org.apache.oozie.command.wf.DefinitionCommand;
039    import org.apache.oozie.command.wf.ExternalIdCommand;
040    import org.apache.oozie.command.wf.WorkflowActionInfoCommand;
041    import org.apache.oozie.service.Services;
042    import org.apache.oozie.service.CallableQueueService;
043    import org.apache.oozie.util.ParamChecker;
044    import org.apache.oozie.util.XConfiguration;
045    import org.apache.oozie.util.XLog;
046    
047    import java.io.Writer;
048    import java.util.List;
049    import java.util.Properties;
050    import java.util.Set;
051    import java.util.HashSet;
052    import java.util.StringTokenizer;
053    import java.util.Map;
054    import java.util.HashMap;
055    import java.util.ArrayList;
056    import java.io.IOException;
057    
058    /**
059     * The DagEngine bean provides all the DAG engine functionality for WS calls.
060     */
061    public class DagEngine extends BaseEngine {
062    
063        private static final int HIGH_PRIORITY = 2;
064    
065        /**
066         * Create a system Dag engine, with no user and no group.
067         */
068        public DagEngine() {
069        }
070    
071        /**
072         * Create a Dag engine to perform operations on behave of a user.
073         *
074         * @param user user name.
075         * @param authToken the authentication token.
076         */
077        public DagEngine(String user, String authToken) {
078            this.user = ParamChecker.notEmpty(user, "user");
079            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
080        }
081    
082        /**
083         * Submit a workflow job. <p/> It validates configuration properties.
084         *
085         * @param conf job configuration.
086         * @param startJob indicates if the job should be started or not.
087         * @return the job Id.
088         * @throws DagEngineException thrown if the job could not be created.
089         */
090        @Override
091        public String submitJob(Configuration conf, boolean startJob) throws DagEngineException {
092            validateSubmitConfiguration(conf);
093            SubmitCommand submit = new SubmitCommand(conf, getAuthToken());
094            try {
095                String jobId = submit.call();
096                if (startJob) {
097                    start(jobId);
098                }
099                return jobId;
100            }
101            catch (CommandException ex) {
102                throw new DagEngineException(ex);
103            }
104        }
105    
106        /**
107         * Submit a pig/mapreduce job through HTTP.
108         * <p/>
109         * It validates configuration properties.
110         *
111         * @param conf job configuration.
112         * @param jobType job type - can be "pig" or "mapreduce".
113         * @return the job Id.
114         * @throws DagEngineException thrown if the job could not be created.
115         */
116        public String submitHttpJob(Configuration conf, String jobType) throws DagEngineException {
117            validateSubmitConfiguration(conf);
118    
119            SubmitHttpCommand submit = null;
120            if (jobType.equals("pig")) {
121                submit = new SubmitPigCommand(conf, getAuthToken());
122            }
123            else if (jobType.equals("mapreduce")) {
124                submit = new SubmitMRCommand(conf, getAuthToken());
125            }
126    
127            try {
128                String jobId = submit.call();
129                start(jobId);
130                return jobId;
131            }
132            catch (CommandException ex) {
133                throw new DagEngineException(ex);
134            }
135        }
136    
137        public static void main(String[] args) throws Exception {
138            // Configuration conf = new XConfiguration(IOUtils.getResourceAsReader(
139            // "org/apache/oozie/coord/conf.xml", -1));
140    
141            Configuration conf = new XConfiguration();
142    
143            // String appXml =
144            // IOUtils.getResourceAsString("org/apache/oozie/coord/test1.xml", -1);
145            conf.set(OozieClient.APP_PATH, "file:///Users/danielwo/oozie/workflows/examples/seed/workflows/map-reduce");
146            conf.set(OozieClient.USER_NAME, "danielwo");
147            conf.set(OozieClient.GROUP_NAME, "other");
148    
149            conf.set("inputDir", "  blah   ");
150    
151            // System.out.println("appXml :"+ appXml + "\n conf :"+ conf);
152            new Services().init();
153            try {
154                DagEngine de = new DagEngine("me", "TESTING_WF");
155                String jobId = de.submitJob(conf, true);
156                System.out.println("WF Job Id " + jobId);
157    
158                Thread.sleep(20000);
159            }
160            finally {
161                Services.get().destroy();
162            }
163        }
164    
165        private void validateSubmitConfiguration(Configuration conf) throws DagEngineException {
166            if (conf.get(OozieClient.APP_PATH) == null) {
167                throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
168            }
169        }
170    
171        /**
172         * Start a job.
173         *
174         * @param jobId job Id.
175         * @throws DagEngineException thrown if the job could not be started.
176         */
177        @Override
178        public void start(String jobId) throws DagEngineException {
179            // Changing to synchronous call from asynchronous queuing to prevent the
180            // loss of command if the queue is full or the queue is lost in case of
181            // failure.
182            try {
183                new StartCommand(jobId).call();
184            }
185            catch (CommandException e) {
186                throw new DagEngineException(e);
187            }
188        }
189    
190        /**
191         * Resume a job.
192         *
193         * @param jobId job Id.
194         * @throws DagEngineException thrown if the job could not be resumed.
195         */
196        @Override
197        public void resume(String jobId) throws DagEngineException {
198            // Changing to synchronous call from asynchronous queuing to prevent the
199            // loss of command if the queue is full or the queue is lost in case of
200            // failure.
201            try {
202                new ResumeCommand(jobId).call();
203            }
204            catch (CommandException e) {
205                throw new DagEngineException(e);
206            }
207        }
208    
209        /**
210         * Suspend a job.
211         *
212         * @param jobId job Id.
213         * @throws DagEngineException thrown if the job could not be suspended.
214         */
215        @Override
216        public void suspend(String jobId) throws DagEngineException {
217            // Changing to synchronous call from asynchronous queuing to prevent the
218            // loss of command if the queue is full or the queue is lost in case of
219            // failure.
220            try {
221                new SuspendCommand(jobId).call();
222            }
223            catch (CommandException e) {
224                throw new DagEngineException(e);
225            }
226        }
227    
228        /**
229         * Kill a job.
230         *
231         * @param jobId job Id.
232         * @throws DagEngineException thrown if the job could not be killed.
233         */
234        @Override
235        public void kill(String jobId) throws DagEngineException {
236            // Changing to synchronous call from asynchronous queuing to prevent the
237            // loss of command if the queue is full or the queue is lost in case of
238            // failure.
239            try {
240                new KillCommand(jobId).call();
241                XLog.getLog(getClass()).info("User " + user + " killed the WF job " + jobId);
242            }
243            catch (CommandException e) {
244                throw new DagEngineException(e);
245            }
246        }
247    
248        /* (non-Javadoc)
249         * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
250         */
251        @Override
252        public void change(String jobId, String changeValue) throws DagEngineException {
253            // This code should not be reached.
254            throw new DagEngineException(ErrorCode.E1017);
255        }
256    
257        /**
258         * Rerun a job.
259         *
260         * @param jobId job Id to rerun.
261         * @param conf configuration information for the rerun.
262         * @throws DagEngineException thrown if the job could not be rerun.
263         */
264        @Override
265        public void reRun(String jobId, Configuration conf) throws DagEngineException {
266            try {
267                validateReRunConfiguration(conf);
268                new ReRunCommand(jobId, conf, getAuthToken()).call();
269                start(jobId);
270            }
271            catch (CommandException ex) {
272                throw new DagEngineException(ex);
273            }
274        }
275    
276        private void validateReRunConfiguration(Configuration conf) throws DagEngineException {
277            if (conf.get(OozieClient.APP_PATH) == null) {
278                throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
279            }
280            if (conf.get(OozieClient.RERUN_SKIP_NODES) == null) {
281                throw new DagEngineException(ErrorCode.E0401, OozieClient.RERUN_SKIP_NODES);
282            }
283        }
284    
285        /**
286         * Process an action callback.
287         *
288         * @param actionId the action Id.
289         * @param externalStatus the action external status.
290         * @param actionData the action output data, <code>null</code> if none.
291         * @throws DagEngineException thrown if the callback could not be processed.
292         */
293        public void processCallback(String actionId, String externalStatus, Properties actionData)
294                throws DagEngineException {
295            XLog.Info.get().clearParameter(XLogService.GROUP);
296            XLog.Info.get().clearParameter(XLogService.USER);
297            Command<Void, ?> command = new CompletedActionCommand(actionId, externalStatus, actionData, HIGH_PRIORITY);
298            if (!Services.get().get(CallableQueueService.class).queue(command)) {
299                XLog.getLog(this.getClass()).warn(XLog.OPS, "queue is full or system is in SAFEMODE, ignoring callback");
300            }
301        }
302    
303        /**
304         * Return the info about a job.
305         *
306         * @param jobId job Id.
307         * @return the workflow job info.
308         * @throws DagEngineException thrown if the job info could not be obtained.
309         */
310        @Override
311        public WorkflowJob getJob(String jobId) throws DagEngineException {
312            try {
313                return new JobCommand(jobId).call();
314            }
315            catch (CommandException ex) {
316                throw new DagEngineException(ex);
317            }
318        }
319    
320        /**
321         * Return the info about a job with actions subset.
322         *
323         * @param jobId job Id
324         * @param start starting from this index in the list of actions belonging to the job
325         * @param length number of actions to be returned
326         * @return the workflow job info.
327         * @throws DagEngineException thrown if the job info could not be obtained.
328         */
329        @Override
330        public WorkflowJob getJob(String jobId, int start, int length) throws DagEngineException {
331            try {
332                return new JobCommand(jobId, start, length).call();
333            }
334            catch (CommandException ex) {
335                throw new DagEngineException(ex);
336            }
337        }
338    
339        /**
340         * Return the a job definition.
341         *
342         * @param jobId job Id.
343         * @return the job definition.
344         * @throws DagEngineException thrown if the job definition could no be obtained.
345         */
346        @Override
347        public String getDefinition(String jobId) throws DagEngineException {
348            try {
349                return new DefinitionCommand(jobId).call();
350            }
351            catch (CommandException ex) {
352                throw new DagEngineException(ex);
353            }
354        }
355    
356        /**
357         * Stream the log of a job.
358         *
359         * @param jobId job Id.
360         * @param writer writer to stream the log to.
361         * @throws IOException thrown if the log cannot be streamed.
362         * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId.
363         */
364        @Override
365        public void streamLog(String jobId, Writer writer) throws IOException, DagEngineException {
366            XLogStreamer.Filter filter = new XLogStreamer.Filter();
367            filter.setParameter(DagXLogInfoService.JOB, jobId);
368            WorkflowJob job = getJob(jobId);
369            Services.get().get(XLogService.class).streamLog(filter, job.getStartTime(), job.getEndTime(), writer);
370        }
371    
372        private static final Set<String> FILTER_NAMES = new HashSet<String>();
373    
374        static {
375            FILTER_NAMES.add(OozieClient.FILTER_USER);
376            FILTER_NAMES.add(OozieClient.FILTER_NAME);
377            FILTER_NAMES.add(OozieClient.FILTER_GROUP);
378            FILTER_NAMES.add(OozieClient.FILTER_STATUS);
379        }
380    
381        /**
382         * Validate a jobs filter.
383         *
384         * @param filter filter to validate.
385         * @return the parsed filter.
386         * @throws DagEngineException thrown if the filter is invalid.
387         */
388        protected Map<String, List<String>> parseFilter(String filter) throws DagEngineException {
389            Map<String, List<String>> map = new HashMap<String, List<String>>();
390            if (filter != null) {
391                StringTokenizer st = new StringTokenizer(filter, ";");
392                while (st.hasMoreTokens()) {
393                    String token = st.nextToken();
394                    if (token.contains("=")) {
395                        String[] pair = token.split("=");
396                        if (pair.length != 2) {
397                            throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
398                        }
399                        if (!FILTER_NAMES.contains(pair[0])) {
400                            throw new DagEngineException(ErrorCode.E0420, filter, XLog
401                                    .format("invalid name [{0}]", pair[0]));
402                        }
403                        if (pair[0].equals("status")) {
404                            try {
405                                WorkflowJob.Status.valueOf(pair[1]);
406                            }
407                            catch (IllegalArgumentException ex) {
408                                throw new DagEngineException(ErrorCode.E0420, filter, XLog.format("invalid status [{0}]",
409                                                                                                  pair[1]));
410                            }
411                        }
412                        List<String> list = map.get(pair[0]);
413                        if (list == null) {
414                            list = new ArrayList<String>();
415                            map.put(pair[0], list);
416                        }
417                        list.add(pair[1]);
418                    }
419                    else {
420                        throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
421                    }
422                }
423            }
424            return map;
425        }
426    
427        /**
428         * Return the info about a set of jobs.
429         *
430         * @param filterStr job filter. Refer to the {@link org.apache.oozie.client.OozieClient} for the filter syntax.
431         * @param start offset, base 1.
432         * @param len number of jobs to return.
433         * @return job info for all matching jobs, the jobs don't contain node action information.
434         * @throws DagEngineException thrown if the jobs info could not be obtained.
435         */
436        @SuppressWarnings("unchecked")
437        public WorkflowsInfo getJobs(String filterStr, int start, int len) throws DagEngineException {
438            Map<String, List<String>> filter = parseFilter(filterStr);
439            try {
440                return new JobsCommand(filter, start, len).call();
441            }
442            catch (CommandException dce) {
443                throw new DagEngineException(dce);
444            }
445        }
446    
447        /**
448         * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes.
449         *
450         * @param externalId external ID provided at job submission time.
451         * @return the associated workflow job ID if any, <code>null</code> if none.
452         * @throws DagEngineException thrown if the lookup could not be done.
453         */
454        @Override
455        public String getJobIdForExternalId(String externalId) throws DagEngineException {
456            try {
457                return new ExternalIdCommand(externalId).call();
458            }
459            catch (CommandException dce) {
460                throw new DagEngineException(dce);
461            }
462        }
463    
464        @Override
465        public CoordinatorJob getCoordJob(String jobId) throws BaseEngineException {
466            throw new BaseEngineException(new XException(ErrorCode.E0301));
467        }
468    
469        @Override
470        public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BaseEngineException {
471            throw new BaseEngineException(new XException(ErrorCode.E0301));
472        }
473    
474        public WorkflowActionBean getWorkflowAction(String actionId) throws BaseEngineException {
475            try {
476                return new WorkflowActionInfoCommand(actionId).call();
477            }
478            catch (CommandException ex) {
479                throw new BaseEngineException(ex);
480            }
481        }
482    
483        @Override
484        public String dryrunSubmit(Configuration conf, boolean startJob) throws BaseEngineException {
485            return null;
486        }
487    }