001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie;
016    
017    import org.apache.oozie.util.XLogStreamer;
018    import org.apache.oozie.service.XLogService;
019    import org.apache.oozie.service.DagXLogInfoService;
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.client.CoordinatorJob;
022    import org.apache.oozie.client.WorkflowJob;
023    import org.apache.oozie.client.OozieClient;
024    import org.apache.oozie.command.wf.CompletedActionCommand;
025    import org.apache.oozie.command.CommandException;
026    import org.apache.oozie.command.Command;
027    import org.apache.oozie.command.wf.JobCommand;
028    import org.apache.oozie.command.wf.JobsCommand;
029    import org.apache.oozie.command.wf.KillCommand;
030    import org.apache.oozie.command.wf.ReRunCommand;
031    import org.apache.oozie.command.wf.ResumeCommand;
032    import org.apache.oozie.command.wf.SubmitCommand;
033    import org.apache.oozie.command.wf.SubmitHttpCommand;
034    import org.apache.oozie.command.wf.SubmitPigCommand;
035    import org.apache.oozie.command.wf.SubmitMRCommand;
036    import org.apache.oozie.command.wf.StartCommand;
037    import org.apache.oozie.command.wf.SuspendCommand;
038    import org.apache.oozie.command.wf.DefinitionCommand;
039    import org.apache.oozie.command.wf.ExternalIdCommand;
040    import org.apache.oozie.command.wf.WorkflowActionInfoCommand;
041    import org.apache.oozie.service.Services;
042    import org.apache.oozie.service.CallableQueueService;
043    import org.apache.oozie.util.ParamChecker;
044    import org.apache.oozie.util.XConfiguration;
045    import org.apache.oozie.util.XLog;
046    
047    import java.io.Writer;
048    import java.util.Date;
049    import java.util.List;
050    import java.util.Properties;
051    import java.util.Set;
052    import java.util.HashSet;
053    import java.util.StringTokenizer;
054    import java.util.Map;
055    import java.util.HashMap;
056    import java.util.ArrayList;
057    import java.io.IOException;
058    
059    /**
060     * The DagEngine bean provides all the DAG engine functionality for WS calls.
061     */
062    public class DagEngine extends BaseEngine {
063    
064        private static final int HIGH_PRIORITY = 2;
065    
066        /**
067         * Create a system Dag engine, with no user and no group.
068         */
069        public DagEngine() {
070        }
071    
072        /**
073         * Create a Dag engine to perform operations on behave of a user.
074         *
075         * @param user user name.
076         * @param authToken the authentication token.
077         */
078        public DagEngine(String user, String authToken) {
079            this.user = ParamChecker.notEmpty(user, "user");
080            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
081        }
082    
083        /**
084         * Submit a workflow job. <p/> It validates configuration properties.
085         *
086         * @param conf job configuration.
087         * @param startJob indicates if the job should be started or not.
088         * @return the job Id.
089         * @throws DagEngineException thrown if the job could not be created.
090         */
091        @Override
092        public String submitJob(Configuration conf, boolean startJob) throws DagEngineException {
093            validateSubmitConfiguration(conf);
094            SubmitCommand submit = new SubmitCommand(conf, getAuthToken());
095            try {
096                String jobId = submit.call();
097                if (startJob) {
098                    start(jobId);
099                }
100                return jobId;
101            }
102            catch (CommandException ex) {
103                throw new DagEngineException(ex);
104            }
105        }
106    
107        /**
108         * Submit a pig/mapreduce job through HTTP.
109         * <p/>
110         * It validates configuration properties.
111         *
112         * @param conf job configuration.
113         * @param jobType job type - can be "pig" or "mapreduce".
114         * @return the job Id.
115         * @throws DagEngineException thrown if the job could not be created.
116         */
117        public String submitHttpJob(Configuration conf, String jobType) throws DagEngineException {
118            validateSubmitConfiguration(conf);
119    
120            SubmitHttpCommand submit = null;
121            if (jobType.equals("pig")) {
122                submit = new SubmitPigCommand(conf, getAuthToken());
123            }
124            else if (jobType.equals("mapreduce")) {
125                submit = new SubmitMRCommand(conf, getAuthToken());
126            }
127    
128            try {
129                String jobId = submit.call();
130                start(jobId);
131                return jobId;
132            }
133            catch (CommandException ex) {
134                throw new DagEngineException(ex);
135            }
136        }
137    
138        public static void main(String[] args) throws Exception {
139            // Configuration conf = new XConfiguration(IOUtils.getResourceAsReader(
140            // "org/apache/oozie/coord/conf.xml", -1));
141    
142            Configuration conf = new XConfiguration();
143    
144            // String appXml =
145            // IOUtils.getResourceAsString("org/apache/oozie/coord/test1.xml", -1);
146            conf.set(OozieClient.APP_PATH, "file:///Users/danielwo/oozie/workflows/examples/seed/workflows/map-reduce");
147            conf.set(OozieClient.USER_NAME, "danielwo");
148            conf.set(OozieClient.GROUP_NAME, "other");
149    
150            conf.set("inputDir", "  blah   ");
151    
152            // System.out.println("appXml :"+ appXml + "\n conf :"+ conf);
153            new Services().init();
154            try {
155                DagEngine de = new DagEngine("me", "TESTING_WF");
156                String jobId = de.submitJob(conf, true);
157                System.out.println("WF Job Id " + jobId);
158    
159                Thread.sleep(20000);
160            }
161            finally {
162                Services.get().destroy();
163            }
164        }
165    
166        private void validateSubmitConfiguration(Configuration conf) throws DagEngineException {
167            if (conf.get(OozieClient.APP_PATH) == null) {
168                throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
169            }
170        }
171    
172        /**
173         * Start a job.
174         *
175         * @param jobId job Id.
176         * @throws DagEngineException thrown if the job could not be started.
177         */
178        @Override
179        public void start(String jobId) throws DagEngineException {
180            // Changing to synchronous call from asynchronous queuing to prevent the
181            // loss of command if the queue is full or the queue is lost in case of
182            // failure.
183            try {
184                new StartCommand(jobId).call();
185            }
186            catch (CommandException e) {
187                throw new DagEngineException(e);
188            }
189        }
190    
191        /**
192         * Resume a job.
193         *
194         * @param jobId job Id.
195         * @throws DagEngineException thrown if the job could not be resumed.
196         */
197        @Override
198        public void resume(String jobId) throws DagEngineException {
199            // Changing to synchronous call from asynchronous queuing to prevent the
200            // loss of command if the queue is full or the queue is lost in case of
201            // failure.
202            try {
203                new ResumeCommand(jobId).call();
204            }
205            catch (CommandException e) {
206                throw new DagEngineException(e);
207            }
208        }
209    
210        /**
211         * Suspend a job.
212         *
213         * @param jobId job Id.
214         * @throws DagEngineException thrown if the job could not be suspended.
215         */
216        @Override
217        public void suspend(String jobId) throws DagEngineException {
218            // Changing to synchronous call from asynchronous queuing to prevent the
219            // loss of command if the queue is full or the queue is lost in case of
220            // failure.
221            try {
222                new SuspendCommand(jobId).call();
223            }
224            catch (CommandException e) {
225                throw new DagEngineException(e);
226            }
227        }
228    
229        /**
230         * Kill a job.
231         *
232         * @param jobId job Id.
233         * @throws DagEngineException thrown if the job could not be killed.
234         */
235        @Override
236        public void kill(String jobId) throws DagEngineException {
237            // Changing to synchronous call from asynchronous queuing to prevent the
238            // loss of command if the queue is full or the queue is lost in case of
239            // failure.
240            try {
241                new KillCommand(jobId).call();
242                XLog.getLog(getClass()).info("User " + user + " killed the WF job " + jobId);
243            }
244            catch (CommandException e) {
245                throw new DagEngineException(e);
246            }
247        }
248    
249        /* (non-Javadoc)
250         * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
251         */
252        @Override
253        public void change(String jobId, String changeValue) throws DagEngineException {
254            // This code should not be reached.
255            throw new DagEngineException(ErrorCode.E1017);
256        }
257    
258        /**
259         * Rerun a job.
260         *
261         * @param jobId job Id to rerun.
262         * @param conf configuration information for the rerun.
263         * @throws DagEngineException thrown if the job could not be rerun.
264         */
265        @Override
266        public void reRun(String jobId, Configuration conf) throws DagEngineException {
267            try {
268                validateReRunConfiguration(conf);
269                new ReRunCommand(jobId, conf, getAuthToken()).call();
270                start(jobId);
271            }
272            catch (CommandException ex) {
273                throw new DagEngineException(ex);
274            }
275        }
276    
277        private void validateReRunConfiguration(Configuration conf) throws DagEngineException {
278            if (conf.get(OozieClient.APP_PATH) == null) {
279                throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
280            }
281            if (conf.get(OozieClient.RERUN_SKIP_NODES) == null) {
282                throw new DagEngineException(ErrorCode.E0401, OozieClient.RERUN_SKIP_NODES);
283            }
284        }
285    
286        /**
287         * Process an action callback.
288         *
289         * @param actionId the action Id.
290         * @param externalStatus the action external status.
291         * @param actionData the action output data, <code>null</code> if none.
292         * @throws DagEngineException thrown if the callback could not be processed.
293         */
294        public void processCallback(String actionId, String externalStatus, Properties actionData)
295                throws DagEngineException {
296            XLog.Info.get().clearParameter(XLogService.GROUP);
297            XLog.Info.get().clearParameter(XLogService.USER);
298            Command<Void, ?> command = new CompletedActionCommand(actionId, externalStatus, actionData, HIGH_PRIORITY);
299            if (!Services.get().get(CallableQueueService.class).queue(command)) {
300                XLog.getLog(this.getClass()).warn(XLog.OPS, "queue is full or system is in SAFEMODE, ignoring callback");
301            }
302        }
303    
304        /**
305         * Return the info about a job.
306         *
307         * @param jobId job Id.
308         * @return the workflow job info.
309         * @throws DagEngineException thrown if the job info could not be obtained.
310         */
311        @Override
312        public WorkflowJob getJob(String jobId) throws DagEngineException {
313            try {
314                return new JobCommand(jobId).call();
315            }
316            catch (CommandException ex) {
317                throw new DagEngineException(ex);
318            }
319        }
320    
321        /**
322         * Return the info about a job with actions subset.
323         *
324         * @param jobId job Id
325         * @param start starting from this index in the list of actions belonging to the job
326         * @param length number of actions to be returned
327         * @return the workflow job info.
328         * @throws DagEngineException thrown if the job info could not be obtained.
329         */
330        @Override
331        public WorkflowJob getJob(String jobId, int start, int length) throws DagEngineException {
332            try {
333                return new JobCommand(jobId, start, length).call();
334            }
335            catch (CommandException ex) {
336                throw new DagEngineException(ex);
337            }
338        }
339    
340        /**
341         * Return the a job definition.
342         *
343         * @param jobId job Id.
344         * @return the job definition.
345         * @throws DagEngineException thrown if the job definition could no be obtained.
346         */
347        @Override
348        public String getDefinition(String jobId) throws DagEngineException {
349            try {
350                return new DefinitionCommand(jobId).call();
351            }
352            catch (CommandException ex) {
353                throw new DagEngineException(ex);
354            }
355        }
356    
357        /**
358         * Stream the log of a job.
359         *
360         * @param jobId job Id.
361         * @param writer writer to stream the log to.
362         * @throws IOException thrown if the log cannot be streamed.
363         * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId.
364         */
365        @Override
366        public void streamLog(String jobId, Writer writer) throws IOException, DagEngineException {
367            XLogStreamer.Filter filter = new XLogStreamer.Filter();
368            filter.setParameter(DagXLogInfoService.JOB, jobId);
369            WorkflowJob job = getJob(jobId);
370            Date lastTime = job.getEndTime();
371            if (lastTime == null) {
372                lastTime = job.getLastModifiedTime();
373            }
374            Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), lastTime, writer);
375        }
376    
377        private static final Set<String> FILTER_NAMES = new HashSet<String>();
378    
379        static {
380            FILTER_NAMES.add(OozieClient.FILTER_USER);
381            FILTER_NAMES.add(OozieClient.FILTER_NAME);
382            FILTER_NAMES.add(OozieClient.FILTER_GROUP);
383            FILTER_NAMES.add(OozieClient.FILTER_STATUS);
384        }
385    
386        /**
387         * Validate a jobs filter.
388         *
389         * @param filter filter to validate.
390         * @return the parsed filter.
391         * @throws DagEngineException thrown if the filter is invalid.
392         */
393        protected Map<String, List<String>> parseFilter(String filter) throws DagEngineException {
394            Map<String, List<String>> map = new HashMap<String, List<String>>();
395            if (filter != null) {
396                StringTokenizer st = new StringTokenizer(filter, ";");
397                while (st.hasMoreTokens()) {
398                    String token = st.nextToken();
399                    if (token.contains("=")) {
400                        String[] pair = token.split("=");
401                        if (pair.length != 2) {
402                            throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
403                        }
404                        if (!FILTER_NAMES.contains(pair[0])) {
405                            throw new DagEngineException(ErrorCode.E0420, filter, XLog
406                                    .format("invalid name [{0}]", pair[0]));
407                        }
408                        if (pair[0].equals("status")) {
409                            try {
410                                WorkflowJob.Status.valueOf(pair[1]);
411                            }
412                            catch (IllegalArgumentException ex) {
413                                throw new DagEngineException(ErrorCode.E0420, filter, XLog.format("invalid status [{0}]",
414                                                                                                  pair[1]));
415                            }
416                        }
417                        List<String> list = map.get(pair[0]);
418                        if (list == null) {
419                            list = new ArrayList<String>();
420                            map.put(pair[0], list);
421                        }
422                        list.add(pair[1]);
423                    }
424                    else {
425                        throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
426                    }
427                }
428            }
429            return map;
430        }
431    
432        /**
433         * Return the info about a set of jobs.
434         *
435         * @param filterStr job filter. Refer to the {@link org.apache.oozie.client.OozieClient} for the filter syntax.
436         * @param start offset, base 1.
437         * @param len number of jobs to return.
438         * @return job info for all matching jobs, the jobs don't contain node action information.
439         * @throws DagEngineException thrown if the jobs info could not be obtained.
440         */
441        @SuppressWarnings("unchecked")
442        public WorkflowsInfo getJobs(String filterStr, int start, int len) throws DagEngineException {
443            Map<String, List<String>> filter = parseFilter(filterStr);
444            try {
445                return new JobsCommand(filter, start, len).call();
446            }
447            catch (CommandException dce) {
448                throw new DagEngineException(dce);
449            }
450        }
451    
452        /**
453         * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes.
454         *
455         * @param externalId external ID provided at job submission time.
456         * @return the associated workflow job ID if any, <code>null</code> if none.
457         * @throws DagEngineException thrown if the lookup could not be done.
458         */
459        @Override
460        public String getJobIdForExternalId(String externalId) throws DagEngineException {
461            try {
462                return new ExternalIdCommand(externalId).call();
463            }
464            catch (CommandException dce) {
465                throw new DagEngineException(dce);
466            }
467        }
468    
469        @Override
470        public CoordinatorJob getCoordJob(String jobId) throws BaseEngineException {
471            throw new BaseEngineException(new XException(ErrorCode.E0301));
472        }
473    
474        @Override
475        public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BaseEngineException {
476            throw new BaseEngineException(new XException(ErrorCode.E0301));
477        }
478    
479        public WorkflowActionBean getWorkflowAction(String actionId) throws BaseEngineException {
480            try {
481                return new WorkflowActionInfoCommand(actionId).call();
482            }
483            catch (CommandException ex) {
484                throw new BaseEngineException(ex);
485            }
486        }
487    
488        @Override
489        public String dryrunSubmit(Configuration conf, boolean startJob) throws BaseEngineException {
490            return null;
491        }
492    }