001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.mapred.JobClient;
019    import org.apache.hadoop.mapred.JobConf;
020    import org.apache.hadoop.mapred.RunningJob;
021    
022    public class StreamingMain extends MapReduceMain {
023    
024        public static void main(String[] args) throws Exception {
025            run(StreamingMain.class, args);
026        }
027    
028        protected RunningJob submitJob(Configuration actionConf) throws Exception {
029            JobConf jobConf = new JobConf();
030    
031            jobConf.set("mapred.mapper.class", "org.apache.hadoop.streaming.PipeMapper");
032            jobConf.set("mapred.reducer.class", "org.apache.hadoop.streaming.PipeReducer");
033            jobConf.set("mapred.map.runner.class", "org.apache.hadoop.streaming.PipeMapRunner");
034    
035            jobConf.set("mapred.input.format.class", "org.apache.hadoop.mapred.TextInputFormat");
036            jobConf.set("mapred.output.format.class", "org.apache.hadoop.mapred.TextOutputFormat");
037            jobConf.set("mapred.output.value.class", "org.apache.hadoop.io.Text");
038            jobConf.set("mapred.output.key.class", "org.apache.hadoop.io.Text");
039    
040            jobConf.set("mapred.create.symlink", "yes");
041            jobConf.set("mapred.used.genericoptionsparser", "true");
042    
043            jobConf.set("stream.addenvironment", "");
044    
045            String value = actionConf.get("oozie.streaming.mapper");
046            if (value != null) {
047                jobConf.set("stream.map.streamprocessor", value);
048            }
049            value = actionConf.get("oozie.streaming.reducer");
050            if (value != null) {
051                jobConf.set("stream.reduce.streamprocessor", value);
052            }
053            value = actionConf.get("oozie.streaming.record-reader");
054            if (value != null) {
055                jobConf.set("stream.recordreader.class", value);
056            }
057            String[] values = getStrings(actionConf, "oozie.streaming.record-reader-mapping");
058            for (String s : values) {
059                String[] kv = s.split("=");
060                jobConf.set("stream.recordreader." + kv[0], kv[1]);
061            }
062            values = getStrings(actionConf, "oozie.streaming.env");
063            value = jobConf.get("stream.addenvironment", "");
064            if (value.length() > 0) {
065                value = value + " ";
066            }
067            for (String s : values) {
068                value = value + s + " ";
069            }
070            jobConf.set("stream.addenvironment", value);
071    
072            addActionConf(jobConf, actionConf);
073    
074            // propagate delegation related props from launcher job to MR job
075            if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
076                jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
077            }
078    
079            JobClient jobClient = null;
080            RunningJob runJob = null;
081            boolean exception = false;
082            try {
083                jobClient = createJobClient(jobConf);
084                runJob = jobClient.submitJob(jobConf);
085            }
086            catch (Exception ex) {
087                exception = true;
088                throw ex;
089            }
090            finally {
091                try {
092                    if (jobClient != null) {
093                        jobClient.close();
094                    }
095                }
096                catch (Exception ex) {
097                    if (exception) {
098                        System.out.println("JobClient Error: " + ex);
099                    }
100                    else {
101                        throw ex;
102                    }
103                }
104            }
105            return runJob;
106        }
107    
108        public static void setStreaming(Configuration conf, String mapper, String reducer, String recordReader,
109                                        String[] recordReaderMapping, String[] env) {
110            if (mapper != null) {
111                conf.set("oozie.streaming.mapper", mapper);
112            }
113            if (reducer != null) {
114                conf.set("oozie.streaming.reducer", reducer);
115            }
116            if (recordReader != null) {
117                conf.set("oozie.streaming.record-reader", recordReader);
118            }
119            setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping);
120            setStrings(conf, "oozie.streaming.env", env);
121        }
122    
123    }