001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.mapred.JobConf;
019    import org.apache.hadoop.mapred.RunningJob;
020    import org.apache.hadoop.mapred.pipes.Submitter;
021    import org.apache.hadoop.filecache.DistributedCache;
022    
023    public class PipesMain extends MapReduceMain {
024    
025        public static void main(String[] args) throws Exception {
026            run(PipesMain.class, args);
027        }
028    
029        protected RunningJob submitJob(Configuration actionConf) throws Exception {
030            JobConf jobConf = new JobConf();
031    
032            String value = actionConf.get("oozie.pipes.map");
033            if (value != null) {
034                jobConf.setBoolean("hadoop.pipes.java.mapper", true);
035                jobConf.set("mapred.mapper.class", value);
036            }
037            value = actionConf.get("oozie.pipes.reduce");
038            if (value != null) {
039                jobConf.setBoolean("hadoop.pipes.java.reducer", true);
040                jobConf.set("mapred.reducer.class", value);
041            }
042            value = actionConf.get("oozie.pipes.inputformat");
043            if (value != null) {
044                jobConf.setBoolean("hadoop.pipes.java.recordreader", true);
045                jobConf.set("mapred.input.format.class", value);
046            }
047            value = actionConf.get("oozie.pipes.partitioner");
048            if (value != null) {
049                jobConf.set("mapred.partitioner.class", value);
050            }
051            value = actionConf.get("oozie.pipes.writer");
052            if (value != null) {
053                jobConf.setBoolean("hadoop.pipes.java.recordwriter", true);
054                jobConf.set("mapred.output.format.class", value);
055            }
056            value = actionConf.get("oozie.pipes.program");
057            if (value != null) {
058                jobConf.set("hadoop.pipes.executable", value);
059                if (value.contains("#")) {
060                    DistributedCache.createSymlink(jobConf);
061                }
062            }
063    
064            addActionConf(jobConf, actionConf);
065    
066            //propagate delegation related props from launcher job to MR job
067            if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
068                jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
069            }
070    
071            return Submitter.jobSubmit(jobConf);
072        }
073    
074        public static void setPipes(Configuration conf, String map, String reduce, String inputFormat, String partitioner,
075                                    String writer, String program) {
076            if (map != null) {
077                conf.set("oozie.pipes.map", map);
078            }
079            if (reduce != null) {
080                conf.set("oozie.pipes.reduce", reduce);
081            }
082            if (inputFormat != null) {
083                conf.set("oozie.pipes.inputformat", inputFormat);
084            }
085            if (partitioner != null) {
086                conf.set("oozie.pipes.partitioner", partitioner);
087            }
088            if (writer != null) {
089                conf.set("oozie.pipes.writer", writer);
090            }
091            if (program != null) {
092                conf.set("oozie.pipes.program", program);
093            }
094        }
095    
096    }