001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.hadoop.mapred.JobConf; 019 import org.apache.hadoop.mapred.RunningJob; 020 import org.apache.hadoop.mapred.pipes.Submitter; 021 import org.apache.hadoop.filecache.DistributedCache; 022 023 public class PipesMain extends MapReduceMain { 024 025 public static void main(String[] args) throws Exception { 026 run(PipesMain.class, args); 027 } 028 029 protected RunningJob submitJob(Configuration actionConf) throws Exception { 030 JobConf jobConf = new JobConf(); 031 032 String value = actionConf.get("oozie.pipes.map"); 033 if (value != null) { 034 jobConf.setBoolean("hadoop.pipes.java.mapper", true); 035 jobConf.set("mapred.mapper.class", value); 036 } 037 value = actionConf.get("oozie.pipes.reduce"); 038 if (value != null) { 039 jobConf.setBoolean("hadoop.pipes.java.reducer", true); 040 jobConf.set("mapred.reducer.class", value); 041 } 042 value = actionConf.get("oozie.pipes.inputformat"); 043 if (value != null) { 044 jobConf.setBoolean("hadoop.pipes.java.recordreader", true); 045 jobConf.set("mapred.input.format.class", value); 046 } 047 value = actionConf.get("oozie.pipes.partitioner"); 048 if (value != null) { 049 jobConf.set("mapred.partitioner.class", value); 050 } 051 value = actionConf.get("oozie.pipes.writer"); 052 if (value != null) { 053 jobConf.setBoolean("hadoop.pipes.java.recordwriter", true); 054 jobConf.set("mapred.output.format.class", value); 055 } 056 value = actionConf.get("oozie.pipes.program"); 057 if (value != null) { 058 jobConf.set("hadoop.pipes.executable", value); 059 if (value.contains("#")) { 060 DistributedCache.createSymlink(jobConf); 061 } 062 } 063 064 addActionConf(jobConf, actionConf); 065 066 //propagate delegation related props from launcher job to MR job 067 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 068 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 069 } 070 071 return Submitter.jobSubmit(jobConf); 072 } 073 074 public static void setPipes(Configuration conf, String map, String reduce, String inputFormat, String partitioner, 075 String writer, String program) { 076 if (map != null) { 077 conf.set("oozie.pipes.map", map); 078 } 079 if (reduce != null) { 080 conf.set("oozie.pipes.reduce", reduce); 081 } 082 if (inputFormat != null) { 083 conf.set("oozie.pipes.inputformat", inputFormat); 084 } 085 if (partitioner != null) { 086 conf.set("oozie.pipes.partitioner", partitioner); 087 } 088 if (writer != null) { 089 conf.set("oozie.pipes.writer", writer); 090 } 091 if (program != null) { 092 conf.set("oozie.pipes.program", program); 093 } 094 } 095 096 }