001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.hadoop.mapred.JobClient; 019 import org.apache.hadoop.mapred.JobConf; 020 import org.apache.hadoop.mapred.RunningJob; 021 022 public class StreamingMain extends MapReduceMain { 023 024 public static void main(String[] args) throws Exception { 025 run(StreamingMain.class, args); 026 } 027 028 protected RunningJob submitJob(Configuration actionConf) throws Exception { 029 JobConf jobConf = new JobConf(); 030 031 jobConf.set("mapred.mapper.class", "org.apache.hadoop.streaming.PipeMapper"); 032 jobConf.set("mapred.reducer.class", "org.apache.hadoop.streaming.PipeReducer"); 033 jobConf.set("mapred.map.runner.class", "org.apache.hadoop.streaming.PipeMapRunner"); 034 035 jobConf.set("mapred.input.format.class", "org.apache.hadoop.mapred.TextInputFormat"); 036 jobConf.set("mapred.output.format.class", "org.apache.hadoop.mapred.TextOutputFormat"); 037 jobConf.set("mapred.output.value.class", "org.apache.hadoop.io.Text"); 038 jobConf.set("mapred.output.key.class", "org.apache.hadoop.io.Text"); 039 040 jobConf.set("mapred.create.symlink", "yes"); 041 jobConf.set("mapred.used.genericoptionsparser", "true"); 042 043 jobConf.set("stream.addenvironment", ""); 044 045 String value = actionConf.get("oozie.streaming.mapper"); 046 if (value != null) { 047 jobConf.set("stream.map.streamprocessor", value); 048 } 049 value = actionConf.get("oozie.streaming.reducer"); 050 if (value != null) { 051 jobConf.set("stream.reduce.streamprocessor", value); 052 } 053 value = actionConf.get("oozie.streaming.record-reader"); 054 if (value != null) { 055 jobConf.set("stream.recordreader.class", value); 056 } 057 String[] values = getStrings(actionConf, "oozie.streaming.record-reader-mapping"); 058 for (String s : values) { 059 String[] kv = s.split("="); 060 jobConf.set("stream.recordreader." + kv[0], kv[1]); 061 } 062 values = getStrings(actionConf, "oozie.streaming.env"); 063 value = jobConf.get("stream.addenvironment", ""); 064 if (value.length() > 0) { 065 value = value + " "; 066 } 067 for (String s : values) { 068 value = value + s + " "; 069 } 070 jobConf.set("stream.addenvironment", value); 071 072 addActionConf(jobConf, actionConf); 073 074 // propagate delegation related props from launcher job to MR job 075 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 076 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 077 } 078 079 JobClient jobClient = null; 080 RunningJob runJob = null; 081 boolean exception = false; 082 try { 083 jobClient = createJobClient(jobConf); 084 runJob = jobClient.submitJob(jobConf); 085 } 086 catch (Exception ex) { 087 exception = true; 088 throw ex; 089 } 090 finally { 091 try { 092 if (jobClient != null) { 093 jobClient.close(); 094 } 095 } 096 catch (Exception ex) { 097 if (exception) { 098 System.out.println("JobClient Error: " + ex); 099 } 100 else { 101 throw ex; 102 } 103 } 104 } 105 return runJob; 106 } 107 108 public static void setStreaming(Configuration conf, String mapper, String reducer, String recordReader, 109 String[] recordReaderMapping, String[] env) { 110 if (mapper != null) { 111 conf.set("oozie.streaming.mapper", mapper); 112 } 113 if (reducer != null) { 114 conf.set("oozie.streaming.reducer", reducer); 115 } 116 if (recordReader != null) { 117 conf.set("oozie.streaming.record-reader", recordReader); 118 } 119 setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping); 120 setStrings(conf, "oozie.streaming.env", env); 121 } 122 123 }