001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.hadoop.fs.Path; 019 import org.apache.hadoop.fs.FileSystem; 020 import org.apache.hadoop.mapred.JobClient; 021 import org.apache.hadoop.mapred.JobConf; 022 import org.apache.hadoop.mapred.RunningJob; 023 import org.apache.hadoop.security.UserGroupInformation; 024 025 import java.util.Map; 026 import java.util.Properties; 027 import java.io.ByteArrayOutputStream; 028 import java.io.IOException; 029 import java.io.FileOutputStream; 030 import java.io.OutputStream; 031 import java.io.File; 032 033 public class MapReduceMain extends LauncherMain { 034 035 public static void main(String[] args) throws Exception { 036 run(MapReduceMain.class, args); 037 } 038 039 protected void run(String[] args) throws Exception { 040 System.out.println(); 041 System.out.println("Oozie Map-Reduce action configuration"); 042 System.out.println("======================="); 043 044 // loading action conf prepared by Oozie 045 Configuration actionConf = new Configuration(false); 046 actionConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml"))); 047 048 System.out.println("Oozie Map-Reduce Configuration: "); 049 System.out.println("------------------------"); 050 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 051 actionConf.writeXml(baos); 052 baos.close(); 053 System.out.println(new String(baos.toByteArray())); 054 System.out.println("------------------------"); 055 System.out.println(); 056 057 System.out.println("Submitting Oozie action Map-Reduce job"); 058 System.out.println(); 059 // submitting job 060 RunningJob runningJob = submitJob(actionConf); 061 062 // propagating job id back to Oozie 063 String jobId = runningJob.getID().toString(); 064 Properties props = new Properties(); 065 props.setProperty("id", jobId); 066 File idFile = new File(System.getProperty("oozie.action.newId.properties")); 067 OutputStream os = new FileOutputStream(idFile); 068 props.store(os, ""); 069 os.close(); 070 071 System.out.println("======================="); 072 System.out.println(); 073 } 074 075 protected void addActionConf(JobConf jobConf, Configuration actionConf) { 076 for (Map.Entry<String, String> entry : actionConf) { 077 jobConf.set(entry.getKey(), entry.getValue()); 078 } 079 } 080 081 protected RunningJob submitJob(Configuration actionConf) throws Exception { 082 JobConf jobConf = new JobConf(); 083 addActionConf(jobConf, actionConf); 084 085 // propagate delegation related props from launcher job to MR job 086 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 087 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 088 } 089 JobClient jobClient = null; 090 RunningJob runJob = null; 091 boolean exception = false; 092 try { 093 jobClient = createJobClient(jobConf); 094 runJob = jobClient.submitJob(jobConf); 095 } 096 catch (Exception ex) { 097 exception = true; 098 throw ex; 099 } 100 finally { 101 try { 102 if (jobClient != null) { 103 jobClient.close(); 104 } 105 } 106 catch (Exception ex) { 107 if (exception) { 108 System.out.println("JobClient Error: " + ex); 109 } 110 else { 111 throw ex; 112 } 113 } 114 } 115 return runJob; 116 } 117 118 @SuppressWarnings("unchecked") 119 protected JobClient createJobClient(JobConf jobConf) throws IOException { 120 return new JobClient(jobConf); 121 } 122 123 // allows any character in the value, the conf.setStrings() does not allow 124 // commas 125 public static void setStrings(Configuration conf, String key, String[] values) { 126 if (values != null) { 127 conf.setInt(key + ".size", values.length); 128 for (int i = 0; i < values.length; i++) { 129 conf.set(key + "." + i, values[i]); 130 } 131 } 132 } 133 134 public static String[] getStrings(Configuration conf, String key) { 135 String[] values = new String[conf.getInt(key + ".size", 0)]; 136 for (int i = 0; i < values.length; i++) { 137 values[i] = conf.get(key + "." + i); 138 } 139 return values; 140 } 141 142 }