001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.hadoop.fs.Path; 019 import org.apache.hadoop.fs.FileSystem; 020 import org.apache.hadoop.mapred.JobClient; 021 import org.apache.hadoop.mapred.JobConf; 022 import org.apache.hadoop.mapred.RunningJob; 023 import org.apache.hadoop.security.UserGroupInformation; 024 025 import java.util.HashSet; 026 import java.util.Map; 027 import java.util.Properties; 028 import java.io.ByteArrayOutputStream; 029 import java.io.IOException; 030 import java.io.FileOutputStream; 031 import java.io.OutputStream; 032 import java.io.File; 033 034 public class MapReduceMain extends LauncherMain { 035 036 public static void main(String[] args) throws Exception { 037 run(MapReduceMain.class, args); 038 } 039 040 protected void run(String[] args) throws Exception { 041 System.out.println(); 042 System.out.println("Oozie Map-Reduce action configuration"); 043 System.out.println("======================="); 044 045 // loading action conf prepared by Oozie 046 Configuration actionConf = new Configuration(false); 047 actionConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml"))); 048 049 logMasking("Map-Reduce job configuration:", new HashSet<String>(), actionConf); 050 051 System.out.println("Submitting Oozie action Map-Reduce job"); 052 System.out.println(); 053 // submitting job 054 RunningJob runningJob = submitJob(actionConf); 055 056 // propagating job id back to Oozie 057 String jobId = runningJob.getID().toString(); 058 Properties props = new Properties(); 059 props.setProperty("id", jobId); 060 File idFile = new File(System.getProperty("oozie.action.newId.properties")); 061 OutputStream os = new FileOutputStream(idFile); 062 props.store(os, ""); 063 os.close(); 064 065 System.out.println("======================="); 066 System.out.println(); 067 } 068 069 protected void addActionConf(JobConf jobConf, Configuration actionConf) { 070 for (Map.Entry<String, String> entry : actionConf) { 071 jobConf.set(entry.getKey(), entry.getValue()); 072 } 073 } 074 075 protected RunningJob submitJob(Configuration actionConf) throws Exception { 076 JobConf jobConf = new JobConf(); 077 addActionConf(jobConf, actionConf); 078 079 // propagate delegation related props from launcher job to MR job 080 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 081 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 082 } 083 JobClient jobClient = null; 084 RunningJob runJob = null; 085 boolean exception = false; 086 try { 087 jobClient = createJobClient(jobConf); 088 runJob = jobClient.submitJob(jobConf); 089 } 090 catch (Exception ex) { 091 exception = true; 092 throw ex; 093 } 094 finally { 095 try { 096 if (jobClient != null) { 097 jobClient.close(); 098 } 099 } 100 catch (Exception ex) { 101 if (exception) { 102 System.out.println("JobClient Error: " + ex); 103 } 104 else { 105 throw ex; 106 } 107 } 108 } 109 return runJob; 110 } 111 112 @SuppressWarnings("unchecked") 113 protected JobClient createJobClient(JobConf jobConf) throws IOException { 114 return new JobClient(jobConf); 115 } 116 117 // allows any character in the value, the conf.setStrings() does not allow 118 // commas 119 public static void setStrings(Configuration conf, String key, String[] values) { 120 if (values != null) { 121 conf.setInt(key + ".size", values.length); 122 for (int i = 0; i < values.length; i++) { 123 conf.set(key + "." + i, values[i]); 124 } 125 } 126 } 127 128 public static String[] getStrings(Configuration conf, String key) { 129 String[] values = new String[conf.getInt(key + ".size", 0)]; 130 for (int i = 0; i < values.length; i++) { 131 values[i] = conf.get(key + "." + i); 132 if (values[i] == null) { 133 values[i] = ""; 134 } 135 } 136 return values; 137 } 138 139 }