001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import java.util.List; 018 019 import org.apache.hadoop.conf.Configuration; 020 import org.apache.hadoop.fs.Path; 021 import org.apache.hadoop.mapred.Counters; 022 import org.apache.hadoop.mapred.JobClient; 023 import org.apache.hadoop.mapred.JobConf; 024 import org.apache.hadoop.mapred.JobID; 025 import org.apache.hadoop.mapred.RunningJob; 026 import org.apache.oozie.action.ActionExecutorException; 027 import org.apache.oozie.client.WorkflowAction; 028 import org.apache.oozie.util.XConfiguration; 029 import org.apache.oozie.util.XLog; 030 import org.apache.oozie.util.XmlUtils; 031 import org.jdom.Element; 032 import org.jdom.Namespace; 033 import org.json.simple.JSONObject; 034 035 public class MapReduceActionExecutor extends JavaActionExecutor { 036 037 public static final String HADOOP_COUNTERS = "hadoop.counters"; 038 private XLog log = XLog.getLog(getClass()); 039 040 public MapReduceActionExecutor() { 041 super("map-reduce"); 042 } 043 044 protected List<Class> getLauncherClasses() { 045 List<Class> classes = super.getLauncherClasses(); 046 classes.add(LauncherMain.class); 047 classes.add(MapReduceMain.class); 048 classes.add(StreamingMain.class); 049 classes.add(PipesMain.class); 050 return classes; 051 } 052 053 protected String getLauncherMain(Configuration launcherConf, Element actionXml) { 054 String mainClass; 055 Namespace ns = actionXml.getNamespace(); 056 if (actionXml.getChild("streaming", ns) != null) { 057 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, StreamingMain.class.getName()); 058 } 059 else { 060 if (actionXml.getChild("pipes", ns) != null) { 061 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName()); 062 } 063 else { 064 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName()); 065 } 066 } 067 return mainClass; 068 } 069 070 @Override 071 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException { 072 super.setupLauncherConf(conf, actionXml, appPath, context); 073 conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); 074 return conf; 075 } 076 077 @SuppressWarnings("unchecked") 078 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) 079 throws ActionExecutorException { 080 Namespace ns = actionXml.getNamespace(); 081 if (actionXml.getChild("streaming", ns) != null) { 082 Element streamingXml = actionXml.getChild("streaming", ns); 083 String mapper = streamingXml.getChildTextTrim("mapper", ns); 084 String reducer = streamingXml.getChildTextTrim("reducer", ns); 085 String recordReader = streamingXml.getChildTextTrim("record-reader", ns); 086 List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns); 087 String[] recordReaderMapping = new String[list.size()]; 088 for (int i = 0; i < list.size(); i++) { 089 recordReaderMapping[i] = list.get(i).getTextTrim(); 090 } 091 list = (List<Element>) streamingXml.getChildren("env", ns); 092 String[] env = new String[list.size()]; 093 for (int i = 0; i < list.size(); i++) { 094 env[i] = list.get(i).getTextTrim(); 095 } 096 StreamingMain.setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env); 097 } 098 else { 099 if (actionXml.getChild("pipes", ns) != null) { 100 Element pipesXml = actionXml.getChild("pipes", ns); 101 String map = pipesXml.getChildTextTrim("map", ns); 102 String reduce = pipesXml.getChildTextTrim("reduce", ns); 103 String inputFormat = pipesXml.getChildTextTrim("inputformat", ns); 104 String partitioner = pipesXml.getChildTextTrim("partitioner", ns); 105 String writer = pipesXml.getChildTextTrim("writer", ns); 106 String program = pipesXml.getChildTextTrim("program", ns); 107 PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program); 108 } 109 } 110 actionConf = super.setupActionConf(actionConf, context, actionXml, appPath); 111 return actionConf; 112 } 113 114 @Override 115 public void end(Context context, WorkflowAction action) throws ActionExecutorException { 116 super.end(context, action); 117 JobClient jobClient = null; 118 boolean exception = false; 119 try { 120 if (action.getStatus() == WorkflowAction.Status.OK) { 121 Element actionXml = XmlUtils.parseXml(action.getConf()); 122 Configuration conf = createBaseHadoopConf(context, actionXml); 123 JobConf jobConf = new JobConf(); 124 XConfiguration.copy(conf, jobConf); 125 jobClient = createJobClient(context, jobConf); 126 RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId())); 127 if (runningJob == null) { 128 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002", 129 "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!", action 130 .getExternalId(), action.getId()); 131 } 132 133 // TODO this has to be done in a better way 134 if (!runningJob.getJobName().startsWith("oozie:action:")) { 135 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR001", 136 "ID swap should have happened in launcher job [{0}]", action.getExternalId()); 137 } 138 Counters counters = runningJob.getCounters(); 139 if (counters != null) { 140 JSONObject json = counterstoJson(counters); 141 context.setVar(HADOOP_COUNTERS, json.toJSONString()); 142 } 143 else { 144 145 context.setVar(HADOOP_COUNTERS, ""); 146 147 XLog.getLog(getClass()).warn("Could not find Hadoop Counters for: [{0}]", action.getExternalId()); 148 } 149 } 150 } 151 catch (Exception ex) { 152 exception = true; 153 throw convertException(ex); 154 } 155 finally { 156 if (jobClient != null) { 157 try { 158 jobClient.close(); 159 } 160 catch (Exception e) { 161 if (exception) { 162 log.error("JobClient error: ", e); 163 } 164 else { 165 throw convertException(e); 166 } 167 } 168 } 169 } 170 } 171 172 @SuppressWarnings("unchecked") 173 private JSONObject counterstoJson(Counters counters) { 174 175 if (counters == null) { 176 return null; 177 } 178 179 JSONObject groups = new JSONObject(); 180 for (String gName : counters.getGroupNames()) { 181 JSONObject group = new JSONObject(); 182 for (Counters.Counter counter : counters.getGroup(gName)) { 183 String cName = counter.getName(); 184 Long cValue = counter.getCounter(); 185 group.put(cName, cValue); 186 } 187 groups.put(gName, group); 188 } 189 return groups; 190 } 191 192 }