001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import java.util.List;
018    
019    import org.apache.hadoop.conf.Configuration;
020    import org.apache.hadoop.fs.Path;
021    import org.apache.hadoop.mapred.Counters;
022    import org.apache.hadoop.mapred.JobClient;
023    import org.apache.hadoop.mapred.JobConf;
024    import org.apache.hadoop.mapred.JobID;
025    import org.apache.hadoop.mapred.RunningJob;
026    import org.apache.oozie.action.ActionExecutorException;
027    import org.apache.oozie.client.WorkflowAction;
028    import org.apache.oozie.util.XConfiguration;
029    import org.apache.oozie.util.XLog;
030    import org.apache.oozie.util.XmlUtils;
031    import org.jdom.Element;
032    import org.jdom.Namespace;
033    import org.json.simple.JSONObject;
034    
035    public class MapReduceActionExecutor extends JavaActionExecutor {
036    
037        public static final String HADOOP_COUNTERS = "hadoop.counters";
038        private XLog log = XLog.getLog(getClass());
039    
040        public MapReduceActionExecutor() {
041            super("map-reduce");
042        }
043    
044        protected List<Class> getLauncherClasses() {
045            List<Class> classes = super.getLauncherClasses();
046            classes.add(LauncherMain.class);
047            classes.add(MapReduceMain.class);
048            classes.add(StreamingMain.class);
049            classes.add(PipesMain.class);
050            return classes;
051        }
052    
053        protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
054            String mainClass;
055            Namespace ns = actionXml.getNamespace();
056            if (actionXml.getChild("streaming", ns) != null) {
057                mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, StreamingMain.class.getName());
058            }
059            else {
060                if (actionXml.getChild("pipes", ns) != null) {
061                    mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName());
062                }
063                else {
064                    mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName());
065                }
066            }
067            return mainClass;
068        }
069    
070        @Override
071        Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
072            super.setupLauncherConf(conf, actionXml, appPath, context);
073            conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
074            return conf;
075        }
076    
077        @SuppressWarnings("unchecked")
078        Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
079                throws ActionExecutorException {
080            Namespace ns = actionXml.getNamespace();
081            if (actionXml.getChild("streaming", ns) != null) {
082                Element streamingXml = actionXml.getChild("streaming", ns);
083                String mapper = streamingXml.getChildTextTrim("mapper", ns);
084                String reducer = streamingXml.getChildTextTrim("reducer", ns);
085                String recordReader = streamingXml.getChildTextTrim("record-reader", ns);
086                List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns);
087                String[] recordReaderMapping = new String[list.size()];
088                for (int i = 0; i < list.size(); i++) {
089                    recordReaderMapping[i] = list.get(i).getTextTrim();
090                }
091                list = (List<Element>) streamingXml.getChildren("env", ns);
092                String[] env = new String[list.size()];
093                for (int i = 0; i < list.size(); i++) {
094                    env[i] = list.get(i).getTextTrim();
095                }
096                StreamingMain.setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env);
097            }
098            else {
099                if (actionXml.getChild("pipes", ns) != null) {
100                    Element pipesXml = actionXml.getChild("pipes", ns);
101                    String map = pipesXml.getChildTextTrim("map", ns);
102                    String reduce = pipesXml.getChildTextTrim("reduce", ns);
103                    String inputFormat = pipesXml.getChildTextTrim("inputformat", ns);
104                    String partitioner = pipesXml.getChildTextTrim("partitioner", ns);
105                    String writer = pipesXml.getChildTextTrim("writer", ns);
106                    String program = pipesXml.getChildTextTrim("program", ns);
107                    PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program);
108                }
109            }
110            actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
111            return actionConf;
112        }
113    
114        @Override
115        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
116            super.end(context, action);
117            JobClient jobClient = null;
118            boolean exception = false;
119            try {
120                if (action.getStatus() == WorkflowAction.Status.OK) {
121                    Element actionXml = XmlUtils.parseXml(action.getConf());
122                    Configuration conf = createBaseHadoopConf(context, actionXml);
123                    JobConf jobConf = new JobConf();
124                    XConfiguration.copy(conf, jobConf);
125                    jobClient = createJobClient(context, jobConf);
126                    RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId()));
127                    if (runningJob == null) {
128                        throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002",
129                                                          "Unknown hadoop job [{0}] associated with action [{1}].  Failing this action!", action
130                                .getExternalId(), action.getId());
131                    }
132    
133                    // TODO this has to be done in a better way
134                    if (!runningJob.getJobName().startsWith("oozie:action:")) {
135                        throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR001",
136                                                          "ID swap should have happened in launcher job [{0}]", action.getExternalId());
137                    }
138                    Counters counters = runningJob.getCounters();
139                    if (counters != null) {
140                        JSONObject json = counterstoJson(counters);
141                        context.setVar(HADOOP_COUNTERS, json.toJSONString());
142                    }
143                    else {
144    
145                        context.setVar(HADOOP_COUNTERS, "");
146    
147                        XLog.getLog(getClass()).warn("Could not find Hadoop Counters for: [{0}]", action.getExternalId());
148                    }
149                }
150            }
151            catch (Exception ex) {
152                exception = true;
153                throw convertException(ex);
154            }
155            finally {
156                if (jobClient != null) {
157                    try {
158                        jobClient.close();
159                    }
160                    catch (Exception e) {
161                        if (exception) {
162                            log.error("JobClient error: ", e);
163                        }
164                        else {
165                            throw convertException(e);
166                        }
167                    }
168                }
169            }
170        }
171    
172        @SuppressWarnings("unchecked")
173        private JSONObject counterstoJson(Counters counters) {
174    
175            if (counters == null) {
176                return null;
177            }
178    
179            JSONObject groups = new JSONObject();
180            for (String gName : counters.getGroupNames()) {
181                JSONObject group = new JSONObject();
182                for (Counters.Counter counter : counters.getGroup(gName)) {
183                    String cName = counter.getName();
184                    Long cValue = counter.getCounter();
185                    group.put(cName, cValue);
186                }
187                groups.put(gName, group);
188            }
189            return groups;
190        }
191    
192    }