001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.fs.Path;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.mapred.JobClient;
021    import org.apache.hadoop.mapred.JobConf;
022    import org.apache.hadoop.mapred.RunningJob;
023    import org.apache.hadoop.security.UserGroupInformation;
024    
025    import java.util.Map;
026    import java.util.Properties;
027    import java.io.ByteArrayOutputStream;
028    import java.io.IOException;
029    import java.io.FileOutputStream;
030    import java.io.OutputStream;
031    import java.io.File;
032    
033    public class MapReduceMain extends LauncherMain {
034    
035        public static void main(String[] args) throws Exception {
036            run(MapReduceMain.class, args);
037        }
038    
039        protected void run(String[] args) throws Exception {
040            System.out.println();
041            System.out.println("Oozie Map-Reduce action configuration");
042            System.out.println("=======================");
043    
044            // loading action conf prepared by Oozie
045            Configuration actionConf = new Configuration(false);
046            actionConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
047    
048            System.out.println("Oozie Map-Reduce Configuration: ");
049            System.out.println("------------------------");
050            ByteArrayOutputStream baos = new ByteArrayOutputStream();
051            actionConf.writeXml(baos);
052            baos.close();
053            System.out.println(new String(baos.toByteArray()));
054            System.out.println("------------------------");
055            System.out.println();
056    
057            System.out.println("Submitting Oozie action Map-Reduce job");
058            System.out.println();
059            // submitting job
060            RunningJob runningJob = submitJob(actionConf);
061    
062            // propagating job id back to Oozie
063            String jobId = runningJob.getID().toString();
064            Properties props = new Properties();
065            props.setProperty("id", jobId);
066            File idFile = new File(System.getProperty("oozie.action.newId.properties"));
067            OutputStream os = new FileOutputStream(idFile);
068            props.store(os, "");
069            os.close();
070    
071            System.out.println("=======================");
072            System.out.println();
073        }
074    
075        protected void addActionConf(JobConf jobConf, Configuration actionConf) {
076            for (Map.Entry<String, String> entry : actionConf) {
077                jobConf.set(entry.getKey(), entry.getValue());
078            }
079        }
080    
081        protected RunningJob submitJob(Configuration actionConf) throws Exception {
082            JobConf jobConf = new JobConf();
083            addActionConf(jobConf, actionConf);
084    
085            // propagate delegation related props from launcher job to MR job
086            if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
087                jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
088            }
089            JobClient jobClient = null;
090            RunningJob runJob = null;
091            boolean exception = false;
092            try {
093                jobClient = createJobClient(jobConf);
094                runJob = jobClient.submitJob(jobConf);
095            }
096            catch (Exception ex) {
097                exception = true;
098                throw ex;
099            }
100            finally {
101                try {
102                    if (jobClient != null) {
103                        jobClient.close();
104                    }
105                }
106                catch (Exception ex) {
107                    if (exception) {
108                        System.out.println("JobClient Error: " + ex);
109                    }
110                    else {
111                        throw ex;
112                    }
113                }
114            }
115            return runJob;
116        }
117    
118        @SuppressWarnings("unchecked")
119        protected JobClient createJobClient(JobConf jobConf) throws IOException {
120            return new JobClient(jobConf);
121        }
122    
123        // allows any character in the value, the conf.setStrings() does not allow
124        // commas
125        public static void setStrings(Configuration conf, String key, String[] values) {
126            if (values != null) {
127                conf.setInt(key + ".size", values.length);
128                for (int i = 0; i < values.length; i++) {
129                    conf.set(key + "." + i, values[i]);
130                }
131            }
132        }
133    
134        public static String[] getStrings(Configuration conf, String key) {
135            String[] values = new String[conf.getInt(key + ".size", 0)];
136            for (int i = 0; i < values.length; i++) {
137                values[i] = conf.get(key + "." + i);
138            }
139            return values;
140        }
141    
142    }