001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.fs.Path;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.mapred.JobClient;
021    import org.apache.hadoop.mapred.JobConf;
022    import org.apache.hadoop.mapred.RunningJob;
023    import org.apache.hadoop.security.UserGroupInformation;
024    
025    import java.util.HashSet;
026    import java.util.Map;
027    import java.util.Properties;
028    import java.io.ByteArrayOutputStream;
029    import java.io.IOException;
030    import java.io.FileOutputStream;
031    import java.io.OutputStream;
032    import java.io.File;
033    
034    public class MapReduceMain extends LauncherMain {
035    
036        public static void main(String[] args) throws Exception {
037            run(MapReduceMain.class, args);
038        }
039    
040        protected void run(String[] args) throws Exception {
041            System.out.println();
042            System.out.println("Oozie Map-Reduce action configuration");
043            System.out.println("=======================");
044    
045            // loading action conf prepared by Oozie
046            Configuration actionConf = new Configuration(false);
047            actionConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
048    
049            logMasking("Map-Reduce job configuration:", new HashSet<String>(), actionConf);
050    
051            System.out.println("Submitting Oozie action Map-Reduce job");
052            System.out.println();
053            // submitting job
054            RunningJob runningJob = submitJob(actionConf);
055    
056            // propagating job id back to Oozie
057            String jobId = runningJob.getID().toString();
058            Properties props = new Properties();
059            props.setProperty("id", jobId);
060            File idFile = new File(System.getProperty("oozie.action.newId.properties"));
061            OutputStream os = new FileOutputStream(idFile);
062            props.store(os, "");
063            os.close();
064    
065            System.out.println("=======================");
066            System.out.println();
067        }
068    
069        protected void addActionConf(JobConf jobConf, Configuration actionConf) {
070            for (Map.Entry<String, String> entry : actionConf) {
071                jobConf.set(entry.getKey(), entry.getValue());
072            }
073        }
074    
075        protected RunningJob submitJob(Configuration actionConf) throws Exception {
076            JobConf jobConf = new JobConf();
077            addActionConf(jobConf, actionConf);
078    
079            // propagate delegation related props from launcher job to MR job
080            if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
081                jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
082            }
083            JobClient jobClient = null;
084            RunningJob runJob = null;
085            boolean exception = false;
086            try {
087                jobClient = createJobClient(jobConf);
088                runJob = jobClient.submitJob(jobConf);
089            }
090            catch (Exception ex) {
091                exception = true;
092                throw ex;
093            }
094            finally {
095                try {
096                    if (jobClient != null) {
097                        jobClient.close();
098                    }
099                }
100                catch (Exception ex) {
101                    if (exception) {
102                        System.out.println("JobClient Error: " + ex);
103                    }
104                    else {
105                        throw ex;
106                    }
107                }
108            }
109            return runJob;
110        }
111    
112        @SuppressWarnings("unchecked")
113        protected JobClient createJobClient(JobConf jobConf) throws IOException {
114            return new JobClient(jobConf);
115        }
116    
117        // allows any character in the value, the conf.setStrings() does not allow
118        // commas
119        public static void setStrings(Configuration conf, String key, String[] values) {
120            if (values != null) {
121                conf.setInt(key + ".size", values.length);
122                for (int i = 0; i < values.length; i++) {
123                    conf.set(key + "." + i, values[i]);
124                }
125            }
126        }
127    
128        public static String[] getStrings(Configuration conf, String key) {
129            String[] values = new String[conf.getInt(key + ".size", 0)];
130            for (int i = 0; i < values.length; i++) {
131                values[i] = conf.get(key + "." + i);
132                if (values[i] == null) {
133                    values[i] = "";
134                }
135            }
136            return values;
137        }
138    
139    }