001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import org.apache.pig.Main; 018 import org.apache.hadoop.conf.Configuration; 019 import org.apache.hadoop.fs.Path; 020 import org.apache.hadoop.mapred.JobClient; 021 022 import java.io.FileNotFoundException; 023 import java.io.OutputStream; 024 import java.io.FileOutputStream; 025 import java.io.BufferedReader; 026 import java.io.FileReader; 027 import java.io.File; 028 import java.io.IOException; 029 import java.util.HashSet; 030 import java.util.Map; 031 import java.util.List; 032 import java.util.ArrayList; 033 import java.util.Properties; 034 import java.util.Set; 035 import java.net.URL; 036 037 public class PigMainWithOldAPI extends LauncherMain { 038 private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>(); 039 040 static { 041 DISALLOWED_PIG_OPTIONS.add("-4"); 042 DISALLOWED_PIG_OPTIONS.add("-log4jconf"); 043 DISALLOWED_PIG_OPTIONS.add("-e"); 044 DISALLOWED_PIG_OPTIONS.add("-execute"); 045 DISALLOWED_PIG_OPTIONS.add("-f"); 046 DISALLOWED_PIG_OPTIONS.add("-file"); 047 DISALLOWED_PIG_OPTIONS.add("-l"); 048 DISALLOWED_PIG_OPTIONS.add("-logfile"); 049 DISALLOWED_PIG_OPTIONS.add("-r"); 050 DISALLOWED_PIG_OPTIONS.add("-dryrun"); 051 DISALLOWED_PIG_OPTIONS.add("-x"); 052 DISALLOWED_PIG_OPTIONS.add("-exectype"); 053 DISALLOWED_PIG_OPTIONS.add("-P"); 054 DISALLOWED_PIG_OPTIONS.add("-propertyFile"); 055 } 056 057 public static void main(String[] args) throws Exception { 058 run(PigMainWithOldAPI.class, args); 059 } 060 061 protected void run(String[] args) throws Exception { 062 System.out.println(); 063 System.out.println("Oozie Pig action configuration"); 064 System.out.println("================================================================="); 065 066 // loading action conf prepared by Oozie 067 Configuration actionConf = new Configuration(false); 068 069 String actionXml = System.getProperty("oozie.action.conf.xml"); 070 071 if (actionXml == null) { 072 throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]"); 073 } 074 if (!new File(actionXml).exists()) { 075 throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist"); 076 } 077 078 actionConf.addResource(new Path("file:///", actionXml)); 079 080 Properties pigProperties = new Properties(); 081 for (Map.Entry<String, String> entry : actionConf) { 082 pigProperties.setProperty(entry.getKey(), entry.getValue()); 083 } 084 085 //propagate delegation related props from launcher job to Pig job 086 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 087 pigProperties.setProperty("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 088 System.out.println("------------------------"); 089 System.out.println("Setting env property for mapreduce.job.credentials.binary to:" 090 + System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 091 System.out.println("------------------------"); 092 System.setProperty("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 093 } 094 else { 095 System.out.println("Non-kerberoes execution"); 096 } 097 098 OutputStream os = new FileOutputStream("pig.properties"); 099 pigProperties.store(os, ""); 100 os.close(); 101 102 System.out.println(); 103 System.out.println("pig.properties content:"); 104 System.out.println("------------------------"); 105 pigProperties.store(System.out, ""); 106 System.out.flush(); 107 System.out.println("------------------------"); 108 System.out.println(); 109 110 List<String> arguments = new ArrayList<String>(); 111 String script = actionConf.get("oozie.pig.script"); 112 113 if (script == null) { 114 throw new RuntimeException("Action Configuration does not have [oozie.pig.script] property"); 115 } 116 117 if (!new File(script).exists()) { 118 throw new RuntimeException("Error: Pig script file [" + script + "] does not exist"); 119 } 120 121 System.out.println("Pig script [" + script + "] content: "); 122 System.out.println("------------------------"); 123 BufferedReader br = new BufferedReader(new FileReader(script)); 124 String line = br.readLine(); 125 while (line != null) { 126 System.out.println(line); 127 line = br.readLine(); 128 } 129 br.close(); 130 System.out.println("------------------------"); 131 System.out.println(); 132 133 arguments.add("-file"); 134 arguments.add(script); 135 String[] params = MapReduceMain.getStrings(actionConf, "oozie.pig.params"); 136 for (String param : params) { 137 arguments.add("-param"); 138 arguments.add(param); 139 } 140 141 String hadoopJobId = System.getProperty("oozie.launcher.job.id"); 142 if (hadoopJobId == null) { 143 throw new RuntimeException("Launcher Hadoop Job ID system property not set"); 144 } 145 146 String logFile = new File("pig-oozie-" + hadoopJobId + ".log").getAbsolutePath(); 147 148 URL log4jFile = Thread.currentThread().getContextClassLoader().getResource("log4j.properties"); 149 if (log4jFile != null) { 150 151 String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO"); 152 153 // append required PIG properties to the default hadoop log4j file 154 Properties hadoopProps = new Properties(); 155 hadoopProps.load(log4jFile.openStream()); 156 hadoopProps.setProperty("log4j.logger.org.apache.pig", pigLogLevel + ", A, B"); 157 hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); 158 hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout"); 159 hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); 160 hadoopProps.setProperty("log4j.appender.B", "org.apache.log4j.FileAppender"); 161 hadoopProps.setProperty("log4j.appender.B.file", logFile); 162 hadoopProps.setProperty("log4j.appender.B.layout", "org.apache.log4j.PatternLayout"); 163 hadoopProps.setProperty("log4j.appender.B.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); 164 165 String localProps = new File("piglog4j.properties").getAbsolutePath(); 166 OutputStream os1 = new FileOutputStream(localProps); 167 hadoopProps.store(os1, ""); 168 os1.close(); 169 170 arguments.add("-log4jconf"); 171 arguments.add(localProps); 172 173 // print out current directory 174 File localDir = new File(localProps).getParentFile(); 175 System.out.println("Current (local) dir = " + localDir.getAbsolutePath()); 176 } 177 else { 178 System.out.println("log4jfile is null"); 179 } 180 181 String pigLog = "pig-" + hadoopJobId + ".log"; 182 arguments.add("-logfile"); 183 arguments.add(pigLog); 184 185 String[] pigArgs = MapReduceMain.getStrings(actionConf, "oozie.pig.args"); 186 for (String pigArg : pigArgs) { 187 if (DISALLOWED_PIG_OPTIONS.contains(pigArg)) { 188 throw new RuntimeException("Error: Pig argument " + pigArg + " is not supported"); 189 } 190 arguments.add(pigArg); 191 } 192 193 System.out.println("Pig command arguments :"); 194 for (String arg : arguments) { 195 System.out.println(" " + arg); 196 } 197 198 System.out.println("================================================================="); 199 System.out.println(); 200 System.out.println(">>> Invoking Pig command line now >>>"); 201 System.out.println(); 202 System.out.flush(); 203 204 try { 205 System.out.println(); 206 runPigJob(new String[] { "-version" }); 207 } 208 catch (SecurityException ex) { 209 LauncherSecurityManager.reset(); 210 } 211 System.out.println(); 212 System.out.flush(); 213 try { 214 runPigJob(arguments.toArray(new String[arguments.size()])); 215 } 216 catch (SecurityException ex) { 217 if (LauncherSecurityManager.getExitInvoked()) { 218 if (LauncherSecurityManager.getExitCode() != 0) { 219 System.err.println(); 220 System.err.println("Pig logfile dump:"); 221 System.err.println(); 222 try { 223 BufferedReader reader = new BufferedReader(new FileReader(pigLog)); 224 line = reader.readLine(); 225 while (line != null) { 226 System.err.println(line); 227 line = reader.readLine(); 228 } 229 reader.close(); 230 } 231 catch (FileNotFoundException e) { 232 System.err.println("pig log file: " + pigLog + " not found."); 233 } 234 throw ex; 235 } 236 } 237 } 238 239 System.out.println(); 240 System.out.println("<<< Invocation of Pig command completed <<<"); 241 System.out.println(); 242 243 // harvesting and recording Hadoop Job IDs 244 Properties jobIds = getHadoopJobIds(logFile); 245 File file = new File(System.getProperty("oozie.action.output.properties")); 246 os = new FileOutputStream(file); 247 jobIds.store(os, ""); 248 os.close(); 249 System.out.println(" Hadoop Job IDs executed by Pig: " + jobIds.getProperty("hadoopJobs")); 250 System.out.println(); 251 } 252 253 protected void runPigJob(String[] args) throws Exception { 254 // running as from the command line 255 Main.main(args); 256 } 257 258 public static void setPigScript(Configuration conf, String script, String[] params, String[] args) { 259 conf.set("oozie.pig.script", script); 260 MapReduceMain.setStrings(conf, "oozie.pig.params", params); 261 MapReduceMain.setStrings(conf, "oozie.pig.args", args); 262 } 263 264 private static final String JOB_ID_LOG_PREFIX = "HadoopJobId: "; 265 266 protected Properties getHadoopJobIds(String logFile) throws IOException { 267 int jobCount = 0; 268 Properties props = new Properties(); 269 StringBuffer sb = new StringBuffer(100); 270 if (new File(logFile).exists() == false) { 271 System.err.println("pig log file: " + logFile + " not present. Therefore no Hadoop jobids found"); 272 props.setProperty("hadoopJobs", ""); 273 } 274 else { 275 BufferedReader br = new BufferedReader(new FileReader(logFile)); 276 String line = br.readLine(); 277 String separator = ""; 278 while (line != null) { 279 if (line.contains(JOB_ID_LOG_PREFIX)) { 280 int jobIdStarts = line.indexOf(JOB_ID_LOG_PREFIX) + JOB_ID_LOG_PREFIX.length(); 281 String jobId = line.substring(jobIdStarts); 282 int jobIdEnds = jobId.indexOf(" "); 283 if (jobIdEnds > -1) { 284 jobId = jobId.substring(0, jobId.indexOf(" ")); 285 } 286 sb.append(separator).append(jobId); 287 separator = ","; 288 } 289 line = br.readLine(); 290 } 291 br.close(); 292 props.setProperty("hadoopJobs", sb.toString()); 293 } 294 return props; 295 } 296 297 }