001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.oozie.action.hadoop; 020 021 import java.io.BufferedReader; 022 import java.io.BufferedWriter; 023 import java.io.File; 024 import java.io.FileWriter; 025 import java.io.IOException; 026 import java.io.InputStreamReader; 027 import java.util.ArrayList; 028 import java.util.HashSet; 029 import java.util.List; 030 import java.util.Map; 031 032 import org.apache.hadoop.conf.Configuration; 033 import org.apache.hadoop.fs.Path; 034 035 public class ShellMain extends LauncherMain { 036 public static final String CONF_OOZIE_SHELL_ARGS = "oozie.shell.args"; 037 public static final String CONF_OOZIE_SHELL_EXEC = "oozie.shell.exec"; 038 public static final String CONF_OOZIE_SHELL_ENVS = "oozie.shell.envs"; 039 public static final String CONF_OOZIE_SHELL_CAPTURE_OUTPUT = "oozie.shell.capture-output"; 040 public static final String OOZIE_ACTION_CONF_XML = "OOZIE_ACTION_CONF_XML"; 041 042 /** 043 * @param args Invoked from LauncherMapper:map() 044 * @throws Exception 045 */ 046 public static void main(String[] args) throws Exception { 047 run(ShellMain.class, args); 048 } 049 050 @Override 051 protected void run(String[] args) throws Exception { 052 053 Configuration actionConf = loadActionConf(); 054 055 int exitCode = execute(actionConf); 056 if (exitCode != 0) { 057 // Shell command failed. therefore make the action failed 058 throw new LauncherMainException(1); 059 } 060 061 } 062 063 /** 064 * Execute the shell command 065 * 066 * @param actionConf 067 * @return command exit value 068 * @throws IOException 069 */ 070 private int execute(Configuration actionConf) throws Exception { 071 String exec = getExec(actionConf); 072 List<String> args = getShellArguments(actionConf); 073 ArrayList<String> cmdArray = getCmdList(exec, args.toArray(new String[args.size()])); 074 ProcessBuilder builder = new ProcessBuilder(cmdArray); 075 Map<String, String> envp = getEnvMap(builder.environment(), actionConf); 076 077 // Getting the Ccurrent working dir and setting it to processbuilder 078 File currDir = new File("dummy").getAbsoluteFile().getParentFile(); 079 System.out.println("Current working dir " + currDir); 080 builder.directory(currDir); 081 082 printCommand(cmdArray, envp); // For debugging purpose 083 084 System.out.println("================================================================="); 085 System.out.println(); 086 System.out.println(">>> Invoking Shell command line now >>"); 087 System.out.println(); 088 System.out.flush(); 089 090 boolean captureOutput = actionConf.getBoolean(CONF_OOZIE_SHELL_CAPTURE_OUTPUT, false); 091 092 // Execute the Command 093 Process p = builder.start(); 094 095 Thread[] thrArray = handleShellOutput(p, captureOutput); 096 097 int exitValue = p.waitFor(); 098 // Wait for both the thread to exit 099 if (thrArray != null) { 100 for (Thread thr : thrArray) { 101 thr.join(); 102 } 103 } 104 System.out.println("Exit code of the Shell command " + exitValue); 105 System.out.println("<<< Invocation of Shell command completed <<<"); 106 System.out.println(); 107 return exitValue; 108 } 109 110 /** 111 * Return the environment variable to pass to in shell command execution. 112 * 113 */ 114 private Map<String, String> getEnvMap(Map<String, String> envp, Configuration actionConf) { 115 // Adding user-specified environments 116 String[] envs = MapReduceMain.getStrings(actionConf, CONF_OOZIE_SHELL_ENVS); 117 for (String env : envs) { 118 String[] varValue = env.split("="); // Error case is handled in 119 // ShellActionExecutor 120 envp.put(varValue[0], varValue[1]); 121 } 122 // Adding action.xml to env 123 envp.put(OOZIE_ACTION_CONF_XML, System.getProperty("oozie.action.conf.xml", "")); 124 return envp; 125 } 126 127 /** 128 * Get the shell commands with the arguments 129 * 130 * @param exec 131 * @param args 132 * @return command and list of args 133 */ 134 private ArrayList<String> getCmdList(String exec, String[] args) { 135 ArrayList<String> cmdArray = new ArrayList<String>(); 136 cmdArray.add(exec); // Main executable 137 for (String arg : args) { // Adding rest of the arguments 138 cmdArray.add(arg); 139 } 140 return cmdArray; 141 } 142 143 /** 144 * Print the output written by the Shell execution in its stdout/stderr. 145 * Also write the stdout output to a file for capturing. 146 * 147 * @param p process 148 * @param captureOutput indicates if STDOUT should be captured or not. 149 * @return Array of threads (one for stdout and another one for stderr 150 * processing 151 * @throws IOException thrown if an IO error occurrs. 152 */ 153 protected Thread[] handleShellOutput(Process p, boolean captureOutput) 154 throws IOException { 155 BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); 156 BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream())); 157 158 OutputWriteThread thrStdout = new OutputWriteThread(input, true, captureOutput); 159 thrStdout.setDaemon(true); 160 thrStdout.start(); 161 162 OutputWriteThread thrStderr = new OutputWriteThread(error, false, false); 163 thrStderr.setDaemon(true); 164 thrStderr.start(); 165 166 return new Thread[]{ thrStdout, thrStderr }; 167 } 168 169 /** 170 * Thread to write output to LM stdout/stderr. Also write the content for 171 * capture-output. 172 */ 173 class OutputWriteThread extends Thread { 174 BufferedReader reader = null; 175 boolean isStdout = false; 176 boolean needCaptured = false; 177 178 public OutputWriteThread(BufferedReader reader, boolean isStdout, boolean needCaptured) { 179 this.reader = reader; 180 this.isStdout = isStdout; 181 this.needCaptured = needCaptured; 182 } 183 184 @Override 185 public void run() { 186 String line; 187 BufferedWriter os = null; 188 189 try { 190 if (needCaptured) { 191 File file = new File(System.getProperty("oozie.action.output.properties")); 192 os = new BufferedWriter(new FileWriter(file)); 193 } 194 while ((line = reader.readLine()) != null) { 195 if (isStdout) { // For stdout 196 System.out.println(line); // 1. Writing to LM STDOUT 197 // 2. Writing for capture output 198 if (os != null) { 199 os.write(line); 200 } 201 } 202 else { 203 System.err.println(line); // 1. Writing to LM STDERR 204 } 205 } 206 } 207 catch (IOException e) { 208 e.printStackTrace(); 209 throw new RuntimeException("Stdout/Stderr read/write error :" + e); 210 }finally { 211 try { 212 reader.close(); 213 } 214 catch (IOException ex) { 215 //NOP ignoring error on close of STDOUT/STDERR 216 } 217 if (os != null) { 218 try { 219 os.close(); 220 } 221 catch (IOException e) { 222 e.printStackTrace(); 223 throw new RuntimeException("Unable to close the file stream :" + e); 224 } 225 } 226 } 227 } 228 } 229 230 /** 231 * Print the command including the arguments as well as the environment 232 * setup 233 * 234 * @param cmdArray :Command Array 235 * @param envp :Environment array 236 */ 237 protected void printCommand(ArrayList<String> cmdArray, Map<String, String> envp) { 238 int i = 0; 239 System.out.println("Full Command .. "); 240 System.out.println("-------------------------"); 241 for (String arg : cmdArray) { 242 System.out.println(i++ + ":" + arg + ":"); 243 } 244 245 if (envp != null) { 246 System.out.println("List of passing environment"); 247 System.out.println("-------------------------"); 248 for (Map.Entry<String, String> entry : envp.entrySet()) { 249 System.out.println(entry.getKey() + "=" + entry.getValue() + ":"); 250 } 251 } 252 253 } 254 255 /** 256 * Retrieve the list of arguments that were originally specified to 257 * Workflow.xml. 258 * 259 * @param actionConf 260 * @return argument list 261 */ 262 protected List<String> getShellArguments(Configuration actionConf) { 263 List<String> arguments = new ArrayList<String>(); 264 String[] scrArgs = MapReduceMain.getStrings(actionConf, CONF_OOZIE_SHELL_ARGS); 265 for (String scrArg : scrArgs) { 266 arguments.add(scrArg); 267 } 268 return arguments; 269 } 270 271 /** 272 * Retrieve the executable name that was originally specified to 273 * Workflow.xml. 274 * 275 * @param actionConf 276 * @return executable 277 */ 278 protected String getExec(Configuration actionConf) { 279 String exec = actionConf.get(CONF_OOZIE_SHELL_EXEC); 280 281 if (exec == null) { 282 throw new RuntimeException("Action Configuration does not have " + CONF_OOZIE_SHELL_EXEC + " property"); 283 } 284 return exec; 285 } 286 287 /** 288 * Read action configuration passes through action xml file. 289 * 290 * @return action Configuration 291 * @throws IOException 292 */ 293 protected Configuration loadActionConf() throws IOException { 294 System.out.println(); 295 System.out.println("Oozie Shell action configuration"); 296 System.out.println("================================================================="); 297 298 // loading action conf prepared by Oozie 299 Configuration actionConf = new Configuration(false); 300 301 String actionXml = System.getProperty("oozie.action.conf.xml"); 302 303 if (actionXml == null) { 304 throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]"); 305 } 306 if (!new File(actionXml).exists()) { 307 throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist"); 308 } 309 310 actionConf.addResource(new Path("file:///", actionXml)); 311 logMasking("Shell configuration:", new HashSet<String>(), actionConf); 312 return actionConf; 313 } 314 }