001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import java.io.BufferedReader;
018    import java.io.File;
019    import java.io.FileReader;
020    import java.io.IOException;
021    import java.io.InputStream;
022    import java.io.InputStreamReader;
023    import java.io.OutputStream;
024    import java.io.OutputStreamWriter;
025    import java.io.PrintWriter;
026    import java.io.StringWriter;
027    import java.io.Writer;
028    import java.lang.reflect.InvocationTargetException;
029    import java.lang.reflect.Method;
030    import java.security.Permission;
031    import java.text.MessageFormat;
032    import java.util.Properties;
033    import java.util.concurrent.ScheduledThreadPoolExecutor;
034    import java.util.concurrent.TimeUnit;
035    
036    import org.apache.hadoop.conf.Configuration;
037    import org.apache.hadoop.fs.FileSystem;
038    import org.apache.hadoop.fs.Path;
039    import org.apache.hadoop.mapred.Counters;
040    import org.apache.hadoop.mapred.JobConf;
041    import org.apache.hadoop.mapred.Mapper;
042    import org.apache.hadoop.mapred.OutputCollector;
043    import org.apache.hadoop.mapred.Reporter;
044    import org.apache.hadoop.mapred.RunningJob;
045    import org.apache.oozie.service.HadoopAccessorException;
046    import org.apache.oozie.service.HadoopAccessorService;
047    import org.apache.oozie.service.Services;
048    import org.apache.oozie.util.XLog;
049    
050    public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, Runnable {
051    
052        public static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class";
053    
054        private static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = "oozie.action.main.arg.count";
055        private static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = "oozie.action.main.arg.";
056        private static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = "oozie.action.max.output.data";
057    
058        private static final String COUNTER_GROUP = "oozie.launcher";
059        private static final String COUNTER_DO_ID_SWAP = "oozie.do.id.swap";
060        private static final String COUNTER_OUTPUT_DATA = "oozie.output.data";
061        private static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error";
062    
063        private static final String OOZIE_JOB_ID = "oozie.job.id";
064        private static final String OOZIE_ACTION_ID = "oozie.action.id";
065    
066        private static final String OOZIE_ACTION_DIR_PATH = "oozie.action.dir.path";
067        private static final String OOZIE_ACTION_RECOVERY_ID = "oozie.action.recovery.id";
068    
069        static final String ACTION_CONF_XML = "action.xml";
070        private static final String ACTION_OUTPUT_PROPS = "output.properties";
071        private static final String ACTION_NEW_ID_PROPS = "newId.properties";
072        private static final String ACTION_ERROR_PROPS = "error.properties";
073    
074        private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException {
075            try {
076                FileSystem fs = FileSystem.get(launcherConf);
077                String jobId = launcherConf.get("mapred.job.id");
078                Path path = new Path(actionDir, recoveryId);
079                if (!fs.exists(path)) {
080                    try {
081                        Writer writer = new OutputStreamWriter(fs.create(path));
082                        writer.write(jobId);
083                        writer.close();
084                    }
085                    catch (IOException ex) {
086                        failLauncher("IO error", ex);
087                    }
088                }
089                else {
090                    InputStream is = fs.open(path);
091                    BufferedReader reader = new BufferedReader(new InputStreamReader(is));
092                    String id = reader.readLine();
093                    reader.close();
094                    if (!jobId.equals(id)) {
095                        failLauncher(MessageFormat.format(
096                                "Hadoop job Id mismatch, action file [{0}] declares Id [{1}] current Id [{2}]", path, id,
097                                jobId), null);
098                    }
099    
100                }
101            }
102            catch (IOException ex) {
103                failLauncher("IO error", ex);
104            }
105        }
106    
107        /**
108         * @param launcherConf
109         * @param actionDir
110         * @param recoveryId
111         * @return
112         * @throws HadoopAccessorException
113         * @throws IOException
114         */
115        public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
116                throws HadoopAccessorException, IOException {
117            String jobId = null;
118            Path recoveryFile = new Path(actionDir, recoveryId);
119            //FileSystem fs = FileSystem.get(launcherConf);
120            FileSystem fs = Services.get().get(HadoopAccessorService.class)
121                    .createFileSystem(launcherConf.get("user.name"),
122                                      launcherConf.get("group.name"), launcherConf);
123    
124            if (fs.exists(recoveryFile)) {
125                InputStream is = fs.open(recoveryFile);
126                BufferedReader reader = new BufferedReader(new InputStreamReader(is));
127                jobId = reader.readLine();
128                reader.close();
129            }
130            return jobId;
131    
132        }
133    
134        public static void setupMainClass(Configuration launcherConf, String javaMainClass) {
135            launcherConf.set(CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
136        }
137    
138        public static void setupMainArguments(Configuration launcherConf, String[] args) {
139            launcherConf.setInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
140            for (int i = 0; i < args.length; i++) {
141                launcherConf.set(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
142            }
143        }
144    
145        public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) {
146            launcherConf.setInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
147        }
148    
149        /**
150         * @param launcherConf
151         * @param jobId
152         * @param actionId
153         * @param actionDir
154         * @param recoveryId
155         * @param actionConf
156         * @throws IOException
157         * @throws HadoopAccessorException
158         */
159        public static void setupLauncherInfo(JobConf launcherConf, String jobId, String actionId, Path actionDir,
160                String recoveryId, Configuration actionConf) throws IOException, HadoopAccessorException {
161    
162            launcherConf.setMapperClass(LauncherMapper.class);
163            launcherConf.setSpeculativeExecution(false);
164            launcherConf.setNumMapTasks(1);
165            launcherConf.setNumReduceTasks(0);
166    
167            launcherConf.set(OOZIE_JOB_ID, jobId);
168            launcherConf.set(OOZIE_ACTION_ID, actionId);
169            launcherConf.set(OOZIE_ACTION_DIR_PATH, actionDir.toString());
170            launcherConf.set(OOZIE_ACTION_RECOVERY_ID, recoveryId);
171    
172            FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(launcherConf.get("user.name"),
173                    launcherConf.get("group.name"), launcherConf);
174            fs.mkdirs(actionDir);
175    
176            OutputStream os = fs.create(new Path(actionDir, ACTION_CONF_XML));
177            actionConf.writeXml(os);
178            os.close();
179    
180            Path inputDir = new Path(actionDir, "input");
181            fs.mkdirs(inputDir);
182            Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "dummy.txt")));
183            writer.write("dummy");
184            writer.close();
185    
186            launcherConf.set("mapred.input.dir", inputDir.toString());
187            launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString());
188        }
189    
190        public static boolean isMainDone(RunningJob runningJob) throws IOException {
191            return runningJob.isComplete();
192        }
193    
194        public static boolean isMainSuccessful(RunningJob runningJob) throws IOException {
195            boolean succeeded = runningJob.isSuccessful();
196            if (succeeded) {
197                Counters counters = runningJob.getCounters();
198                if (counters != null) {
199                    Counters.Group group = counters.getGroup(COUNTER_GROUP);
200                    if (group != null) {
201                        succeeded = group.getCounter(COUNTER_LAUNCHER_ERROR) == 0;
202                    }
203                }
204            }
205            return succeeded;
206        }
207    
208        public static boolean hasOutputData(RunningJob runningJob) throws IOException {
209            boolean output = false;
210            Counters counters = runningJob.getCounters();
211            if (counters != null) {
212                Counters.Group group = counters.getGroup(COUNTER_GROUP);
213                if (group != null) {
214                    output = group.getCounter(COUNTER_OUTPUT_DATA) == 1;
215                }
216            }
217            return output;
218        }
219    
220        /**
221         * @param runningJob
222         * @return
223         * @throws IOException
224         */
225        public static boolean hasIdSwap(RunningJob runningJob) throws IOException {
226            boolean swap = false;
227            Counters counters = runningJob.getCounters();
228            if (counters != null) {
229                Counters.Group group = counters.getGroup(COUNTER_GROUP);
230                if (group != null) {
231                    swap = group.getCounter(COUNTER_DO_ID_SWAP) == 1;
232                }
233            }
234            return swap;
235        }
236    
237        /**
238         * @param runningJob
239         * @param user
240         * @param group
241         * @param actionDir
242         * @return
243         * @throws IOException
244         * @throws HadoopAccessorException
245         */
246        public static boolean hasIdSwap(RunningJob runningJob, String user, String group, Path actionDir)
247                throws IOException, HadoopAccessorException {
248            boolean swap = false;
249    
250            XLog log = XLog.getLog("org.apache.oozie.action.hadoop.LauncherMapper");
251    
252            Counters counters = runningJob.getCounters();
253            if (counters != null) {
254                Counters.Group counterGroup = counters.getGroup(COUNTER_GROUP);
255                if (counterGroup != null) {
256                    swap = counterGroup.getCounter(COUNTER_DO_ID_SWAP) == 1;
257                }
258            }
259            // additional check for swapped hadoop ID
260            // Can't rely on hadoop counters existing
261            // we'll check for the newID file in hdfs if the hadoop counters is null
262            else {
263    
264                Path p = getIdSwapPath(actionDir);
265                // log.debug("Checking for newId file in: [{0}]", p);
266    
267                FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, p.toUri(),
268                                                                                                 new Configuration());
269                if (fs.exists(p)) {
270                    log.debug("Hadoop Counters is null, but found newID file.");
271    
272                    swap = true;
273                }
274                else {
275                    log.debug("Hadoop Counters is null, and newID file doesn't exist at: [{0}]", p);
276                }
277            }
278            return swap;
279        }
280    
281        public static Path getOutputDataPath(Path actionDir) {
282            return new Path(actionDir, ACTION_OUTPUT_PROPS);
283        }
284    
285        public static Path getErrorPath(Path actionDir) {
286            return new Path(actionDir, ACTION_ERROR_PROPS);
287        }
288    
289        public static Path getIdSwapPath(Path actionDir) {
290            return new Path(actionDir, ACTION_NEW_ID_PROPS);
291        }
292    
293        private JobConf jobConf;
294        private Path actionDir;
295        private ScheduledThreadPoolExecutor timer;
296    
297        private boolean configFailure = false;
298    
299        public LauncherMapper() {
300        }
301    
302        public void configure(JobConf jobConf) {
303            System.out.println();
304            System.out.println("Oozie Launcher starts");
305            System.out.println();
306            this.jobConf = jobConf;
307            actionDir = new Path(getJobConf().get(OOZIE_ACTION_DIR_PATH));
308            String recoveryId = jobConf.get(OOZIE_ACTION_RECOVERY_ID, null);
309            try {
310                setRecoveryId(jobConf, actionDir, recoveryId);
311            }
312            catch (LauncherException ex) {
313                configFailure = true;
314            }
315        }
316    
317        public void map(K1 key, V1 value, OutputCollector<K2, V2> collector, Reporter reporter) throws IOException {
318            try {
319                if (configFailure) {
320                    throw new LauncherException();
321                }
322                else {
323                    String mainClass = getJobConf().get(CONF_OOZIE_ACTION_MAIN_CLASS);
324                    String msgPrefix = "Main class [" + mainClass + "], ";
325                    Throwable errorCause = null;
326                    String errorMessage = null;
327    
328                    try {
329                        new LauncherSecurityManager();
330                    }
331                    catch (SecurityException ex) {
332                        errorMessage = "Could not set LauncherSecurityManager";
333                        errorCause = ex;
334                    }
335    
336                    try {
337                        setupHeartBeater(reporter);
338    
339                        setupMainConfiguration();
340    
341                        String[] args = getMainArguments(getJobConf());
342    
343                        printContentsOfCurrentDir();
344    
345                        System.out.println();
346                        System.out.println("Oozie Java/Map-Reduce/Pig action launcher-job configuration");
347                        System.out.println("=================================================================");
348                        System.out.println("Workflow job id   : " + System.getProperty("oozie.job.id"));
349                        System.out.println("Workflow action id: " + System.getProperty("oozie.action.id"));
350                        System.out.println();
351                        System.out.println("Main class        : " + mainClass);
352                        System.out.println();
353                        System.out.println("Maximum output    : "
354                                + getJobConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024));
355                        System.out.println();
356                        System.out.println("Arguments         :");
357                        for (String arg : args) {
358                            System.out.println("                    " + arg);
359                        }
360    
361                        System.out.println();
362                        System.out.println("Java System Properties:");
363                        System.out.println("------------------------");
364                        System.getProperties().store(System.out, "");
365                        System.out.flush();
366                        System.out.println("------------------------");
367                        System.out.println();
368    
369                        System.out.println("=================================================================");
370                        System.out.println();
371                        System.out.println(">>> Invoking Main class now >>>");
372                        System.out.println();
373                        System.out.flush();
374    
375                        try {
376                            Class klass = getJobConf().getClass(CONF_OOZIE_ACTION_MAIN_CLASS, Object.class);
377                            Method mainMethod = klass.getMethod("main", String[].class);
378                            mainMethod.invoke(null, (Object) args);
379                        }
380                        catch (InvocationTargetException ex) {
381                            if (SecurityException.class.isInstance(ex.getCause())) {
382                                if (LauncherSecurityManager.getExitInvoked()) {
383                                    System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode()
384                                            + ")");
385                                    System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode()
386                                            + ")");
387                                    // if 0 main() method finished successfully
388                                    // ignoring
389                                    if (LauncherSecurityManager.getExitCode() != 0) {
390                                        errorMessage = msgPrefix + "exit code [" + LauncherSecurityManager.getExitCode()
391                                                + "]";
392                                        errorCause = null;
393                                    }
394                                }
395                            }
396                            else {
397                                throw ex;
398                            }
399                        }
400                        finally {
401                            System.out.println();
402                            System.out.println("<<< Invocation of Main class completed <<<");
403                            System.out.println();
404                        }
405                        if (errorMessage == null) {
406                            File outputData = new File(System.getProperty("oozie.action.output.properties"));
407                            if (outputData.exists()) {
408                                FileSystem fs = FileSystem.get(getJobConf());
409                                fs.copyFromLocalFile(new Path(outputData.toString()), new Path(actionDir,
410                                                                                               ACTION_OUTPUT_PROPS));
411                                reporter.incrCounter(COUNTER_GROUP, COUNTER_OUTPUT_DATA, 1);
412    
413                                int maxOutputData = getJobConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024);
414                                if (outputData.length() > maxOutputData) {
415                                    String msg = MessageFormat.format("Output data size [{0}] exceeds maximum [{1}]",
416                                                                      outputData.length(), maxOutputData);
417                                    failLauncher(msg, null);
418                                }
419                                System.out.println();
420                                System.out.println("Oozie Launcher, capturing output data:");
421                                System.out.println("=======================");
422                                Properties props = new Properties();
423                                props.load(new FileReader(outputData));
424                                props.store(System.out, "");
425                                System.out.println();
426                                System.out.println("=======================");
427                                System.out.println();
428                            }
429                            File newId = new File(System.getProperty("oozie.action.newId.properties"));
430                            if (newId.exists()) {
431                                Properties props = new Properties();
432                                props.load(new FileReader(newId));
433                                if (props.getProperty("id") == null) {
434                                    throw new IllegalStateException("ID swap file does not have [id] property");
435                                }
436                                FileSystem fs = FileSystem.get(getJobConf());
437                                fs.copyFromLocalFile(new Path(newId.toString()), new Path(actionDir, ACTION_NEW_ID_PROPS));
438                                reporter.incrCounter(COUNTER_GROUP, COUNTER_DO_ID_SWAP, 1);
439    
440                                System.out.println("Oozie Launcher, copying new Hadoop job id to file: "
441                                        + new Path(actionDir, ACTION_NEW_ID_PROPS).toUri());
442    
443                                System.out.println();
444                                System.out.println("Oozie Launcher, propagating new Hadoop job id to Oozie");
445                                System.out.println("=======================");
446                                System.out.println("id: " + props.getProperty("id"));
447                                System.out.println("=======================");
448                                System.out.println();
449                            }
450                        }
451                    }
452                    catch (NoSuchMethodException ex) {
453                        errorMessage = msgPrefix + "main() method not found";
454                        errorCause = ex;
455                    }
456                    catch (InvocationTargetException ex) {
457                        errorMessage = msgPrefix + "main() threw exception";
458                        errorCause = ex.getTargetException();
459                    }
460                    catch (Throwable ex) {
461                        errorMessage = msgPrefix + "exception invoking main()";
462                        errorCause = ex;
463                    }
464                    finally {
465                        destroyHeartBeater();
466                        if (errorMessage != null) {
467                            failLauncher(errorMessage, errorCause);
468                        }
469                    }
470                }
471            }
472            catch (LauncherException ex) {
473                reporter.incrCounter(COUNTER_GROUP, COUNTER_LAUNCHER_ERROR, 1);
474                System.out.println();
475                System.out.println("Oozie Launcher failed, finishing Hadoop job gracefully");
476                System.out.println();
477            }
478        }
479    
480        public void close() throws IOException {
481            System.out.println();
482            System.out.println("Oozie Launcher ends");
483            System.out.println();
484        }
485    
486        protected JobConf getJobConf() {
487            return jobConf;
488        }
489    
490        private void setupMainConfiguration() throws IOException {
491            FileSystem fs = FileSystem.get(getJobConf());
492            fs.copyToLocalFile(new Path(getJobConf().get(OOZIE_ACTION_DIR_PATH), ACTION_CONF_XML), new Path(new File(
493                    ACTION_CONF_XML).getAbsolutePath()));
494    
495            System.setProperty("oozie.launcher.job.id", getJobConf().get("mapred.job.id"));
496            System.setProperty("oozie.job.id", getJobConf().get(OOZIE_JOB_ID));
497            System.setProperty("oozie.action.id", getJobConf().get(OOZIE_ACTION_ID));
498            System.setProperty("oozie.action.conf.xml", new File(ACTION_CONF_XML).getAbsolutePath());
499            System.setProperty("oozie.action.output.properties", new File(ACTION_OUTPUT_PROPS).getAbsolutePath());
500            System.setProperty("oozie.action.newId.properties", new File(ACTION_NEW_ID_PROPS).getAbsolutePath());
501        }
502    
503        public static String[] getMainArguments(Configuration conf) {
504            String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)];
505            for (int i = 0; i < args.length; i++) {
506                args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
507            }
508            return args;
509        }
510    
511        private void setupHeartBeater(Reporter reporter) {
512            timer = new ScheduledThreadPoolExecutor(1);
513            timer.scheduleAtFixedRate(new LauncherMapper(reporter), 0, 30, TimeUnit.SECONDS);
514        }
515    
516        private void destroyHeartBeater() {
517            timer.shutdownNow();
518        }
519    
520        private Reporter reporter;
521    
522        private LauncherMapper(Reporter reporter) {
523            this.reporter = reporter;
524        }
525    
526        public void run() {
527            System.out.println("Heart beat");
528            reporter.progress();
529        }
530    
531        private void failLauncher(String reason, Throwable ex) throws LauncherException {
532            try {
533                if (ex != null) {
534                    reason += ", " + ex.getMessage();
535                }
536                Properties errorProps = new Properties();
537                errorProps.setProperty("error.reason", reason);
538                if (ex != null) {
539                    if (ex.getMessage() != null) {
540                        errorProps.setProperty("exception.message", ex.getMessage());
541                    }
542                    StringWriter sw = new StringWriter();
543                    PrintWriter pw = new PrintWriter(sw);
544                    ex.printStackTrace(pw);
545                    pw.close();
546                    errorProps.setProperty("exception.stacktrace", sw.toString());
547                }
548                FileSystem fs = FileSystem.get(getJobConf());
549                OutputStream os = fs.create(new Path(actionDir, ACTION_ERROR_PROPS));
550                errorProps.store(os, "");
551                os.close();
552    
553                System.out.print("Failing Oozie Launcher, " + reason + "\n");
554                System.err.print("Failing Oozie Launcher, " + reason + "\n");
555                if (ex != null) {
556                    ex.printStackTrace(System.out);
557                    ex.printStackTrace(System.err);
558                }
559                throw new LauncherException();
560            }
561            catch (IOException rex) {
562                throw new RuntimeException("Error while failing launcher, " + rex.getMessage(), rex);
563            }
564        }
565    
566        /**
567         * Print files and directories in current directory. Will list files in the sub-directory (only 1 level deep)
568         */
569        protected void printContentsOfCurrentDir() {
570            File folder = new File(".");
571            System.out.println();
572            System.out.println("Files in current dir:" + folder.getAbsolutePath());
573            System.out.println("======================");
574    
575            File[] listOfFiles = folder.listFiles();
576            for (File fileName : listOfFiles) {
577                if (fileName.isFile()) {
578                    System.out.println("File: " + fileName.getName());
579                }
580                else if (fileName.isDirectory()) {
581                    System.out.println("Dir: " + fileName.getName());
582                    File subDir = new File(fileName.getName());
583                    File[] moreFiles = subDir.listFiles();
584                    for (File subFileName : moreFiles) {
585                        if (subFileName.isFile()) {
586                            System.out.println("  File: " + subFileName.getName());
587                        }
588                        else if (subFileName.isDirectory()) {
589                            System.out.println("  Dir: " + subFileName.getName());
590                        }
591                    }
592                }
593            }
594        }
595    
596    }
597    
598    class LauncherSecurityManager extends SecurityManager {
599        private static boolean exitInvoked;
600        private static int exitCode;
601        private SecurityManager securityManager;
602    
603        public LauncherSecurityManager() {
604            reset();
605            securityManager = System.getSecurityManager();
606            System.setSecurityManager(this);
607        }
608    
609        @Override
610        public void checkPermission(Permission perm) {
611            if (securityManager != null) {
612                // check everything with the original SecurityManager
613                securityManager.checkPermission(perm);
614            }
615        }
616    
617        @Override
618        public void checkExit(int status) throws SecurityException {
619            exitInvoked = true;
620            exitCode = status;
621            throw new SecurityException("Intercepted System.exit(" + status + ")");
622        }
623    
624        public static boolean getExitInvoked() {
625            return exitInvoked;
626        }
627    
628        public static int getExitCode() {
629            return exitCode;
630        }
631    
632        public static void reset() {
633            exitInvoked = false;
634            exitCode = 0;
635        }
636    }
637    
638    class LauncherException extends Exception {
639    }