001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.hadoop;
019    
020    import java.io.File;
021    import java.io.FileOutputStream;
022    import java.io.IOException;
023    import java.io.OutputStream;
024    import java.net.URL;
025    import java.util.Map;
026    import java.util.Properties;
027    
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.hadoop.fs.Path;
030    import com.cloudera.sqoop.Sqoop;
031    import org.apache.log4j.PropertyConfigurator;
032    
033    public class SqoopMain extends LauncherMain {
034    
035        private static final String SQOOP_ARGS = "oozie.sqoop.args";
036    
037        public static final String SQOOP_SITE_CONF = "sqoop-site.xml";
038    
039        private static final String JOB_ID_LOG_PREFIX = "Job complete: ";
040        private static final String SQOOP_LOG4J_PROPS = "sqoop-log4j.properties";
041    
042        public static void main(String[] args) throws Exception {
043            run(SqoopMain.class, args);
044        }
045    
046        private static Configuration initActionConf() {
047            // loading action conf prepared by Oozie
048            Configuration sqoopConf = new Configuration(false);
049    
050            String actionXml = System.getProperty("oozie.action.conf.xml");
051    
052            if (actionXml == null) {
053                throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
054            }
055            if (!new File(actionXml).exists()) {
056                throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
057            }
058    
059            sqoopConf.addResource(new Path("file:///", actionXml));
060    
061            String delegationToken = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
062            if (delegationToken != null) {
063                sqoopConf.set("mapreduce.job.credentials.binary", delegationToken);
064                System.out.println("------------------------");
065                System.out.println("Setting env property for mapreduce.job.credentials.binary to: " + delegationToken);
066                System.out.println("------------------------");
067                System.setProperty("mapreduce.job.credentials.binary", delegationToken);
068            } else {
069                System.out.println("Non-Kerberos execution");
070            }
071    
072            return sqoopConf;
073        }
074    
075        public static Configuration setUpSqoopSite() throws Exception {
076            Configuration sqoopConf = initActionConf();
077    
078            // Write the action configuration out to hive-site.xml
079            OutputStream os = new FileOutputStream(SQOOP_SITE_CONF);
080            sqoopConf.writeXml(os);
081            os.close();
082    
083            System.out.println();
084            System.out.println("Sqoop Configuration Properties:");
085            System.out.println("------------------------");
086            for (Map.Entry<String, String> entry : sqoopConf) {
087                System.out.println(entry.getKey() + "=" + entry.getValue());
088            }
089            System.out.flush();
090            System.out.println("------------------------");
091            System.out.println();
092            return sqoopConf;
093        }
094    
095        public static String setUpSqoopLog4J(Configuration sqoopConf) throws IOException {
096            //Logfile to capture job IDs
097            String hadoopJobId = System.getProperty("oozie.launcher.job.id");
098            if (hadoopJobId == null) {
099                throw new RuntimeException("Launcher Hadoop Job ID system property not set");
100            }
101    
102            String logFile = new File("sqoop-oozie-" + hadoopJobId + ".log").getAbsolutePath();
103    
104            Properties hadoopProps = new Properties();
105    
106            // Preparing log4j configuration
107            URL log4jFile = Thread.currentThread().getContextClassLoader().getResource("log4j.properties");
108            if (log4jFile != null) {
109                // getting hadoop log4j configuration
110                hadoopProps.load(log4jFile.openStream());
111            }
112    
113            String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO");
114    
115            hadoopProps.setProperty("log4j.logger.com.cloudera.sqoop", logLevel + ", A");
116            hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
117            hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");
118            hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
119    
120            hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender");
121            hadoopProps.setProperty("log4j.appender.jobid.file", logFile);
122            hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout");
123            hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
124            hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid");
125    
126            String localProps = new File(SQOOP_LOG4J_PROPS).getAbsolutePath();
127            OutputStream os1 = new FileOutputStream(localProps);
128            hadoopProps.store(os1, "");
129            os1.close();
130    
131            PropertyConfigurator.configure(SQOOP_LOG4J_PROPS);
132    
133            localProps = new File(HiveMain.HIVE_L4J_PROPS).getAbsolutePath();
134            os1 = new FileOutputStream(localProps);
135            hadoopProps.store(os1, "");
136            os1.close();
137    
138            localProps = new File(HiveMain.HIVE_EXEC_L4J_PROPS).getAbsolutePath();
139            os1 = new FileOutputStream(localProps);
140            hadoopProps.store(os1, "");
141            os1.close();
142            return logFile;
143        }
144    
145        protected void run(String[] args) throws Exception {
146            System.out.println();
147            System.out.println("Oozie Sqoop action configuration");
148            System.out.println("=================================================================");
149    
150            Configuration sqoopConf = setUpSqoopSite();
151            String logFile = setUpSqoopLog4J(sqoopConf);
152    
153            String[] sqoopArgs = MapReduceMain.getStrings(sqoopConf, SQOOP_ARGS);
154            if (sqoopArgs == null) {
155                throw new RuntimeException("Action Configuration does not have [" + SQOOP_ARGS + "] property");
156            }
157    
158            System.out.println("Sqoop command arguments :");
159            for (String arg : sqoopArgs) {
160                System.out.println("             " + arg);
161            }
162    
163            HiveMain.setUpHiveSite();
164    
165            System.out.println("=================================================================");
166            System.out.println();
167            System.out.println(">>> Invoking Sqoop command line now >>>");
168            System.out.println();
169            System.out.flush();
170    
171            try {
172                runSqoopJob(sqoopArgs);
173            }
174            catch (SecurityException ex) {
175                if (LauncherSecurityManager.getExitInvoked()) {
176                    if (LauncherSecurityManager.getExitCode() != 0) {
177                        throw ex;
178                    }
179                }
180            }
181    
182            System.out.println();
183            System.out.println("<<< Invocation of Sqoop command completed <<<");
184            System.out.println();
185    
186            // harvesting and recording Hadoop Job IDs
187            Properties jobIds = HiveMain.getHadoopJobIds(logFile, JOB_ID_LOG_PREFIX);
188            File file = new File(System.getProperty("oozie.action.output.properties"));
189            OutputStream os = new FileOutputStream(file);
190            jobIds.store(os, "");
191            os.close();
192            System.out.println(" Hadoop Job IDs executed by Sqoop: " + jobIds.getProperty("hadoopJobs"));
193            System.out.println();
194        }
195    
196        protected void runSqoopJob(String[] args) throws Exception {
197            // running as from the command line
198            Sqoop.main(args);
199        }
200    
201        public static void setSqoopCommand(Configuration conf, String[] args) {
202            MapReduceMain.setStrings(conf, SQOOP_ARGS, args);
203        }
204    }