001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.oozie.action.hadoop; 019 020 import java.io.File; 021 import java.io.FileOutputStream; 022 import java.io.IOException; 023 import java.io.OutputStream; 024 import java.net.URL; 025 import java.util.Map; 026 import java.util.Properties; 027 028 import org.apache.hadoop.conf.Configuration; 029 import org.apache.hadoop.fs.Path; 030 import com.cloudera.sqoop.Sqoop; 031 import org.apache.log4j.PropertyConfigurator; 032 033 public class SqoopMain extends LauncherMain { 034 035 private static final String SQOOP_ARGS = "oozie.sqoop.args"; 036 037 public static final String SQOOP_SITE_CONF = "sqoop-site.xml"; 038 039 private static final String JOB_ID_LOG_PREFIX = "Job complete: "; 040 private static final String SQOOP_LOG4J_PROPS = "sqoop-log4j.properties"; 041 042 public static void main(String[] args) throws Exception { 043 run(SqoopMain.class, args); 044 } 045 046 private static Configuration initActionConf() { 047 // loading action conf prepared by Oozie 048 Configuration sqoopConf = new Configuration(false); 049 050 String actionXml = System.getProperty("oozie.action.conf.xml"); 051 052 if (actionXml == null) { 053 throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]"); 054 } 055 if (!new File(actionXml).exists()) { 056 throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist"); 057 } 058 059 sqoopConf.addResource(new Path("file:///", actionXml)); 060 061 String delegationToken = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); 062 if (delegationToken != null) { 063 sqoopConf.set("mapreduce.job.credentials.binary", delegationToken); 064 System.out.println("------------------------"); 065 System.out.println("Setting env property for mapreduce.job.credentials.binary to: " + delegationToken); 066 System.out.println("------------------------"); 067 System.setProperty("mapreduce.job.credentials.binary", delegationToken); 068 } else { 069 System.out.println("Non-Kerberos execution"); 070 } 071 072 return sqoopConf; 073 } 074 075 public static Configuration setUpSqoopSite() throws Exception { 076 Configuration sqoopConf = initActionConf(); 077 078 // Write the action configuration out to hive-site.xml 079 OutputStream os = new FileOutputStream(SQOOP_SITE_CONF); 080 sqoopConf.writeXml(os); 081 os.close(); 082 083 System.out.println(); 084 System.out.println("Sqoop Configuration Properties:"); 085 System.out.println("------------------------"); 086 for (Map.Entry<String, String> entry : sqoopConf) { 087 System.out.println(entry.getKey() + "=" + entry.getValue()); 088 } 089 System.out.flush(); 090 System.out.println("------------------------"); 091 System.out.println(); 092 return sqoopConf; 093 } 094 095 public static String setUpSqoopLog4J(Configuration sqoopConf) throws IOException { 096 //Logfile to capture job IDs 097 String hadoopJobId = System.getProperty("oozie.launcher.job.id"); 098 if (hadoopJobId == null) { 099 throw new RuntimeException("Launcher Hadoop Job ID system property not set"); 100 } 101 102 String logFile = new File("sqoop-oozie-" + hadoopJobId + ".log").getAbsolutePath(); 103 104 Properties hadoopProps = new Properties(); 105 106 // Preparing log4j configuration 107 URL log4jFile = Thread.currentThread().getContextClassLoader().getResource("log4j.properties"); 108 if (log4jFile != null) { 109 // getting hadoop log4j configuration 110 hadoopProps.load(log4jFile.openStream()); 111 } 112 113 String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO"); 114 115 hadoopProps.setProperty("log4j.logger.com.cloudera.sqoop", logLevel + ", A"); 116 hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); 117 hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout"); 118 hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); 119 120 hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender"); 121 hadoopProps.setProperty("log4j.appender.jobid.file", logFile); 122 hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout"); 123 hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); 124 hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid"); 125 126 String localProps = new File(SQOOP_LOG4J_PROPS).getAbsolutePath(); 127 OutputStream os1 = new FileOutputStream(localProps); 128 hadoopProps.store(os1, ""); 129 os1.close(); 130 131 PropertyConfigurator.configure(SQOOP_LOG4J_PROPS); 132 133 localProps = new File(HiveMain.HIVE_L4J_PROPS).getAbsolutePath(); 134 os1 = new FileOutputStream(localProps); 135 hadoopProps.store(os1, ""); 136 os1.close(); 137 138 localProps = new File(HiveMain.HIVE_EXEC_L4J_PROPS).getAbsolutePath(); 139 os1 = new FileOutputStream(localProps); 140 hadoopProps.store(os1, ""); 141 os1.close(); 142 return logFile; 143 } 144 145 protected void run(String[] args) throws Exception { 146 System.out.println(); 147 System.out.println("Oozie Sqoop action configuration"); 148 System.out.println("================================================================="); 149 150 Configuration sqoopConf = setUpSqoopSite(); 151 String logFile = setUpSqoopLog4J(sqoopConf); 152 153 String[] sqoopArgs = MapReduceMain.getStrings(sqoopConf, SQOOP_ARGS); 154 if (sqoopArgs == null) { 155 throw new RuntimeException("Action Configuration does not have [" + SQOOP_ARGS + "] property"); 156 } 157 158 System.out.println("Sqoop command arguments :"); 159 for (String arg : sqoopArgs) { 160 System.out.println(" " + arg); 161 } 162 163 HiveMain.setUpHiveSite(); 164 165 System.out.println("================================================================="); 166 System.out.println(); 167 System.out.println(">>> Invoking Sqoop command line now >>>"); 168 System.out.println(); 169 System.out.flush(); 170 171 try { 172 runSqoopJob(sqoopArgs); 173 } 174 catch (SecurityException ex) { 175 if (LauncherSecurityManager.getExitInvoked()) { 176 if (LauncherSecurityManager.getExitCode() != 0) { 177 throw ex; 178 } 179 } 180 } 181 182 System.out.println(); 183 System.out.println("<<< Invocation of Sqoop command completed <<<"); 184 System.out.println(); 185 186 // harvesting and recording Hadoop Job IDs 187 Properties jobIds = HiveMain.getHadoopJobIds(logFile, JOB_ID_LOG_PREFIX); 188 File file = new File(System.getProperty("oozie.action.output.properties")); 189 OutputStream os = new FileOutputStream(file); 190 jobIds.store(os, ""); 191 os.close(); 192 System.out.println(" Hadoop Job IDs executed by Sqoop: " + jobIds.getProperty("hadoopJobs")); 193 System.out.println(); 194 } 195 196 protected void runSqoopJob(String[] args) throws Exception { 197 // running as from the command line 198 Sqoop.main(args); 199 } 200 201 public static void setSqoopCommand(Configuration conf, String[] args) { 202 MapReduceMain.setStrings(conf, SQOOP_ARGS, args); 203 } 204 }