001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.oozie.action.hadoop; 019 020 import java.io.IOException; 021 import java.io.StringReader; 022 import java.util.List; 023 024 import org.apache.hadoop.conf.Configuration; 025 import org.apache.hadoop.fs.FileStatus; 026 import org.apache.hadoop.fs.FileSystem; 027 import org.apache.hadoop.fs.Path; 028 import org.apache.oozie.action.ActionExecutorException; 029 import org.apache.oozie.client.WorkflowAction; 030 import org.apache.oozie.util.XConfiguration; 031 import org.apache.oozie.util.XmlUtils; 032 import org.jdom.Element; 033 import org.jdom.JDOMException; 034 import org.jdom.Namespace; 035 036 import static org.apache.oozie.action.hadoop.SqoopMain.SQOOP_DEFAULT_CONF; 037 import static org.apache.oozie.action.hadoop.SqoopMain.SQOOP_SITE_CONF; 038 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_DEFAULT_CONF; 039 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_EXEC_L4J_PROPS; 040 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_L4J_PROPS; 041 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_SITE_CONF; 042 import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; 043 044 public class SqoopActionExecutor extends JavaActionExecutor { 045 046 047 public SqoopActionExecutor() { 048 super("sqoop"); 049 } 050 051 @Override 052 protected List<Class> getLauncherClasses() { 053 List<Class> classes = super.getLauncherClasses(); 054 classes.add(LauncherMain.class); 055 classes.add(MapReduceMain.class); 056 classes.add(SqoopMain.class); 057 return classes; 058 } 059 060 @Override 061 protected String getLauncherMain(Configuration launcherConf, Element actionXml) { 062 return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, SqoopMain.class.getName()); 063 } 064 065 @Override 066 void injectActionCallback(Context context, Configuration launcherConf) { 067 } 068 069 protected void addResourceToCache(Configuration conf, FileSystem fs, Path resourcePath) 070 throws ActionExecutorException { 071 try { 072 if (!fs.exists(resourcePath) || !fs.isFile(resourcePath)) { 073 return; 074 } 075 Path parentPath = resourcePath.getParent(); 076 String resourceName = resourcePath.getName(); 077 addToCache(conf, parentPath, resourceName + "#" + resourceName, false); 078 } catch (Exception ex) { 079 throw convertException(ex); 080 } 081 } 082 083 @Override 084 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) 085 throws ActionExecutorException { 086 try { 087 super.setupLauncherConf(conf, actionXml, appPath, context); 088 Namespace ns = actionXml.getNamespace(); 089 090 // Look for hive-log4j.properties and hive-exec-log4j.properties 091 // in the configuration directory and add them to the distributed 092 // cache. 093 Element e = actionXml.getChild("conf-dir", ns); 094 if (e != null) { 095 Path hiveConfPath = new Path(appPath, e.getTextTrim()); 096 FileSystem fs = getActionFileSystem(context, actionXml); 097 FileStatus status = fs.getFileStatus(hiveConfPath); 098 if (status.isDir()) { 099 addResourceToCache(conf, fs, new Path(hiveConfPath,HIVE_L4J_PROPS)); 100 addResourceToCache(conf, fs, new Path(hiveConfPath, HIVE_EXEC_L4J_PROPS)); 101 } 102 } 103 return conf; 104 } catch (IOException ex) { 105 throw convertException(ex); 106 } 107 } 108 109 @Override 110 @SuppressWarnings("unchecked") 111 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) 112 throws ActionExecutorException { 113 super.setupActionConf(actionConf, context, actionXml, appPath); 114 Namespace ns = actionXml.getNamespace(); 115 116 // Read sqoop-default.xml, sqoop-site.xml, hive-default.xml, hive-site.xml, 117 // and any configuration properties defined in the workflow and overlay 118 // these properties onto the action configuration in this order. 119 Element e = actionXml.getChild("conf-dir", ns); 120 try { 121 if (e != null) { 122 Path sqoopConfPath = new Path(appPath, e.getTextTrim()); 123 FileSystem fs = getActionFileSystem(context, actionXml); 124 FileStatus status = fs.getFileStatus(sqoopConfPath); 125 if (status.isDir()) { 126 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, SQOOP_DEFAULT_CONF)); 127 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, SQOOP_SITE_CONF)); 128 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, HIVE_DEFAULT_CONF)); 129 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, HIVE_SITE_CONF)); 130 } 131 } 132 133 e = actionXml.getChild("configuration", ns); 134 if (e != null) { 135 String strConf = XmlUtils.prettyPrint(e).toString(); 136 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf)); 137 checkForDisallowedProps(inlineConf, "inline configuration"); 138 XConfiguration.copy(inlineConf, actionConf); 139 } 140 } catch (IOException ex) { 141 throw convertException(ex); 142 } 143 144 String command = actionXml.getChild("command", ns).getTextTrim(); 145 146 SqoopMain.setSqoopCommand(actionConf, command); 147 return actionConf; 148 } 149 150 private void loadConfiguration(Configuration baseConf, FileSystem fs, Path confPath) 151 throws ActionExecutorException { 152 try { 153 if (!fs.exists(confPath) || !fs.isFile(confPath)) { 154 return; 155 } 156 Configuration newConf = new XConfiguration(fs.open(confPath)); 157 checkForDisallowedProps(newConf, confPath.getName()); 158 XConfiguration.copy(newConf, baseConf); 159 } catch (IOException ex) { 160 throw convertException(ex); 161 } 162 } 163 164 @Override 165 protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { 166 return true; 167 } 168 }