001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.hadoop;
019    
020    import java.io.IOException;
021    import java.io.StringReader;
022    import java.util.List;
023    
024    import org.apache.hadoop.conf.Configuration;
025    import org.apache.hadoop.fs.FileStatus;
026    import org.apache.hadoop.fs.FileSystem;
027    import org.apache.hadoop.fs.Path;
028    import org.apache.oozie.action.ActionExecutorException;
029    import org.apache.oozie.client.WorkflowAction;
030    import org.apache.oozie.util.XConfiguration;
031    import org.apache.oozie.util.XmlUtils;
032    import org.jdom.Element;
033    import org.jdom.JDOMException;
034    import org.jdom.Namespace;
035    
036    import static org.apache.oozie.action.hadoop.SqoopMain.SQOOP_DEFAULT_CONF;
037    import static org.apache.oozie.action.hadoop.SqoopMain.SQOOP_SITE_CONF;
038    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_DEFAULT_CONF;
039    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_EXEC_L4J_PROPS;
040    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_L4J_PROPS;
041    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_SITE_CONF;
042    import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
043    
044    public class SqoopActionExecutor extends JavaActionExecutor {
045    
046    
047        public SqoopActionExecutor() {
048            super("sqoop");
049        }
050    
051        @Override
052        protected List<Class> getLauncherClasses() {
053            List<Class> classes = super.getLauncherClasses();
054            classes.add(LauncherMain.class);
055            classes.add(MapReduceMain.class);
056            classes.add(SqoopMain.class);
057            return classes;
058        }
059    
060        @Override
061        protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
062            return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, SqoopMain.class.getName());
063        }
064    
065        @Override
066        void injectActionCallback(Context context, Configuration launcherConf) {
067        }
068    
069        protected void addResourceToCache(Configuration conf, FileSystem fs, Path resourcePath)
070        throws ActionExecutorException {
071            try {
072                if (!fs.exists(resourcePath) || !fs.isFile(resourcePath)) {
073                    return;
074                }
075                Path parentPath = resourcePath.getParent();
076                String resourceName = resourcePath.getName();
077                addToCache(conf, parentPath, resourceName + "#" + resourceName, false);
078            } catch (Exception ex) {
079                throw convertException(ex);
080            }
081        }
082    
083        @Override
084        Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
085                throws ActionExecutorException {
086            try {
087                super.setupLauncherConf(conf, actionXml, appPath, context);
088                Namespace ns = actionXml.getNamespace();
089    
090                // Look for hive-log4j.properties and hive-exec-log4j.properties
091                // in the configuration directory and add them to the distributed
092                // cache.
093                Element e = actionXml.getChild("conf-dir", ns);
094                if (e != null) {
095                    Path hiveConfPath = new Path(appPath, e.getTextTrim());
096                    FileSystem fs = getActionFileSystem(context, actionXml);
097                    FileStatus status = fs.getFileStatus(hiveConfPath);
098                    if (status.isDir()) {
099                        addResourceToCache(conf, fs, new Path(hiveConfPath,HIVE_L4J_PROPS));
100                        addResourceToCache(conf, fs, new Path(hiveConfPath, HIVE_EXEC_L4J_PROPS));
101                    }
102                }
103                return conf;
104            } catch (IOException ex) {
105                throw convertException(ex);
106            }
107        }
108    
109        @Override
110        @SuppressWarnings("unchecked")
111        Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
112                throws ActionExecutorException {
113            super.setupActionConf(actionConf, context, actionXml, appPath);
114            Namespace ns = actionXml.getNamespace();
115    
116            // Read sqoop-default.xml, sqoop-site.xml, hive-default.xml, hive-site.xml,
117            // and any configuration properties defined in the workflow and overlay
118            // these properties onto the action configuration in this order.
119            Element e = actionXml.getChild("conf-dir", ns);
120            try {
121                if (e != null) {
122                    Path sqoopConfPath = new Path(appPath, e.getTextTrim());
123                    FileSystem fs = getActionFileSystem(context, actionXml);
124                    FileStatus status = fs.getFileStatus(sqoopConfPath);
125                    if (status.isDir()) {
126                        loadConfiguration(actionConf, fs, new Path(sqoopConfPath, SQOOP_DEFAULT_CONF));
127                        loadConfiguration(actionConf, fs, new Path(sqoopConfPath, SQOOP_SITE_CONF));
128                        loadConfiguration(actionConf, fs, new Path(sqoopConfPath, HIVE_DEFAULT_CONF));
129                        loadConfiguration(actionConf, fs, new Path(sqoopConfPath, HIVE_SITE_CONF));
130                    }
131                }
132    
133                e = actionXml.getChild("configuration", ns);
134                if (e != null) {
135                    String strConf = XmlUtils.prettyPrint(e).toString();
136                    XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
137                    checkForDisallowedProps(inlineConf, "inline configuration");
138                    XConfiguration.copy(inlineConf, actionConf);
139                }
140            } catch (IOException ex) {
141                throw convertException(ex);
142            }
143    
144            String command = actionXml.getChild("command", ns).getTextTrim();
145    
146            SqoopMain.setSqoopCommand(actionConf, command);
147            return actionConf;
148        }
149    
150        private void loadConfiguration(Configuration baseConf, FileSystem fs, Path confPath)
151        throws ActionExecutorException {
152            try {
153                if (!fs.exists(confPath) || !fs.isFile(confPath)) {
154                    return;
155                }
156                Configuration newConf = new XConfiguration(fs.open(confPath));
157                checkForDisallowedProps(newConf, confPath.getName());
158                XConfiguration.copy(newConf, baseConf);
159            } catch (IOException ex) {
160                throw convertException(ex);
161            }
162        }
163    
164        @Override
165        protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
166            return true;
167        }
168    }