001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.hadoop;
019    
020    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_DEFAULT_CONF;
021    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_EXEC_L4J_PROPS;
022    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_L4J_PROPS;
023    import static org.apache.oozie.action.hadoop.HiveMain.HIVE_SITE_CONF;
024    import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
025    
026    import java.io.IOException;
027    import java.io.StringReader;
028    import java.util.List;
029    
030    import org.apache.hadoop.conf.Configuration;
031    import org.apache.hadoop.fs.FileStatus;
032    import org.apache.hadoop.fs.FileSystem;
033    import org.apache.hadoop.fs.Path;
034    import org.apache.oozie.action.ActionExecutorException;
035    import org.apache.oozie.client.WorkflowAction;
036    import org.apache.oozie.util.XConfiguration;
037    import org.apache.oozie.util.XmlUtils;
038    import org.jdom.Element;
039    import org.jdom.JDOMException;
040    import org.jdom.Namespace;
041    
042    public class HiveActionExecutor extends JavaActionExecutor {
043    
044        public HiveActionExecutor() {
045            super("hive");
046        }
047    
048        @Override
049        protected List<Class> getLauncherClasses() {
050            List<Class> classes = super.getLauncherClasses();
051            classes.add(LauncherMain.class);
052            classes.add(MapReduceMain.class);
053            classes.add(HiveMain.class);
054            return classes;
055        }
056    
057        @Override
058        protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
059            return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, HiveMain.class.getName());
060        }
061    
062        @Override
063        void injectActionCallback(Context context, Configuration launcherConf) {
064        }
065    
066        protected void addResourceToCache(Configuration conf, FileSystem fs, Path resourcePath)
067        throws ActionExecutorException {
068            try {
069                if (!fs.exists(resourcePath) || !fs.isFile(resourcePath)) {
070                    return;
071                }
072                Path parentPath = resourcePath.getParent();
073                String resourceName = resourcePath.getName();
074                addToCache(conf, parentPath, resourceName + "#" + resourceName, false);
075            } catch (Exception ex) {
076                throw convertException(ex);
077            }
078        }
079    
080        @Override
081        protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
082                throws ActionExecutorException {
083            try {
084                super.setupLauncherConf(conf, actionXml, appPath, context);
085                Namespace ns = actionXml.getNamespace();
086    
087                // Look for hive-log4j.properties and hive-exec-log4j.properties
088                // in the configuration directory and add them to the distributed
089                // cache.
090                Element e = actionXml.getChild("conf-dir", ns);
091                if (e != null) {
092                    Path hiveConfPath = new Path(appPath, e.getTextTrim());
093                    FileSystem fs = getActionFileSystem(context, actionXml);
094                    FileStatus status = fs.getFileStatus(hiveConfPath);
095                    if (status.isDir()) {
096                        addResourceToCache(conf, fs, new Path(hiveConfPath,HIVE_L4J_PROPS));
097                        addResourceToCache(conf, fs, new Path(hiveConfPath, HIVE_EXEC_L4J_PROPS));
098                    }
099                }
100    
101                // Make the query file available to Hive.
102                String script = actionXml.getChild("script", ns).getTextTrim();
103                String scriptName = new Path(script).getName();
104                addToCache(conf, appPath, script + "#" + scriptName, false);
105    
106                return conf;
107            } catch (IOException ex) {
108                throw convertException(ex);
109            }
110        }
111    
112        @Override
113        @SuppressWarnings("unchecked")
114        Configuration setupActionConf(Configuration actionConf, Context context,
115                Element actionXml, Path appPath) throws ActionExecutorException {
116    
117            try {
118                // Read hive-default.xml, hive-site.xml, and any configuration
119                // properties defined in the workflow and overlay these properties
120                // onto the action configuration in this order.
121                Namespace ns = actionXml.getNamespace();
122                Element e = actionXml.getChild("conf-dir", ns);
123                if (e != null) {
124                    Path hiveConfPath = new Path(appPath, e.getTextTrim());
125                    FileSystem fs = getActionFileSystem(context, actionXml);
126                    FileStatus status = fs.getFileStatus(hiveConfPath);
127                    if (status.isDir()) {
128                        loadConfiguration(actionConf, fs, new Path(hiveConfPath, HIVE_DEFAULT_CONF));
129                        loadConfiguration(actionConf, fs, new Path(hiveConfPath, HIVE_SITE_CONF));
130                    }
131                }
132    
133                e = actionXml.getChild("configuration", ns);
134                if (e != null) {
135                    String strConf = XmlUtils.prettyPrint(e).toString();
136                    XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
137                    checkForDisallowedProps(inlineConf, "inline configuration");
138                    XConfiguration.copy(inlineConf, actionConf);
139                }
140    
141                String script = actionXml.getChild("script", ns).getTextTrim();
142                String scriptName = new Path(script).getName();
143    
144                List<Element> params = actionXml.getChildren("param", ns);
145                String[] strParams = new String[params.size()];
146                for (int i = 0; i < params.size(); i++) {
147                    strParams[i] = params.get(i).getTextTrim();
148                }
149    
150                HiveMain.setHiveScript(actionConf, scriptName, strParams);
151                return actionConf;
152            } catch (IOException e) {
153                throw convertException(e);
154            }
155        }
156    
157        private void loadConfiguration(Configuration baseConf, FileSystem fs, Path confPath)
158        throws ActionExecutorException {
159            try {
160                if (!fs.exists(confPath) || !fs.isFile(confPath)) {
161                    return;
162                }
163                Configuration newConf = new XConfiguration(fs.open(confPath));
164                checkForDisallowedProps(newConf, confPath.getName());
165                XConfiguration.copy(newConf, baseConf);
166            } catch (IOException ex) {
167                throw convertException(ex);
168            }
169        }
170    
171        @Override
172        protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
173            return true;
174        }
175    
176    }