001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.oozie.action.hadoop; 019 020 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_DEFAULT_CONF; 021 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_EXEC_L4J_PROPS; 022 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_L4J_PROPS; 023 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_SITE_CONF; 024 import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; 025 026 import java.io.IOException; 027 import java.io.StringReader; 028 import java.util.List; 029 030 import org.apache.hadoop.conf.Configuration; 031 import org.apache.hadoop.fs.FileStatus; 032 import org.apache.hadoop.fs.FileSystem; 033 import org.apache.hadoop.fs.Path; 034 import org.apache.oozie.action.ActionExecutorException; 035 import org.apache.oozie.client.WorkflowAction; 036 import org.apache.oozie.util.XConfiguration; 037 import org.apache.oozie.util.XmlUtils; 038 import org.jdom.Element; 039 import org.jdom.JDOMException; 040 import org.jdom.Namespace; 041 042 public class HiveActionExecutor extends JavaActionExecutor { 043 044 public HiveActionExecutor() { 045 super("hive"); 046 } 047 048 @Override 049 protected List<Class> getLauncherClasses() { 050 List<Class> classes = super.getLauncherClasses(); 051 classes.add(LauncherMain.class); 052 classes.add(MapReduceMain.class); 053 classes.add(HiveMain.class); 054 return classes; 055 } 056 057 @Override 058 protected String getLauncherMain(Configuration launcherConf, Element actionXml) { 059 return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, HiveMain.class.getName()); 060 } 061 062 @Override 063 void injectActionCallback(Context context, Configuration launcherConf) { 064 } 065 066 protected void addResourceToCache(Configuration conf, FileSystem fs, Path resourcePath) 067 throws ActionExecutorException { 068 try { 069 if (!fs.exists(resourcePath) || !fs.isFile(resourcePath)) { 070 return; 071 } 072 Path parentPath = resourcePath.getParent(); 073 String resourceName = resourcePath.getName(); 074 addToCache(conf, parentPath, resourceName + "#" + resourceName, false); 075 } catch (Exception ex) { 076 throw convertException(ex); 077 } 078 } 079 080 @Override 081 protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) 082 throws ActionExecutorException { 083 try { 084 super.setupLauncherConf(conf, actionXml, appPath, context); 085 Namespace ns = actionXml.getNamespace(); 086 087 // Look for hive-log4j.properties and hive-exec-log4j.properties 088 // in the configuration directory and add them to the distributed 089 // cache. 090 Element e = actionXml.getChild("conf-dir", ns); 091 if (e != null) { 092 Path hiveConfPath = new Path(appPath, e.getTextTrim()); 093 FileSystem fs = getActionFileSystem(context, actionXml); 094 FileStatus status = fs.getFileStatus(hiveConfPath); 095 if (status.isDir()) { 096 addResourceToCache(conf, fs, new Path(hiveConfPath,HIVE_L4J_PROPS)); 097 addResourceToCache(conf, fs, new Path(hiveConfPath, HIVE_EXEC_L4J_PROPS)); 098 } 099 } 100 101 // Make the query file available to Hive. 102 String script = actionXml.getChild("script", ns).getTextTrim(); 103 String scriptName = new Path(script).getName(); 104 addToCache(conf, appPath, script + "#" + scriptName, false); 105 106 return conf; 107 } catch (IOException ex) { 108 throw convertException(ex); 109 } 110 } 111 112 @Override 113 @SuppressWarnings("unchecked") 114 Configuration setupActionConf(Configuration actionConf, Context context, 115 Element actionXml, Path appPath) throws ActionExecutorException { 116 117 try { 118 // Read hive-default.xml, hive-site.xml, and any configuration 119 // properties defined in the workflow and overlay these properties 120 // onto the action configuration in this order. 121 Namespace ns = actionXml.getNamespace(); 122 Element e = actionXml.getChild("conf-dir", ns); 123 if (e != null) { 124 Path hiveConfPath = new Path(appPath, e.getTextTrim()); 125 FileSystem fs = getActionFileSystem(context, actionXml); 126 FileStatus status = fs.getFileStatus(hiveConfPath); 127 if (status.isDir()) { 128 loadConfiguration(actionConf, fs, new Path(hiveConfPath, HIVE_DEFAULT_CONF)); 129 loadConfiguration(actionConf, fs, new Path(hiveConfPath, HIVE_SITE_CONF)); 130 } 131 } 132 133 e = actionXml.getChild("configuration", ns); 134 if (e != null) { 135 String strConf = XmlUtils.prettyPrint(e).toString(); 136 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf)); 137 checkForDisallowedProps(inlineConf, "inline configuration"); 138 XConfiguration.copy(inlineConf, actionConf); 139 } 140 141 String script = actionXml.getChild("script", ns).getTextTrim(); 142 String scriptName = new Path(script).getName(); 143 144 List<Element> params = actionXml.getChildren("param", ns); 145 String[] strParams = new String[params.size()]; 146 for (int i = 0; i < params.size(); i++) { 147 strParams[i] = params.get(i).getTextTrim(); 148 } 149 150 HiveMain.setHiveScript(actionConf, scriptName, strParams); 151 return actionConf; 152 } catch (IOException e) { 153 throw convertException(e); 154 } 155 } 156 157 private void loadConfiguration(Configuration baseConf, FileSystem fs, Path confPath) 158 throws ActionExecutorException { 159 try { 160 if (!fs.exists(confPath) || !fs.isFile(confPath)) { 161 return; 162 } 163 Configuration newConf = new XConfiguration(fs.open(confPath)); 164 checkForDisallowedProps(newConf, confPath.getName()); 165 XConfiguration.copy(newConf, baseConf); 166 } catch (IOException ex) { 167 throw convertException(ex); 168 } 169 } 170 171 @Override 172 protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { 173 return true; 174 } 175 176 }