001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.oozie.action.hadoop; 019 020 import java.io.BufferedReader; 021 import java.io.BufferedWriter; 022 import java.io.File; 023 import java.io.FileOutputStream; 024 import java.io.FileReader; 025 import java.io.FileWriter; 026 import java.io.IOException; 027 import java.io.OutputStream; 028 import java.util.ArrayList; 029 import java.util.HashMap; 030 import java.util.HashSet; 031 import java.util.List; 032 import java.util.Map; 033 import java.util.Map.Entry; 034 import java.util.Set; 035 import java.util.regex.Matcher; 036 import java.util.regex.Pattern; 037 038 import org.apache.hadoop.conf.Configuration; 039 import org.apache.hadoop.fs.Path; 040 import org.apache.hadoop.hive.cli.CliDriver; 041 042 public class HiveMain extends LauncherMain { 043 044 public static final String HIVE_L4J_PROPS = "hive-log4j.properties"; 045 public static final String HIVE_EXEC_L4J_PROPS = "hive-exec-log4j.properties"; 046 public static final String HIVE_DEFAULT_CONF = "hive-default.xml"; 047 public static final String HIVE_SITE_CONF = "hive-site.xml"; 048 public static final String HIVE_SCRIPT = "oozie.hive.script"; 049 public static final String HIVE_PARAMS = "oozie.hive.params"; 050 051 public static void main(String[] args) throws Exception { 052 run(HiveMain.class, args); 053 } 054 055 private Configuration initActionConf() { 056 // Loading action conf prepared by Oozie 057 Configuration hiveConf = new Configuration(false); 058 059 String actionXml = System.getProperty("oozie.action.conf.xml"); 060 061 if (actionXml == null) { 062 throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]"); 063 } 064 if (!new File(actionXml).exists()) { 065 throw new RuntimeException("Action Configuration XML file [" 066 + actionXml + "] does not exist"); 067 } else { 068 System.out.println("Using action configuration file " + actionXml); 069 } 070 071 hiveConf.addResource(new Path("file:///", actionXml)); 072 073 // Propagate delegation related props from launcher job to Hive job 074 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 075 hiveConf.set("mapreduce.job.credentials.binary", 076 System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 077 System.out.println("------------------------"); 078 System.out.println("Setting env property for mapreduce.job.credentials.binary to:" 079 + System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 080 System.out.println("------------------------"); 081 System.setProperty("mapreduce.job.credentials.binary", 082 System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 083 } else { 084 System.out.println("Non-Kerberos execution"); 085 } 086 087 // Have to explicitly unset this property or Hive will not set it. 088 hiveConf.set("mapred.job.name", ""); 089 090 // See https://issues.apache.org/jira/browse/HIVE-1411 091 hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG"); 092 093 return hiveConf; 094 } 095 096 protected void run(String[] args) throws Exception { 097 System.out.println(); 098 System.out.println("Oozie Hive action configuration"); 099 System.out.println("================================================================="); 100 101 Configuration hiveConf = initActionConf(); 102 103 // Write the action configuration out to hive-site.xml 104 OutputStream os = new FileOutputStream(HIVE_SITE_CONF); 105 hiveConf.writeXml(os); 106 os.close(); 107 108 System.out.println(); 109 System.out.println("Hive Configuration Properties:"); 110 System.out.println("------------------------"); 111 for (Entry<String, String> entry : hiveConf) { 112 System.out.println(entry.getKey() + "=" + entry.getValue()); 113 } 114 System.out.flush(); 115 System.out.println("------------------------"); 116 System.out.println(); 117 118 List<String> arguments = new ArrayList<String>(); 119 String scriptPath = hiveConf.get(HIVE_SCRIPT); 120 121 if (scriptPath == null) { 122 throw new RuntimeException("Action Configuration does not have [oozie.hive.script] property"); 123 } 124 125 if (!new File(scriptPath).exists()) { 126 throw new RuntimeException("Hive script file [" + scriptPath + "] does not exist"); 127 } 128 129 // 130 // Prepare the Hive Script 131 // 132 System.out.println("Hive script [" + scriptPath + "] content: "); 133 System.out.println("------------------------"); 134 System.out.println(readStringFromFile(scriptPath)); 135 System.out.println("------------------------"); 136 System.out.println(); 137 138 Map<String, String> varMap = new HashMap<String, String>(); 139 140 for (String param : MapReduceMain.getStrings(hiveConf, HIVE_PARAMS)) { 141 System.out.println("Param Expression: '" + param + "'"); 142 143 int idx = param.indexOf('='); 144 if (idx == -1) { 145 throw new RuntimeException("Parameter expression must contain an assignment: " 146 + param); 147 } else if (idx == 0) { 148 throw new RuntimeException("Parameter value not specified: " + param); 149 } 150 String var = param.substring(0, idx); 151 String val = param.substring(idx + 1, param.length()); 152 System.out.println("var = " + var + ", val = " + val); 153 varMap.put(var, val); 154 } 155 156 if (varMap.size() > 0) { 157 System.out.println("Substituting parameter values..."); 158 String subScript = substituteVars(varMap, readStringFromFile(scriptPath)); 159 scriptPath = scriptPath + ".sub"; 160 writeStringToFile(scriptPath, subScript); 161 162 System.out.println("Substituted script [" + scriptPath + "] content: "); 163 System.out.println("------------------------"); 164 System.out.println(readStringFromFile(scriptPath)); 165 System.out.println("------------------------"); 166 System.out.println(); 167 } 168 169 arguments.add("-f"); 170 arguments.add(scriptPath); 171 172 173 System.out.println("Hive command arguments :"); 174 for (String arg : arguments) { 175 System.out.println(" " + arg); 176 } 177 178 System.out.println("================================================================="); 179 System.out.println(); 180 System.out.println(">>> Invoking Hive command line now >>>"); 181 System.out.println(); 182 System.out.flush(); 183 184 runHive(arguments.toArray(new String[arguments.size()])); 185 186 System.out.println("\n<<< Invocation of Hive command completed <<<\n"); 187 } 188 189 private void runHive(String[] args) throws Exception { 190 CliDriver.main(args); 191 } 192 193 public static void setHiveScript(Configuration conf, String script, String[] params) { 194 conf.set(HIVE_SCRIPT, script); 195 MapReduceMain.setStrings(conf, HIVE_PARAMS, params); 196 } 197 198 private static String readStringFromFile(String filePath) throws IOException { 199 String line; 200 BufferedReader br = new BufferedReader(new FileReader(filePath)); 201 StringBuilder sb = new StringBuilder(); 202 String sep = System.getProperty("line.separator"); 203 while ((line = br.readLine()) != null) { 204 sb.append(line).append(sep); 205 } 206 return sb.toString(); 207 } 208 209 private static void writeStringToFile(String filePath, String str) throws IOException { 210 BufferedWriter out = new BufferedWriter(new FileWriter(filePath)); 211 out.write(str); 212 out.close(); 213 } 214 215 216 // Ripped from oahc.Configuration with slight modifications. 217 // Note: 218 // Rather than break at a small pattern limit, we need Hive action to 219 // allow large substitution limit but break if there is cycle 220 // in substitution. 221 private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); 222 static int MAX_VARIABLE_SUBST_LIMIT = 10000; 223 224 static String substituteVars(Map<String, String> varMap, String expr) { 225 Matcher match = varPat.matcher(""); 226 String eval = expr; 227 Set<String> evalSet = new HashSet<String>(); 228 for(int s = 0; s < MAX_VARIABLE_SUBST_LIMIT; s++) { 229 if (evalSet.contains(eval)) { 230 //Cyclic resolution pattern detected. Return current expression. 231 return eval; 232 } 233 evalSet.add(eval); 234 match.reset(eval); 235 if (!match.find()) { 236 return eval; 237 } 238 String var = match.group(); 239 var = var.substring(2, var.length()-1); // remove ${ .. } 240 String val = varMap.get(var); 241 if (null == val) { 242 return eval; // return literal ${var}: var is unbound 243 } 244 // substitute 245 eval = eval.substring(0, match.start())+val+eval.substring(match.end()); 246 } 247 throw new RuntimeException( 248 "Maximum variable substitution limit exceeded: " 249 + MAX_VARIABLE_SUBST_LIMIT + " " + expr); 250 } 251 }