001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.hadoop;
019    
020    import java.io.BufferedReader;
021    import java.io.BufferedWriter;
022    import java.io.File;
023    import java.io.FileOutputStream;
024    import java.io.FileReader;
025    import java.io.FileWriter;
026    import java.io.IOException;
027    import java.io.OutputStream;
028    import java.util.ArrayList;
029    import java.util.HashMap;
030    import java.util.HashSet;
031    import java.util.List;
032    import java.util.Map;
033    import java.util.Map.Entry;
034    import java.util.Set;
035    import java.util.regex.Matcher;
036    import java.util.regex.Pattern;
037    
038    import org.apache.hadoop.conf.Configuration;
039    import org.apache.hadoop.fs.Path;
040    import org.apache.hadoop.hive.cli.CliDriver;
041    
042    public class HiveMain extends LauncherMain {
043    
044        public static final String HIVE_L4J_PROPS = "hive-log4j.properties";
045        public static final String HIVE_EXEC_L4J_PROPS = "hive-exec-log4j.properties";
046        public static final String HIVE_DEFAULT_CONF = "hive-default.xml";
047        public static final String HIVE_SITE_CONF = "hive-site.xml";
048        public static final String HIVE_SCRIPT = "oozie.hive.script";
049        public static final String HIVE_PARAMS = "oozie.hive.params";
050    
051        public static void main(String[] args) throws Exception {
052            run(HiveMain.class, args);
053        }
054    
055        private Configuration initActionConf() {
056            // Loading action conf prepared by Oozie
057            Configuration hiveConf = new Configuration(false);
058    
059            String actionXml = System.getProperty("oozie.action.conf.xml");
060    
061            if (actionXml == null) {
062                throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
063            }
064            if (!new File(actionXml).exists()) {
065                throw new RuntimeException("Action Configuration XML file ["
066                        + actionXml + "] does not exist");
067            } else {
068                System.out.println("Using action configuration file " + actionXml);
069            }
070    
071            hiveConf.addResource(new Path("file:///", actionXml));
072    
073            // Propagate delegation related props from launcher job to Hive job
074            if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
075                hiveConf.set("mapreduce.job.credentials.binary",
076                        System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
077                System.out.println("------------------------");
078                System.out.println("Setting env property for mapreduce.job.credentials.binary to:"
079                        + System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
080                System.out.println("------------------------");
081                System.setProperty("mapreduce.job.credentials.binary",
082                        System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
083            } else {
084                System.out.println("Non-Kerberos execution");
085            }
086    
087            // Have to explicitly unset this property or Hive will not set it.
088            hiveConf.set("mapred.job.name", "");
089    
090            // See https://issues.apache.org/jira/browse/HIVE-1411
091            hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
092    
093            return hiveConf;
094        }
095    
096        protected void run(String[] args) throws Exception {
097            System.out.println();
098            System.out.println("Oozie Hive action configuration");
099            System.out.println("=================================================================");
100    
101            Configuration hiveConf = initActionConf();
102    
103            // Write the action configuration out to hive-site.xml
104            OutputStream os = new FileOutputStream(HIVE_SITE_CONF);
105            hiveConf.writeXml(os);
106            os.close();
107    
108            System.out.println();
109            System.out.println("Hive Configuration Properties:");
110            System.out.println("------------------------");
111            for (Entry<String, String> entry : hiveConf) {
112                System.out.println(entry.getKey() + "=" + entry.getValue());
113            }
114            System.out.flush();
115            System.out.println("------------------------");
116            System.out.println();
117    
118            List<String> arguments = new ArrayList<String>();
119            String scriptPath = hiveConf.get(HIVE_SCRIPT);
120    
121            if (scriptPath == null) {
122                throw new RuntimeException("Action Configuration does not have [oozie.hive.script] property");
123            }
124    
125            if (!new File(scriptPath).exists()) {
126                throw new RuntimeException("Hive script file [" + scriptPath + "] does not exist");
127            }
128    
129            //
130            // Prepare the Hive Script
131            //
132            System.out.println("Hive script [" + scriptPath + "] content: ");
133            System.out.println("------------------------");
134            System.out.println(readStringFromFile(scriptPath));
135            System.out.println("------------------------");
136            System.out.println();
137    
138            Map<String, String> varMap = new HashMap<String, String>();
139    
140            for (String param : MapReduceMain.getStrings(hiveConf, HIVE_PARAMS)) {
141                System.out.println("Param Expression: '" + param + "'");
142    
143                int idx = param.indexOf('=');
144                if (idx == -1) {
145                    throw new RuntimeException("Parameter expression must contain an assignment: "
146                            + param);
147                } else if (idx == 0) {
148                    throw new RuntimeException("Parameter value not specified: " + param);
149                }
150                String var = param.substring(0, idx);
151                String val = param.substring(idx + 1, param.length());
152                System.out.println("var = " + var + ", val = " + val);
153                varMap.put(var, val);
154            }
155    
156            if (varMap.size() > 0) {
157                System.out.println("Substituting parameter values...");
158                String subScript = substituteVars(varMap, readStringFromFile(scriptPath));
159                scriptPath = scriptPath + ".sub";
160                writeStringToFile(scriptPath, subScript);
161    
162                System.out.println("Substituted script [" + scriptPath + "] content: ");
163                System.out.println("------------------------");
164                System.out.println(readStringFromFile(scriptPath));
165                System.out.println("------------------------");
166                System.out.println();
167            }
168    
169            arguments.add("-f");
170            arguments.add(scriptPath);
171    
172    
173            System.out.println("Hive command arguments :");
174            for (String arg : arguments) {
175                System.out.println("             " + arg);
176            }
177    
178            System.out.println("=================================================================");
179            System.out.println();
180            System.out.println(">>> Invoking Hive command line now >>>");
181            System.out.println();
182            System.out.flush();
183    
184            runHive(arguments.toArray(new String[arguments.size()]));
185    
186            System.out.println("\n<<< Invocation of Hive command completed <<<\n");
187        }
188    
189        private void runHive(String[] args) throws Exception {
190            CliDriver.main(args);
191        }
192    
193        public static void setHiveScript(Configuration conf, String script, String[] params) {
194            conf.set(HIVE_SCRIPT, script);
195            MapReduceMain.setStrings(conf, HIVE_PARAMS, params);
196        }
197    
198        private static String readStringFromFile(String filePath) throws IOException {
199            String line;
200            BufferedReader br = new BufferedReader(new FileReader(filePath));
201            StringBuilder sb = new StringBuilder();
202            String sep = System.getProperty("line.separator");
203            while ((line = br.readLine()) != null) {
204                sb.append(line).append(sep);
205            }
206            return sb.toString();
207         }
208    
209        private static void writeStringToFile(String filePath, String str) throws IOException {
210            BufferedWriter out = new BufferedWriter(new FileWriter(filePath));
211            out.write(str);
212            out.close();
213        }
214    
215    
216        // Ripped from oahc.Configuration with slight modifications.
217        // Note:
218        //   Rather than break at a small pattern limit, we need Hive action to
219        //   allow large substitution limit but break if there is cycle
220        //   in substitution.
221        private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
222        static int MAX_VARIABLE_SUBST_LIMIT = 10000;
223    
224        static String substituteVars(Map<String, String> varMap, String expr) {
225            Matcher match = varPat.matcher("");
226            String eval = expr;
227            Set<String> evalSet = new HashSet<String>();
228            for(int s = 0; s < MAX_VARIABLE_SUBST_LIMIT; s++) {
229                if (evalSet.contains(eval)) {
230                    //Cyclic resolution pattern detected. Return current expression.
231                    return eval;
232                }
233                evalSet.add(eval);
234                match.reset(eval);
235                if (!match.find()) {
236                    return eval;
237                }
238                String var = match.group();
239                var = var.substring(2, var.length()-1); // remove ${ .. }
240                String val = varMap.get(var);
241                if (null == val) {
242                    return eval; // return literal ${var}: var is unbound
243                }
244                // substitute
245                eval = eval.substring(0, match.start())+val+eval.substring(match.end());
246            }
247            throw new RuntimeException(
248                    "Maximum variable substitution limit exceeded: "
249                    + MAX_VARIABLE_SUBST_LIMIT + " " + expr);
250        }
251    }