001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.client; 016 017 import java.io.BufferedReader; 018 import java.io.File; 019 import java.io.FileReader; 020 import java.io.IOException; 021 import java.io.InputStreamReader; 022 import java.net.HttpURLConnection; 023 import java.util.Properties; 024 025 import org.apache.oozie.client.rest.JsonTags; 026 import org.apache.oozie.client.rest.RestConstants; 027 import org.json.simple.JSONObject; 028 import org.json.simple.JSONValue; 029 030 public class XOozieClient extends OozieClient { 031 032 public static final String JT = "mapred.job.tracker"; 033 034 public static final String NN = "fs.default.name"; 035 036 public static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal"; 037 038 public static final String NN_PRINCIPAL = "dfs.namenode.kerberos.principal"; 039 040 public static final String PIG_SCRIPT = "oozie.pig.script"; 041 042 public static final String PIG_OPTIONS = "oozie.pig.options"; 043 044 public static final String FILES = "oozie.files"; 045 046 public static final String ARCHIVES = "oozie.archives"; 047 048 public static final String IS_PROXY_SUBMISSION = "oozie.proxysubmission"; 049 050 protected XOozieClient() { 051 } 052 053 /** 054 * Create an eXtended Workflow client instance. 055 * 056 * @param oozieUrl URL of the Oozie instance it will interact with. 057 */ 058 public XOozieClient(String oozieUrl) { 059 super(oozieUrl); 060 } 061 062 private String readPigScript(String script) throws IOException { 063 if (!new File(script).exists()) { 064 throw new IOException("Error: Pig script file [" + script + "] does not exist"); 065 } 066 067 BufferedReader br = null; 068 try { 069 br = new BufferedReader(new FileReader(script)); 070 StringBuilder sb = new StringBuilder(); 071 String line; 072 while ((line = br.readLine()) != null) { 073 sb.append(line + "\n"); 074 } 075 return sb.toString(); 076 } 077 finally { 078 try { 079 br.close(); 080 } 081 catch (IOException ex) { 082 System.err.println("Error: " + ex.getMessage()); 083 } 084 } 085 } 086 087 static void setStrings(Properties conf, String key, String[] values) { 088 if (values != null) { 089 conf.setProperty(key + ".size", (new Integer(values.length)).toString()); 090 for (int i = 0; i < values.length; i++) { 091 conf.setProperty(key + "." + i, values[i]); 092 } 093 } 094 } 095 096 private void validateHttpSubmitConf(Properties conf) { 097 String JT = conf.getProperty(XOozieClient.JT); 098 if (JT == null) { 099 throw new RuntimeException("jobtracker is not specified in conf"); 100 } 101 102 String NN = conf.getProperty(XOozieClient.NN); 103 if (NN == null) { 104 throw new RuntimeException("namenode is not specified in conf"); 105 } 106 107 String libPath = conf.getProperty(LIBPATH); 108 if (libPath == null) { 109 throw new RuntimeException("libpath is not specified in conf"); 110 } 111 if (!libPath.startsWith("hdfs://")) { 112 String newLibPath = NN + libPath; 113 conf.setProperty(LIBPATH, newLibPath); 114 } 115 116 conf.setProperty(IS_PROXY_SUBMISSION, "true"); 117 } 118 119 /** 120 * Submit a Pig job via HTTP. 121 * 122 * @param conf job configuration. 123 * @param pigScriptFile pig script file. 124 * @param pigArgs pig arguments string. 125 * @return the job Id. 126 * @throws OozieClientException thrown if the job could not be submitted. 127 */ 128 public String submitPig(Properties conf, String pigScriptFile, String[] pigArgs) throws IOException, OozieClientException { 129 if (conf == null) { 130 throw new IllegalArgumentException("conf cannot be null"); 131 } 132 if (pigScriptFile == null) { 133 throw new IllegalArgumentException("pigScriptFile cannot be null"); 134 } 135 136 validateHttpSubmitConf(conf); 137 138 conf.setProperty(XOozieClient.PIG_SCRIPT, readPigScript(pigScriptFile)); 139 setStrings(conf, XOozieClient.PIG_OPTIONS, pigArgs); 140 141 return (new HttpJobSubmit(conf, "pig")).call(); 142 } 143 144 /** 145 * Submit a Map/Reduce job via HTTP. 146 * 147 * @param conf job configuration. 148 * @return the job Id. 149 * @throws OozieClientException thrown if the job could not be submitted. 150 */ 151 public String submitMapReduce(Properties conf) throws OozieClientException { 152 if (conf == null) { 153 throw new IllegalArgumentException("conf cannot be null"); 154 } 155 156 validateHttpSubmitConf(conf); 157 158 return (new HttpJobSubmit(conf, "mapreduce")).call(); 159 } 160 161 private class HttpJobSubmit extends ClientCallable<String> { 162 private Properties conf; 163 164 HttpJobSubmit(Properties conf, String jobType) { 165 super("POST", RestConstants.JOBS, "", prepareParams(RestConstants.JOBTYPE_PARAM, jobType)); 166 this.conf = notNull(conf, "conf"); 167 } 168 169 @Override 170 protected String call(HttpURLConnection conn) throws IOException, OozieClientException { 171 conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); 172 writeToXml(conf, conn.getOutputStream()); 173 if (conn.getResponseCode() == HttpURLConnection.HTTP_CREATED) { 174 JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); 175 return (String) json.get(JsonTags.JOB_ID); 176 } 177 if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { 178 handleError(conn); 179 } 180 return null; 181 } 182 } 183 184 /** 185 * set LIBPATH for HTTP submission job. 186 * 187 * @param conf Configuration object. 188 * @param path lib HDFS path. 189 */ 190 public void setLib(Properties conf, String pathStr) { 191 conf.setProperty(LIBPATH, pathStr); 192 } 193 194 /** 195 * The equivalent to <file> tag in oozie's workflow xml. 196 * 197 * @param conf Configuration object. 198 * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name. 199 * For example, "/user/oozie/parameter_file#myparams". If no "#..." is specified, file name will be used as 200 * symbolic link name. 201 */ 202 public void addFile(Properties conf, String file) { 203 if (file == null || file.length() == 0) { 204 throw new IllegalArgumentException("file cannot be null or empty"); 205 } 206 String files = conf.getProperty(FILES); 207 conf.setProperty(FILES, files == null ? file : files + "," + file); 208 } 209 210 /** 211 * The equivalent to <archive> tag in oozie's workflow xml. 212 * 213 * @param conf Configuration object. 214 * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name. 215 * For example, "/user/oozie/udf1.jar#my.jar". If no "#..." is specified, file name will be used as 216 * symbolic link name. 217 */ 218 public void addArchive(Properties conf, String file) { 219 if (file == null || file.length() == 0) { 220 throw new IllegalArgumentException("file cannot be null or empty"); 221 } 222 String files = conf.getProperty(ARCHIVES); 223 conf.setProperty(ARCHIVES, files == null ? file : files + "," + file); 224 } 225 }