001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.client;
016    
017    import java.io.BufferedReader;
018    import java.io.File;
019    import java.io.FileReader;
020    import java.io.IOException;
021    import java.io.InputStreamReader;
022    import java.net.HttpURLConnection;
023    import java.util.Properties;
024    
025    import org.apache.oozie.client.rest.JsonTags;
026    import org.apache.oozie.client.rest.RestConstants;
027    import org.json.simple.JSONObject;
028    import org.json.simple.JSONValue;
029    
030    public class XOozieClient extends OozieClient {
031    
032        public static final String JT = "mapred.job.tracker";
033    
034        public static final String NN = "fs.default.name";
035    
036        public static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
037    
038        public static final String NN_PRINCIPAL = "dfs.namenode.kerberos.principal";
039    
040        public static final String PIG_SCRIPT = "oozie.pig.script";
041    
042        public static final String PIG_OPTIONS = "oozie.pig.options";
043    
044        public static final String FILES = "oozie.files";
045    
046        public static final String ARCHIVES = "oozie.archives";
047        
048        public static final String IS_PROXY_SUBMISSION = "oozie.proxysubmission";
049    
050        protected XOozieClient() {
051        }
052    
053        /**
054         * Create an eXtended Workflow client instance.
055         *
056         * @param oozieUrl URL of the Oozie instance it will interact with.
057         */
058        public XOozieClient(String oozieUrl) {
059            super(oozieUrl);
060        }
061    
062        private String readPigScript(String script) throws IOException {
063            if (!new File(script).exists()) {
064                throw new IOException("Error: Pig script file [" + script + "] does not exist");
065            }
066    
067            BufferedReader br = null;
068            try {
069                br = new BufferedReader(new FileReader(script));
070                StringBuilder sb = new StringBuilder();
071                String line;
072                while ((line = br.readLine()) != null) {
073                    sb.append(line + "\n");
074                }
075                return sb.toString();
076            }
077            finally {
078                try {
079                    br.close();
080                }
081                catch (IOException ex) {
082                    System.err.println("Error: " + ex.getMessage());
083                }
084            }
085        }
086    
087        static void setStrings(Properties conf, String key, String[] values) {
088            if (values != null) {
089                conf.setProperty(key + ".size", (new Integer(values.length)).toString());
090                for (int i = 0; i < values.length; i++) {
091                    conf.setProperty(key + "." + i, values[i]);
092                }
093            }
094        }
095    
096        private void validateHttpSubmitConf(Properties conf) {
097            String JT = conf.getProperty(XOozieClient.JT);
098            if (JT == null) {
099                throw new RuntimeException("jobtracker is not specified in conf");
100            }
101    
102            String NN = conf.getProperty(XOozieClient.NN);
103            if (NN == null) {
104                throw new RuntimeException("namenode is not specified in conf");
105            }
106    
107            String libPath = conf.getProperty(LIBPATH);
108            if (libPath == null) {
109                throw new RuntimeException("libpath is not specified in conf");
110            }
111            if (!libPath.startsWith("hdfs://")) {
112                String newLibPath = NN + libPath;
113                conf.setProperty(LIBPATH, newLibPath);
114            }
115            
116            conf.setProperty(IS_PROXY_SUBMISSION, "true");
117        }
118    
119        /**
120         * Submit a Pig job via HTTP.
121         *
122         * @param conf job configuration.
123         * @param pigScriptFile pig script file.
124         * @param pigArgs pig arguments string.
125         * @return the job Id.
126         * @throws OozieClientException thrown if the job could not be submitted.
127         */
128        public String submitPig(Properties conf, String pigScriptFile, String[] pigArgs) throws IOException, OozieClientException {
129            if (conf == null) {
130                throw new IllegalArgumentException("conf cannot be null");
131            }
132            if (pigScriptFile == null) {
133                throw new IllegalArgumentException("pigScriptFile cannot be null");
134            }
135    
136            validateHttpSubmitConf(conf);
137    
138            conf.setProperty(XOozieClient.PIG_SCRIPT, readPigScript(pigScriptFile));
139            setStrings(conf, XOozieClient.PIG_OPTIONS, pigArgs);
140    
141            return (new HttpJobSubmit(conf, "pig")).call();
142        }
143    
144        /**
145         * Submit a Map/Reduce job via HTTP.
146         *
147         * @param conf job configuration.
148         * @return the job Id.
149         * @throws OozieClientException thrown if the job could not be submitted.
150         */
151        public String submitMapReduce(Properties conf) throws OozieClientException {
152            if (conf == null) {
153                throw new IllegalArgumentException("conf cannot be null");
154            }
155    
156            validateHttpSubmitConf(conf);
157    
158            return (new HttpJobSubmit(conf, "mapreduce")).call();
159        }
160    
161        private class HttpJobSubmit extends ClientCallable<String> {
162            private Properties conf;
163    
164            HttpJobSubmit(Properties conf, String jobType) {
165                super("POST", RestConstants.JOBS, "", prepareParams(RestConstants.JOBTYPE_PARAM, jobType));
166                this.conf = notNull(conf, "conf");
167            }
168    
169            @Override
170            protected String call(HttpURLConnection conn) throws IOException, OozieClientException {
171                conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE);
172                writeToXml(conf, conn.getOutputStream());
173                if (conn.getResponseCode() == HttpURLConnection.HTTP_CREATED) {
174                    JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
175                    return (String) json.get(JsonTags.JOB_ID);
176                }
177                if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
178                    handleError(conn);
179                }
180                return null;
181            }
182        }
183    
184        /**
185         * set LIBPATH for HTTP submission job.
186         *
187         * @param conf Configuration object.
188         * @param path lib HDFS path.
189         */
190        public void setLib(Properties conf, String pathStr) {
191            conf.setProperty(LIBPATH, pathStr);
192        }
193    
194        /**
195         * The equivalent to <file> tag in oozie's workflow xml.
196         *
197         * @param conf Configuration object.
198         * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
199         *             For example, "/user/oozie/parameter_file#myparams". If no "#..." is specified, file name will be used as
200         *             symbolic link name.
201         */
202        public void addFile(Properties conf, String file) {
203            if (file == null || file.length() == 0) {
204                throw new IllegalArgumentException("file cannot be null or empty");
205            }
206            String files = conf.getProperty(FILES);
207            conf.setProperty(FILES, files == null ? file : files + "," + file);
208        }
209    
210        /**
211         * The equivalent to <archive> tag in oozie's workflow xml.
212         *
213         * @param conf Configuration object.
214         * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
215         *             For example, "/user/oozie/udf1.jar#my.jar". If no "#..." is specified, file name will be used as
216         *             symbolic link name.
217         */
218        public void addArchive(Properties conf, String file) {
219            if (file == null || file.length() == 0) {
220                throw new IllegalArgumentException("file cannot be null or empty");
221            }
222            String files = conf.getProperty(ARCHIVES);
223            conf.setProperty(ARCHIVES, files == null ? file : files + "," + file);
224        }
225    }