001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.fs.FileStatus;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.fs.Path;
021    import org.apache.hadoop.fs.PathFilter;
022    import org.apache.oozie.client.OozieClient;
023    import org.apache.oozie.workflow.WorkflowApp;
024    import org.apache.oozie.workflow.WorkflowException;
025    import org.apache.oozie.util.IOUtils;
026    import org.apache.oozie.util.XConfiguration;
027    import org.apache.oozie.util.XLog;
028    import org.apache.oozie.ErrorCode;
029    
030    import java.io.IOException;
031    import java.io.InputStreamReader;
032    import java.io.Reader;
033    import java.io.StringWriter;
034    import java.net.URI;
035    import java.net.URISyntaxException;
036    import java.util.ArrayList;
037    import java.util.List;
038    import java.util.Map;
039    
040    /**
041     * Service that provides application workflow definition reading from the path and creation of the proto configuration.
042     */
043    public abstract class WorkflowAppService implements Service {
044    
045        public static final String CONF_PREFIX = Service.CONF_PREFIX + "WorkflowAppService.";
046    
047        public static final String SYSTEM_LIB_PATH = CONF_PREFIX + "system.libpath";
048    
049        public static final String APP_LIB_PATH_LIST = "oozie.wf.application.lib";
050    
051        public static final String HADOOP_UGI = "hadoop.job.ugi";
052    
053        public static final String HADOOP_USER = "user.name";
054    
055        public static final String HADOOP_JT_KERBEROS_NAME = "mapreduce.jobtracker.kerberos.principal";
056    
057        public static final String HADOOP_NN_KERBEROS_NAME = "dfs.namenode.kerberos.principal";
058    
059        private Path systemLibPath;
060    
061        /**
062         * Initialize the workflow application service.
063         *
064         * @param services services instance.
065         */
066        public void init(Services services) {
067            String path = services.getConf().get(SYSTEM_LIB_PATH, " ");
068            if (path.trim().length() > 0) {
069                systemLibPath = new Path(path.trim());
070            }
071        }
072    
073        /**
074         * Destroy the workflow application service.
075         */
076        public void destroy() {
077        }
078    
079        /**
080         * Return the public interface for workflow application service.
081         *
082         * @return {@link WorkflowAppService}.
083         */
084        public Class<? extends Service> getInterface() {
085            return WorkflowAppService.class;
086        }
087    
088        /**
089         * Read workflow definition.
090         *
091         * @param appPath application path.
092         * @param user user name.
093         * @param group group name.
094         * @param autToken authentication token.
095         * @return workflow definition.
096         * @throws WorkflowException thrown if the definition could not be read.
097         */
098        protected String readDefinition(String appPath, String user, String group, String autToken, Configuration conf)
099                throws WorkflowException {
100            try {
101                URI uri = new URI(appPath);
102                FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri, conf);
103                Reader reader = new InputStreamReader(fs.open(new Path(uri.getPath())));
104                StringWriter writer = new StringWriter();
105                IOUtils.copyCharStream(reader, writer);
106                return writer.toString();
107    
108            }
109            catch (IOException ex) {
110                throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
111            }
112            catch (URISyntaxException ex) {
113                throw new WorkflowException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
114            }
115            catch (HadoopAccessorException ex) {
116                throw new WorkflowException(ex);
117            }
118            catch (Exception ex) {
119                throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
120            }
121        }
122    
123        /**
124         * Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be
125         * added to distributed cache. These paths include .jar,.so and the resource file paths.
126         *
127         * @param jobConf job configuration.
128         * @param authToken authentication token.
129         * @param isWorkflowJob indicates if the job is a workflow job or not.
130         * @return proto configuration.
131         * @throws WorkflowException thrown if the proto action configuration could not be created.
132         */
133        public XConfiguration createProtoActionConf(Configuration jobConf, String authToken, boolean isWorkflowJob)
134                throws WorkflowException {
135            XConfiguration conf = new XConfiguration();
136            try {
137                String user = jobConf.get(OozieClient.USER_NAME);
138                String group = jobConf.get(OozieClient.GROUP_NAME);
139                String hadoopUgi = user + "," + group;
140    
141                conf.set(OozieClient.USER_NAME, user);
142                conf.set(OozieClient.GROUP_NAME, group);
143                conf.set(HADOOP_UGI, hadoopUgi);
144    
145                conf.set(HADOOP_JT_KERBEROS_NAME, jobConf.get(HADOOP_JT_KERBEROS_NAME));
146                conf.set(HADOOP_NN_KERBEROS_NAME, jobConf.get(HADOOP_NN_KERBEROS_NAME));
147    
148                URI uri = new URI(jobConf.get(OozieClient.APP_PATH));
149    
150                FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri, conf);
151    
152                Path appPath = new Path(uri.getPath());
153                XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(OozieClient.LIBPATH));
154                XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath);
155    
156                List<String> filePaths;
157                if (isWorkflowJob) {
158                    filePaths = getLibFiles(fs, new Path(appPath.getParent(), "lib"));
159                }
160                else {
161                    filePaths = new ArrayList<String>();
162                }
163    
164                String[] libPaths = jobConf.getStrings(OozieClient.LIBPATH);
165                if (libPaths != null && libPaths.length > 0) {
166                    for (int i = 0; i < libPaths.length; i++) {
167                        if (libPaths[i].trim().length() > 0) {
168                            Path libPath = new Path(libPaths[i].trim());
169                            List<String> libFilePaths = getLibFiles(fs, libPath);
170                            filePaths.addAll(libFilePaths);
171                        }
172                    }
173                }
174    
175                if (systemLibPath != null && jobConf.getBoolean(OozieClient.USE_SYSTEM_LIBPATH, false)) {
176                    List<String> libFilePaths = getLibFiles(fs, systemLibPath);
177                    filePaths.addAll(libFilePaths);
178                }
179    
180                conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
181    
182                //Add all properties start with 'oozie.'
183                for (Map.Entry<String, String> entry : jobConf) {
184                    if (entry.getKey().startsWith("oozie.")) {
185                        String name = entry.getKey();
186                        String value = entry.getValue();
187                        conf.set(name, value);
188                    }
189                }
190                return conf;
191            }
192            catch (IOException ex) {
193                throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
194            }
195            catch (URISyntaxException ex) {
196                throw new WorkflowException(ErrorCode.E0711, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
197            }
198            catch (HadoopAccessorException ex) {
199                throw new WorkflowException(ex);
200            }
201            catch (Exception ex) {
202                throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH),
203                                            ex.getMessage(), ex);
204            }
205        }
206    
207        /**
208         * Parse workflow definition.
209         *
210         * @param jobConf job configuration.
211         * @param authToken authentication token.
212         * @return workflow application.
213         * @throws WorkflowException thrown if the workflow application could not be parsed.
214         */
215        public abstract WorkflowApp parseDef(Configuration jobConf, String authToken) throws WorkflowException;
216    
217        /**
218         * Parse workflow definition.
219         * @param wfXml workflow.
220         * @return workflow application.
221         * @throws WorkflowException thrown if the workflow application could not be parsed.
222         */
223        public abstract WorkflowApp parseDef(String wfXml) throws WorkflowException;
224    
225        /**
226         * Get all library paths.
227         *
228         * @param fs file system object.
229         * @param libPath hdfs library path.
230         * @return list of paths.
231         * @throws IOException thrown if the lib paths could not be obtained.
232         */
233        private List<String> getLibFiles(FileSystem fs, Path libPath) throws IOException {
234            List<String> libPaths = new ArrayList<String>();
235            if (fs.exists(libPath)) {
236                FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
237    
238                for (FileStatus file : files) {
239                    libPaths.add(file.getPath().toUri().getPath().trim());
240                }
241            }
242            else {
243                XLog.getLog(getClass()).warn("libpath [{0}] does not exists", libPath);
244            }
245            return libPaths;
246        }
247    
248        /*
249         * Filter class doing no filtering.
250         * We dont need define this class, but seems fs.listStatus() is not working properly without this.
251         * So providing this dummy no filtering Filter class.
252         */
253        private class NoPathFilter implements PathFilter {
254            @Override
255            public boolean accept(Path path) {
256                return true;
257            }
258        }
259    }