001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.wf;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.oozie.WorkflowJobBean;
019    import org.apache.oozie.ErrorCode;
020    import org.apache.oozie.service.WorkflowStoreService;
021    import org.apache.oozie.service.WorkflowAppService;
022    import org.apache.oozie.service.Services;
023    import org.apache.oozie.service.DagXLogInfoService;
024    import org.apache.oozie.util.XLog;
025    import org.apache.oozie.util.ParamChecker;
026    import org.apache.oozie.util.XConfiguration;
027    import org.apache.oozie.util.XmlUtils;
028    import org.apache.oozie.command.CommandException;
029    import org.apache.oozie.store.StoreException;
030    import org.apache.oozie.store.WorkflowStore;
031    import org.apache.oozie.workflow.WorkflowApp;
032    import org.apache.oozie.workflow.WorkflowException;
033    import org.apache.oozie.workflow.WorkflowInstance;
034    import org.apache.oozie.workflow.WorkflowLib;
035    import org.apache.oozie.util.PropertiesUtils;
036    import org.apache.oozie.client.OozieClient;
037    import org.apache.oozie.client.WorkflowJob;
038    import org.apache.oozie.client.XOozieClient;
039    import org.jdom.Element;
040    import org.jdom.Namespace;
041    
042    import java.util.Date;
043    import java.util.Map;
044    import java.util.Set;
045    import java.util.HashSet;
046    
047    public abstract class SubmitHttpCommand extends WorkflowCommand<String> {
048        protected static final Set<String> MANDATORY_OOZIE_CONFS = new HashSet<String>();
049        protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>();
050    
051        static {
052            MANDATORY_OOZIE_CONFS.add(XOozieClient.JT);
053            MANDATORY_OOZIE_CONFS.add(XOozieClient.NN);
054            MANDATORY_OOZIE_CONFS.add(XOozieClient.LIBPATH);
055    
056            OPTIONAL_OOZIE_CONFS.add(XOozieClient.FILES);
057            OPTIONAL_OOZIE_CONFS.add(XOozieClient.ARCHIVES);
058        }
059    
060        private Configuration conf;
061        private String authToken;
062    
063        public SubmitHttpCommand(String name, String type, Configuration conf, String authToken) {
064            super(name, type, 1, XLog.STD);
065            this.conf = ParamChecker.notNull(conf, "conf");
066            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
067        }
068    
069        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
070        private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
071    
072        static {
073            String[] badUserProps = { PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
074                    PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
075                    PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
076                    PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
077            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
078    
079            String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
080                    WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME };
081            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
082            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
083        }
084    
085        /**
086         * Generate workflow xml from conf object
087         *
088         * @param conf the configuration object
089         * @return workflow xml def string representation
090         */
091        abstract protected String getWorkflowXml(Configuration conf);
092    
093        @Override
094        protected String call(WorkflowStore store) throws StoreException, CommandException {
095            incrJobCounter(1);
096            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
097            try {
098                XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
099                String wfXml = getWorkflowXml(conf);
100                XLog.getLog(getClass()).debug("workflow xml created on the server side is :\n");
101                XLog.getLog(getClass()).debug(wfXml);
102                WorkflowApp app = wps.parseDef(wfXml);
103                XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken);
104                WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
105    
106                PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
107    
108                // Resolving all variables in the job properties.
109                // This ensures the Hadoop Configuration semantics is preserved.
110                XConfiguration resolvedVarsConf = new XConfiguration();
111                for (Map.Entry<String, String> entry : conf) {
112                    resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
113                }
114                conf = resolvedVarsConf;
115    
116                WorkflowInstance wfInstance;
117                try {
118                    wfInstance = workflowLib.createInstance(app, conf);
119                }
120                catch (WorkflowException e) {
121                    throw new StoreException(e);
122                }
123    
124                Configuration conf = wfInstance.getConf();
125    
126                WorkflowJobBean workflow = new WorkflowJobBean();
127                workflow.setId(wfInstance.getId());
128                workflow.setAppName(app.getName());
129                workflow.setAppPath(conf.get(OozieClient.APP_PATH));
130                workflow.setConf(XmlUtils.prettyPrint(conf).toString());
131                workflow.setProtoActionConf(protoActionConf.toXmlString());
132                workflow.setCreatedTime(new Date());
133                workflow.setLastModifiedTime(new Date());
134                workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
135                workflow.setStatus(WorkflowJob.Status.PREP);
136                workflow.setRun(0);
137                workflow.setUser(conf.get(OozieClient.USER_NAME));
138                workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
139                workflow.setAuthToken(authToken);
140                workflow.setWorkflowInstance(wfInstance);
141                workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
142    
143                setLogInfo(workflow);
144                store.insertWorkflow(workflow);
145    
146                return workflow.getId();
147            }
148            catch (WorkflowException ex) {
149                throw new CommandException(ex);
150            }
151            catch (Exception ex) {
152                throw new CommandException(ErrorCode.E0803, ex);
153            }
154        }
155    
156        static private void addSection(Element X, Namespace ns, String filesStr, String tagName) {
157            if (filesStr != null) {
158                String[] files = filesStr.split(",");
159                for (String f : files) {
160                    Element tagElement = new Element(tagName, ns);
161                    if (f.contains("#")) {
162                        tagElement.addContent(f);
163                    }
164                    else {
165                        String filename = f.substring(f.lastIndexOf("/") + 1, f.length());
166                        if (filename == null || filename.isEmpty()) {
167                            tagElement.addContent(f);
168                        }
169                        else {
170                            tagElement.addContent(f + "#" + filename);
171                        }
172                    }
173                    X.addContent(tagElement);
174                }
175            }
176        }
177    
178        /**
179         * Add file section in X.
180         *
181         * @param parent XML element to be appended
182         * @param conf Configuration object
183         * @param ns XML element namespace
184         */
185        static void addFileSection(Element X, Configuration conf, Namespace ns) {
186            String filesStr = conf.get(XOozieClient.FILES);
187            addSection(X, ns, filesStr, "file");
188        }
189    
190        /**
191         * Add archive section in X.
192         *
193         * @param parent XML element to be appended
194         * @param conf Configuration object
195         * @param ns XML element namespace
196         */
197        static void addArchiveSection(Element X, Configuration conf, Namespace ns) {
198            String archivesStr = conf.get(XOozieClient.ARCHIVES);
199            addSection(X, ns, archivesStr, "archive");
200        }
201    }