001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.wf;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.oozie.WorkflowJobBean;
019    import org.apache.oozie.ErrorCode;
020    import org.apache.oozie.service.WorkflowStoreService;
021    import org.apache.oozie.service.WorkflowAppService;
022    import org.apache.oozie.service.Services;
023    import org.apache.oozie.service.DagXLogInfoService;
024    import org.apache.oozie.util.XLog;
025    import org.apache.oozie.util.ParamChecker;
026    import org.apache.oozie.util.XConfiguration;
027    import org.apache.oozie.util.XmlUtils;
028    import org.apache.oozie.command.CommandException;
029    import org.apache.oozie.store.StoreException;
030    import org.apache.oozie.store.WorkflowStore;
031    import org.apache.oozie.workflow.WorkflowApp;
032    import org.apache.oozie.workflow.WorkflowException;
033    import org.apache.oozie.workflow.WorkflowInstance;
034    import org.apache.oozie.workflow.WorkflowLib;
035    import org.apache.oozie.util.PropertiesUtils;
036    import org.apache.oozie.client.OozieClient;
037    import org.apache.oozie.client.WorkflowJob;
038    import org.apache.oozie.client.XOozieClient;
039    import org.jdom.Element;
040    import org.jdom.Namespace;
041    
042    import java.util.Date;
043    import java.util.Map;
044    import java.util.Set;
045    import java.util.HashSet;
046    
047    public abstract class SubmitHttpCommand extends WorkflowCommand<String> {
048        public static final String USE_SYSTEM_LIBPATH_FOR_MR_PIG_JOBS ="use.system.libpath.for.mapreduce.and.pig.jobs";
049    
050        protected static final Set<String> MANDATORY_OOZIE_CONFS = new HashSet<String>();
051        protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>();
052    
053        static {
054            MANDATORY_OOZIE_CONFS.add(XOozieClient.JT);
055            MANDATORY_OOZIE_CONFS.add(XOozieClient.NN);
056    
057            OPTIONAL_OOZIE_CONFS.add(XOozieClient.FILES);
058            OPTIONAL_OOZIE_CONFS.add(XOozieClient.ARCHIVES);
059        }
060    
061        private Configuration conf;
062        private String authToken;
063    
064        public SubmitHttpCommand(String name, String type, Configuration conf, String authToken) {
065            super(name, type, 1, XLog.STD);
066            this.conf = ParamChecker.notNull(conf, "conf");
067            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
068        }
069    
070        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
071        private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
072    
073        static {
074            String[] badUserProps = { PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
075                    PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
076                    PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
077                    PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
078            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
079    
080            String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
081                    WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME };
082            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
083            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
084        }
085    
086    
087        /**
088         * Generate workflow xml from conf object
089         *
090         * @param conf the configuration object
091         * @return workflow xml def string representation
092         */
093        abstract protected String getWorkflowXml(Configuration conf);
094    
095        @Override
096        protected String call(WorkflowStore store) throws StoreException, CommandException {
097            incrJobCounter(1);
098            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
099            try {
100                XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
101                String wfXml = getWorkflowXml(conf);
102                XLog.getLog(getClass()).debug("workflow xml created on the server side is :\n");
103                XLog.getLog(getClass()).debug(wfXml);
104                WorkflowApp app = wps.parseDef(wfXml);
105    
106                if (conf.get(OozieClient.USE_SYSTEM_LIBPATH) == null) {
107                    if (Services.get().getConf().getBoolean(USE_SYSTEM_LIBPATH_FOR_MR_PIG_JOBS, false)) {
108                        conf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
109                    }
110                }
111    
112                XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken, false);
113                WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
114    
115                PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
116    
117                // Resolving all variables in the job properties.
118                // This ensures the Hadoop Configuration semantics is preserved.
119                XConfiguration resolvedVarsConf = new XConfiguration();
120                for (Map.Entry<String, String> entry : conf) {
121                    resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
122                }
123                conf = resolvedVarsConf;
124    
125                WorkflowInstance wfInstance;
126                try {
127                    wfInstance = workflowLib.createInstance(app, conf);
128                }
129                catch (WorkflowException e) {
130                    throw new StoreException(e);
131                }
132    
133                Configuration conf = wfInstance.getConf();
134    
135                WorkflowJobBean workflow = new WorkflowJobBean();
136                workflow.setId(wfInstance.getId());
137                workflow.setAppName(app.getName());
138                workflow.setAppPath(conf.get(OozieClient.APP_PATH));
139                workflow.setConf(XmlUtils.prettyPrint(conf).toString());
140                workflow.setProtoActionConf(protoActionConf.toXmlString());
141                workflow.setCreatedTime(new Date());
142                workflow.setLastModifiedTime(new Date());
143                workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
144                workflow.setStatus(WorkflowJob.Status.PREP);
145                workflow.setRun(0);
146                workflow.setUser(conf.get(OozieClient.USER_NAME));
147                workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
148                workflow.setAuthToken(authToken);
149                workflow.setWorkflowInstance(wfInstance);
150                workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
151    
152                setLogInfo(workflow);
153                store.insertWorkflow(workflow);
154    
155                return workflow.getId();
156            }
157            catch (WorkflowException ex) {
158                throw new CommandException(ex);
159            }
160            catch (Exception ex) {
161                throw new CommandException(ErrorCode.E0803, ex);
162            }
163        }
164    
165        static private void addSection(Element X, Namespace ns, String filesStr, String tagName) {
166            if (filesStr != null) {
167                String[] files = filesStr.split(",");
168                for (String f : files) {
169                    Element tagElement = new Element(tagName, ns);
170                    if (f.contains("#")) {
171                        tagElement.addContent(f);
172                    }
173                    else {
174                        String filename = f.substring(f.lastIndexOf("/") + 1, f.length());
175                        if (filename == null || filename.isEmpty()) {
176                            tagElement.addContent(f);
177                        }
178                        else {
179                            tagElement.addContent(f + "#" + filename);
180                        }
181                    }
182                    X.addContent(tagElement);
183                }
184            }
185        }
186    
187        /**
188         * Add file section in X.
189         *
190         * @param parent XML element to be appended
191         * @param conf Configuration object
192         * @param ns XML element namespace
193         */
194        static void addFileSection(Element X, Configuration conf, Namespace ns) {
195            String filesStr = conf.get(XOozieClient.FILES);
196            addSection(X, ns, filesStr, "file");
197        }
198    
199        /**
200         * Add archive section in X.
201         *
202         * @param parent XML element to be appended
203         * @param conf Configuration object
204         * @param ns XML element namespace
205         */
206        static void addArchiveSection(Element X, Configuration conf, Namespace ns) {
207            String archivesStr = conf.get(XOozieClient.ARCHIVES);
208            addSection(X, ns, archivesStr, "archive");
209        }
210    }