001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.wf; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.oozie.WorkflowJobBean; 019 import org.apache.oozie.ErrorCode; 020 import org.apache.oozie.service.WorkflowStoreService; 021 import org.apache.oozie.service.WorkflowAppService; 022 import org.apache.oozie.service.Services; 023 import org.apache.oozie.service.DagXLogInfoService; 024 import org.apache.oozie.util.XLog; 025 import org.apache.oozie.util.ParamChecker; 026 import org.apache.oozie.util.XConfiguration; 027 import org.apache.oozie.util.XmlUtils; 028 import org.apache.oozie.command.CommandException; 029 import org.apache.oozie.store.StoreException; 030 import org.apache.oozie.store.WorkflowStore; 031 import org.apache.oozie.workflow.WorkflowApp; 032 import org.apache.oozie.workflow.WorkflowException; 033 import org.apache.oozie.workflow.WorkflowInstance; 034 import org.apache.oozie.workflow.WorkflowLib; 035 import org.apache.oozie.util.PropertiesUtils; 036 import org.apache.oozie.client.OozieClient; 037 import org.apache.oozie.client.WorkflowJob; 038 import org.apache.oozie.client.XOozieClient; 039 import org.jdom.Element; 040 import org.jdom.Namespace; 041 042 import java.util.Date; 043 import java.util.Map; 044 import java.util.Set; 045 import java.util.HashSet; 046 047 public abstract class SubmitHttpCommand extends WorkflowCommand<String> { 048 public static final String USE_SYSTEM_LIBPATH_FOR_MR_PIG_JOBS ="use.system.libpath.for.mapreduce.and.pig.jobs"; 049 050 protected static final Set<String> MANDATORY_OOZIE_CONFS = new HashSet<String>(); 051 protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>(); 052 053 static { 054 MANDATORY_OOZIE_CONFS.add(XOozieClient.JT); 055 MANDATORY_OOZIE_CONFS.add(XOozieClient.NN); 056 057 OPTIONAL_OOZIE_CONFS.add(XOozieClient.FILES); 058 OPTIONAL_OOZIE_CONFS.add(XOozieClient.ARCHIVES); 059 } 060 061 private Configuration conf; 062 private String authToken; 063 064 public SubmitHttpCommand(String name, String type, Configuration conf, String authToken) { 065 super(name, type, 1, XLog.STD); 066 this.conf = ParamChecker.notNull(conf, "conf"); 067 this.authToken = ParamChecker.notEmpty(authToken, "authToken"); 068 } 069 070 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>(); 071 private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>(); 072 073 static { 074 String[] badUserProps = { PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES, 075 PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB, 076 PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN, 077 PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS }; 078 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES); 079 080 String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI, 081 WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME }; 082 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES); 083 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES); 084 } 085 086 087 /** 088 * Generate workflow xml from conf object 089 * 090 * @param conf the configuration object 091 * @return workflow xml def string representation 092 */ 093 abstract protected String getWorkflowXml(Configuration conf); 094 095 @Override 096 protected String call(WorkflowStore store) throws StoreException, CommandException { 097 incrJobCounter(1); 098 WorkflowAppService wps = Services.get().get(WorkflowAppService.class); 099 try { 100 XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN)); 101 String wfXml = getWorkflowXml(conf); 102 XLog.getLog(getClass()).debug("workflow xml created on the server side is :\n"); 103 XLog.getLog(getClass()).debug(wfXml); 104 WorkflowApp app = wps.parseDef(wfXml); 105 106 if (conf.get(OozieClient.USE_SYSTEM_LIBPATH) == null) { 107 if (Services.get().getConf().getBoolean(USE_SYSTEM_LIBPATH_FOR_MR_PIG_JOBS, false)) { 108 conf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); 109 } 110 } 111 112 XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken, false); 113 WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB(); 114 115 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES); 116 117 // Resolving all variables in the job properties. 118 // This ensures the Hadoop Configuration semantics is preserved. 119 XConfiguration resolvedVarsConf = new XConfiguration(); 120 for (Map.Entry<String, String> entry : conf) { 121 resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey())); 122 } 123 conf = resolvedVarsConf; 124 125 WorkflowInstance wfInstance; 126 try { 127 wfInstance = workflowLib.createInstance(app, conf); 128 } 129 catch (WorkflowException e) { 130 throw new StoreException(e); 131 } 132 133 Configuration conf = wfInstance.getConf(); 134 135 WorkflowJobBean workflow = new WorkflowJobBean(); 136 workflow.setId(wfInstance.getId()); 137 workflow.setAppName(app.getName()); 138 workflow.setAppPath(conf.get(OozieClient.APP_PATH)); 139 workflow.setConf(XmlUtils.prettyPrint(conf).toString()); 140 workflow.setProtoActionConf(protoActionConf.toXmlString()); 141 workflow.setCreatedTime(new Date()); 142 workflow.setLastModifiedTime(new Date()); 143 workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, "")); 144 workflow.setStatus(WorkflowJob.Status.PREP); 145 workflow.setRun(0); 146 workflow.setUser(conf.get(OozieClient.USER_NAME)); 147 workflow.setGroup(conf.get(OozieClient.GROUP_NAME)); 148 workflow.setAuthToken(authToken); 149 workflow.setWorkflowInstance(wfInstance); 150 workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID)); 151 152 setLogInfo(workflow); 153 store.insertWorkflow(workflow); 154 155 return workflow.getId(); 156 } 157 catch (WorkflowException ex) { 158 throw new CommandException(ex); 159 } 160 catch (Exception ex) { 161 throw new CommandException(ErrorCode.E0803, ex); 162 } 163 } 164 165 static private void addSection(Element X, Namespace ns, String filesStr, String tagName) { 166 if (filesStr != null) { 167 String[] files = filesStr.split(","); 168 for (String f : files) { 169 Element tagElement = new Element(tagName, ns); 170 if (f.contains("#")) { 171 tagElement.addContent(f); 172 } 173 else { 174 String filename = f.substring(f.lastIndexOf("/") + 1, f.length()); 175 if (filename == null || filename.isEmpty()) { 176 tagElement.addContent(f); 177 } 178 else { 179 tagElement.addContent(f + "#" + filename); 180 } 181 } 182 X.addContent(tagElement); 183 } 184 } 185 } 186 187 /** 188 * Add file section in X. 189 * 190 * @param parent XML element to be appended 191 * @param conf Configuration object 192 * @param ns XML element namespace 193 */ 194 static void addFileSection(Element X, Configuration conf, Namespace ns) { 195 String filesStr = conf.get(XOozieClient.FILES); 196 addSection(X, ns, filesStr, "file"); 197 } 198 199 /** 200 * Add archive section in X. 201 * 202 * @param parent XML element to be appended 203 * @param conf Configuration object 204 * @param ns XML element namespace 205 */ 206 static void addArchiveSection(Element X, Configuration conf, Namespace ns) { 207 String archivesStr = conf.get(XOozieClient.ARCHIVES); 208 addSection(X, ns, archivesStr, "archive"); 209 } 210 }