001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.wf; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.oozie.service.WorkflowAppService; 019 import org.apache.oozie.util.XmlUtils; 020 import org.jdom.Element; 021 import org.jdom.Namespace; 022 import org.apache.oozie.client.XOozieClient; 023 024 import java.util.HashSet; 025 import java.util.Iterator; 026 import java.util.Map; 027 import java.util.Set; 028 029 public class SubmitMRCommand extends SubmitHttpCommand { 030 private static final Set<String> SKIPPED_CONFS = new HashSet<String>(); 031 032 public SubmitMRCommand(Configuration conf, String authToken) { 033 super("submitMR", "submitMR", conf, authToken); 034 } 035 036 static { 037 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER); 038 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_UGI); 039 SKIPPED_CONFS.add(XOozieClient.JT); 040 SKIPPED_CONFS.add(XOozieClient.NN); 041 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_JT_KERBEROS_NAME); 042 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_NN_KERBEROS_NAME); 043 } 044 045 private Element generateConfigurationSection(Configuration conf, Namespace ns) { 046 Element configuration = null; 047 Iterator<Map.Entry<String, String>> iter = conf.iterator(); 048 while (iter.hasNext()) { 049 Map.Entry<String, String> entry = iter.next(); 050 String name = entry.getKey(); 051 if (MANDATORY_OOZIE_CONFS.contains(name) || OPTIONAL_OOZIE_CONFS.contains(name) 052 || SKIPPED_CONFS.contains(name)) { 053 continue; 054 } 055 056 if (configuration == null) { 057 configuration = new Element("configuration", ns); 058 } 059 060 String value = entry.getValue(); 061 Element property = new Element("property", ns); 062 Element nameElement = new Element("name", ns); 063 nameElement.addContent(name != null ? name : ""); 064 property.addContent(nameElement); 065 Element valueElement = new Element("value", ns); 066 valueElement.addContent(value != null ? value : ""); 067 property.addContent(valueElement); 068 configuration.addContent(property); 069 } 070 071 return configuration; 072 } 073 074 private Element generateMRSection(Configuration conf, Namespace ns) { 075 Element mapreduce = new Element("map-reduce", ns); 076 Element jt = new Element("job-tracker", ns); 077 jt.addContent(conf.get(XOozieClient.JT)); 078 mapreduce.addContent(jt); 079 Element nn = new Element("name-node", ns); 080 nn.addContent(conf.get(XOozieClient.NN)); 081 mapreduce.addContent(nn); 082 083 if (conf.size() > MANDATORY_OOZIE_CONFS.size()) { // excluding JT, NN, 084 // LIBPATH 085 // configuration section 086 Element configuration = generateConfigurationSection(conf, ns); 087 if (configuration != null) { 088 mapreduce.addContent(configuration); 089 } 090 091 // file section 092 addFileSection(mapreduce, conf, ns); 093 094 // archive section 095 addArchiveSection(mapreduce, conf, ns); 096 } 097 098 return mapreduce; 099 } 100 101 /* 102 * (non-Javadoc) 103 * 104 * @see 105 * org.apache.oozie.command.wf.SubmitHttpCommand#getWorkflowXml(org.apache 106 * .hadoop.conf.Configuration) 107 */ 108 @Override 109 protected String getWorkflowXml(Configuration conf) { 110 for (String key : MANDATORY_OOZIE_CONFS) { 111 String value = conf.get(key); 112 if (value == null) { 113 throw new RuntimeException(key + " is not specified"); 114 } 115 } 116 117 Namespace ns = Namespace.getNamespace("uri:oozie:workflow:0.2"); 118 Element root = new Element("workflow-app", ns); 119 root.setAttribute("name", "oozie-mapreduce"); 120 121 Element start = new Element("start", ns); 122 start.setAttribute("to", "hadoop1"); 123 root.addContent(start); 124 125 Element action = new Element("action", ns); 126 action.setAttribute("name", "hadoop1"); 127 128 Element mapreduce = generateMRSection(conf, ns); 129 action.addContent(mapreduce); 130 131 Element ok = new Element("ok", ns); 132 ok.setAttribute("to", "end"); 133 action.addContent(ok); 134 135 Element error = new Element("error", ns); 136 error.setAttribute("to", "fail"); 137 action.addContent(error); 138 139 root.addContent(action); 140 141 Element kill = new Element("kill", ns); 142 kill.setAttribute("name", "fail"); 143 Element message = new Element("message", ns); 144 message.addContent("Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]"); 145 kill.addContent(message); 146 root.addContent(kill); 147 148 Element end = new Element("end", ns); 149 end.setAttribute("name", "end"); 150 root.addContent(end); 151 152 return XmlUtils.prettyPrint(root).toString(); 153 } 154 }