001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.coord; 016 017 import java.util.Calendar; 018 import java.util.Date; 019 import java.util.HashMap; 020 import java.util.Iterator; 021 import java.util.List; 022 import java.util.Map; 023 import java.util.TimeZone; 024 025 import org.apache.hadoop.conf.Configuration; 026 import org.apache.oozie.service.ELService; 027 import org.apache.oozie.service.Services; 028 import org.apache.oozie.util.DateUtils; 029 import org.apache.oozie.util.ELEvaluator; 030 import org.apache.oozie.util.XLog; 031 import org.apache.oozie.util.XmlUtils; 032 import org.jdom.Element; 033 034 /** 035 * This class provide different evaluators required at different stages 036 */ 037 public class CoordELEvaluator { 038 public static final Integer MINUTE = 1; 039 public static final Integer HOUR = 60 * MINUTE; 040 041 /** 042 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage 043 * 1) 044 * 045 * @param conf : Configuration containing property variables 046 * @return configured ELEvaluator 047 */ 048 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) { 049 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group); 050 setConfigToEval(eval, conf); 051 return eval; 052 } 053 054 /** 055 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2) 056 * 057 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag 058 * @param appInst : Application Instance related information such as Action creation Time 059 * @param conf :Configuration to substitute any variables 060 * @return configured ELEvaluator 061 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown 062 */ 063 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf) 064 throws Exception { 065 return createInstancesELEvaluator("coord-action-create", event, appInst, conf); 066 } 067 068 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst, 069 Configuration conf) throws Exception { 070 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag); 071 setConfigToEval(eval, conf); 072 SyncCoordDataset ds = getDSObject(event); 073 CoordELFunctions.configureEvaluator(eval, ds, appInst); 074 return eval; 075 } 076 077 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group, 078 HashMap<String, String> dataNameList) throws Exception { 079 ELEvaluator eval = createELEvaluatorForGroup(conf, group); 080 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) { 081 String key = it.next(); 082 String value = dataNameList.get(key); 083 eval.setVariable("oozie.dataname." + key, value); 084 } 085 return eval; 086 } 087 088 /** 089 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function 090 * should be resolved when all other data dependencies are met. 091 * 092 * @param actualTime : Action start time 093 * @param nominalTime : Action creation time 094 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag 095 * @param conf :Configuration to substitute any variables 096 * @return configured ELEvaluator 097 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown 098 */ 099 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf) 100 throws Exception { 101 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start"); 102 setConfigToEval(eval, conf); 103 SyncCoordDataset ds = getDSObject(dEvent); 104 SyncCoordAction appInst = new SyncCoordAction();// TODO: 105 appInst.setNominalTime(nominalTime); 106 appInst.setActualTime(actualTime);// TODO: 107 CoordELFunctions.configureEvaluator(eval, ds, appInst); 108 eval.setVariable(CoordELFunctions.CONFIGURATION, conf); 109 return eval; 110 } 111 112 /** 113 * Create a SLA evaluator to be used during Materialization 114 * 115 * @param nominalTime 116 * @param conf 117 * @return 118 * @throws Exception 119 */ 120 public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception { 121 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create"); 122 setConfigToEval(eval, conf); 123 SyncCoordAction appInst = new SyncCoordAction();// TODO: 124 appInst.setNominalTime(nominalTime); 125 CoordELFunctions.configureEvaluator(eval, null, appInst); 126 return eval; 127 } 128 129 /** 130 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3) 131 * 132 * @param eJob : XML element for the application instance 133 * @param conf :Configuration to substitute any variables 134 * @return configured ELEvaluator 135 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown 136 */ 137 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception { 138 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start"); 139 setConfigToEval(e, conf); 140 SyncCoordAction appInst = new SyncCoordAction(); 141 String strNominalTime = eJob.getAttributeValue("action-nominal-time"); 142 if (strNominalTime != null) { 143 appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime)); 144 appInst.setActionId(actionId); 145 appInst.setName(eJob.getAttributeValue("name")); 146 } 147 CoordELFunctions.configureEvaluator(e, null, appInst); 148 Element events = eJob.getChild("input-events", eJob.getNamespace()); 149 if (events != null) { 150 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) { 151 if (data.getChild("uris", data.getNamespace()) != null) { 152 String uris = data.getChild("uris", data.getNamespace()).getTextTrim(); 153 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR); 154 e.setVariable(".datain." + data.getAttributeValue("name"), uris); 155 } 156 else { 157 } 158 if (data.getChild("unresolved-instances", data.getNamespace()) != null) { 159 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO: 160 // check 161 // null 162 } 163 } 164 } 165 events = eJob.getChild("output-events", eJob.getNamespace()); 166 if (events != null) { 167 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) { 168 if (data.getChild("uris", data.getNamespace()) != null) { 169 String uris = data.getChild("uris", data.getNamespace()).getTextTrim(); 170 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR); 171 e.setVariable(".dataout." + data.getAttributeValue("name"), uris); 172 } 173 else { 174 }// TODO 175 if (data.getChild("unresolved-instances", data.getNamespace()) != null) { 176 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO: 177 // check 178 // null 179 } 180 } 181 } 182 return e; 183 } 184 185 /** 186 * Create a new Evaluator to resolve URI temple with time specific constant 187 * 188 * @param strDate : Date-time 189 * @return configured ELEvaluator 190 * @throws Exception If there is any date-time string in wrong format, the exception is thrown 191 */ 192 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception { 193 ELEvaluator eval = new ELEvaluator(); 194 Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC 195 // always??? 196 date.setTime(DateUtils.parseDateUTC(strDate)); 197 eval.setVariable("YEAR", date.get(Calendar.YEAR)); 198 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1)); 199 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH))); 200 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY))); 201 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE))); 202 return eval; 203 } 204 205 /** 206 * Create Dataset object using the Dataset XML information 207 * 208 * @param eData 209 * @return 210 * @throws Exception 211 */ 212 private static SyncCoordDataset getDSObject(Element eData) throws Exception { 213 SyncCoordDataset ds = new SyncCoordDataset(); 214 Element eDataset = eData.getChild("dataset", eData.getNamespace()); 215 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData)); 216 Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance")); 217 ds.setInitInstance(initInstance); 218 if (eDataset.getAttributeValue("frequency") != null) { 219 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency")); 220 ds.setFrequency(frequency); 221 ds.setType("SYNC"); 222 if (eDataset.getAttributeValue("freq_timeunit") == null) { 223 throw new RuntimeException("No freq_timeunit defined in data set definition\n" 224 + XmlUtils.prettyPrint(eDataset)); 225 } 226 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit"))); 227 if (eDataset.getAttributeValue("timezone") == null) { 228 throw new RuntimeException("No timezone defined in data set definition\n" 229 + XmlUtils.prettyPrint(eDataset)); 230 } 231 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone"))); 232 if (eDataset.getAttributeValue("end_of_duration") == null) { 233 throw new RuntimeException("No end_of_duration defined in data set definition\n" 234 + XmlUtils.prettyPrint(eDataset)); 235 } 236 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration"))); 237 238 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace()); 239 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement); 240 ds.setDoneFlag(doneFlag); 241 } 242 else { 243 ds.setType("ASYNC"); 244 } 245 String name = eDataset.getAttributeValue("name"); 246 ds.setName(name); 247 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template", 248 // eData.getNamespace())); 249 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim(); 250 ds.setUriTemplate(uriTemplate); 251 // ds.setTimeUnit(TimeUnit.MINUTES); 252 return ds; 253 } 254 255 /** 256 * Set all job configurations properties into evaluator. 257 * 258 * @param eval : Evaluator to set variables 259 * @param conf : configurations to set Evaluator 260 */ 261 private static void setConfigToEval(ELEvaluator eval, Configuration conf) { 262 for (Map.Entry<String, String> entry : conf) { 263 eval.setVariable(entry.getKey(), entry.getValue()); 264 } 265 } 266 267 /** 268 * make any one digit number to two digit string pre-appending a"0" 269 * 270 * @param num : number to make sting 271 * @return :String of length at least two digit. 272 */ 273 private static String make2Digits(int num) { 274 String ret = "" + num; 275 if (num <= 9) { 276 ret = "0" + ret; 277 } 278 return ret; 279 } 280 }