001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.coord; 016 017 import java.util.Calendar; 018 import java.util.Date; 019 import java.util.HashMap; 020 import java.util.Iterator; 021 import java.util.List; 022 import java.util.Map; 023 import java.util.TimeZone; 024 025 import org.apache.hadoop.conf.Configuration; 026 import org.apache.oozie.service.ELService; 027 import org.apache.oozie.service.Services; 028 import org.apache.oozie.util.DateUtils; 029 import org.apache.oozie.util.ELEvaluator; 030 import org.apache.oozie.util.XLog; 031 import org.apache.oozie.util.XmlUtils; 032 import org.jdom.Element; 033 034 /** 035 * This class provide different evaluators required at different stages 036 */ 037 public class CoordELEvaluator { 038 public static final Integer MINUTE = 1; 039 public static final Integer HOUR = 60 * MINUTE; 040 041 /** 042 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage 043 * 1) 044 * 045 * @param conf : Configuration containing property variables 046 * @return configured ELEvaluator 047 */ 048 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) { 049 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group); 050 setConfigToEval(eval, conf); 051 return eval; 052 } 053 054 /** 055 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2) 056 * 057 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag 058 * @param appInst : Application Instance related information such as Action creation Time 059 * @param conf :Configuration to substitute any variables 060 * @return configured ELEvaluator 061 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown 062 */ 063 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf) 064 throws Exception { 065 return createInstancesELEvaluator("coord-action-create", event, appInst, conf); 066 } 067 068 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst, 069 Configuration conf) throws Exception { 070 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag); 071 setConfigToEval(eval, conf); 072 SyncCoordDataset ds = getDSObject(event); 073 CoordELFunctions.configureEvaluator(eval, ds, appInst); 074 return eval; 075 } 076 077 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group, 078 HashMap<String, String> dataNameList) throws Exception { 079 ELEvaluator eval = createELEvaluatorForGroup(conf, group); 080 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) { 081 String key = it.next(); 082 String value = dataNameList.get(key); 083 eval.setVariable("oozie.dataname." + key, value); 084 } 085 return eval; 086 } 087 088 /** 089 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function 090 * should be resolved when all other data dependencies are met. 091 * 092 * @param actualTime : Action start time 093 * @param nominalTime : Action creation time 094 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag 095 * @param conf :Configuration to substitute any variables 096 * @return configured ELEvaluator 097 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown 098 */ 099 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf) 100 throws Exception { 101 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start"); 102 setConfigToEval(eval, conf); 103 SyncCoordDataset ds = getDSObject(dEvent); 104 SyncCoordAction appInst = new SyncCoordAction();// TODO: 105 appInst.setNominalTime(nominalTime); 106 appInst.setActualTime(actualTime);// TODO: 107 CoordELFunctions.configureEvaluator(eval, ds, appInst); 108 // Configuration tmpConf = new Configuration(); 109 Configuration tmpConf = CoordUtils.getHadoopConf(conf); 110 // TODO:Set hadoop properties 111 eval.setVariable(CoordELFunctions.CONFIGURATION, tmpConf); 112 return eval; 113 } 114 115 /** 116 * Create a SLA evaluator to be used during Materialization 117 * 118 * @param nominalTime 119 * @param conf 120 * @return 121 * @throws Exception 122 */ 123 public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception { 124 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create"); 125 setConfigToEval(eval, conf); 126 SyncCoordAction appInst = new SyncCoordAction();// TODO: 127 appInst.setNominalTime(nominalTime); 128 CoordELFunctions.configureEvaluator(eval, null, appInst); 129 return eval; 130 } 131 132 /** 133 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3) 134 * 135 * @param eJob : XML element for the application instance 136 * @param conf :Configuration to substitute any variables 137 * @return configured ELEvaluator 138 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown 139 */ 140 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception { 141 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start"); 142 setConfigToEval(e, conf); 143 SyncCoordAction appInst = new SyncCoordAction(); 144 String strNominalTime = eJob.getAttributeValue("action-nominal-time"); 145 if (strNominalTime != null) { 146 appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime)); 147 appInst.setActionId(actionId); 148 appInst.setName(eJob.getAttributeValue("name")); 149 } 150 CoordELFunctions.configureEvaluator(e, null, appInst); 151 Element events = eJob.getChild("input-events", eJob.getNamespace()); 152 if (events != null) { 153 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) { 154 if (data.getChild("uris", data.getNamespace()) != null) { 155 String uris = data.getChild("uris", data.getNamespace()).getTextTrim(); 156 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR); 157 e.setVariable(".datain." + data.getAttributeValue("name"), uris); 158 } 159 else { 160 } 161 if (data.getChild("unresolved-instances", data.getNamespace()) != null) { 162 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO: 163 // check 164 // null 165 } 166 } 167 } 168 events = eJob.getChild("output-events", eJob.getNamespace()); 169 if (events != null) { 170 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) { 171 if (data.getChild("uris", data.getNamespace()) != null) { 172 String uris = data.getChild("uris", data.getNamespace()).getTextTrim(); 173 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR); 174 e.setVariable(".dataout." + data.getAttributeValue("name"), uris); 175 } 176 else { 177 }// TODO 178 if (data.getChild("unresolved-instances", data.getNamespace()) != null) { 179 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO: 180 // check 181 // null 182 } 183 } 184 } 185 return e; 186 } 187 188 /** 189 * Create a new Evaluator to resolve URI temple with time specific constant 190 * 191 * @param strDate : Date-time 192 * @return configured ELEvaluator 193 * @throws Exception If there is any date-time string in wrong format, the exception is thrown 194 */ 195 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception { 196 ELEvaluator eval = new ELEvaluator(); 197 Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC 198 // always??? 199 date.setTime(DateUtils.parseDateUTC(strDate)); 200 eval.setVariable("YEAR", date.get(Calendar.YEAR)); 201 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1)); 202 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH))); 203 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY))); 204 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE))); 205 return eval; 206 } 207 208 /** 209 * Create Dataset object using the Dataset XML information 210 * 211 * @param eData 212 * @return 213 * @throws Exception 214 */ 215 private static SyncCoordDataset getDSObject(Element eData) throws Exception { 216 SyncCoordDataset ds = new SyncCoordDataset(); 217 Element eDataset = eData.getChild("dataset", eData.getNamespace()); 218 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData)); 219 Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance")); 220 ds.setInitInstance(initInstance); 221 if (eDataset.getAttributeValue("frequency") != null) { 222 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency")); 223 ds.setFrequency(frequency); 224 ds.setType("SYNC"); 225 if (eDataset.getAttributeValue("freq_timeunit") == null) { 226 throw new RuntimeException("No freq_timeunit defined in data set definition\n" 227 + XmlUtils.prettyPrint(eDataset)); 228 } 229 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit"))); 230 if (eDataset.getAttributeValue("timezone") == null) { 231 throw new RuntimeException("No timezone defined in data set definition\n" 232 + XmlUtils.prettyPrint(eDataset)); 233 } 234 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone"))); 235 if (eDataset.getAttributeValue("end_of_duration") == null) { 236 throw new RuntimeException("No end_of_duration defined in data set definition\n" 237 + XmlUtils.prettyPrint(eDataset)); 238 } 239 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration"))); 240 241 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace()); 242 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement); 243 ds.setDoneFlag(doneFlag); 244 } 245 else { 246 ds.setType("ASYNC"); 247 } 248 String name = eDataset.getAttributeValue("name"); 249 ds.setName(name); 250 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template", 251 // eData.getNamespace())); 252 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim(); 253 ds.setUriTemplate(uriTemplate); 254 // ds.setTimeUnit(TimeUnit.MINUTES); 255 return ds; 256 } 257 258 /** 259 * Set all job configurations properties into evaluator. 260 * 261 * @param eval : Evaluator to set variables 262 * @param conf : configurations to set Evaluator 263 */ 264 private static void setConfigToEval(ELEvaluator eval, Configuration conf) { 265 for (Map.Entry<String, String> entry : conf) { 266 eval.setVariable(entry.getKey(), entry.getValue()); 267 } 268 } 269 270 /** 271 * make any one digit number to two digit string pre-appending a"0" 272 * 273 * @param num : number to make sting 274 * @return :String of length at least two digit. 275 */ 276 private static String make2Digits(int num) { 277 String ret = "" + num; 278 if (num <= 9) { 279 ret = "0" + ret; 280 } 281 return ret; 282 } 283 }