001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.util; 016 017 import java.util.HashMap; 018 import java.util.List; 019 import java.util.Map; 020 import java.util.regex.Matcher; 021 import java.util.regex.Pattern; 022 023 import java.util.Date; 024 import java.io.File; 025 import java.io.FileInputStream; 026 import java.io.IOException; 027 import java.io.InputStream; 028 import java.io.Writer; 029 import java.util.ArrayList; 030 import java.util.Collections; 031 032 import org.apache.oozie.util.XLog; 033 import org.apache.oozie.util.XLogReader; 034 035 /** 036 * XLogStreamer streams the given log file to logWriter after applying the given filter. 037 */ 038 public class XLogStreamer { 039 040 /** 041 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks 042 * if the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by 043 * "|") jobId appName actionId token 044 */ 045 public static class Filter { 046 private Map<String, Integer> logLevels; 047 private Map<String, String> filterParams; 048 private static List<String> parameters = new ArrayList<String>(); 049 private boolean noFilter; 050 private Pattern filterPattern; 051 052 //TODO Patterns to be read from config file 053 private static final String DEFAULT_REGEX = "[^\\]]*"; 054 055 public static final String ALLOW_ALL_REGEX = "(.*)"; 056 private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)"; 057 private static final String WHITE_SPACE_REGEX = "\\s+"; 058 private static final String LOG_LEVEL_REGEX = "(\\w+)"; 059 private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX 060 + WHITE_SPACE_REGEX; 061 private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX); 062 063 public Filter() { 064 filterParams = new HashMap<String, String>(); 065 for (int i = 0; i < parameters.size(); i++) { 066 filterParams.put(parameters.get(i), DEFAULT_REGEX); 067 } 068 logLevels = null; 069 noFilter = true; 070 filterPattern = null; 071 } 072 073 public void setLogLevel(String logLevel) { 074 if (logLevel != null && logLevel.trim().length() > 0) { 075 this.logLevels = new HashMap<String, Integer>(); 076 String[] levels = logLevel.split("\\|"); 077 for (int i = 0; i < levels.length; i++) { 078 String s = levels[i].trim().toUpperCase(); 079 try { 080 XLog.Level.valueOf(s); 081 } 082 catch (Exception ex) { 083 continue; 084 } 085 this.logLevels.put(levels[i].toUpperCase(), 1); 086 } 087 } 088 } 089 090 public void setParameter(String filterParam, String value) { 091 if (filterParams.containsKey(filterParam)) { 092 noFilter = false; 093 filterParams.put(filterParam, value); 094 } 095 } 096 097 public static void defineParameter(String filterParam) { 098 parameters.add(filterParam); 099 } 100 101 public boolean isFilterPresent() { 102 if (noFilter && logLevels == null) { 103 return false; 104 } 105 return true; 106 } 107 108 /** 109 * Checks if the logLevel and logMessage goes through the logFilter. 110 * 111 * @param logParts 112 * @return 113 */ 114 public boolean matches(ArrayList<String> logParts) { 115 String logLevel = logParts.get(0); 116 String logMessage = logParts.get(1); 117 if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) { 118 Matcher logMatcher = filterPattern.matcher(logMessage); 119 return logMatcher.matches(); 120 } 121 else { 122 return false; 123 } 124 } 125 126 /** 127 * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing logLevel and 128 * logMessage if the pattern matches i.e A new log statement, else returns null. 129 * 130 * @param logLine 131 * @return Array containing log level and log message 132 */ 133 public ArrayList<String> splitLogMessage(String logLine) { 134 Matcher splitter = SPLITTER_PATTERN.matcher(logLine); 135 if (splitter.matches()) { 136 ArrayList<String> logParts = new ArrayList<String>(); 137 logParts.add(splitter.group(2));// log level 138 logParts.add(splitter.group(3));// Log Message 139 return logParts; 140 } 141 else { 142 return null; 143 } 144 } 145 146 /** 147 * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be 148 * assigned if no filters are set. 149 */ 150 public void constructPattern() { 151 if (noFilter && logLevels == null) { 152 filterPattern = Pattern.compile(ALLOW_ALL_REGEX); 153 return; 154 } 155 StringBuilder sb = new StringBuilder(); 156 if (noFilter) { 157 sb.append("(.*)"); 158 } 159 else { 160 sb.append("(.* - "); 161 for (int i = 0; i < parameters.size(); i++) { 162 sb.append(parameters.get(i) + "\\["); 163 sb.append(filterParams.get(parameters.get(i)) + "\\] "); 164 } 165 sb.append(".*)"); 166 } 167 filterPattern = Pattern.compile(sb.toString()); 168 } 169 170 public static void reset() { 171 parameters.clear(); 172 } 173 } 174 175 private String logFile; 176 private String logPath; 177 private Filter logFilter; 178 private Writer logWriter; 179 private long logRotation; 180 181 public XLogStreamer(Filter logFilter, Writer logWriter, String logPath, String logFile, long logRotationSecs) { 182 this.logWriter = logWriter; 183 this.logFilter = logFilter; 184 if (logFile == null) { 185 logFile = "oozie-app.log"; 186 } 187 this.logFile = logFile; 188 this.logPath = logPath; 189 this.logRotation = logRotationSecs * 1000l; 190 } 191 192 /** 193 * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after 194 * applying the filters. 195 * 196 * @param startTime 197 * @param endTime 198 * @throws IOException 199 */ 200 public void streamLog(Date startTime, Date endTime) throws IOException { 201 long startTimeMillis = 0; 202 long endTimeMillis; 203 if (startTime != null) { 204 startTimeMillis = startTime.getTime(); 205 } 206 if (endTime == null) { 207 endTimeMillis = System.currentTimeMillis(); 208 } 209 else { 210 endTimeMillis = endTime.getTime(); 211 } 212 File dir = new File(logPath); 213 ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile); 214 for (int i = 0; i < fileList.size(); i++) { 215 InputStream ifs; 216 ifs = new FileInputStream(fileList.get(i).getFileName()); 217 XLogReader logReader = new XLogReader(ifs, logFilter, logWriter); 218 logReader.processLog(); 219 } 220 } 221 222 /** 223 * File name along with the modified time which will be used to sort later. 224 */ 225 class FileInfo implements Comparable<FileInfo> { 226 String fileName; 227 long modTime; 228 229 public FileInfo(String fileName, long modTime) { 230 this.fileName = fileName; 231 this.modTime = modTime; 232 } 233 234 public String getFileName() { 235 return fileName; 236 } 237 238 public long getModTime() { 239 return modTime; 240 } 241 242 public int compareTo(FileInfo fileInfo) { 243 return (int) (this.modTime - fileInfo.modTime); 244 } 245 } 246 247 /** 248 * Gets the file list that will have the logs between startTime and endTime. 249 * 250 * @param dir 251 * @param startTime 252 * @param endTime 253 * @param logRotationTime 254 * @param logFile 255 * @return List of files to be streamed 256 */ 257 private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime, 258 String logFile) { 259 String[] children = dir.list(); 260 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 261 if (children == null) { 262 return fileList; 263 } 264 else { 265 for (int i = 0; i < children.length; i++) { 266 String filename = children[i]; 267 if (!filename.startsWith(logFile) && !filename.equals(logFile)) { 268 continue; 269 } 270 File file = new File(dir.getAbsolutePath(), filename); 271 long modTime = file.lastModified(); 272 if (modTime < startTime) { 273 continue; 274 } 275 if (modTime / logRotationTime > (endTime / logRotationTime + 1)) { 276 continue; 277 } 278 fileList.add(new FileInfo(file.getAbsolutePath(), modTime)); 279 } 280 } 281 Collections.sort(fileList); 282 return fileList; 283 } 284 }