001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.util; 016 017 import java.io.File; 018 import java.io.FileInputStream; 019 import java.io.IOException; 020 import java.io.InputStream; 021 import java.io.Writer; 022 import java.util.ArrayList; 023 import java.util.Collections; 024 import java.util.Date; 025 import java.util.HashMap; 026 import java.util.List; 027 import java.util.Map; 028 import java.util.regex.Matcher; 029 import java.util.regex.Pattern; 030 031 /** 032 * XLogStreamer streams the given log file to logWriter after applying the given filter. 033 */ 034 public class XLogStreamer { 035 036 /** 037 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks 038 * if the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by 039 * "|") jobId appName actionId token 040 */ 041 public static class Filter { 042 private Map<String, Integer> logLevels; 043 private Map<String, String> filterParams; 044 private static List<String> parameters = new ArrayList<String>(); 045 private boolean noFilter; 046 private Pattern filterPattern; 047 048 //TODO Patterns to be read from config file 049 private static final String DEFAULT_REGEX = "[^\\]]*"; 050 051 public static final String ALLOW_ALL_REGEX = "(.*)"; 052 private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)"; 053 private static final String WHITE_SPACE_REGEX = "\\s+"; 054 private static final String LOG_LEVEL_REGEX = "(\\w+)"; 055 private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX 056 + WHITE_SPACE_REGEX; 057 private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX); 058 059 public Filter() { 060 filterParams = new HashMap<String, String>(); 061 for (int i = 0; i < parameters.size(); i++) { 062 filterParams.put(parameters.get(i), DEFAULT_REGEX); 063 } 064 logLevels = null; 065 noFilter = true; 066 filterPattern = null; 067 } 068 069 public void setLogLevel(String logLevel) { 070 if (logLevel != null && logLevel.trim().length() > 0) { 071 this.logLevels = new HashMap<String, Integer>(); 072 String[] levels = logLevel.split("\\|"); 073 for (int i = 0; i < levels.length; i++) { 074 String s = levels[i].trim().toUpperCase(); 075 try { 076 XLog.Level.valueOf(s); 077 } 078 catch (Exception ex) { 079 continue; 080 } 081 this.logLevels.put(levels[i].toUpperCase(), 1); 082 } 083 } 084 } 085 086 public void setParameter(String filterParam, String value) { 087 if (filterParams.containsKey(filterParam)) { 088 noFilter = false; 089 filterParams.put(filterParam, value); 090 } 091 } 092 093 public static void defineParameter(String filterParam) { 094 parameters.add(filterParam); 095 } 096 097 public boolean isFilterPresent() { 098 if (noFilter && logLevels == null) { 099 return false; 100 } 101 return true; 102 } 103 104 /** 105 * Checks if the logLevel and logMessage goes through the logFilter. 106 * 107 * @param logParts 108 * @return 109 */ 110 public boolean matches(ArrayList<String> logParts) { 111 String logLevel = logParts.get(0); 112 String logMessage = logParts.get(1); 113 if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) { 114 Matcher logMatcher = filterPattern.matcher(logMessage); 115 return logMatcher.matches(); 116 } 117 else { 118 return false; 119 } 120 } 121 122 /** 123 * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing logLevel and 124 * logMessage if the pattern matches i.e A new log statement, else returns null. 125 * 126 * @param logLine 127 * @return Array containing log level and log message 128 */ 129 public ArrayList<String> splitLogMessage(String logLine) { 130 Matcher splitter = SPLITTER_PATTERN.matcher(logLine); 131 if (splitter.matches()) { 132 ArrayList<String> logParts = new ArrayList<String>(); 133 logParts.add(splitter.group(2));// log level 134 logParts.add(splitter.group(3));// Log Message 135 return logParts; 136 } 137 else { 138 return null; 139 } 140 } 141 142 /** 143 * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be 144 * assigned if no filters are set. 145 */ 146 public void constructPattern() { 147 if (noFilter && logLevels == null) { 148 filterPattern = Pattern.compile(ALLOW_ALL_REGEX); 149 return; 150 } 151 StringBuilder sb = new StringBuilder(); 152 if (noFilter) { 153 sb.append("(.*)"); 154 } 155 else { 156 sb.append("(.* - "); 157 for (int i = 0; i < parameters.size(); i++) { 158 sb.append(parameters.get(i) + "\\["); 159 sb.append(filterParams.get(parameters.get(i)) + "\\] "); 160 } 161 sb.append(".*)"); 162 } 163 filterPattern = Pattern.compile(sb.toString()); 164 } 165 166 public static void reset() { 167 parameters.clear(); 168 } 169 } 170 171 private String logFile; 172 private String logPath; 173 private Filter logFilter; 174 private Writer logWriter; 175 private long logRotation; 176 177 public XLogStreamer(Filter logFilter, Writer logWriter, String logPath, String logFile, long logRotationSecs) { 178 this.logWriter = logWriter; 179 this.logFilter = logFilter; 180 if (logFile == null) { 181 logFile = "oozie-app.log"; 182 } 183 this.logFile = logFile; 184 this.logPath = logPath; 185 this.logRotation = logRotationSecs * 1000l; 186 } 187 188 /** 189 * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after 190 * applying the filters. 191 * 192 * @param startTime 193 * @param endTime 194 * @throws IOException 195 */ 196 public void streamLog(Date startTime, Date endTime) throws IOException { 197 long startTimeMillis = 0; 198 long endTimeMillis; 199 if (startTime != null) { 200 startTimeMillis = startTime.getTime(); 201 } 202 if (endTime == null) { 203 endTimeMillis = System.currentTimeMillis(); 204 } 205 else { 206 endTimeMillis = endTime.getTime(); 207 } 208 File dir = new File(logPath); 209 ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile); 210 for (int i = 0; i < fileList.size(); i++) { 211 InputStream ifs; 212 ifs = new FileInputStream(fileList.get(i).getFileName()); 213 XLogReader logReader = new XLogReader(ifs, logFilter, logWriter); 214 logReader.processLog(); 215 } 216 } 217 218 /** 219 * File name along with the modified time which will be used to sort later. 220 */ 221 class FileInfo implements Comparable<FileInfo> { 222 String fileName; 223 long modTime; 224 225 public FileInfo(String fileName, long modTime) { 226 this.fileName = fileName; 227 this.modTime = modTime; 228 } 229 230 public String getFileName() { 231 return fileName; 232 } 233 234 public long getModTime() { 235 return modTime; 236 } 237 238 public int compareTo(FileInfo fileInfo) { 239 long diff = this.modTime - fileInfo.modTime; 240 if(diff > 0) { 241 return 1; 242 } else if(diff < 0) { 243 return -1; 244 } else { 245 return 0; 246 } 247 } 248 } 249 250 /** 251 * Gets the file list that will have the logs between startTime and endTime. 252 * 253 * @param dir 254 * @param startTime 255 * @param endTime 256 * @param logRotationTime 257 * @param logFile 258 * @return List of files to be streamed 259 */ 260 private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime, 261 String logFile) { 262 String[] children = dir.list(); 263 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 264 if (children == null) { 265 return fileList; 266 } 267 else { 268 for (int i = 0; i < children.length; i++) { 269 String filename = children[i]; 270 if (!filename.startsWith(logFile) && !filename.equals(logFile)) { 271 continue; 272 } 273 File file = new File(dir.getAbsolutePath(), filename); 274 long modTime = file.lastModified(); 275 if (modTime < startTime) { 276 continue; 277 } 278 if (modTime / logRotationTime > (endTime / logRotationTime + 1)) { 279 continue; 280 } 281 fileList.add(new FileInfo(file.getAbsolutePath(), modTime)); 282 } 283 } 284 Collections.sort(fileList); 285 return fileList; 286 } 287 }