001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.util;
016    
017    import java.io.File;
018    import java.io.FileInputStream;
019    import java.io.IOException;
020    import java.io.InputStream;
021    import java.io.Writer;
022    import java.util.ArrayList;
023    import java.util.Collections;
024    import java.util.Date;
025    import java.util.HashMap;
026    import java.util.List;
027    import java.util.Map;
028    import java.util.regex.Matcher;
029    import java.util.regex.Pattern;
030    
031    /**
032     * XLogStreamer streams the given log file to logWriter after applying the given filter.
033     */
034    public class XLogStreamer {
035    
036        /**
037         * Filter that will construct the regular expression that will be used to filter the log statement. And also checks
038         * if the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by
039         * "|") jobId appName actionId token
040         */
041        public static class Filter {
042            private Map<String, Integer> logLevels;
043            private Map<String, String> filterParams;
044            private static List<String> parameters = new ArrayList<String>();
045            private boolean noFilter;
046            private Pattern filterPattern;
047    
048            //TODO Patterns to be read from config file
049            private static final String DEFAULT_REGEX = "[^\\]]*";
050    
051            public static final String ALLOW_ALL_REGEX = "(.*)";
052            private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
053            private static final String WHITE_SPACE_REGEX = "\\s+";
054            private static final String LOG_LEVEL_REGEX = "(\\w+)";
055            private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
056                    + WHITE_SPACE_REGEX;
057            private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
058    
059            public Filter() {
060                filterParams = new HashMap<String, String>();
061                for (int i = 0; i < parameters.size(); i++) {
062                    filterParams.put(parameters.get(i), DEFAULT_REGEX);
063                }
064                logLevels = null;
065                noFilter = true;
066                filterPattern = null;
067            }
068    
069            public void setLogLevel(String logLevel) {
070                if (logLevel != null && logLevel.trim().length() > 0) {
071                    this.logLevels = new HashMap<String, Integer>();
072                    String[] levels = logLevel.split("\\|");
073                    for (int i = 0; i < levels.length; i++) {
074                        String s = levels[i].trim().toUpperCase();
075                        try {
076                            XLog.Level.valueOf(s);
077                        }
078                        catch (Exception ex) {
079                            continue;
080                        }
081                        this.logLevels.put(levels[i].toUpperCase(), 1);
082                    }
083                }
084            }
085    
086            public void setParameter(String filterParam, String value) {
087                if (filterParams.containsKey(filterParam)) {
088                    noFilter = false;
089                    filterParams.put(filterParam, value);
090                }
091            }
092    
093            public static void defineParameter(String filterParam) {
094                parameters.add(filterParam);
095            }
096    
097            public boolean isFilterPresent() {
098                if (noFilter && logLevels == null) {
099                    return false;
100                }
101                return true;
102            }
103    
104            /**
105             * Checks if the logLevel and logMessage goes through the logFilter.
106             *
107             * @param logParts
108             * @return
109             */
110            public boolean matches(ArrayList<String> logParts) {
111                String logLevel = logParts.get(0);
112                String logMessage = logParts.get(1);
113                if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
114                    Matcher logMatcher = filterPattern.matcher(logMessage);
115                    return logMatcher.matches();
116                }
117                else {
118                    return false;
119                }
120            }
121    
122            /**
123             * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing logLevel and
124             * logMessage if the pattern matches i.e A new log statement, else returns null.
125             *
126             * @param logLine
127             * @return Array containing log level and log message
128             */
129            public ArrayList<String> splitLogMessage(String logLine) {
130                Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
131                if (splitter.matches()) {
132                    ArrayList<String> logParts = new ArrayList<String>();
133                    logParts.add(splitter.group(2));// log level
134                    logParts.add(splitter.group(3));// Log Message
135                    return logParts;
136                }
137                else {
138                    return null;
139                }
140            }
141    
142            /**
143             * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be
144             * assigned if no filters are set.
145             */
146            public void constructPattern() {
147                if (noFilter && logLevels == null) {
148                    filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
149                    return;
150                }
151                StringBuilder sb = new StringBuilder();
152                if (noFilter) {
153                    sb.append("(.*)");
154                }
155                else {
156                    sb.append("(.* - ");
157                    for (int i = 0; i < parameters.size(); i++) {
158                        sb.append(parameters.get(i) + "\\[");
159                        sb.append(filterParams.get(parameters.get(i)) + "\\] ");
160                    }
161                    sb.append(".*)");
162                }
163                filterPattern = Pattern.compile(sb.toString());
164            }
165    
166            public static void reset() {
167                parameters.clear();
168            }
169        }
170    
171        private String logFile;
172        private String logPath;
173        private Filter logFilter;
174        private Writer logWriter;
175        private long logRotation;
176    
177        public XLogStreamer(Filter logFilter, Writer logWriter, String logPath, String logFile, long logRotationSecs) {
178            this.logWriter = logWriter;
179            this.logFilter = logFilter;
180            if (logFile == null) {
181                logFile = "oozie-app.log";
182            }
183            this.logFile = logFile;
184            this.logPath = logPath;
185            this.logRotation = logRotationSecs * 1000l;
186        }
187    
188        /**
189         * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
190         * applying the filters.
191         *
192         * @param startTime
193         * @param endTime
194         * @throws IOException
195         */
196        public void streamLog(Date startTime, Date endTime) throws IOException {
197            long startTimeMillis = 0;
198            long endTimeMillis;
199            if (startTime != null) {
200                startTimeMillis = startTime.getTime();
201            }
202            if (endTime == null) {
203                endTimeMillis = System.currentTimeMillis();
204            }
205            else {
206                endTimeMillis = endTime.getTime();
207            }
208            File dir = new File(logPath);
209            ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
210            for (int i = 0; i < fileList.size(); i++) {
211                InputStream ifs;
212                ifs = new FileInputStream(fileList.get(i).getFileName());
213                XLogReader logReader = new XLogReader(ifs, logFilter, logWriter);
214                logReader.processLog();
215            }
216        }
217    
218        /**
219         * File name along with the modified time which will be used to sort later.
220         */
221        class FileInfo implements Comparable<FileInfo> {
222            String fileName;
223            long modTime;
224    
225            public FileInfo(String fileName, long modTime) {
226                this.fileName = fileName;
227                this.modTime = modTime;
228            }
229    
230            public String getFileName() {
231                return fileName;
232            }
233    
234            public long getModTime() {
235                return modTime;
236            }
237    
238            public int compareTo(FileInfo fileInfo) {
239                long diff = this.modTime - fileInfo.modTime;
240                if(diff > 0) {
241                    return 1;
242                } else if(diff < 0) {
243                    return -1;
244                } else {
245                    return 0;
246                }
247            }
248        }
249    
250        /**
251         * Gets the file list that will have the logs between startTime and endTime.
252         *
253         * @param dir
254         * @param startTime
255         * @param endTime
256         * @param logRotationTime
257         * @param logFile
258         * @return List of files to be streamed
259         */
260        private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime,
261                                                String logFile) {
262            String[] children = dir.list();
263            ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
264            if (children == null) {
265                return fileList;
266            }
267            else {
268                for (int i = 0; i < children.length; i++) {
269                    String filename = children[i];
270                    if (!filename.startsWith(logFile) && !filename.equals(logFile)) {
271                        continue;
272                    }
273                    File file = new File(dir.getAbsolutePath(), filename);
274                    long modTime = file.lastModified();
275                    if (modTime < startTime) {
276                        continue;
277                    }
278                    if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
279                        continue;
280                    }
281                    fileList.add(new FileInfo(file.getAbsolutePath(), modTime));
282                }
283            }
284            Collections.sort(fileList);
285            return fileList;
286        }
287    }