001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.util;
016    
017    import java.util.HashMap;
018    import java.util.List;
019    import java.util.Map;
020    import java.util.regex.Matcher;
021    import java.util.regex.Pattern;
022    
023    import java.util.Date;
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.Writer;
029    import java.util.ArrayList;
030    import java.util.Collections;
031    
032    import org.apache.oozie.util.XLog;
033    import org.apache.oozie.util.XLogReader;
034    
035    /**
036     * XLogStreamer streams the given log file to logWriter after applying the given filter.
037     */
038    public class XLogStreamer {
039    
040        /**
041         * Filter that will construct the regular expression that will be used to filter the log statement. And also checks
042         * if the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by
043         * "|") jobId appName actionId token
044         */
045        public static class Filter {
046            private Map<String, Integer> logLevels;
047            private Map<String, String> filterParams;
048            private static List<String> parameters = new ArrayList<String>();
049            private boolean noFilter;
050            private Pattern filterPattern;
051    
052            //TODO Patterns to be read from config file
053            private static final String DEFAULT_REGEX = "[^\\]]*";
054    
055            public static final String ALLOW_ALL_REGEX = "(.*)";
056            private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
057            private static final String WHITE_SPACE_REGEX = "\\s+";
058            private static final String LOG_LEVEL_REGEX = "(\\w+)";
059            private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
060                    + WHITE_SPACE_REGEX;
061            private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
062    
063            public Filter() {
064                filterParams = new HashMap<String, String>();
065                for (int i = 0; i < parameters.size(); i++) {
066                    filterParams.put(parameters.get(i), DEFAULT_REGEX);
067                }
068                logLevels = null;
069                noFilter = true;
070                filterPattern = null;
071            }
072    
073            public void setLogLevel(String logLevel) {
074                if (logLevel != null && logLevel.trim().length() > 0) {
075                    this.logLevels = new HashMap<String, Integer>();
076                    String[] levels = logLevel.split("\\|");
077                    for (int i = 0; i < levels.length; i++) {
078                        String s = levels[i].trim().toUpperCase();
079                        try {
080                            XLog.Level.valueOf(s);
081                        }
082                        catch (Exception ex) {
083                            continue;
084                        }
085                        this.logLevels.put(levels[i].toUpperCase(), 1);
086                    }
087                }
088            }
089    
090            public void setParameter(String filterParam, String value) {
091                if (filterParams.containsKey(filterParam)) {
092                    noFilter = false;
093                    filterParams.put(filterParam, value);
094                }
095            }
096    
097            public static void defineParameter(String filterParam) {
098                parameters.add(filterParam);
099            }
100    
101            public boolean isFilterPresent() {
102                if (noFilter && logLevels == null) {
103                    return false;
104                }
105                return true;
106            }
107    
108            /**
109             * Checks if the logLevel and logMessage goes through the logFilter.
110             *
111             * @param logParts
112             * @return
113             */
114            public boolean matches(ArrayList<String> logParts) {
115                String logLevel = logParts.get(0);
116                String logMessage = logParts.get(1);
117                if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
118                    Matcher logMatcher = filterPattern.matcher(logMessage);
119                    return logMatcher.matches();
120                }
121                else {
122                    return false;
123                }
124            }
125    
126            /**
127             * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing logLevel and
128             * logMessage if the pattern matches i.e A new log statement, else returns null.
129             *
130             * @param logLine
131             * @return Array containing log level and log message
132             */
133            public ArrayList<String> splitLogMessage(String logLine) {
134                Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
135                if (splitter.matches()) {
136                    ArrayList<String> logParts = new ArrayList<String>();
137                    logParts.add(splitter.group(2));// log level
138                    logParts.add(splitter.group(3));// Log Message
139                    return logParts;
140                }
141                else {
142                    return null;
143                }
144            }
145    
146            /**
147             * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be
148             * assigned if no filters are set.
149             */
150            public void constructPattern() {
151                if (noFilter && logLevels == null) {
152                    filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
153                    return;
154                }
155                StringBuilder sb = new StringBuilder();
156                if (noFilter) {
157                    sb.append("(.*)");
158                }
159                else {
160                    sb.append("(.* - ");
161                    for (int i = 0; i < parameters.size(); i++) {
162                        sb.append(parameters.get(i) + "\\[");
163                        sb.append(filterParams.get(parameters.get(i)) + "\\] ");
164                    }
165                    sb.append(".*)");
166                }
167                filterPattern = Pattern.compile(sb.toString());
168            }
169    
170            public static void reset() {
171                parameters.clear();
172            }
173        }
174    
175        private String logFile;
176        private String logPath;
177        private Filter logFilter;
178        private Writer logWriter;
179        private long logRotation;
180    
181        public XLogStreamer(Filter logFilter, Writer logWriter, String logPath, String logFile, long logRotationSecs) {
182            this.logWriter = logWriter;
183            this.logFilter = logFilter;
184            if (logFile == null) {
185                logFile = "oozie-app.log";
186            }
187            this.logFile = logFile;
188            this.logPath = logPath;
189            this.logRotation = logRotationSecs * 1000l;
190        }
191    
192        /**
193         * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
194         * applying the filters.
195         *
196         * @param startTime
197         * @param endTime
198         * @throws IOException
199         */
200        public void streamLog(Date startTime, Date endTime) throws IOException {
201            long startTimeMillis = 0;
202            long endTimeMillis;
203            if (startTime != null) {
204                startTimeMillis = startTime.getTime();
205            }
206            if (endTime == null) {
207                endTimeMillis = System.currentTimeMillis();
208            }
209            else {
210                endTimeMillis = endTime.getTime();
211            }
212            File dir = new File(logPath);
213            ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
214            for (int i = 0; i < fileList.size(); i++) {
215                InputStream ifs;
216                ifs = new FileInputStream(fileList.get(i).getFileName());
217                XLogReader logReader = new XLogReader(ifs, logFilter, logWriter);
218                logReader.processLog();
219            }
220        }
221    
222        /**
223         * File name along with the modified time which will be used to sort later.
224         */
225        class FileInfo implements Comparable<FileInfo> {
226            String fileName;
227            long modTime;
228    
229            public FileInfo(String fileName, long modTime) {
230                this.fileName = fileName;
231                this.modTime = modTime;
232            }
233    
234            public String getFileName() {
235                return fileName;
236            }
237    
238            public long getModTime() {
239                return modTime;
240            }
241    
242            public int compareTo(FileInfo fileInfo) {
243                return (int) (this.modTime - fileInfo.modTime);
244            }
245        }
246    
247        /**
248         * Gets the file list that will have the logs between startTime and endTime.
249         *
250         * @param dir
251         * @param startTime
252         * @param endTime
253         * @param logRotationTime
254         * @param logFile
255         * @return List of files to be streamed
256         */
257        private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime,
258                                                String logFile) {
259            String[] children = dir.list();
260            ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
261            if (children == null) {
262                return fileList;
263            }
264            else {
265                for (int i = 0; i < children.length; i++) {
266                    String filename = children[i];
267                    if (!filename.startsWith(logFile) && !filename.equals(logFile)) {
268                        continue;
269                    }
270                    File file = new File(dir.getAbsolutePath(), filename);
271                    long modTime = file.lastModified();
272                    if (modTime < startTime) {
273                        continue;
274                    }
275                    if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
276                        continue;
277                    }
278                    fileList.add(new FileInfo(file.getAbsolutePath(), modTime));
279                }
280            }
281            Collections.sort(fileList);
282            return fileList;
283        }
284    }