001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.servlet;
016    
017    import org.apache.oozie.client.OozieClient.SYSTEM_MODE;
018    import org.apache.oozie.client.rest.JsonBean;
019    import org.apache.oozie.client.rest.RestConstants;
020    import org.apache.oozie.service.DagXLogInfoService;
021    import org.apache.oozie.service.InstrumentationService;
022    import org.apache.oozie.service.Services;
023    import org.apache.oozie.service.XLogService;
024    import org.apache.oozie.util.Instrumentation;
025    import org.apache.oozie.util.ParamChecker;
026    import org.apache.oozie.util.XLog;
027    import org.apache.oozie.ErrorCode;
028    import org.json.simple.JSONObject;
029    import org.json.simple.JSONStreamAware;
030    
031    import javax.servlet.ServletConfig;
032    import javax.servlet.ServletException;
033    import javax.servlet.http.HttpServlet;
034    import javax.servlet.http.HttpServletRequest;
035    import javax.servlet.http.HttpServletResponse;
036    import java.io.IOException;
037    import java.util.ArrayList;
038    import java.util.Arrays;
039    import java.util.HashMap;
040    import java.util.List;
041    import java.util.Map;
042    import java.util.concurrent.atomic.AtomicLong;
043    
044    /**
045     * Base class for Oozie web service API Servlets. <p/> This class provides common instrumentation, error logging and
046     * other common functionality.
047     */
048    public abstract class JsonRestServlet extends HttpServlet {
049    
050        private static final String JSTON_UTF8 = RestConstants.JSON_CONTENT_TYPE + "; charset=\"UTF-8\"";
051    
052        protected static final String XML_UTF8 = RestConstants.XML_CONTENT_TYPE + "; charset=\"UTF-8\"";
053    
054        protected static final String TEXT_UTF8 = RestConstants.TEXT_CONTENT_TYPE + "; charset=\"UTF-8\"";
055    
056        protected static final String AUDIT_OPERATION = "audit.operation";
057        protected static final String AUDIT_PARAM = "audit.param";
058        protected static final String AUDIT_ERROR_CODE = "audit.error.code";
059        protected static final String AUDIT_ERROR_MESSAGE = "audit.error.message";
060        protected static final String AUDIT_HTTP_STATUS_CODE = "audit.http.status.code";
061    
062        private XLog auditLog;
063        XLog.Info logInfo;
064    
065        /**
066         * This bean defines a query string parameter.
067         */
068        public static class ParameterInfo {
069            private String name;
070            private Class type;
071            private List<String> methods;
072            private boolean required;
073    
074            /**
075             * Creates a ParameterInfo with querystring parameter definition.
076             *
077             * @param name querystring parameter name.
078             * @param type type for the parameter value, valid types are: <code>Integer, Boolean and String</code>
079             * @param required indicates if the parameter is required.
080             * @param methods HTTP methods the parameter is used by.
081             */
082            public ParameterInfo(String name, Class type, boolean required, List<String> methods) {
083                this.name = ParamChecker.notEmpty(name, "name");
084                if (type != Integer.class && type != Boolean.class && type != String.class) {
085                    throw new IllegalArgumentException("Type must be integer, boolean or string");
086                }
087                this.type = ParamChecker.notNull(type, "type");
088                this.required = required;
089                this.methods = ParamChecker.notNull(methods, "methods");
090            }
091    
092        }
093    
094        /**
095         * This bean defines a REST resource.
096         */
097        public static class ResourceInfo {
098            private String name;
099            private boolean wildcard;
100            private List<String> methods;
101            private Map<String, ParameterInfo> parameters = new HashMap<String, ParameterInfo>();
102    
103            /**
104             * Creates a ResourceInfo with a REST resource definition.
105             *
106             * @param name name of the REST resource, it can be an fixed resource name, empty or a wildcard ('*').
107             * @param methods HTTP methods supported by the resource.
108             * @param parameters parameters supported by the resource.
109             */
110            public ResourceInfo(String name, List<String> methods, List<ParameterInfo> parameters) {
111                this.name = name;
112                wildcard = name.equals("*");
113                for (ParameterInfo parameter : parameters) {
114                    this.parameters.put(parameter.name, parameter);
115                }
116                this.methods = ParamChecker.notNull(methods, "methods");
117            }
118        }
119    
120        /**
121         * Name of the instrumentation group for the WS layer, value is 'webservices'.
122         */
123        protected static final String INSTRUMENTATION_GROUP = "webservices";
124    
125        private static final String INSTR_TOTAL_REQUESTS_SAMPLER = "requests";
126        private static final String INSTR_TOTAL_REQUESTS_COUNTER = "requests";
127        private static final String INSTR_TOTAL_FAILED_REQUESTS_COUNTER = "failed";
128        private static AtomicLong TOTAL_REQUESTS_SAMPLER_COUNTER;
129    
130        private Instrumentation instrumentation;
131        private String instrumentationName;
132        private AtomicLong samplerCounter = new AtomicLong();
133        private ThreadLocal<Instrumentation.Cron> requestCron = new ThreadLocal<Instrumentation.Cron>();
134        private List<ResourceInfo> resourcesInfo = new ArrayList<ResourceInfo>();
135        private boolean allowSafeModeChanges;
136    
137        /**
138         * Creates a servlet with a specified instrumentation sampler name for its requests.
139         *
140         * @param instrumentationName instrumentation name for timer and samplers for the servlet.
141         * @param resourcesInfo list of resource definitions supported by the servlet, empty and wildcard resources must be
142         * the last ones, in that order, first empty and the wildcard.
143         */
144        public JsonRestServlet(String instrumentationName, ResourceInfo... resourcesInfo) {
145            this.instrumentationName = ParamChecker.notEmpty(instrumentationName, "instrumentationName");
146            if (resourcesInfo.length == 0) {
147                throw new IllegalArgumentException("There must be at least one ResourceInfo");
148            }
149            this.resourcesInfo = Arrays.asList(resourcesInfo);
150            auditLog = XLog.getLog("oozieaudit");
151            auditLog.setMsgPrefix("");
152            logInfo = new XLog.Info(XLog.Info.get());
153        }
154    
155        /**
156         * Enable HTTP POST/PUT/DELETE methods while in safe mode.
157         *
158         * @param allow <code>true</code> enabled safe mode changes, <code>false</code> disable safe mode changes
159         * (default).
160         */
161        protected void setAllowSafeModeChanges(boolean allow) {
162            allowSafeModeChanges = allow;
163        }
164    
165        /**
166         * Define an instrumentation sampler. <p/> Sampling period is 60 seconds, the sampling frequency is 1 second. <p/>
167         * The instrumentation group used is {@link #INSTRUMENTATION_GROUP}.
168         *
169         * @param samplerName sampler name.
170         * @param samplerCounter sampler counter.
171         */
172        private void defineSampler(String samplerName, final AtomicLong samplerCounter) {
173            instrumentation.addSampler(INSTRUMENTATION_GROUP, samplerName, 60, 1, new Instrumentation.Variable<Long>() {
174                public Long getValue() {
175                    return samplerCounter.get();
176                }
177            });
178        }
179    
180        /**
181         * Add an instrumentation cron.
182         *
183         * @param name name of the timer for the cron.
184         * @param cron cron to add to a instrumentation timer.
185         */
186        private void addCron(String name, Instrumentation.Cron cron) {
187            instrumentation.addCron(INSTRUMENTATION_GROUP, name, cron);
188        }
189    
190        /**
191         * Start the request instrumentation cron.
192         */
193        protected void startCron() {
194            requestCron.get().start();
195        }
196    
197        /**
198         * Stop the request instrumentation cron.
199         */
200        protected void stopCron() {
201            requestCron.get().stop();
202        }
203    
204        /**
205         * Initializes total request and servlet request samplers.
206         */
207        public void init(ServletConfig servletConfig) throws ServletException {
208            super.init(servletConfig);
209            instrumentation = Services.get().get(InstrumentationService.class).get();
210            synchronized (JsonRestServlet.class) {
211                if (TOTAL_REQUESTS_SAMPLER_COUNTER == null) {
212                    TOTAL_REQUESTS_SAMPLER_COUNTER = new AtomicLong();
213                    defineSampler(INSTR_TOTAL_REQUESTS_SAMPLER, TOTAL_REQUESTS_SAMPLER_COUNTER);
214                }
215            }
216            defineSampler(instrumentationName, samplerCounter);
217        }
218    
219        /**
220         * Convenience method for instrumentation counters.
221         *
222         * @param name counter name.
223         * @param count count to increment the counter.
224         */
225        private void incrCounter(String name, int count) {
226            if (instrumentation != null) {
227                instrumentation.incr(INSTRUMENTATION_GROUP, name, count);
228            }
229        }
230    
231        /**
232         * Logs audit information for write requests to the audit log.
233         *
234         * @param request the http request.
235         */
236        private void logAuditInfo(HttpServletRequest request) {
237            if (request.getAttribute(AUDIT_OPERATION) != null) {
238                Integer httpStatusCode = (Integer) request.getAttribute(AUDIT_HTTP_STATUS_CODE);
239                httpStatusCode = (httpStatusCode != null) ? httpStatusCode : HttpServletResponse.SC_OK;
240                String status = (httpStatusCode == HttpServletResponse.SC_OK) ? "SUCCESS" : "FAILED";
241                String operation = (String) request.getAttribute(AUDIT_OPERATION);
242                String param = (String) request.getAttribute(AUDIT_PARAM);
243                String user = XLog.Info.get().getParameter(XLogService.USER);
244                String group = XLog.Info.get().getParameter(XLogService.GROUP);
245                String jobId = XLog.Info.get().getParameter(DagXLogInfoService.JOB);
246                String app = XLog.Info.get().getParameter(DagXLogInfoService.APP);
247    
248                String errorCode = (String) request.getAttribute(AUDIT_ERROR_CODE);
249                String errorMessage = (String) request.getAttribute(AUDIT_ERROR_MESSAGE);
250    
251                auditLog
252                        .info(
253                                "USER [{0}], GROUP [{1}], APP [{2}], JOBID [{3}], OPERATION [{4}], PARAMETER [{5}], STATUS [{6}], HTTPCODE [{7}], ERRORCODE [{8}], ERRORMESSAGE [{9}]",
254                                user, group, app, jobId, operation, param, status, httpStatusCode, errorCode, errorMessage);
255            }
256        }
257    
258        /**
259         * Dispatches to super after loginfo and intrumentation handling. In case of errors dispatches error response codes
260         * and does error logging.
261         */
262        @SuppressWarnings("unchecked")
263        protected final void service(HttpServletRequest request, HttpServletResponse response) throws ServletException,
264                IOException {
265            //if (Services.get().isSafeMode() && !request.getMethod().equals("GET") && !allowSafeModeChanges) {
266            if (Services.get().getSystemMode() != SYSTEM_MODE.NORMAL && !request.getMethod().equals("GET") && !allowSafeModeChanges) {
267                sendErrorResponse(response, HttpServletResponse.SC_SERVICE_UNAVAILABLE, ErrorCode.E0002.toString(),
268                                  ErrorCode.E0002.getTemplate());
269                return;
270            }
271            Instrumentation.Cron cron = new Instrumentation.Cron();
272            requestCron.set(cron);
273            try {
274                cron.start();
275                validateRestUrl(request.getMethod(), getResourceName(request), request.getParameterMap());
276                XLog.Info.get().clear();
277                String user = getUser(request);
278                XLog.Info.get().setParameter(XLogService.USER, user);
279                TOTAL_REQUESTS_SAMPLER_COUNTER.incrementAndGet();
280                samplerCounter.incrementAndGet();
281                super.service(request, response);
282            }
283            catch (XServletException ex) {
284                XLog log = XLog.getLog(getClass());
285                log.warn("URL[{0} {1}] error[{2}], {3}", request.getMethod(), getRequestUrl(request), ex.getErrorCode(), ex
286                        .getMessage(), ex);
287                request.setAttribute(AUDIT_ERROR_MESSAGE, ex.getMessage());
288                request.setAttribute(AUDIT_ERROR_CODE, ex.getErrorCode().toString());
289                request.setAttribute(AUDIT_HTTP_STATUS_CODE, ex.getHttpStatusCode());
290                incrCounter(INSTR_TOTAL_FAILED_REQUESTS_COUNTER, 1);
291                sendErrorResponse(response, ex.getHttpStatusCode(), ex.getErrorCode().toString(), ex.getMessage());
292            }
293            catch (RuntimeException ex) {
294                XLog log = XLog.getLog(getClass());
295                log.error("URL[{0} {1}] error, {2}", request.getMethod(), getRequestUrl(request), ex.getMessage(), ex);
296                incrCounter(INSTR_TOTAL_FAILED_REQUESTS_COUNTER, 1);
297                throw ex;
298            }
299            finally {
300                logAuditInfo(request);
301                TOTAL_REQUESTS_SAMPLER_COUNTER.decrementAndGet();
302                incrCounter(INSTR_TOTAL_REQUESTS_COUNTER, 1);
303                samplerCounter.decrementAndGet();
304                XLog.Info.remove();
305                cron.stop();
306                // TODO
307                incrCounter(instrumentationName, 1);
308                incrCounter(instrumentationName + "-" + request.getMethod(), 1);
309                addCron(instrumentationName, cron);
310                addCron(instrumentationName + "-" + request.getMethod(), cron);
311                requestCron.remove();
312            }
313        }
314    
315        private String getRequestUrl(HttpServletRequest request) {
316            StringBuffer url = request.getRequestURL();
317            if (request.getQueryString() != null) {
318                url.append("?").append(request.getQueryString());
319            }
320            return url.toString();
321        }
322    
323        /**
324         * Sends a JSON response.
325         *
326         * @param response servlet response.
327         * @param statusCode HTTP status code.
328         * @param bean bean to send as JSON response.
329         * @throws java.io.IOException thrown if the bean could not be serialized to the response output stream.
330         */
331        protected void sendJsonResponse(HttpServletResponse response, int statusCode, JsonBean bean) throws IOException {
332            response.setStatus(statusCode);
333            JSONObject json = bean.toJSONObject();
334            response.setContentType(JSTON_UTF8);
335            json.writeJSONString(response.getWriter());
336        }
337    
338        /**
339         * Sends a error response.
340         *
341         * @param response servlet response.
342         * @param statusCode HTTP status code.
343         * @param error error code.
344         * @param message error message.
345         * @throws java.io.IOException thrown if the error response could not be set.
346         */
347        protected void sendErrorResponse(HttpServletResponse response, int statusCode, String error, String message)
348                throws IOException {
349            response.setHeader(RestConstants.OOZIE_ERROR_CODE, error);
350            response.setHeader(RestConstants.OOZIE_ERROR_MESSAGE, message);
351            response.sendError(statusCode);
352        }
353    
354        protected void sendJsonResponse(HttpServletResponse response, int statusCode, JSONStreamAware json)
355                throws IOException {
356            if (statusCode == HttpServletResponse.SC_OK || statusCode == HttpServletResponse.SC_CREATED) {
357                response.setStatus(statusCode);
358            }
359            else {
360                response.sendError(statusCode);
361            }
362            response.setStatus(statusCode);
363            response.setContentType(JSTON_UTF8);
364            json.writeJSONString(response.getWriter());
365        }
366    
367        /**
368         * Validates REST URL using the ResourceInfos of the servlet.
369         *
370         * @param method HTTP method.
371         * @param resourceName resource name.
372         * @param queryStringParams query string parameters.
373         * @throws javax.servlet.ServletException thrown if the resource name or parameters are incorrect.
374         */
375        @SuppressWarnings("unchecked")
376        protected void validateRestUrl(String method, String resourceName, Map<String, String[]> queryStringParams)
377                throws ServletException {
378    
379            if (resourceName.contains("/")) {
380                throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0301, resourceName);
381            }
382    
383            boolean valid = false;
384            for (int i = 0; !valid && i < resourcesInfo.size(); i++) {
385                ResourceInfo resourceInfo = resourcesInfo.get(i);
386                if (resourceInfo.name.equals(resourceName) || resourceInfo.wildcard) {
387                    if (!resourceInfo.methods.contains(method)) {
388                        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0301, resourceName);
389                    }
390                    for (Map.Entry<String, String[]> entry : queryStringParams.entrySet()) {
391                        String name = entry.getKey();
392                        ParameterInfo parameterInfo = resourceInfo.parameters.get(name);
393                        if (parameterInfo != null) {
394                            if (!parameterInfo.methods.contains(method)) {
395                                throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, name);
396                            }
397                            String value = entry.getValue()[0].trim();
398                            if (parameterInfo.type.equals(Boolean.class)) {
399                                value = value.toLowerCase();
400                                if (!value.equals("true") && !value.equals("false")) {
401                                    throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0304, name,
402                                                                "boolean");
403                                }
404                            }
405                            if (parameterInfo.type.equals(Integer.class)) {
406                                try {
407                                    Integer.parseInt(value);
408                                }
409                                catch (NumberFormatException ex) {
410                                    throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0304, name,
411                                                                "integer");
412                                }
413                            }
414                        }
415                    }
416                    for (ParameterInfo parameterInfo : resourceInfo.parameters.values()) {
417                        if (parameterInfo.methods.contains(method) && parameterInfo.required
418                                && queryStringParams.get(parameterInfo.name) == null) {
419                            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0305,
420                                                        parameterInfo.name);
421                        }
422                    }
423                    valid = true;
424                }
425            }
426            if (!valid) {
427                throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0301, resourceName);
428            }
429        }
430    
431        /**
432         * Return the resource name of the request. <p/> The resource name is the whole extra path. If the extra path starts
433         * with '/', the first '/' is trimmed.
434         *
435         * @param request request instance
436         * @return the resource name, <code>null</code> if none.
437         */
438        protected String getResourceName(HttpServletRequest request) {
439            String requestPath = request.getPathInfo();
440            if (requestPath != null) {
441                while (requestPath.startsWith("/")) {
442                    requestPath = requestPath.substring(1);
443                }
444                requestPath = requestPath.trim();
445            }
446            else {
447                requestPath = "";
448            }
449            return requestPath;
450        }
451    
452        /**
453         * Return the request content type, lowercase and without attributes.
454         *
455         * @param request servlet request.
456         * @return the request content type, <code>null</code> if none.
457         */
458        protected String getContentType(HttpServletRequest request) {
459            String contentType = request.getContentType();
460            if (contentType != null) {
461                int index = contentType.indexOf(";");
462                if (index > -1) {
463                    contentType = contentType.substring(0, index);
464                }
465                contentType = contentType.toLowerCase();
466            }
467            return contentType;
468        }
469    
470        /**
471         * Validate and return the content type of the request.
472         *
473         * @param request servlet request.
474         * @param expected expected contentType.
475         * @return the normalized content type (lowercase and without modifiers).
476         * @throws XServletException thrown if the content type is invalid.
477         */
478        protected String validateContentType(HttpServletRequest request, String expected) throws XServletException {
479            String contentType = getContentType(request);
480            if (contentType == null || contentType.trim().length() == 0) {
481                throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0300, contentType);
482            }
483            if (!contentType.equals(expected)) {
484                throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0300, contentType);
485            }
486            return contentType;
487        }
488    
489        /**
490         * Request attribute constant for the authenticatio token.
491         */
492        public static final String AUTH_TOKEN = "oozie.auth.token";
493    
494        /**
495         * Request attribute constant for the user name.
496         */
497        public static final String USER_NAME = "oozie.user.name";
498    
499        protected static final String UNDEF = "?";
500    
501        /**
502         * Return the authentication token of the request if any.
503         *
504         * @param request request.
505         * @return the authentication token, <code>null</code> if there is none.
506         */
507        protected String getAuthToken(HttpServletRequest request) {
508            String authToken = (String) request.getAttribute(AUTH_TOKEN);
509            return (authToken != null) ? authToken : UNDEF;
510        }
511    
512        /**
513         * Return the user name of the request if any.
514         *
515         * @param request request.
516         * @return the user name, <code>null</code> if there is none.
517         */
518        protected String getUser(HttpServletRequest request) {
519            String userName = (String) request.getAttribute(USER_NAME);
520            return (userName != null) ? userName : UNDEF;
521        }
522    
523        /**
524         * Set the log info with the given information.
525         *
526         * @param jobid job ID.
527         * @param actionid action ID.
528         */
529        protected void setLogInfo(String jobid, String actionid) {
530            logInfo.setParameter(DagXLogInfoService.JOB, jobid);
531            logInfo.setParameter(DagXLogInfoService.ACTION, actionid);
532    
533            XLog.Info.get().setParameters(logInfo);
534        }
535    }