[PATCH] [Wok 0/4] Implement User Request Logger
User request logs are saved by default at log/wok-req.log. The maximum file size is 3 MB and there is rotation with log/wok-req.log.1, which means it will spend at most 6 MB of disk space. The log format is JSON, since the web user interface will provide a log utility to easily search and filter log results. Sample contents of log/wok-req.log: {"date": "2016-02-23", "app": "gingerbase", "req": "DELETE", "user": "lucio", "time": "15:08:02"} >>> Delete host debug report 'test1' {"date": "2016-02-23", "app": "gingerbase", "req": "POST", "user": "lucio", "time": "15:41:07"} >>> Enable host software repository 'rhel7.2' {"date": "2016-02-23", "app": "gingerbase", "req": "POST", "user": "lucio", "time": "15:42:07"} >>> Disable host software repository 'rhel7.2' There will be a download function for search results, which will generate a more log-like text file based on the search results. Request log results can be searched using the following parameters: * app: filter by application that received the request (wok, kimchi, etc.) * user: filter by user that performed the request * req: filter by request type: POST, DELETE, PUT. GET requests are not logged. * date: filter by request date in format YYYY-MM-DD Sample search on user request log using parameters user and app: curl -u lucio -H "Content-Type: application/json" -H "Accept: application/json" "http://localhost:8010/logs?app=wok&user=root" -X GET -d '{}' Lucio Correia (4): Add User Request Logger Log user requests Implement User Request Logger API Fix tests docs/API/logs.md | 21 ++++++++++ src/wok/control/base.py | 80 ++++++++++++++++++++++++++++++++++--- src/wok/control/logs.py | 43 ++++++++++++++++++++ src/wok/model/logs.py | 31 +++++++++++++++ src/wok/reqlogger.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++++ src/wok/server.py | 10 +++++ src/wok/utils.py | 12 ++++++ src/wokd.in | 4 ++ tests/utils.py | 1 + 9 files changed, 299 insertions(+), 6 deletions(-) create mode 100644 docs/API/logs.md create mode 100644 src/wok/control/logs.py create mode 100644 src/wok/model/logs.py create mode 100644 src/wok/reqlogger.py -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/reqlogger.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/wok/reqlogger.py diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py new file mode 100644 index 0000000..642add4 --- /dev/null +++ b/src/wok/reqlogger.py @@ -0,0 +1,103 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# + +import json +import logging +import os.path + +from datetime import datetime + + +MAX_FILE_SIZE = 3072000 +NUM_BACKUP_FILES = 1 +WOK_REQUEST_LOGGER = 'wok_request_logger' + + +class RequestParser(object): + def __init__(self): + logger = logging.getLogger(WOK_REQUEST_LOGGER) + self.baseFile = logger.handlers[0].baseFilename + + def getRecords(self): + records = self.getRecordsFromFile(self.baseFile) + + for count in range(NUM_BACKUP_FILES): + filename = ".".join([self.baseFile, str(count + 1)]) + records.extend(self.getRecordsFromFile(filename)) + + return records + + def getRecordsFromFile(self, filename): + """ + Returns a list of dict, where each dict corresponds to a request + record. + """ + records = [] + + if not os.path.exists(filename): + return [] + + # read records from file + with open(filename) as f: + line = f.readline() + while line != "": + data = line.split(">>>") + if len(data) > 1: + record = json.JSONDecoder().decode(data[0]) + record['message'] = data[1].strip() + records.append(record) + + line = f.readline() + + return records + + def getFilteredRecords(self, filter_params): + """ + Returns a filtered list of dict, where each dict corresponds to a User + Log record. + """ + results = [] + records = self.getRecords() + + for record in records: + if all(key in record and record[key] == val + for key, val in filter_params.iteritems()): + results.append(record) + + return results + + +class RequestRecord(object): + def __init__(self, message, **kwargs): + self.message = message + self.kwargs = kwargs + + # register timestamp + timestamp = datetime.today() + self.kwargs['date'] = timestamp.strftime('%Y-%m-%d') + self.kwargs['time'] = timestamp.strftime('%H:%M:%S') + + def __str__(self): + info = json.JSONEncoder().encode(self.kwargs) + return '%s >>> %s' % (info, self.message) + + def log(self): + reqLogger = logging.getLogger(WOK_REQUEST_LOGGER) + reqLogger.info(self) -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/control/base.py | 80 +++++++++++++++++++++++++++++++++++++++++++++---- src/wok/server.py | 10 +++++++ src/wok/utils.py | 12 ++++++++ src/wokd.in | 4 +++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 363fd60..d08f519 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -32,6 +32,15 @@ from wok.control.utils import validate_params from wok.exception import InvalidOperation, InvalidParameter from wok.exception import MissingParameter, NotFoundError from wok.exception import OperationFailed, UnauthorizedError, WokException +from wok.reqlogger import RequestRecord +from wok.utils import get_plugin_from_request + + +LOG_DISABLED_METHODS = ['GET'] + +# Default request log messages +COLLECTION_DEFAULT_LOG = "request on collection" +RESOURCE_DEFAULT_LOG = "request on resource" class Resource(object): @@ -58,6 +67,7 @@ class Resource(object): self.model_args = (ident,) self.role_key = None self.admin_methods = [] + self.log_map = {} def _redirect(self, action_result, code=303): uri_params = [] @@ -102,7 +112,8 @@ class Resource(object): def _generate_action_handler_base(self, action_name, render_fn, destructive=False, action_args=None): def wrapper(*args, **kwargs): - validate_method(('POST'), self.role_key, self.admin_methods) + method = 'POST' + validate_method((method), self.role_key, self.admin_methods) try: self.lookup() if not self.is_authorized(): @@ -137,6 +148,17 @@ class Resource(object): raise cherrypy.HTTPError(500, e.message) except WokException, e: raise cherrypy.HTTPError(500, e.message) + finally: + params = {} + if model_args: + params = {'ident': str(model_args[0])} + + RequestRecord( + self.getRequestMessage(method, action_name) % params, + app=get_plugin_from_request(), + req=method, + user=cherrypy.session.get(USER_NAME, 'N/A') + ).log() wrapper.__name__ = action_name wrapper.exposed = True @@ -162,6 +184,18 @@ class Resource(object): raise cherrypy.HTTPError(500, e.message) except InvalidOperation, e: raise cherrypy.HTTPError(400, e.message) + finally: + method = 'DELETE' + params = {} + if self.model_args: + params = {'ident': str(self.model_args[0])} + + RequestRecord( + self.getRequestMessage(method, 'default') % params, + app=get_plugin_from_request(), + req=method, + user=cherrypy.session.get(USER_NAME, 'N/A') + ).log() @cherrypy.expose def index(self, *args, **kargs): @@ -203,14 +237,23 @@ class Resource(object): return user_name in users or len(set(user_groups) & set(groups)) > 0 def update(self, *args, **kargs): + params = parse_request() + try: update = getattr(self.model, model_fn(self, 'update')) except AttributeError: e = InvalidOperation('WOKAPI0003E', {'resource': get_class_name(self)}) raise cherrypy.HTTPError(405, e.message) + finally: + method = 'PUT' + RequestRecord( + self.getRequestMessage(method) % params, + app=get_plugin_from_request(), + req=method, + user=cherrypy.session.get(USER_NAME, 'N/A') + ).log() - params = parse_request() validate_params(params, self, 'update') args = list(self.model_args) + [params] @@ -222,6 +265,13 @@ class Resource(object): def get(self): return wok.template.render(get_class_name(self), self.data) + def getRequestMessage(self, method, action='default'): + """ + Provide customized user activity log message in inherited classes + through log_map attribute. + """ + return self.log_map.get(method, {}).get(action, RESOURCE_DEFAULT_LOG) + @property def data(self): """ @@ -273,6 +323,7 @@ class Collection(object): self.model_args = [] self.role_key = None self.admin_methods = [] + self.log_map = {} def create(self, params, *args): try: @@ -341,18 +392,27 @@ class Collection(object): data = self.filter_data(resources, fields_filter) return wok.template.render(get_class_name(self), data) + def getRequestMessage(self, method): + """ + Provide customized user activity log message in inherited classes + through log_map attribute. + """ + return self.log_map.get(method, COLLECTION_DEFAULT_LOG) + @cherrypy.expose def index(self, *args, **kwargs): + params = {} method = validate_method(('GET', 'POST'), self.role_key, self.admin_methods) try: if method == 'GET': - filter_params = cherrypy.request.params - validate_params(filter_params, self, 'get_list') - return self.get(filter_params) + params = cherrypy.request.params + validate_params(params, self, 'get_list') + return self.get(params) elif method == 'POST': - return self.create(parse_request(), *args) + params = parse_request() + return self.create(params, *args) except InvalidOperation, e: raise cherrypy.HTTPError(400, e.message) except InvalidParameter, e: @@ -365,6 +425,14 @@ class Collection(object): raise cherrypy.HTTPError(500, e.message) except WokException, e: raise cherrypy.HTTPError(500, e.message) + finally: + if method not in LOG_DISABLED_METHODS: + RequestRecord( + self.getRequestMessage(method) % params, + app=get_plugin_from_request(), + req=method, + user=cherrypy.session.get(USER_NAME, 'N/A') + ).log() class AsyncCollection(Collection): diff --git a/src/wok/server.py b/src/wok/server.py index 75b41d5..1414f2e 100644 --- a/src/wok/server.py +++ b/src/wok/server.py @@ -36,6 +36,7 @@ from wok.model import model from wok.proxy import start_proxy, terminate_proxy from wok.root import WokRoot from wok.safewatchedfilehandler import SafeWatchedFileHandler +from wok.reqlogger import MAX_FILE_SIZE, NUM_BACKUP_FILES, WOK_REQUEST_LOGGER from wok.utils import get_enabled_plugins, import_class LOGGING_LEVEL = {"debug": logging.DEBUG, @@ -130,6 +131,15 @@ class Server(object): # Add error log file to cherrypy configuration cherrypy.log.error_log.addHandler(h) + # Request logger setup + h = logging.handlers.RotatingFileHandler(options.req_log, 'a', + maxBytes=MAX_FILE_SIZE, + backupCount=NUM_BACKUP_FILES) + h.setFormatter(logging.Formatter('%(message)s')) + reqLogger = logging.getLogger(WOK_REQUEST_LOGGER) + reqLogger.setLevel(logging.INFO) + reqLogger.addHandler(h) + # only add logrotate if wok is installed if paths.installed: diff --git a/src/wok/utils.py b/src/wok/utils.py index e2f1d8e..7b1aa06 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -120,6 +120,18 @@ def get_all_tabs(): return tabs +def get_plugin_from_request(): + """ + Returns name of plugin being requested. If no plugin, returns 'wok'. + """ + script_name = cherrypy.request.script_name + split = script_name.split('/') + if len(split) > 2 and split[1] == 'plugins': + return split[2] + + return 'wok' + + def import_class(class_path): module_name, class_name = class_path.rsplit('.', 1) try: diff --git a/src/wokd.in b/src/wokd.in index 7255d3c..c91f07a 100644 --- a/src/wokd.in +++ b/src/wokd.in @@ -36,6 +36,7 @@ if not config.paths.installed: ACCESS_LOG = "wok-access.log" ERROR_LOG = "wok-error.log" +REQ_LOG = "wok-req.log" def main(options): @@ -76,6 +77,9 @@ def main(options): parser.add_option('--error-log', default=os.path.join(logDir, ERROR_LOG), help="Error log file") + parser.add_option('--req-log', + default=os.path.join(logDir, REQ_LOG), + help="User Request log file") parser.add_option('--environment', default=runningEnv, help="Running environment of wok server") parser.add_option('--test', action='store_true', -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- docs/API/logs.md | 21 +++++++++++++++++++++ src/wok/control/logs.py | 43 +++++++++++++++++++++++++++++++++++++++++++ src/wok/model/logs.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 docs/API/logs.md create mode 100644 src/wok/control/logs.py create mode 100644 src/wok/model/logs.py diff --git a/docs/API/logs.md b/docs/API/logs.md new file mode 100644 index 0000000..4e836e3 --- /dev/null +++ b/docs/API/logs.md @@ -0,0 +1,21 @@ +## REST API Specification for Logs + +### Collection: Logs + +**URI:** /logs + +**Methods:** + +* **GET**: Retrieve a list of entries from User Request Log + * Parameters: + * app: Filter entries by application that received the request. + Use "wok" or any plugin installed, like "kimchi". + * req: Filter entries by type of request: "DELETE", "POST", "PUT". + "GET" requests are not logged. + * user: Filter entries by user that performed the request. + * date: Filter entries by date of record in the format "YYYY-MM-DD" + +#### Examples +GET /logs +[{entry-record1}, {entry-record2}, {entry-record3}, ...] + diff --git a/src/wok/control/logs.py b/src/wok/control/logs.py new file mode 100644 index 0000000..0eb04d7 --- /dev/null +++ b/src/wok/control/logs.py @@ -0,0 +1,43 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import wok.template + +from wok.control.base import SimpleCollection +from wok.control.utils import get_class_name, model_fn +from wok.control.utils import UrlSubNode + + +@UrlSubNode("logs") +class Logs(SimpleCollection): + def __init__(self, model): + super(Logs, self).__init__(model) + self.role_key = 'logs' + self.admin_methods = ['GET'] + + def get(self, filter_params): + res_list = [] + + try: + get_list = getattr(self.model, model_fn(self, 'get_list')) + res_list = get_list(filter_params) + except AttributeError: + pass + + return wok.template.render(get_class_name(self), res_list) diff --git a/src/wok/model/logs.py b/src/wok/model/logs.py new file mode 100644 index 0000000..09d3f47 --- /dev/null +++ b/src/wok/model/logs.py @@ -0,0 +1,33 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import cherrypy + +from wok.reqlogger import RequestParser + + +class LogsModel(object): + def __init__(self, **kargs): + pass + + def get_list(self, filter_params): + if filter_params: + return RequestParser().getFilteredRecords(filter_params) + + return RequestParser().getRecords() -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/model/logs.py | 2 -- tests/utils.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/wok/model/logs.py b/src/wok/model/logs.py index 09d3f47..89a79ab 100644 --- a/src/wok/model/logs.py +++ b/src/wok/model/logs.py @@ -17,8 +17,6 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -import cherrypy - from wok.reqlogger import RequestParser diff --git a/tests/utils.py b/tests/utils.py index d158ba1..249205d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -119,6 +119,7 @@ def run_server(host, port, ssl_port, test_mode, cherrypy_port=None, 'websockets_port': 64667, 'ssl_cert': '', 'ssl_key': '', 'max_body_size': '4*1024', 'test': test_mode, 'access_log': '/dev/null', 'error_log': '/dev/null', + 'req_log': '/dev/null', 'environment': environment, 'log_level': 'debug'})() if model is not None: setattr(args, 'model', model) -- 1.9.1
participants (1)
-
Lucio Correia