9:43 p.m.
User request logs are saved by default at log/wok-req.log. The maximum file
size is 3 MB and there is rotation with log/wok-req.log.1, which means it will
spend at most 6 MB of disk space.
The log format is JSON, since the web user interface will provide a log utility
to easily search and filter log results. Sample contents of log/wok-req.log:
{"date": "2016-02-23", "app": "gingerbase",
"req": "DELETE", "user": "lucio",
"time": "15:08:02"} >>> Delete host debug report
'test1'
{"date": "2016-02-23", "app": "gingerbase",
"req": "POST", "user": "lucio", "time":
"15:41:07"} >>> Enable host software repository 'rhel7.2'
{"date": "2016-02-23", "app": "gingerbase",
"req": "POST", "user": "lucio", "time":
"15:42:07"} >>> Disable host software repository 'rhel7.2'
There will be a download function for search results, which will generate a
more log-like text file based on the search results.
Request log results can be searched using the following parameters:
* app: filter by application that received the request (wok, kimchi, etc.)
* user: filter by user that performed the request
* req: filter by request type: POST, DELETE, PUT. GET requests are not logged.
* date: filter by request date in format YYYY-MM-DD
Sample search on user request log using parameters user and app:
curl -u lucio -H "Content-Type: application/json" -H "Accept:
application/json" "http://localhost:8010/logs?app=wok&user=root" -X GET
-d '{}'
Lucio Correia (4):
Add User Request Logger
Log user requests
Implement User Request Logger API
Fix tests
docs/API/logs.md | 21 ++++++++++
src/wok/control/base.py | 80 ++++++++++++++++++++++++++++++++++---
src/wok/control/logs.py | 43 ++++++++++++++++++++
src/wok/model/logs.py | 31 +++++++++++++++
src/wok/reqlogger.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++++
src/wok/server.py | 10 +++++
src/wok/utils.py | 12 ++++++
src/wokd.in | 4 ++
tests/utils.py | 1 +
9 files changed, 299 insertions(+), 6 deletions(-)
create mode 100644 docs/API/logs.md
create mode 100644 src/wok/control/logs.py
create mode 100644 src/wok/model/logs.py
create mode 100644 src/wok/reqlogger.py
--
1.9.1
9:43 p.m.
New subject: [PATCH] [Wok 1/4] Add User Request Logger
Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com>
---
src/wok/reqlogger.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 103 insertions(+)
create mode 100644 src/wok/reqlogger.py
diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py
new file mode 100644
index 0000000..642add4
--- /dev/null
+++ b/src/wok/reqlogger.py
@@ -0,0 +1,103 @@
+#
+# Project Wok
+#
+# Copyright IBM Corp, 2016
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import json
+import logging
+import os.path
+
+from datetime import datetime
+
+
+MAX_FILE_SIZE = 3072000
+NUM_BACKUP_FILES = 1
+WOK_REQUEST_LOGGER = 'wok_request_logger'
+
+
+class RequestParser(object):
+ def __init__(self):
+ logger = logging.getLogger(WOK_REQUEST_LOGGER)
+ self.baseFile = logger.handlers[0].baseFilename
+
+ def getRecords(self):
+ records = self.getRecordsFromFile(self.baseFile)
+
+ for count in range(NUM_BACKUP_FILES):
+ filename = ".".join([self.baseFile, str(count + 1)])
+ records.extend(self.getRecordsFromFile(filename))
+
+ return records
+
+ def getRecordsFromFile(self, filename):
+ """
+ Returns a list of dict, where each dict corresponds to a request
+ record.
+ """
+ records = []
+
+ if not os.path.exists(filename):
+ return []
+
+ # read records from file
+ with open(filename) as f:
+ line = f.readline()
+ while line != "":
+ data = line.split(">>>")
+ if len(data) > 1:
+ record = json.JSONDecoder().decode(data[0])
+ record['message'] = data[1].strip()
+ records.append(record)
+
+ line = f.readline()
+
+ return records
+
+ def getFilteredRecords(self, filter_params):
+ """
+ Returns a filtered list of dict, where each dict corresponds to a User
+ Log record.
+ """
+ results = []
+ records = self.getRecords()
+
+ for record in records:
+ if all(key in record and record[key] == val
+ for key, val in filter_params.iteritems()):
+ results.append(record)
+
+ return results
+
+
+class RequestRecord(object):
+ def __init__(self, message, **kwargs):
+ self.message = message
+ self.kwargs = kwargs
+
+ # register timestamp
+ timestamp = datetime.today()
+ self.kwargs['date'] = timestamp.strftime('%Y-%m-%d')
+ self.kwargs['time'] = timestamp.strftime('%H:%M:%S')
+
+ def __str__(self):
+ info = json.JSONEncoder().encode(self.kwargs)
+ return '%s >>> %s' % (info, self.message)
+
+ def log(self):
+ reqLogger = logging.getLogger(WOK_REQUEST_LOGGER)
+ reqLogger.info(self)
--
1.9.1
9:43 p.m.
New subject: [PATCH] [Wok 2/4] Log user requests
Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com>
---
src/wok/control/base.py | 80 +++++++++++++++++++++++++++++++++++++++++++++----
src/wok/server.py | 10 +++++++
src/wok/utils.py | 12 ++++++++
src/wokd.in | 4 +++
4 files changed, 100 insertions(+), 6 deletions(-)
diff --git a/src/wok/control/base.py b/src/wok/control/base.py
index 363fd60..d08f519 100644
--- a/src/wok/control/base.py
+++ b/src/wok/control/base.py
@@ -32,6 +32,15 @@ from wok.control.utils import validate_params
from wok.exception import InvalidOperation, InvalidParameter
from wok.exception import MissingParameter, NotFoundError
from wok.exception import OperationFailed, UnauthorizedError, WokException
+from wok.reqlogger import RequestRecord
+from wok.utils import get_plugin_from_request
+
+
+LOG_DISABLED_METHODS = ['GET']
+
+# Default request log messages
+COLLECTION_DEFAULT_LOG = "request on collection"
+RESOURCE_DEFAULT_LOG = "request on resource"
class Resource(object):
@@ -58,6 +67,7 @@ class Resource(object):
self.model_args = (ident,)
self.role_key = None
self.admin_methods = []
+ self.log_map = {}
def _redirect(self, action_result, code=303):
uri_params = []
@@ -102,7 +112,8 @@ class Resource(object):
def _generate_action_handler_base(self, action_name, render_fn,
destructive=False, action_args=None):
def wrapper(*args, **kwargs):
- validate_method(('POST'), self.role_key, self.admin_methods)
+ method = 'POST'
+ validate_method((method), self.role_key, self.admin_methods)
try:
self.lookup()
if not self.is_authorized():
@@ -137,6 +148,17 @@ class Resource(object):
raise cherrypy.HTTPError(500, e.message)
except WokException, e:
raise cherrypy.HTTPError(500, e.message)
+ finally:
+ params = {}
+ if model_args:
+ params = {'ident': str(model_args[0])}
+
+ RequestRecord(
+ self.getRequestMessage(method, action_name) % params,
+ app=get_plugin_from_request(),
+ req=method,
+ user=cherrypy.session.get(USER_NAME, 'N/A')
+ ).log()
wrapper.__name__ = action_name
wrapper.exposed = True
@@ -162,6 +184,18 @@ class Resource(object):
raise cherrypy.HTTPError(500, e.message)
except InvalidOperation, e:
raise cherrypy.HTTPError(400, e.message)
+ finally:
+ method = 'DELETE'
+ params = {}
+ if self.model_args:
+ params = {'ident': str(self.model_args[0])}
+
+ RequestRecord(
+ self.getRequestMessage(method, 'default') % params,
+ app=get_plugin_from_request(),
+ req=method,
+ user=cherrypy.session.get(USER_NAME, 'N/A')
+ ).log()
@cherrypy.expose
def index(self, *args, **kargs):
@@ -203,14 +237,23 @@ class Resource(object):
return user_name in users or len(set(user_groups) & set(groups)) > 0
def update(self, *args, **kargs):
+ params = parse_request()
+
try:
update = getattr(self.model, model_fn(self, 'update'))
except AttributeError:
e = InvalidOperation('WOKAPI0003E', {'resource':
get_class_name(self)})
raise cherrypy.HTTPError(405, e.message)
+ finally:
+ method = 'PUT'
+ RequestRecord(
+ self.getRequestMessage(method) % params,
+ app=get_plugin_from_request(),
+ req=method,
+ user=cherrypy.session.get(USER_NAME, 'N/A')
+ ).log()
- params = parse_request()
validate_params(params, self, 'update')
args = list(self.model_args) + [params]
@@ -222,6 +265,13 @@ class Resource(object):
def get(self):
return wok.template.render(get_class_name(self), self.data)
+ def getRequestMessage(self, method, action='default'):
+ """
+ Provide customized user activity log message in inherited classes
+ through log_map attribute.
+ """
+ return self.log_map.get(method, {}).get(action, RESOURCE_DEFAULT_LOG)
+
@property
def data(self):
"""
@@ -273,6 +323,7 @@ class Collection(object):
self.model_args = []
self.role_key = None
self.admin_methods = []
+ self.log_map = {}
def create(self, params, *args):
try:
@@ -341,18 +392,27 @@ class Collection(object):
data = self.filter_data(resources, fields_filter)
return wok.template.render(get_class_name(self), data)
+ def getRequestMessage(self, method):
+ """
+ Provide customized user activity log message in inherited classes
+ through log_map attribute.
+ """
+ return self.log_map.get(method, COLLECTION_DEFAULT_LOG)
+
@cherrypy.expose
def index(self, *args, **kwargs):
+ params = {}
method = validate_method(('GET', 'POST'),
self.role_key, self.admin_methods)
try:
if method == 'GET':
- filter_params = cherrypy.request.params
- validate_params(filter_params, self, 'get_list')
- return self.get(filter_params)
+ params = cherrypy.request.params
+ validate_params(params, self, 'get_list')
+ return self.get(params)
elif method == 'POST':
- return self.create(parse_request(), *args)
+ params = parse_request()
+ return self.create(params, *args)
except InvalidOperation, e:
raise cherrypy.HTTPError(400, e.message)
except InvalidParameter, e:
@@ -365,6 +425,14 @@ class Collection(object):
raise cherrypy.HTTPError(500, e.message)
except WokException, e:
raise cherrypy.HTTPError(500, e.message)
+ finally:
+ if method not in LOG_DISABLED_METHODS:
+ RequestRecord(
+ self.getRequestMessage(method) % params,
+ app=get_plugin_from_request(),
+ req=method,
+ user=cherrypy.session.get(USER_NAME, 'N/A')
+ ).log()
class AsyncCollection(Collection):
diff --git a/src/wok/server.py b/src/wok/server.py
index 75b41d5..1414f2e 100644
--- a/src/wok/server.py
+++ b/src/wok/server.py
@@ -36,6 +36,7 @@ from wok.model import model
from wok.proxy import start_proxy, terminate_proxy
from wok.root import WokRoot
from wok.safewatchedfilehandler import SafeWatchedFileHandler
+from wok.reqlogger import MAX_FILE_SIZE, NUM_BACKUP_FILES, WOK_REQUEST_LOGGER
from wok.utils import get_enabled_plugins, import_class
LOGGING_LEVEL = {"debug": logging.DEBUG,
@@ -130,6 +131,15 @@ class Server(object):
# Add error log file to cherrypy configuration
cherrypy.log.error_log.addHandler(h)
+ # Request logger setup
+ h = logging.handlers.RotatingFileHandler(options.req_log, 'a',
+ maxBytes=MAX_FILE_SIZE,
+ backupCount=NUM_BACKUP_FILES)
+ h.setFormatter(logging.Formatter('%(message)s'))
+ reqLogger = logging.getLogger(WOK_REQUEST_LOGGER)
+ reqLogger.setLevel(logging.INFO)
+ reqLogger.addHandler(h)
+
# only add logrotate if wok is installed
if paths.installed:
diff --git a/src/wok/utils.py b/src/wok/utils.py
index e2f1d8e..7b1aa06 100644
--- a/src/wok/utils.py
+++ b/src/wok/utils.py
@@ -120,6 +120,18 @@ def get_all_tabs():
return tabs
+def get_plugin_from_request():
+ """
+ Returns name of plugin being requested. If no plugin, returns 'wok'.
+ """
+ script_name = cherrypy.request.script_name
+ split = script_name.split('/')
+ if len(split) > 2 and split[1] == 'plugins':
+ return split[2]
+
+ return 'wok'
+
+
def import_class(class_path):
module_name, class_name = class_path.rsplit('.', 1)
try:
diff --git a/src/wokd.in b/src/wokd.in
index 7255d3c..c91f07a 100644
--- a/src/wokd.in
+++ b/src/wokd.in
@@ -36,6 +36,7 @@ if not config.paths.installed:
ACCESS_LOG = "wok-access.log"
ERROR_LOG = "wok-error.log"
+REQ_LOG = "wok-req.log"
def main(options):
@@ -76,6 +77,9 @@ def main(options):
parser.add_option('--error-log',
default=os.path.join(logDir, ERROR_LOG),
help="Error log file")
+ parser.add_option('--req-log',
+ default=os.path.join(logDir, REQ_LOG),
+ help="User Request log file")
parser.add_option('--environment', default=runningEnv,
help="Running environment of wok server")
parser.add_option('--test', action='store_true',
--
1.9.1
9:43 p.m.
New subject: [PATCH] [Wok 3/4] Implement User Request Logger API
Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com>
---
docs/API/logs.md | 21 +++++++++++++++++++++
src/wok/control/logs.py | 43 +++++++++++++++++++++++++++++++++++++++++++
src/wok/model/logs.py | 33 +++++++++++++++++++++++++++++++++
3 files changed, 97 insertions(+)
create mode 100644 docs/API/logs.md
create mode 100644 src/wok/control/logs.py
create mode 100644 src/wok/model/logs.py
diff --git a/docs/API/logs.md b/docs/API/logs.md
new file mode 100644
index 0000000..4e836e3
--- /dev/null
+++ b/docs/API/logs.md
@@ -0,0 +1,21 @@
+## REST API Specification for Logs
+
+### Collection: Logs
+
+**URI:** /logs
+
+**Methods:**
+
+* **GET**: Retrieve a list of entries from User Request Log
+ * Parameters:
+ * app: Filter entries by application that received the request.
+ Use "wok" or any plugin installed, like "kimchi".
+ * req: Filter entries by type of request: "DELETE", "POST",
"PUT".
+ "GET" requests are not logged.
+ * user: Filter entries by user that performed the request.
+ * date: Filter entries by date of record in the format "YYYY-MM-DD"
+
+#### Examples
+GET /logs
+[{entry-record1}, {entry-record2}, {entry-record3}, ...]
+
diff --git a/src/wok/control/logs.py b/src/wok/control/logs.py
new file mode 100644
index 0000000..0eb04d7
--- /dev/null
+++ b/src/wok/control/logs.py
@@ -0,0 +1,43 @@
+#
+# Project Wok
+#
+# Copyright IBM Corp, 2016
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import wok.template
+
+from wok.control.base import SimpleCollection
+from wok.control.utils import get_class_name, model_fn
+from wok.control.utils import UrlSubNode
+
+
+@UrlSubNode("logs")
+class Logs(SimpleCollection):
+ def __init__(self, model):
+ super(Logs, self).__init__(model)
+ self.role_key = 'logs'
+ self.admin_methods = ['GET']
+
+ def get(self, filter_params):
+ res_list = []
+
+ try:
+ get_list = getattr(self.model, model_fn(self, 'get_list'))
+ res_list = get_list(filter_params)
+ except AttributeError:
+ pass
+
+ return wok.template.render(get_class_name(self), res_list)
diff --git a/src/wok/model/logs.py b/src/wok/model/logs.py
new file mode 100644
index 0000000..09d3f47
--- /dev/null
+++ b/src/wok/model/logs.py
@@ -0,0 +1,33 @@
+#
+# Project Wok
+#
+# Copyright IBM Corp, 2016
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import cherrypy
+
+from wok.reqlogger import RequestParser
+
+
+class LogsModel(object):
+ def __init__(self, **kargs):
+ pass
+
+ def get_list(self, filter_params):
+ if filter_params:
+ return RequestParser().getFilteredRecords(filter_params)
+
+ return RequestParser().getRecords()
--
1.9.1
9:43 p.m.
New subject: [PATCH] [Wok 4/4] Fix tests
Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com>
---
src/wok/model/logs.py | 2 --
tests/utils.py | 1 +
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/wok/model/logs.py b/src/wok/model/logs.py
index 09d3f47..89a79ab 100644
--- a/src/wok/model/logs.py
+++ b/src/wok/model/logs.py
@@ -17,8 +17,6 @@
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-import cherrypy
-
from wok.reqlogger import RequestParser
diff --git a/tests/utils.py b/tests/utils.py
index d158ba1..249205d 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -119,6 +119,7 @@ def run_server(host, port, ssl_port, test_mode, cherrypy_port=None,
'websockets_port': 64667, 'ssl_cert': '',
'ssl_key': '',
'max_body_size': '4*1024', 'test': test_mode,
'access_log': '/dev/null', 'error_log':
'/dev/null',
+ 'req_log': '/dev/null',
'environment': environment, 'log_level':
'debug'})()
if model is not None:
setattr(args, 'model', model)
--
1.9.1
2978
days inactive
2978
days old
4 comments
1 participants
participants (1)
-
Lucio Correia