Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com&gt; --- src/wok/control/base.py | 47 +++++++++++++++++------------------------------ src/wok/root.py | 41 ++++------------------------------------- 2 files changed, 21 insertions(+), 67 deletions(-) diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 04cf2cb..f4443c8 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -43,6 +43,20 @@ RESOURCE_DEFAULT_LOG = "WOKRES0001L" LOG_DISABLED_METHODS = ['GET'] +def log_request(code, params, exception, method, status, user=None): + log_id = RequestRecord( + {'code': code, 'params': params}, + exception, + app=get_plugin_from_request(), + req=method, + status=status, + user=user or cherrypy.session.get(USER_NAME, 'N/A'), + ip=cherrypy.request.remote.ip + ).log() + + return log_id + + class Resource(object): """ A Resource represents a single entity in the API (such as a Virtual @@ -157,16 +171,7 @@ class Resource(object): # log request code = self.getRequestMessage(method, action_name) reqParams = utf8_dict(self.log_args, request) - RequestRecord( - reqParams, - details, - app=get_plugin_from_request(), - msgCode=code, - req=method, - status=status, - user=cherrypy.session.get(USER_NAME, 'N/A'), - ip=cherrypy.request.remote.ip - ).log() + log_request(code, reqParams, details, method, status) wrapper.__name__ = action_name wrapper.exposed = True @@ -220,16 +225,7 @@ class Resource(object): # log request if method not in LOG_DISABLED_METHODS: code = self.getRequestMessage(method) - RequestRecord( - self.log_args, - details, - app=get_plugin_from_request(), - msgCode=code, - req=method, - status=status, - user=cherrypy.session.get(USER_NAME, 'N/A'), - ip=cherrypy.request.remote.ip - ).log() + log_request(code, self.log_args, details, method, status) return result @@ -462,16 +458,7 @@ class Collection(object): # log request code = self.getRequestMessage(method) reqParams = utf8_dict(self.log_args, params) - RequestRecord( - reqParams, - details, - app=get_plugin_from_request(), - msgCode=code, - req=method, - status=status, - user=cherrypy.session.get(USER_NAME, 'N/A'), - ip=cherrypy.request.remote.ip - ).log() + log_request(code, reqParams, details, method, status) class AsyncCollection(Collection): diff --git a/src/wok/root.py b/src/wok/root.py index e54bf38..8f2f764 100644 --- a/src/wok/root.py +++ b/src/wok/root.py @@ -29,10 +29,9 @@ from wok import template from wok.i18n import messages from wok.config import paths as wok_paths from wok.control import sub_nodes -from wok.control.base import Resource +from wok.control.base import log_request, Resource from wok.control.utils import parse_request from wok.exception import MissingParameter -from wok.reqlogger import RequestRecord ROOT_REQUESTS = { @@ -156,8 +155,6 @@ class WokRoot(Root): details = None method = 'POST' code = self.getRequestMessage(method, 'login') - app = 'wok' - ip = cherrypy.request.remote.ip try: params = parse_request() @@ -165,18 +162,7 @@ class WokRoot(Root): password = params['password'] except KeyError, item: details = e = MissingParameter('WOKAUTH0003E', {'item': str(item)}) - - RequestRecord( - params, - details, - app=app, - msgCode=code, - req=method, - status=400, - user='N/A', - ip=ip - ).log() - + log_request(code, params, details, method, 400) raise cherrypy.HTTPError(400, e.message) try: @@ -186,16 +172,7 @@ class WokRoot(Root): status = e.status raise finally: - RequestRecord( - params, - details, - app=app, - msgCode=code, - req=method, - status=status, - user='N/A', - ip=ip - ).log() + log_request(code, params, details, method, status) return json.dumps(user_info) @@ -204,19 +181,9 @@ class WokRoot(Root): method = 'POST' code = self.getRequestMessage(method, 'logout') params = {'username': cherrypy.session.get(auth.USER_NAME, 'N/A')} - ip = cherrypy.request.remote.ip auth.logout() - RequestRecord( - params, - None, - app='wok', - msgCode=code, - req=method, - status=200, - user=params['username'], - ip=ip - ).log() + log_request(code, params, None, method, 200, user=params['username']) return '{}' -- 1.9.1

* Add ID to each entry * Change log entry format to one JSON per line * Modify getRecords() to apply AsyncTask entry status to original request entry Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com&gt; --- src/wok/message.py | 2 +- src/wok/reqlogger.py | 142 +++++++++++++++++++++++++++++---------------------- 2 files changed, 82 insertions(+), 62 deletions(-) diff --git a/src/wok/message.py b/src/wok/message.py index ff4cbc9..28462b7 100644 --- a/src/wok/message.py +++ b/src/wok/message.py @@ -51,7 +51,7 @@ class WokMessage(object): wok_app = cherrypy.tree.apps.get('', None) # get app from plugin path if specified - if self.plugin: + if self.plugin and self.plugin != 'wok': app = cherrypy.tree.apps.get(self.plugin, None) # if on request, try to get app from it elif cherrypy.request.app: diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index 9f1d2c8..2923d7c 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -24,6 +24,7 @@ import logging import logging.handlers import os.path import time +import uuid from cherrypy.process.plugins import BackgroundTask from tempfile import NamedTemporaryFile @@ -32,7 +33,7 @@ from wok.config import config, get_log_download_path from wok.exception import InvalidParameter, OperationFailed from wok.message import WokMessage from wok.stringutils import ascii_dict -from wok.utils import remove_old_files +from wok.utils import plugin_root, remove_old_files # Log search setup @@ -62,6 +63,9 @@ UNSAFE_REQUEST_PARAMETERS = ['password', 'passwd'] REQUEST_LOG_FILE = "wok-req.log" WOK_REQUEST_LOGGER = "wok_request_logger" +# AsyncTask handling +ASYNCTASK_REQUEST_METHOD = 'TASK' + class RequestLogger(object): def __init__(self): @@ -114,19 +118,17 @@ class RequestParser(object): return LOG_DOWNLOAD_URI % os.path.basename(fd.name) - def getTranslatedMessage(self, record, params, detParams): - code = record.get('msgCode', '') - app = record.get('app', 'wok') - plugin = None - if app != 'wok': - plugin = "/plugins/%s" % app - + def getTranslatedMessage(self, message, error, app): + plugin = plugin_root.get(app, 'wok') + code = message.get('code', '') + params = message.get('params', {}) msg = WokMessage(code, params, plugin) text = msg.get_text(prepend_code=False, translate=True) - detCode = record.get('detCode', '') - if detCode: - msg = WokMessage(detCode, detParams, plugin) + if error: + code = error.get('code', '') + params = error.get('params', {}) + msg = WokMessage(code, params, plugin) text += ' ' + msg.get_text(prepend_code=True, translate=True) return text @@ -137,8 +139,40 @@ class RequestParser(object): for filename in glob.glob(self.baseFile + "-*[!.gz]"): records.extend(self.getRecordsFromFile(filename)) - # Return ordered by latest events first - return sorted(records, key=lambda k: k['date'] + k['time'], + # sort records chronologically + sort = sorted(records, key=lambda k: k['info']['date'] + + k['info']['time']) + + # normalize entries + normalized = {} + asynctask_updates = [] + for record in sort: + # async task entries are left to the end + if record['info']['req'] == ASYNCTASK_REQUEST_METHOD: + asynctask_updates.append(record) + continue + + # initial request entry: generate message text + message = record.pop('message') + error = record.pop('error', None) + plugin = record['info']['app'] + text = self.getTranslatedMessage(message, error, plugin) + record['info']['message'] = text + + id = record.pop('id') + normalized[id] = record['info'] + + # async task entries: update only status + for record in asynctask_updates: + id = record.pop('id') + + # task id may not exist, since GET requests are not logged but + # may generate tasks (i.e. AsyncResource) + if id in normalized: + normalized[id]['status'] = record['info']['status'] + + # return results in chronological reverse order + return sorted(normalized.values(), key=lambda k: k['date'] + k['time'], reverse=True) def getRecordsFromFile(self, filename): @@ -151,34 +185,12 @@ class RequestParser(object): if not os.path.exists(filename): return [] - # read records from file try: with open(filename) as f: line = f.readline() while line != "": - data = line.split(">>>") - if len(data) > 1: - record = json.JSONDecoder().decode(data[0]) - - if len(data) > 2: - # new log format: translate message on the fly - params = json.JSONDecoder().decode(data[1]) - detParams = None - - # has original exception details - if len(data) > 3: - detParams = json.JSONDecoder().decode(data[2]) - - msg = self.getTranslatedMessage(record, params, - detParams) - else: - # make it compatible with v2.2 log files, which - # messages are already translated - msg = data[1].strip() - - record['message'] = msg - records.append(record) - + record = json.JSONDecoder().decode(line) + records.append(record) line = f.readline() f. close() @@ -217,20 +229,32 @@ class RequestParser(object): class RequestRecord(object): - def __init__(self, msgParams, details, **kwargs): - # log record data - self.logData = kwargs - self.details = details - + def __init__(self, message, error, id=None, **kwargs): # data for message translation - self.code = self.logData['msgCode'] - self.params = self.getSafeReqParams(msgParams) - - # register timestamp in local time + self.message = { + 'code': message['code'], + 'params': self.getSafeReqParams(message['params']), + } + + # data for error translation (WokException) + self.error = None + if error: + self.error = { + 'code': error.code, + 'params': error.params, + } + + # log entry info + self.id = id or str(uuid.uuid4()) + self.info = kwargs + + # generate timestamp timestamp = time.localtime() - self.logData['date'] = time.strftime(TS_DATE_FORMAT, timestamp) - self.logData['time'] = time.strftime(TS_TIME_FORMAT, timestamp) - self.logData['zone'] = time.strftime(TS_ZONE_FORMAT, timestamp) + self.info.update({ + 'date': time.strftime(TS_DATE_FORMAT, timestamp), + 'time': time.strftime(TS_TIME_FORMAT, timestamp), + 'zone': time.strftime(TS_ZONE_FORMAT, timestamp), + }) def getSafeReqParams(self, params): result = params.copy() @@ -239,20 +263,16 @@ class RequestRecord(object): return result def __str__(self): - # original exception details - details = '' - if self.details: - self.logData['detCode'] = self.details.code - details = ">>> %s" % json.JSONEncoder().encode(self.details.params) - - # request log message - msg = WokMessage(self.code, self.params) - msgText = msg.get_text(prepend_code=False, translate=False) - logData = json.JSONEncoder().encode(self.logData) - msgParams = json.JSONEncoder().encode(self.params) + entry = { + "id": self.id, + "message": self.message, + "error": self.error, + "info": self.info, + } - return '%s >>> %s %s >>> %s' % (logData, msgParams, details, msgText) + return json.JSONEncoder().encode(entry) def log(self): reqLogger = logging.getLogger(WOK_REQUEST_LOGGER) reqLogger.info(self) + return self.id -- 1.9.1

Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com&gt; --- src/wok/control/base.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 213e6b5..06f1f08 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -26,6 +26,7 @@ import urllib2 import wok.template +from wok.asynctask import save_request_log_id from wok.auth import USER_GROUPS, USER_NAME, USER_ROLES from wok.control.utils import get_class_name, internal_redirect, model_fn from wok.control.utils import parse_request, validate_method @@ -162,6 +163,7 @@ class Resource(object): self.info['persistent'] is True): result = render_fn(self, action_result) status = cherrypy.response.status + return result except WokException, e: details = e @@ -171,7 +173,9 @@ class Resource(object): # log request code = self.getRequestMessage(method, action_name) reqParams = utf8_dict(self.log_args, request) - log_request(code, reqParams, details, method, status) + log_id = log_request(code, reqParams, details, method, status) + if status == 202: + save_request_log_id(log_id, action_result['id']) wrapper.__name__ = action_name wrapper.exposed = True @@ -223,7 +227,7 @@ class Resource(object): raise finally: # log request - if method not in LOG_DISABLED_METHODS: + if method not in LOG_DISABLED_METHODS and status != 202: code = self.getRequestMessage(method) log_request(code, self.log_args, details, method, status) @@ -307,6 +311,15 @@ class AsyncResource(Resource): raise cherrypy.HTTPError(405, e.message) cherrypy.response.status = 202 + + # log request + method = 'DELETE' + code = self.getRequestMessage(method) + reqParams = utf8_dict(self.log_args) + log_id = log_request(code, reqParams, None, method, + cherrypy.response.status) + save_request_log_id(log_id, task['id']) + return wok.template.render("Task", task) @@ -454,7 +467,7 @@ class Collection(object): status = e.status raise finally: - if method not in LOG_DISABLED_METHODS: + if method not in LOG_DISABLED_METHODS and status != 202: # log request code = self.getRequestMessage(method) reqParams = utf8_dict(self.log_args, params) @@ -480,6 +493,15 @@ class AsyncCollection(Collection): args = self.model_args + [params] task = create(*args) cherrypy.response.status = 202 + + # log request + method = 'POST' + code = self.getRequestMessage(method) + reqParams = utf8_dict(self.log_args, params) + log_id = log_request(code, reqParams, None, method, + cherrypy.response.status) + save_request_log_id(log_id, task['id']) + return wok.template.render("Task", task) -- 1.9.1

That log file is not intended anymore to be a log file itself. Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com&gt; --- src/wok/config.py.in | 4 ++++ src/wok/reqlogger.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/wok/config.py.in b/src/wok/config.py.in index cbe585c..f82aa0a 100644 --- a/src/wok/config.py.in +++ b/src/wok/config.py.in @@ -66,6 +66,10 @@ def get_object_store(): return os.path.join(paths.state_dir, 'objectstore') +def get_data_dir(): + return paths.state_dir + + def get_version(): return "-".join([__version__, __release__]) diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index 7a95907..92eff83 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -29,7 +29,7 @@ import uuid from cherrypy.process.plugins import BackgroundTask from tempfile import NamedTemporaryFile -from wok.config import config, get_log_download_path +from wok.config import get_data_dir, get_log_download_path from wok.exception import InvalidParameter, OperationFailed from wok.message import WokMessage from wok.stringutils import ascii_dict @@ -60,7 +60,7 @@ TS_ZONE_FORMAT = "%Z" UNSAFE_REQUEST_PARAMETERS = ['password', 'passwd'] # Log handler setup -REQUEST_LOG_FILE = "wok-req.log" +REQUEST_LOG_FILE = "user-requests.data" WOK_REQUEST_LOGGER = "wok_request_logger" # AsyncTask handling @@ -69,7 +69,7 @@ ASYNCTASK_REQUEST_METHOD = 'TASK' class RequestLogger(object): def __init__(self): - log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE) + log = os.path.join(get_data_dir(), REQUEST_LOG_FILE) h = logging.handlers.WatchedFileHandler(log, 'a') h.setFormatter(logging.Formatter('%(message)s')) self.handler = h -- 1.9.1

On 08/31/2016 06:06 PM, Lucio Correia wrote: Maybe you can use "cherrypy.request.app.root.domain" directly or... (see below) It is better to have the uri as the key. So you can easily grab the application name based on uri. plugin_root[uri] = name Replase all the above block to: return plugin_root.get(uri, 'wok')

On 08/31/2016 06:06 PM, Lucio Correia wrote: Why is that needed? Why wok is different from the rest of the application? I am curious to see how this constant is used. Do not set 'wok' as plugin name. It is not true as wok is not a plugin. Why? Could you explain why do we need to add an specific 'req' information for task?

On 08/31/2016 06:06 PM, Lucio Correia wrote: This should be only on generate_action_handler_task(), right? As this function is specific to handle Tasks. Why status 202 has a different behavior? Is this result of the "if status != 202" added above? > # log request Same question I did before. Is this result of the "if status != 202" added above? Unless to save the log_id to the AsyncTask instance, shouldn't the log record be the same of the other HTTP return codes?

On 08/31/2016 06:06 PM, Lucio Correia wrote: You don't need a new function. You can access paths directly wherever you want.

On 01-09-2016 10:35, Aline Manera wrote: This is consequence of returning 'wok' as default instead of None in get_plugin_from_request. I will rethink it in case I can return None there. The var is named plugin but it means app.. wok or any plugin. I may change the variable name. Because they are just updates to the original request log entry. And the task log entry may be written before the task finishes since runs in separate thread. I use it to differentiate task finish log entries from regular request log entries. -- Lucio Correia Software Engineer IBM LTC Brazil

On 01-09-2016 11:09, Aline Manera wrote: Yes, I did it originally. But I need request information and it's not possible to call parse_request() again. So I needed to move it to here. It's logged separately in AsyncResource class methods, since it needs log entry id and task id, which is available only there. Yes this is the "separate log" for requests that generate tasks (return 202). Same answer above :) the task ID is only available here, and the log ID is generated when the message is logged. Here is the only place both are available. -- Lucio Correia Software Engineer IBM LTC Brazil

On 01-09-2016 11:11, Aline Manera wrote: OK -- Lucio Correia Software Engineer IBM LTC Brazil

On 09/01/2016 01:08 PM, Lucio Correia wrote: Ok. I got your point. tasks_queue is a global variable =)

On 09/01/2016 01:15 PM, Lucio Correia wrote: You can change the function to return the log_id as parameter. Having an if statement like when we have specific function to handle tasks looks weird to me. That is not true. Each resource element has an ID, ie, when the resource is a Task, you have the Task ID there too. You can simplify use self.ident in the base Resource class. I don't think we need that. You can access the task ID by self.ident in the base Resource class

On 01-09-2016 13:48, Aline Manera wrote: The TASK type log entries are not in the final log - UI does not see them. They are filtered by backend before, exactly what this code is doing here. In the task log entry, I use status=200 or status=400 (see async task _log() usage) to mean success or failure, and here I use that value to update the original request log entry (which contains 202 until task finishes). -- Lucio Correia Software Engineer IBM LTC Brazil

On 01-09-2016 13:21, Aline Manera wrote: It does not work, any other idea? In worst case I keep it based on target_uri for now and add a TODO note to fix it later? ====================================================================== ERROR: test_async_tasks (test_tasks.AsyncTaskTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "test_tasks.py", line 61, in test_async_tasks taskid = AsyncTask('', self._quick_op, 'Hello').id File "/home/dev/wok/src/wok/asynctask.py", line 61, in __init__ self.plugin = cherrypy.request.app.root.domain AttributeError: 'NoneType' object has no attribute 'root' -- Lucio Correia Software Engineer IBM LTC Brazil