[Kimchi-devel] [PATCH] [Wok 3/7] Update Request Logger to handle AsyncTask status
Lucio Correia
luciojhc at linux.vnet.ibm.com
Wed Aug 31 21:06:29 UTC 2016
* Add ID to each entry
* Change log entry format to one JSON per line
* Modify getRecords() to apply AsyncTask entry status
to original request entry
Signed-off-by: Lucio Correia <luciojhc at linux.vnet.ibm.com>
---
src/wok/message.py | 2 +-
src/wok/reqlogger.py | 142 +++++++++++++++++++++++++++++----------------------
2 files changed, 82 insertions(+), 62 deletions(-)
diff --git a/src/wok/message.py b/src/wok/message.py
index ff4cbc9..28462b7 100644
--- a/src/wok/message.py
+++ b/src/wok/message.py
@@ -51,7 +51,7 @@ class WokMessage(object):
wok_app = cherrypy.tree.apps.get('', None)
# get app from plugin path if specified
- if self.plugin:
+ if self.plugin and self.plugin != 'wok':
app = cherrypy.tree.apps.get(self.plugin, None)
# if on request, try to get app from it
elif cherrypy.request.app:
diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py
index 9f1d2c8..2923d7c 100644
--- a/src/wok/reqlogger.py
+++ b/src/wok/reqlogger.py
@@ -24,6 +24,7 @@ import logging
import logging.handlers
import os.path
import time
+import uuid
from cherrypy.process.plugins import BackgroundTask
from tempfile import NamedTemporaryFile
@@ -32,7 +33,7 @@ from wok.config import config, get_log_download_path
from wok.exception import InvalidParameter, OperationFailed
from wok.message import WokMessage
from wok.stringutils import ascii_dict
-from wok.utils import remove_old_files
+from wok.utils import plugin_root, remove_old_files
# Log search setup
@@ -62,6 +63,9 @@ UNSAFE_REQUEST_PARAMETERS = ['password', 'passwd']
REQUEST_LOG_FILE = "wok-req.log"
WOK_REQUEST_LOGGER = "wok_request_logger"
+# AsyncTask handling
+ASYNCTASK_REQUEST_METHOD = 'TASK'
+
class RequestLogger(object):
def __init__(self):
@@ -114,19 +118,17 @@ class RequestParser(object):
return LOG_DOWNLOAD_URI % os.path.basename(fd.name)
- def getTranslatedMessage(self, record, params, detParams):
- code = record.get('msgCode', '')
- app = record.get('app', 'wok')
- plugin = None
- if app != 'wok':
- plugin = "/plugins/%s" % app
-
+ def getTranslatedMessage(self, message, error, app):
+ plugin = plugin_root.get(app, 'wok')
+ code = message.get('code', '')
+ params = message.get('params', {})
msg = WokMessage(code, params, plugin)
text = msg.get_text(prepend_code=False, translate=True)
- detCode = record.get('detCode', '')
- if detCode:
- msg = WokMessage(detCode, detParams, plugin)
+ if error:
+ code = error.get('code', '')
+ params = error.get('params', {})
+ msg = WokMessage(code, params, plugin)
text += ' ' + msg.get_text(prepend_code=True, translate=True)
return text
@@ -137,8 +139,40 @@ class RequestParser(object):
for filename in glob.glob(self.baseFile + "-*[!.gz]"):
records.extend(self.getRecordsFromFile(filename))
- # Return ordered by latest events first
- return sorted(records, key=lambda k: k['date'] + k['time'],
+ # sort records chronologically
+ sort = sorted(records, key=lambda k: k['info']['date'] +
+ k['info']['time'])
+
+ # normalize entries
+ normalized = {}
+ asynctask_updates = []
+ for record in sort:
+ # async task entries are left to the end
+ if record['info']['req'] == ASYNCTASK_REQUEST_METHOD:
+ asynctask_updates.append(record)
+ continue
+
+ # initial request entry: generate message text
+ message = record.pop('message')
+ error = record.pop('error', None)
+ plugin = record['info']['app']
+ text = self.getTranslatedMessage(message, error, plugin)
+ record['info']['message'] = text
+
+ id = record.pop('id')
+ normalized[id] = record['info']
+
+ # async task entries: update only status
+ for record in asynctask_updates:
+ id = record.pop('id')
+
+ # task id may not exist, since GET requests are not logged but
+ # may generate tasks (i.e. AsyncResource)
+ if id in normalized:
+ normalized[id]['status'] = record['info']['status']
+
+ # return results in chronological reverse order
+ return sorted(normalized.values(), key=lambda k: k['date'] + k['time'],
reverse=True)
def getRecordsFromFile(self, filename):
@@ -151,34 +185,12 @@ class RequestParser(object):
if not os.path.exists(filename):
return []
- # read records from file
try:
with open(filename) as f:
line = f.readline()
while line != "":
- data = line.split(">>>")
- if len(data) > 1:
- record = json.JSONDecoder().decode(data[0])
-
- if len(data) > 2:
- # new log format: translate message on the fly
- params = json.JSONDecoder().decode(data[1])
- detParams = None
-
- # has original exception details
- if len(data) > 3:
- detParams = json.JSONDecoder().decode(data[2])
-
- msg = self.getTranslatedMessage(record, params,
- detParams)
- else:
- # make it compatible with v2.2 log files, which
- # messages are already translated
- msg = data[1].strip()
-
- record['message'] = msg
- records.append(record)
-
+ record = json.JSONDecoder().decode(line)
+ records.append(record)
line = f.readline()
f. close()
@@ -217,20 +229,32 @@ class RequestParser(object):
class RequestRecord(object):
- def __init__(self, msgParams, details, **kwargs):
- # log record data
- self.logData = kwargs
- self.details = details
-
+ def __init__(self, message, error, id=None, **kwargs):
# data for message translation
- self.code = self.logData['msgCode']
- self.params = self.getSafeReqParams(msgParams)
-
- # register timestamp in local time
+ self.message = {
+ 'code': message['code'],
+ 'params': self.getSafeReqParams(message['params']),
+ }
+
+ # data for error translation (WokException)
+ self.error = None
+ if error:
+ self.error = {
+ 'code': error.code,
+ 'params': error.params,
+ }
+
+ # log entry info
+ self.id = id or str(uuid.uuid4())
+ self.info = kwargs
+
+ # generate timestamp
timestamp = time.localtime()
- self.logData['date'] = time.strftime(TS_DATE_FORMAT, timestamp)
- self.logData['time'] = time.strftime(TS_TIME_FORMAT, timestamp)
- self.logData['zone'] = time.strftime(TS_ZONE_FORMAT, timestamp)
+ self.info.update({
+ 'date': time.strftime(TS_DATE_FORMAT, timestamp),
+ 'time': time.strftime(TS_TIME_FORMAT, timestamp),
+ 'zone': time.strftime(TS_ZONE_FORMAT, timestamp),
+ })
def getSafeReqParams(self, params):
result = params.copy()
@@ -239,20 +263,16 @@ class RequestRecord(object):
return result
def __str__(self):
- # original exception details
- details = ''
- if self.details:
- self.logData['detCode'] = self.details.code
- details = ">>> %s" % json.JSONEncoder().encode(self.details.params)
-
- # request log message
- msg = WokMessage(self.code, self.params)
- msgText = msg.get_text(prepend_code=False, translate=False)
- logData = json.JSONEncoder().encode(self.logData)
- msgParams = json.JSONEncoder().encode(self.params)
+ entry = {
+ "id": self.id,
+ "message": self.message,
+ "error": self.error,
+ "info": self.info,
+ }
- return '%s >>> %s %s >>> %s' % (logData, msgParams, details, msgText)
+ return json.JSONEncoder().encode(entry)
def log(self):
reqLogger = logging.getLogger(WOK_REQUEST_LOGGER)
reqLogger.info(self)
+ return self.id
--
1.9.1
More information about the Kimchi-devel
mailing list