[PATCH] [Wok 0/4] Fix issues #140 and #142
In order to run tests successfully, remove your data/logs/wok-req.log. DEPENDS ON: [Wok] Isolate string utils in order to avoid cyclic import Lucio Correia (4): Add option to get untranslated message text Log message code and parameters instead of translated text Fix issue with converting message to unicode Update tests src/wok/control/base.py | 13 ++++++------- src/wok/message.py | 32 ++++++++++++++++++-------------- src/wok/reqlogger.py | 47 ++++++++++++++++++++++++++++++++++++++--------- src/wok/root.py | 12 ++++++------ tests/test_api.py | 2 +- 5 files changed, 69 insertions(+), 37 deletions(-) -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/message.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/wok/message.py b/src/wok/message.py index 4c35747..35f40be 100644 --- a/src/wok/message.py +++ b/src/wok/message.py @@ -46,7 +46,7 @@ class WokMessage(object): self.args = args self.plugin = plugin - def _get_translation(self): + def _get_text(self, translate): wok_app = cherrypy.tree.apps.get('', None) # get app from plugin path if specified @@ -68,20 +68,23 @@ class WokMessage(object): app = wok_app text = app.root.messages.get(self.code, self.code) - # do translation - domain = app.root.domain - paths = app.root.paths - lang = validate_language(get_lang()) + if translate: + # do translation + domain = app.root.domain + paths = app.root.paths + lang = validate_language(get_lang()) - try: - translation = gettext.translation(domain, paths.mo_dir, [lang]) - except: - translation = gettext + try: + translation = gettext.translation(domain, paths.mo_dir, [lang]) + except: + translation = gettext + + return translation.gettext(text) - return translation.gettext(text) + return gettext.gettext(text) - def get_text(self, prepend_code=True): - msg = self._get_translation() + def get_text(self, prepend_code=True, translate=True): + msg = self._get_text(translate) msg = unicode(msg, 'utf-8') % self.args if prepend_code: -- 1.9.1
* Add data needed for translation to user request log entries (issue #140). * In order to translate user request log messages when they are requested (issue #142), save them untranslated to the base log file (wok-req.log). * Keep compatibility to prior format of base log file, which does not have needed information to do translation. Old log entry format: [logrecord] >>> [translated message] New log entry format: [logrecord] >>> [message params] >>> [untranslated message] Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/control/base.py | 13 ++++++------- src/wok/reqlogger.py | 47 ++++++++++++++++++++++++++++++++++++++--------- src/wok/root.py | 12 ++++++------ 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 6dfc977..f563aed 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -31,7 +31,6 @@ from wok.control.utils import get_class_name, internal_redirect, model_fn from wok.control.utils import parse_request, validate_method from wok.control.utils import validate_params from wok.exception import InvalidOperation, UnauthorizedError, WokException -from wok.message import WokMessage from wok.reqlogger import RequestRecord from wok.stringutils import encode_value, utf8_dict from wok.utils import get_plugin_from_request, wok_log @@ -156,10 +155,10 @@ class Resource(object): # log request code = self.getRequestMessage(method, action_name) reqParams = utf8_dict(self.log_args, request) - msg = WokMessage(code, reqParams).get_text(prepend_code=False) RequestRecord( - msg, + reqParams, app=get_plugin_from_request(), + msgCode=code, req=method, status=status, user=cherrypy.session.get(USER_NAME, 'N/A'), @@ -216,10 +215,10 @@ class Resource(object): # log request if method not in LOG_DISABLED_METHODS: code = self.getRequestMessage(method) - msg = WokMessage(code, self.log_args) RequestRecord( - msg.get_text(prepend_code=False), + self.log_args, app=get_plugin_from_request(), + msgCode=code, req=method, status=status, user=cherrypy.session.get(USER_NAME, 'N/A'), @@ -455,10 +454,10 @@ class Collection(object): # log request code = self.getRequestMessage(method) reqParams = utf8_dict(self.log_args, params) - msg = WokMessage(code, reqParams).get_text(prepend_code=False) RequestRecord( - msg, + reqParams, app=get_plugin_from_request(), + msgCode=code, req=method, status=status, user=cherrypy.session.get(USER_NAME, 'N/A'), diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index fd02382..6f3435e 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -30,6 +30,7 @@ from tempfile import NamedTemporaryFile from wok.config import config, get_log_download_path from wok.exception import InvalidParameter, OperationFailed +from wok.message import WokMessage from wok.stringutils import ascii_dict from wok.utils import remove_old_files @@ -112,6 +113,16 @@ class RequestParser(object): return LOG_DOWNLOAD_URI % os.path.basename(fd.name) + def getTranslatedMessage(self, record, params): + code = record.get('msgCode', '') + app = record.get('app', 'wok') + plugin = None + if app != 'wok': + plugin = "/plugins/%s" % app + + msg = WokMessage(code, params, plugin) + return msg.get_text(prepend_code=False, translate=True) + def getRecords(self): records = self.getRecordsFromFile(self.baseFile) @@ -140,7 +151,17 @@ class RequestParser(object): data = line.split(">>>") if len(data) > 1: record = json.JSONDecoder().decode(data[0]) - record['message'] = data[1].strip() + + if len(data) > 2: + # new log format: translate message on the fly + params = json.JSONDecoder().decode(data[1]) + msg = self.getTranslatedMessage(record, params) + else: + # make it compatible with v2.2 log files, which + # messages are already translated + msg = data[1].strip() + + record['message'] = msg records.append(record) line = f.readline() @@ -181,19 +202,27 @@ class RequestParser(object): class RequestRecord(object): - def __init__(self, message, **kwargs): - self.message = message - self.kwargs = kwargs + def __init__(self, msgParams, **kwargs): + # log record data + self.logData = kwargs + + # data for message translation + self.code = self.logData['msgCode'] + self.params = msgParams # register timestamp in local time timestamp = time.localtime() - self.kwargs['date'] = time.strftime(TS_DATE_FORMAT, timestamp) - self.kwargs['time'] = time.strftime(TS_TIME_FORMAT, timestamp) - self.kwargs['zone'] = time.strftime(TS_ZONE_FORMAT, timestamp) + self.logData['date'] = time.strftime(TS_DATE_FORMAT, timestamp) + self.logData['time'] = time.strftime(TS_TIME_FORMAT, timestamp) + self.logData['zone'] = time.strftime(TS_ZONE_FORMAT, timestamp) def __str__(self): - info = json.JSONEncoder().encode(self.kwargs) - return '%s >>> %s' % (info, self.message) + # log untranslated message to base log file + msg = WokMessage(self.code, self.params) + msgText = msg.get_text(prepend_code=False, translate=False) + msgParams = json.JSONEncoder().encode(self.params) + logData = json.JSONEncoder().encode(self.logData) + return '%s >>> %s >>> %s' % (logData, msgParams, msgText) def log(self): reqLogger = logging.getLogger(WOK_REQUEST_LOGGER) diff --git a/src/wok/root.py b/src/wok/root.py index 8601b71..08f4981 100644 --- a/src/wok/root.py +++ b/src/wok/root.py @@ -32,7 +32,6 @@ from wok.control import sub_nodes from wok.control.base import Resource from wok.control.utils import parse_request from wok.exception import MissingParameter -from wok.message import WokMessage from wok.reqlogger import RequestRecord @@ -161,13 +160,13 @@ class WokRoot(Root): try: params = parse_request() - msg = WokMessage(code, params).get_text(prepend_code=False) username = params['username'] password = params['password'] except KeyError, item: RequestRecord( - msg, + params, app=app, + msgCode=code, req=method, status=400, user='N/A', @@ -185,8 +184,9 @@ class WokRoot(Root): raise finally: RequestRecord( - msg, + params, app=app, + msgCode=code, req=method, status=status, user='N/A', @@ -200,14 +200,14 @@ class WokRoot(Root): method = 'POST' code = self.getRequestMessage(method, 'logout') params = {'username': cherrypy.session.get(auth.USER_NAME, 'N/A')} - msg = WokMessage(code, params).get_text(prepend_code=False) ip = cherrypy.request.remote.ip auth.logout() RequestRecord( - msg, + params, app='wok', + msgCode=code, req=method, status=200, user=params['username'], -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/message.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/wok/message.py b/src/wok/message.py index 35f40be..ff4cbc9 100644 --- a/src/wok/message.py +++ b/src/wok/message.py @@ -22,6 +22,7 @@ import cherrypy import gettext +from wok.stringutils import decode_value from wok.template import get_lang, validate_language @@ -36,7 +37,7 @@ class WokMessage(object): try: # In case the value formats itself to an ascii string. - args[key] = unicode(str(value), 'utf-8') + args[key] = decode_value(value) except UnicodeEncodeError: # In case the value is a WokException or it formats # itself to a unicode string. @@ -85,7 +86,7 @@ class WokMessage(object): def get_text(self, prepend_code=True, translate=True): msg = self._get_text(translate) - msg = unicode(msg, 'utf-8') % self.args + msg = decode_value(msg) % self.args if prepend_code: return "%s: %s" % (self.code, msg) -- 1.9.1
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index bcf34cb..c4ca755 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -84,7 +84,7 @@ class APITests(unittest.TestCase): self.assertGreaterEqual(records, 1) for record in records: keys = [u'zone', u'ip', u'app', u'req', u'user', u'time', u'date', - u'message', u'status'] + u'message', u'msgCode', u'status'] self.assertEquals(sorted(keys), sorted(record.keys())) # Test search by app -- 1.9.1
I've talked offline with Lucio and he agreed into sending a v2 where #140 is fixed (at this moment only #142 is being fixed) and creating a 'blacklist' of user log keys that shouldn't be logged in wok-req.log to avoid logging passwords in plain text. Daniel On 08/04/2016 03:15 PM, Lucio Correia wrote:
In order to run tests successfully, remove your data/logs/wok-req.log.
DEPENDS ON: [Wok] Isolate string utils in order to avoid cyclic import
Lucio Correia (4): Add option to get untranslated message text Log message code and parameters instead of translated text Fix issue with converting message to unicode Update tests
src/wok/control/base.py | 13 ++++++------- src/wok/message.py | 32 ++++++++++++++++++-------------- src/wok/reqlogger.py | 47 ++++++++++++++++++++++++++++++++++++++--------- src/wok/root.py | 12 ++++++------ tests/test_api.py | 2 +- 5 files changed, 69 insertions(+), 37 deletions(-)
participants (2)
-
Daniel Henrique Barboza -
Lucio Correia