Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/config.py.in | 10 +++ src/wok/i18n.py | 4 +- src/wok/reqlogger.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/wok/utils.py | 20 ++++++ 4 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 src/wok/reqlogger.py diff --git a/src/wok/config.py.in b/src/wok/config.py.in index 40fbcda..afe0f08 100644 --- a/src/wok/config.py.in +++ b/src/wok/config.py.in @@ -60,6 +60,10 @@ FONTS_PATH = { SESSIONSTIMEOUT = 10 # session time out is 10 minutes +def get_log_download_path(): + return os.path.join(paths.state_dir, 'logs') + + def get_object_store(): return os.path.join(paths.state_dir, 'objectstore') @@ -188,6 +192,12 @@ class WokConfig(dict): 'tools.sessions.timeout': SESSIONSTIMEOUT, 'tools.wokauth.on': False }, + '/data/logs': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': '%s/logs' % paths.state_dir, + 'tools.nocache.on': False, + 'tools.wokauth.on': True, + }, '/base64/jquery.base64.js': { 'tools.staticfile.on': True, 'tools.staticfile.filename': '%s/base64/jquery.base64.js' % diff --git a/src/wok/i18n.py b/src/wok/i18n.py index 82d28d1..e6087f4 100644 --- a/src/wok/i18n.py +++ b/src/wok/i18n.py @@ -37,12 +37,14 @@ messages = { "WOKASYNC0002E": _("Unable to start task due error: %(err)s"), "WOKASYNC0003E": _("Timeout of %(seconds)s seconds expired while running task '%(task)s."), - "WOKAUTH0001E": _("Authentication failed for user '%(username)s'. [Error code: %(code)s]"), "WOKAUTH0002E": _("You are not authorized to access Kimchi"), "WOKAUTH0003E": _("Specify %(item)s to login into Kimchi"), "WOKAUTH0005E": _("Invalid LDAP configuration: %(item)s : %(value)s"), + "WOKLOG0001E": _("Invalid filter parameter. Filter parameters allowed: %(filters)s"), + "WOKLOG0002E": _("Creation of log file failed: %(err)s"), + "WOKOBJST0001E": _("Unable to find %(item)s in datastore"), "WOKUTILS0001E": _("Unable to reach %(url)s. Make sure it is accessible and try again."), diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py new file mode 100644 index 0000000..3e79377 --- /dev/null +++ b/src/wok/reqlogger.py @@ -0,0 +1,184 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# + +import json +import logging +import logging.handlers +import os.path + +from cherrypy.process.plugins import BackgroundTask +from datetime import datetime +from tempfile import NamedTemporaryFile + +from wok.config import config, get_log_download_path +from wok.exception import InvalidParameter, OperationFailed +from wok.utils import remove_old_files + + +# Log search setup +FILTER_FIELDS = ['app', 'date', 'download', 'req', 'user'] +LOG_DOWNLOAD_URI = "/data/logs/%s" +LOG_DOWNLOAD_TIMEOUT = 6 +LOG_FORMAT = "[%(date)s %(time)s] %(req)-6s %(app)-11s %(user)s: %(message)s\n" +SECONDS_PER_HOUR = 360 + +# Log handler setup +MAX_FILE_SIZE = 3072000 +NUM_BACKUP_FILES = 1 +REQUEST_LOG_FILE = "wok-req.log" +WOK_REQUEST_LOGGER = "wok_request_logger" + + +class RequestLogger(object): + def __init__(self): + log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE) + h = logging.handlers.RotatingFileHandler(log, 'a', + maxBytes=MAX_FILE_SIZE, + backupCount=NUM_BACKUP_FILES) + h.setFormatter(logging.Formatter('%(message)s')) + self.handler = h + self.logger = logging.getLogger(WOK_REQUEST_LOGGER) + self.logger.setLevel(logging.INFO) + self.logger.addHandler(self.handler) + + # start request log's downloadable temporary files removal task + interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR + self.clean_task = BackgroundTask(interval, self.cleanLogFiles) + self.clean_task.start() + + def cleanLogFiles(self): + globexpr = "%s/*.txt" % get_log_download_path() + remove_old_files(globexpr, LOG_DOWNLOAD_TIMEOUT) + + +class RequestParser(object): + def __init__(self): + logger = logging.getLogger(WOK_REQUEST_LOGGER) + self.baseFile = logger.handlers[0].baseFilename + self.downloadDir = get_log_download_path() + + def generateLogFile(self, records): + """ + Generates a log-format text file with lines for each record specified. + Returns a download URI for the generated file. + """ + try: + # sort records chronologically + sortedList = sorted(records, key=lambda k: k['date'] + k['time']) + + # generate log file + fd = NamedTemporaryFile(mode='w', dir=self.downloadDir, + suffix='.txt', delete=False) + + with fd: + for record in sortedList: + fd.write(LOG_FORMAT % record) + + fd.close() + except IOError as e: + raise OperationFailed("WOKLOG0002E", {'err': str(e)}) + + return LOG_DOWNLOAD_URI % os.path.basename(fd.name) + + def getRecords(self): + records = self.getRecordsFromFile(self.baseFile) + + for count in range(NUM_BACKUP_FILES): + filename = ".".join([self.baseFile, str(count + 1)]) + records.extend(self.getRecordsFromFile(filename)) + + return records + + def getRecordsFromFile(self, filename): + """ + Returns a list of dict, where each dict corresponds to a request + record. + """ + records = [] + + if not os.path.exists(filename): + return [] + + # read records from file + try: + with open(filename) as f: + line = f.readline() + while line != "": + data = line.split(">>>") + if len(data) > 1: + record = json.JSONDecoder().decode(data[0]) + record['message'] = data[1].strip() + records.append(record) + + line = f.readline() + + f. close() + except IOError as e: + raise OperationFailed("WOKLOG0002E", {'err': str(e)}) + + return records + + def getFilteredRecords(self, filter_params): + """ + Returns a dict containing the filtered list of request log entries + (dicts), and an optional uri for downloading results in a text file. + """ + uri = None + results = [] + records = self.getRecords() + + # fail for unrecognized filter options + for key in filter_params.keys(): + if key not in FILTER_FIELDS: + filters = ", ".join(FILTER_FIELDS) + raise InvalidParameter("WOKLOG0001E", {"filters": filters}) + + download = filter_params.pop('download', False) + + # filter records according to parameters + for record in records: + if all(key in record and record[key] == val + for key, val in filter_params.iteritems()): + results.append(record) + + # download option active: generate text file and provide donwload uri + if download and len(results) > 0: + uri = self.generateLogFile(results) + + return {'uri': uri, 'records': results} + + +class RequestRecord(object): + def __init__(self, message, **kwargs): + self.message = message + self.kwargs = kwargs + + # register timestamp + timestamp = datetime.today() + self.kwargs['date'] = timestamp.strftime('%Y-%m-%d') + self.kwargs['time'] = timestamp.strftime('%H:%M:%S') + + def __str__(self): + info = json.JSONEncoder().encode(self.kwargs) + return '%s >>> %s' % (info, self.message) + + def log(self): + reqLogger = logging.getLogger(WOK_REQUEST_LOGGER) + reqLogger.info(self) diff --git a/src/wok/utils.py b/src/wok/utils.py index e2f1d8e..48fe414 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -21,6 +21,7 @@ # import cherrypy +import glob import grp import os import psutil @@ -31,7 +32,9 @@ import subprocess import sys import traceback import xml.etree.ElementTree as ET + from cherrypy.lib.reprconf import Parser +from datetime import datetime, timedelta from multiprocessing import Process, Queue from threading import Timer @@ -336,6 +339,23 @@ def probe_file_permission_as_user(file, user): return queue.get() +def remove_old_files(globexpr, hours): + """ + Delete files matching globexpr that are older than specified hours. + """ + minTime = datetime.now() - timedelta(hours=hours) + + try: + for f in glob.glob(globexpr): + timestamp = os.path.getmtime(f) + fileTime = datetime.fromtimestamp(timestamp) + + if fileTime < minTime: + os.remove(f) + except (IOError, OSError) as e: + wok_log.error(str(e)) + + def get_next_clone_name(all_names, basename, name_suffix=''): """Find the next available name for a cloned resource. -- 1.9.1