[Kimchi-devel] [PATCH] [Wok 1/4] Add User Request Logger
Lucio Correia
luciojhc at linux.vnet.ibm.com
Fri Feb 26 19:12:52 UTC 2016
Signed-off-by: Lucio Correia <luciojhc at linux.vnet.ibm.com>
---
src/wok/config.py.in | 10 +++
src/wok/i18n.py | 4 +-
src/wok/reqlogger.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++
src/wok/utils.py | 20 ++++++
4 files changed, 217 insertions(+), 1 deletion(-)
create mode 100644 src/wok/reqlogger.py
diff --git a/src/wok/config.py.in b/src/wok/config.py.in
index 40fbcda..afe0f08 100644
--- a/src/wok/config.py.in
+++ b/src/wok/config.py.in
@@ -60,6 +60,10 @@ FONTS_PATH = {
SESSIONSTIMEOUT = 10 # session time out is 10 minutes
+def get_log_download_path():
+ return os.path.join(paths.state_dir, 'logs')
+
+
def get_object_store():
return os.path.join(paths.state_dir, 'objectstore')
@@ -188,6 +192,12 @@ class WokConfig(dict):
'tools.sessions.timeout': SESSIONSTIMEOUT,
'tools.wokauth.on': False
},
+ '/data/logs': {
+ 'tools.staticdir.on': True,
+ 'tools.staticdir.dir': '%s/logs' % paths.state_dir,
+ 'tools.nocache.on': False,
+ 'tools.wokauth.on': True,
+ },
'/base64/jquery.base64.js': {
'tools.staticfile.on': True,
'tools.staticfile.filename': '%s/base64/jquery.base64.js' %
diff --git a/src/wok/i18n.py b/src/wok/i18n.py
index 82d28d1..e6087f4 100644
--- a/src/wok/i18n.py
+++ b/src/wok/i18n.py
@@ -37,12 +37,14 @@ messages = {
"WOKASYNC0002E": _("Unable to start task due error: %(err)s"),
"WOKASYNC0003E": _("Timeout of %(seconds)s seconds expired while running task '%(task)s."),
-
"WOKAUTH0001E": _("Authentication failed for user '%(username)s'. [Error code: %(code)s]"),
"WOKAUTH0002E": _("You are not authorized to access Kimchi"),
"WOKAUTH0003E": _("Specify %(item)s to login into Kimchi"),
"WOKAUTH0005E": _("Invalid LDAP configuration: %(item)s : %(value)s"),
+ "WOKLOG0001E": _("Invalid filter parameter. Filter parameters allowed: %(filters)s"),
+ "WOKLOG0002E": _("Creation of log file failed: %(err)s"),
+
"WOKOBJST0001E": _("Unable to find %(item)s in datastore"),
"WOKUTILS0001E": _("Unable to reach %(url)s. Make sure it is accessible and try again."),
diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py
new file mode 100644
index 0000000..3e79377
--- /dev/null
+++ b/src/wok/reqlogger.py
@@ -0,0 +1,184 @@
+#
+# Project Wok
+#
+# Copyright IBM Corp, 2016
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import json
+import logging
+import logging.handlers
+import os.path
+
+from cherrypy.process.plugins import BackgroundTask
+from datetime import datetime
+from tempfile import NamedTemporaryFile
+
+from wok.config import config, get_log_download_path
+from wok.exception import InvalidParameter, OperationFailed
+from wok.utils import remove_old_files
+
+
+# Log search setup
+FILTER_FIELDS = ['app', 'date', 'download', 'req', 'user']
+LOG_DOWNLOAD_URI = "/data/logs/%s"
+LOG_DOWNLOAD_TIMEOUT = 6
+LOG_FORMAT = "[%(date)s %(time)s] %(req)-6s %(app)-11s %(user)s: %(message)s\n"
+SECONDS_PER_HOUR = 360
+
+# Log handler setup
+MAX_FILE_SIZE = 3072000
+NUM_BACKUP_FILES = 1
+REQUEST_LOG_FILE = "wok-req.log"
+WOK_REQUEST_LOGGER = "wok_request_logger"
+
+
+class RequestLogger(object):
+ def __init__(self):
+ log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE)
+ h = logging.handlers.RotatingFileHandler(log, 'a',
+ maxBytes=MAX_FILE_SIZE,
+ backupCount=NUM_BACKUP_FILES)
+ h.setFormatter(logging.Formatter('%(message)s'))
+ self.handler = h
+ self.logger = logging.getLogger(WOK_REQUEST_LOGGER)
+ self.logger.setLevel(logging.INFO)
+ self.logger.addHandler(self.handler)
+
+ # start request log's downloadable temporary files removal task
+ interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR
+ self.clean_task = BackgroundTask(interval, self.cleanLogFiles)
+ self.clean_task.start()
+
+ def cleanLogFiles(self):
+ globexpr = "%s/*.txt" % get_log_download_path()
+ remove_old_files(globexpr, LOG_DOWNLOAD_TIMEOUT)
+
+
+class RequestParser(object):
+ def __init__(self):
+ logger = logging.getLogger(WOK_REQUEST_LOGGER)
+ self.baseFile = logger.handlers[0].baseFilename
+ self.downloadDir = get_log_download_path()
+
+ def generateLogFile(self, records):
+ """
+ Generates a log-format text file with lines for each record specified.
+ Returns a download URI for the generated file.
+ """
+ try:
+ # sort records chronologically
+ sortedList = sorted(records, key=lambda k: k['date'] + k['time'])
+
+ # generate log file
+ fd = NamedTemporaryFile(mode='w', dir=self.downloadDir,
+ suffix='.txt', delete=False)
+
+ with fd:
+ for record in sortedList:
+ fd.write(LOG_FORMAT % record)
+
+ fd.close()
+ except IOError as e:
+ raise OperationFailed("WOKLOG0002E", {'err': str(e)})
+
+ return LOG_DOWNLOAD_URI % os.path.basename(fd.name)
+
+ def getRecords(self):
+ records = self.getRecordsFromFile(self.baseFile)
+
+ for count in range(NUM_BACKUP_FILES):
+ filename = ".".join([self.baseFile, str(count + 1)])
+ records.extend(self.getRecordsFromFile(filename))
+
+ return records
+
+ def getRecordsFromFile(self, filename):
+ """
+ Returns a list of dict, where each dict corresponds to a request
+ record.
+ """
+ records = []
+
+ if not os.path.exists(filename):
+ return []
+
+ # read records from file
+ try:
+ with open(filename) as f:
+ line = f.readline()
+ while line != "":
+ data = line.split(">>>")
+ if len(data) > 1:
+ record = json.JSONDecoder().decode(data[0])
+ record['message'] = data[1].strip()
+ records.append(record)
+
+ line = f.readline()
+
+ f. close()
+ except IOError as e:
+ raise OperationFailed("WOKLOG0002E", {'err': str(e)})
+
+ return records
+
+ def getFilteredRecords(self, filter_params):
+ """
+ Returns a dict containing the filtered list of request log entries
+ (dicts), and an optional uri for downloading results in a text file.
+ """
+ uri = None
+ results = []
+ records = self.getRecords()
+
+ # fail for unrecognized filter options
+ for key in filter_params.keys():
+ if key not in FILTER_FIELDS:
+ filters = ", ".join(FILTER_FIELDS)
+ raise InvalidParameter("WOKLOG0001E", {"filters": filters})
+
+ download = filter_params.pop('download', False)
+
+ # filter records according to parameters
+ for record in records:
+ if all(key in record and record[key] == val
+ for key, val in filter_params.iteritems()):
+ results.append(record)
+
+ # download option active: generate text file and provide donwload uri
+ if download and len(results) > 0:
+ uri = self.generateLogFile(results)
+
+ return {'uri': uri, 'records': results}
+
+
+class RequestRecord(object):
+ def __init__(self, message, **kwargs):
+ self.message = message
+ self.kwargs = kwargs
+
+ # register timestamp
+ timestamp = datetime.today()
+ self.kwargs['date'] = timestamp.strftime('%Y-%m-%d')
+ self.kwargs['time'] = timestamp.strftime('%H:%M:%S')
+
+ def __str__(self):
+ info = json.JSONEncoder().encode(self.kwargs)
+ return '%s >>> %s' % (info, self.message)
+
+ def log(self):
+ reqLogger = logging.getLogger(WOK_REQUEST_LOGGER)
+ reqLogger.info(self)
diff --git a/src/wok/utils.py b/src/wok/utils.py
index e2f1d8e..48fe414 100644
--- a/src/wok/utils.py
+++ b/src/wok/utils.py
@@ -21,6 +21,7 @@
#
import cherrypy
+import glob
import grp
import os
import psutil
@@ -31,7 +32,9 @@ import subprocess
import sys
import traceback
import xml.etree.ElementTree as ET
+
from cherrypy.lib.reprconf import Parser
+from datetime import datetime, timedelta
from multiprocessing import Process, Queue
from threading import Timer
@@ -336,6 +339,23 @@ def probe_file_permission_as_user(file, user):
return queue.get()
+def remove_old_files(globexpr, hours):
+ """
+ Delete files matching globexpr that are older than specified hours.
+ """
+ minTime = datetime.now() - timedelta(hours=hours)
+
+ try:
+ for f in glob.glob(globexpr):
+ timestamp = os.path.getmtime(f)
+ fileTime = datetime.fromtimestamp(timestamp)
+
+ if fileTime < minTime:
+ os.remove(f)
+ except (IOError, OSError) as e:
+ wok_log.error(str(e))
+
+
def get_next_clone_name(all_names, basename, name_suffix=''):
"""Find the next available name for a cloned resource.
--
1.9.1
More information about the Kimchi-devel
mailing list