[Kimchi-devel] [PATCH 10/17] V7 Ginger Base : base plugin model files

From: chandrureddy <chandra(a)linux.vnet.ibm.com&gt; --- src/wok/plugins/gingerbase/model/Makefile.am | 25 ++ src/wok/plugins/gingerbase/model/__init__.py | 18 + src/wok/plugins/gingerbase/model/config.py | 83 +++++ src/wok/plugins/gingerbase/model/cpuinfo.py | 105 ++++++ src/wok/plugins/gingerbase/model/debugreports.py | 215 +++++++++++ src/wok/plugins/gingerbase/model/host.py | 449 +++++++++++++++++++++++ src/wok/plugins/gingerbase/model/model.py | 68 ++++ src/wok/plugins/kimchi/model/debugreports.py | 213 ----------- 8 files changed, 963 insertions(+), 213 deletions(-) create mode 100644 src/wok/plugins/gingerbase/model/Makefile.am create mode 100644 src/wok/plugins/gingerbase/model/__init__.py create mode 100644 src/wok/plugins/gingerbase/model/config.py create mode 100644 src/wok/plugins/gingerbase/model/cpuinfo.py create mode 100644 src/wok/plugins/gingerbase/model/debugreports.py create mode 100644 src/wok/plugins/gingerbase/model/host.py create mode 100644 src/wok/plugins/gingerbase/model/model.py delete mode 100644 src/wok/plugins/kimchi/model/debugreports.py diff --git a/src/wok/plugins/gingerbase/model/Makefile.am b/src/wok/plugins/gingerbase/model/Makefile.am new file mode 100644 index 0000000..80c4e07 --- /dev/null +++ b/src/wok/plugins/gingerbase/model/Makefile.am @@ -0,0 +1,25 @@ +# +# Ginger Base +# +# Copyright IBM Corp, 2015 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +model_PYTHON = *.py + +modeldir = $(pythondir)/wok/plugins/gingerbase/model + +install-data-local: + $(MKDIR_P) $(DESTDIR)$(modeldir) diff --git a/src/wok/plugins/gingerbase/model/__init__.py b/src/wok/plugins/gingerbase/model/__init__.py new file mode 100644 index 0000000..355a76c --- /dev/null +++ b/src/wok/plugins/gingerbase/model/__init__.py @@ -0,0 +1,18 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/src/wok/plugins/gingerbase/model/config.py b/src/wok/plugins/gingerbase/model/config.py new file mode 100644 index 0000000..8aed61f --- /dev/null +++ b/src/wok/plugins/gingerbase/model/config.py @@ -0,0 +1,83 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2014-2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import cherrypy + +from wok.basemodel import Singleton +from wok.config import config as kconfig +from wok.config import get_version +from wok.utils import wok_log + +from ..repositories import Repositories +from ..swupdate import SoftwareUpdate +from debugreports import DebugReportsModel + + +class ConfigModel(object): + def __init__(self, **kargs): + pass + + def lookup(self, name): + proxy_port = kconfig.get('display', 'display_proxy_port') + return {'display_proxy_port': proxy_port, + 'version': get_version()} + + +class CapabilitiesModel(object): + __metaclass__ = Singleton + + def __init__(self, **kargs): + # Subscribe function to set host capabilities to be run when cherrypy + # server is up + # It is needed because some features tests depends on the server + cherrypy.engine.subscribe('start', self._set_capabilities) + + # Subscribe function to clean any Kimchi leftovers + cherrypy.engine.subscribe('stop', self._clean_leftovers) + + def _clean_leftovers(self): + pass + + def _set_capabilities(self): + wok_log.info("*** Running feature tests ***") + wok_log.info("*** Feature tests completed ***") + _set_capabilities.priority = 90 + + def lookup(self, *ident): + report_tool = DebugReportsModel.get_system_report_tool() + try: + SoftwareUpdate() + except Exception: + update_tool = False + else: + update_tool = True + + try: + repo = Repositories() + except Exception: + repo_mngt_tool = None + else: + repo_mngt_tool = repo._pkg_mnger.TYPE + + return {'system_report_tool': bool(report_tool), + 'update_tool': update_tool, + 'repo_mngt_tool': repo_mngt_tool, + } diff --git a/src/wok/plugins/gingerbase/model/cpuinfo.py b/src/wok/plugins/gingerbase/model/cpuinfo.py new file mode 100644 index 0000000..d28d4e1 --- /dev/null +++ b/src/wok/plugins/gingerbase/model/cpuinfo.py @@ -0,0 +1,105 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2014-2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import platform + +from wok.exception import InvalidParameter, InvalidOperation +from wok.utils import run_command +from wok.plugins.gingerbase.lscpu import LsCpu + + +ARCH = 'power' if platform.machine().startswith('ppc') else 'x86' + + +class CPUInfoModel(object): + """ + Get information about a CPU for hyperthreading (on x86) + or SMT (on POWER) for logic when creating templates and VMs. + """ + + def __init__(self, **kargs): + self.guest_threads_enabled = False + self.sockets = 0 + self.cores_present = 0 + self.cores_available = 0 + self.cores_per_socket = 0 + self.threads_per_core = 0 + self.max_threads = 0 + self.lscpu = LsCpu() + + if ARCH == 'power': + # IBM PowerPC + self.guest_threads_enabled = True + out, error, rc = run_command(['ppc64_cpu', '--smt']) + if rc or 'on' in out: + # SMT has to be disabled for guest to use threads as CPUs. + # rc is always zero, whether SMT is off or on. + self.guest_threads_enabled = False + out, error, rc = run_command(['ppc64_cpu', '--cores-present']) + if not rc: + self.cores_present = int(out.split()[-1]) + out, error, rc = run_command(['ppc64_cpu', '--cores-on']) + if not rc: + self.cores_available = int(out.split()[-1]) + out, error, rc = run_command(['ppc64_cpu', '--threads-per-core']) + if not rc: + self.threads_per_core = int(out.split()[-1]) + self.sockets = self.cores_present/self.threads_per_core + if self.sockets == 0: + self.sockets = 1 + self.cores_per_socket = self.cores_present/self.sockets + else: + # Intel or AMD + self.guest_threads_enabled = True + self.sockets = int(self.lscpu.get_sockets()) + self.cores_per_socket = int(self.lscpu.get_cores_per_socket()) + self.cores_present = self.cores_per_socket * self.sockets + self.cores_available = self.cores_present + self.threads_per_core = self.lscpu.get_threads_per_core() + + def lookup(self, ident): + return { + 'guest_threads_enabled': self.guest_threads_enabled, + 'sockets': self.sockets, + 'cores_per_socket': self.cores_per_socket, + 'cores_present': self.cores_present, + 'cores_available': self.cores_available, + 'threads_per_core': self.threads_per_core, + } + + def check_topology(self, vcpus, topology): + """ + param vcpus: should be an integer + param iso_path: the path of the guest ISO + param topology: {'sockets': x, 'cores': x, 'threads': x} + """ + sockets = topology['sockets'] + cores = topology['cores'] + threads = topology['threads'] + + if not self.guest_threads_enabled: + raise InvalidOperation("GGBCPUINF0003E") + if vcpus != sockets * cores * threads: + raise InvalidParameter("GGBCPUINF0002E") + if vcpus > self.cores_available * self.threads_per_core: + raise InvalidParameter("GGBCPUINF0001E") + if threads > self.threads_per_core: + raise InvalidParameter("GGBCPUINF0002E") diff --git a/src/wok/plugins/gingerbase/model/debugreports.py b/src/wok/plugins/gingerbase/model/debugreports.py new file mode 100644 index 0000000..94ab7fe --- /dev/null +++ b/src/wok/plugins/gingerbase/model/debugreports.py @@ -0,0 +1,215 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2014-2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import fnmatch +import glob +import logging +import os +import shutil +import subprocess +import time + +from wok.exception import InvalidParameter, NotFoundError, OperationFailed +from wok.exception import WokException +from wok.utils import add_task, wok_log +from wok.utils import run_command +from wok.model.tasks import TaskModel + +from wok.plugins.gingerbase import config + + +class DebugReportsModel(object): + def __init__(self, **kargs): + self.objstore = kargs['objstore'] + self.task = TaskModel(**kargs) + + def create(self, params): + ident = params.get('name').strip() + # Generate a name with time and millisec precision, if necessary + if ident is None or ident == "": + ident = 'report-' + str(int(time.time() * 1000)) + else: + if ident in self.get_list(): + raise InvalidParameter("GGBDR0008E", {"name": ident}) + taskid = self._gen_debugreport_file(ident) + return self.task.lookup(taskid) + + def get_list(self): + path = config.get_debugreports_path() + file_pattern = os.path.join(path, '*.*') + file_lists = glob.glob(file_pattern) + file_lists = [os.path.split(file)[1] for file in file_lists] + name_lists = [file.split('.', 1)[0] for file in file_lists] + + return name_lists + + def _gen_debugreport_file(self, name): + gen_cmd = self.get_system_report_tool() + + if gen_cmd is not None: + return add_task('/plugins/gingerbase/debugreports/%s' % name, + gen_cmd, self.objstore, name) + + raise OperationFailed("GGBDR0002E") + + @staticmethod + def sosreport_generate(cb, name): + def log_error(e): + log = logging.getLogger('Model') + log.warning('Exception in generating debug file: %s', e) + + try: + command = ['sosreport', '--batch', '--name=%s' % name] + output, error, retcode = run_command(command) + + if retcode != 0: + raise OperationFailed("GGBDR0003E", {'name': name, + 'err': retcode}) + + # SOSREPORT might create file in /tmp or /var/tmp + # FIXME: The right way should be passing the tar.xz file directory + # though the parameter '--tmp-dir', but it is failing in Fedora 20 + patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*'] + reports = [] + reportFile = None + for p in patterns: + reports = reports + [f for f in glob.glob(p % name)] + for f in reports: + if not fnmatch.fnmatch(f, '*.md5'): + reportFile = f + break + # Some error in sosreport happened + if reportFile is None: + wok_log.error('Debug report file not found. See sosreport ' + 'output for detail:\n%s', output) + fname = (patterns[0] % name).split('/')[-1] + raise OperationFailed('GGBDR0004E', {'name': fname}) + + md5_report_file = reportFile + '.md5' + report_file_extension = '.' + reportFile.split('.', 1)[1] + path = config.get_debugreports_path() + target = os.path.join(path, name + report_file_extension) + # Moving report + msg = 'Moving debug report file "%s" to "%s"' % (reportFile, + target) + wok_log.info(msg) + shutil.move(reportFile, target) + # Deleting md5 + msg = 'Deleting report md5 file: "%s"' % (md5_report_file) + wok_log.info(msg) + with open(md5_report_file) as f: + md5 = f.read().strip() + wok_log.info('Md5 file content: "%s"', md5) + os.remove(md5_report_file) + cb('OK', True) + return + + except WokException as e: + log_error(e) + raise + + except OSError as e: + log_error(e) + raise + + except Exception, e: + # No need to call cb to update the task status here. + # The task object will catch the exception raised here + # and update the task status there + log_error(e) + raise OperationFailed("GGBDR0005E", {'name': name, 'err': e}) + + @staticmethod + def get_system_report_tool(): + # Please add new possible debug report command here + # and implement the report generating function + # based on the new report command + report_tools = ({'cmd': 'sosreport --help', + 'fn': DebugReportsModel.sosreport_generate},) + + # check if the command can be found by shell one by one + for helper_tool in report_tools: + try: + retcode = subprocess.call(helper_tool['cmd'], shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if retcode == 0: + return helper_tool['fn'] + except Exception, e: + wok_log.info('Exception running command: %s', e) + + return None + + +class DebugReportModel(object): + def __init__(self, **kargs): + pass + + def lookup(self, name): + path = config.get_debugreports_path() + file_pattern = os.path.join(path, name) + file_pattern = file_pattern + '.*' + try: + file_target = glob.glob(file_pattern)[0] + except IndexError: + raise NotFoundError("GGBDR0001E", {'name': name}) + + ctime = os.stat(file_target).st_mtime + ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime)) + file_target = os.path.split(file_target)[-1] + file_target = os.path.join("plugins/gingerbase/data/debugreports", + file_target) + return {'uri': file_target, + 'ctime': ctime} + + def update(self, name, params): + path = config.get_debugreports_path() + file_pattern = os.path.join(path, name + '.*') + try: + file_source = glob.glob(file_pattern)[0] + except IndexError: + raise NotFoundError("GGBDR0001E", {'name': name}) + + file_target = file_source.replace(name, params['name']) + if os.path.isfile(file_target): + raise InvalidParameter('GGBDR0008E', {'name': params['name']}) + + shutil.move(file_source, file_target) + wok_log.info('%s renamed to %s' % (file_source, file_target)) + return params['name'] + + def delete(self, name): + path = config.get_debugreports_path() + file_pattern = os.path.join(path, name + '.*') + try: + file_target = glob.glob(file_pattern)[0] + except IndexError: + raise NotFoundError("GGBDR0001E", {'name': name}) + + os.remove(file_target) + + +class DebugReportContentModel(object): + def __init__(self, **kargs): + self._debugreport = DebugReportModel() + + def lookup(self, name): + return self._debugreport.lookup(name) diff --git a/src/wok/plugins/gingerbase/model/host.py b/src/wok/plugins/gingerbase/model/host.py new file mode 100644 index 0000000..062cf24 --- /dev/null +++ b/src/wok/plugins/gingerbase/model/host.py @@ -0,0 +1,449 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import platform +import psutil +import time +from cherrypy.process.plugins import BackgroundTask +from collections import defaultdict +import glob + +from wok.basemodel import Singleton +from wok.exception import InvalidOperation +from wok.exception import OperationFailed +from wok.utils import add_task, wok_log +from wok.model.tasks import TaskModel +from wok.config import config as kconfig + + +from wok.plugins.gingerbase import disks +from wok.plugins.gingerbase.repositories import Repositories +from wok.plugins.gingerbase.swupdate import SoftwareUpdate +from wok.plugins.gingerbase.model.debugreports import DebugReportsModel + + +HOST_STATS_INTERVAL = 1 + + +class HostModel(object): + def __init__(self, **kargs): + # self.conn = kargs['conn'] + self.objstore = kargs['objstore'] + self.task = TaskModel(**kargs) + self.host_info = self._get_host_info() + + def _get_ppc_cpu_info(self): + res = {} + with open('/proc/cpuinfo') as f: + for line in f.xreadlines(): + # Parse CPU, CPU's revision and CPU's clock information + for key in ['cpu', 'revision', 'clock']: + if key in line: + info = line.split(':')[1].strip() + if key == 'clock': + value = float(info.split('MHz')[0].strip()) / 1000 + else: + value = info.split('(')[0].strip() + res[key] = value + + # Power machines show, for each cpu/core, a block with + # all cpu information. Here we control the scan of the + # necessary information (1st block provides + # everything), skipping the function when find all + # information. + if len(res.keys()) == 3: + return "%(cpu)s (%(revision)s) @ %(clock)s GHz\ + " % res + + return "" + + def _get_host_info(self): + res = {} + if platform.machine().startswith('ppc'): + res['cpu_model'] = self._get_ppc_cpu_info() + else: + with open('/proc/cpuinfo') as f: + for line in f.xreadlines(): + if "model name" in line: + res['cpu_model'] = line.split(':')[1].strip() + break + + res['cpus'] = 0 + res['memory'] = 0L + + # Include IBM PowerKVM name to supported distro names + _sup_distros = platform._supported_dists + ('ibm_powerkvm',) + # 'fedora' '17' 'Beefy Miracle' + distro, version, codename = platform.linux_distribution( + supported_dists=_sup_distros) + res['os_distro'] = distro + res['os_version'] = version + res['os_codename'] = unicode(codename, "utf-8") + + return res + + def lookup(self, *name): + cpus = psutil.NUM_CPUS + + # psutil is unstable on how to get the number of + # cpus, different versions call it differently + if hasattr(psutil, 'cpu_count'): + cpus = psutil.cpu_count() + + elif hasattr(psutil, 'NUM_CPUS'): + cpus = psutil.NUM_CPUS + + elif hasattr(psutil, '_psplatform'): + for method_name in ['_get_num_cpus', 'get_num_cpus']: + + method = getattr(psutil._psplatform, method_name, None) + if method is not None: + cpus = method() + break + + self.host_info['cpus'] = cpus + if hasattr(psutil, 'phymem_usage'): + self.host_info['memory'] = psutil.phymem_usage().total + elif hasattr(psutil, 'virtual_memory'): + self.host_info['memory'] = psutil.virtual_memory().total + return self.host_info + + def swupdate(self, *name): + try: + swupdate = SoftwareUpdate() + except: + raise OperationFailed('GGBPKGUPD0004E') + + pkgs = swupdate.getNumOfUpdates() + if pkgs == 0: + raise OperationFailed('GGBPKGUPD0001E') + + wok_log.debug('Host is going to be updated.') + taskid = add_task('/plugins/gingerbase/host/swupdate', + swupdate.doUpdate, + self.objstore, None) + return self.task.lookup(taskid) + + def shutdown(self, args=None): + # Check for running vms before shutdown + # FIXME : Find alternative way to figure out if any vms running + # running_vms = self._get_vms_list_by_state('running') + # if len(running_vms) > 0: + # raise OperationFailed("GGBHOST0001E") + + wok_log.info('Host is going to shutdown.') + os.system('shutdown -h now') + + def reboot(self, args=None): + # Find running VMs + # FIXME : Find alternative way to figure out if any vms running + # running_vms = self._get_vms_list_by_state('running') + # if len(running_vms) > 0: + # raise OperationFailed("GGBHOST0002E") + + wok_log.info('Host is going to reboot.') + os.system('reboot') + + # def _get_vms_list_by_state(self, state): + # conn = self.conn.get() + # return [dom.name().decode('utf-8') + # for dom in conn.listAllDomains(0) + # if (DOM_STATE_MAP[dom.info()[0]]) == state] + + +class SoftwareUpdateProgressModel(object): + def __init__(self, **kargs): + self.task = TaskModel(**kargs) + self.objstore = kargs['objstore'] + + def lookup(self, *name): + try: + swupdate = SoftwareUpdate() + except: + raise OperationFailed('GGBPKGUPD0004E') + + taskid = add_task('/plugins/kimchi/host/swupdateprogress', + swupdate.tailUpdateLogs, self.objstore, None) + return self.task.lookup(taskid) + + +class HostStatsModel(object): + __metaclass__ = Singleton + + def __init__(self, **kargs): + self.host_stats = defaultdict(list) + self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, + self._update_host_stats) + self.host_stats_thread.start() + + def lookup(self, *name): + return {'cpu_utilization': self.host_stats['cpu_utilization'][-1], + 'memory': self.host_stats['memory'][-1], + 'disk_read_rate': self.host_stats['disk_read_rate'][-1], + 'disk_write_rate': self.host_stats['disk_write_rate'][-1], + 'net_recv_rate': self.host_stats['net_recv_rate'][-1], + 'net_sent_rate': self.host_stats['net_sent_rate'][-1]} + + def _update_host_stats(self): + preTimeStamp = self.host_stats['timestamp'] + timestamp = time.time() + # FIXME when we upgrade psutil, we can get uptime by psutil.uptime + # we get uptime by float(open("/proc/uptime").readline().split()[0]) + # and calculate the first io_rate after the OS started. + with open("/proc/uptime") as time_f: + seconds = (timestamp - preTimeStamp if preTimeStamp else + float(time_f.readline().split()[0])) + + self.host_stats['timestamp'] = timestamp + self._get_host_disk_io_rate(seconds) + self._get_host_network_io_rate(seconds) + + self._get_percentage_host_cpu_usage() + self._get_host_memory_stats() + + # store only 60 stats (1 min) + for key, value in self.host_stats.iteritems(): + if isinstance(value, list): + if len(value) == 60: + self.host_stats[key] = value[10:] + + def _get_percentage_host_cpu_usage(self): + # This is cpu usage producer. This producer will calculate the usage + # at an interval of HOST_STATS_INTERVAL. + # The psutil.cpu_percent works as non blocking. + # psutil.cpu_percent maintains a cpu time sample. + # It will update the cpu time sample when it is called. + # So only this producer can call psutil.cpu_percent in kimchi. + self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None)) + + def _get_host_memory_stats(self): + virt_mem = psutil.virtual_memory() + # available: + # the actual amount of available memory that can be given + # instantly to processes that request more memory in bytes; this + # is calculated by summing different memory values depending on + # the platform (e.g. free + buffers + cached on Linux) + memory_stats = {'total': virt_mem.total, + 'free': virt_mem.free, + 'cached': virt_mem.cached, + 'buffers': virt_mem.buffers, + 'avail': virt_mem.available} + self.host_stats['memory'].append(memory_stats) + + def _get_host_disk_io_rate(self, seconds): + disk_read_bytes = self.host_stats['disk_read_bytes'] + disk_write_bytes = self.host_stats['disk_write_bytes'] + prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0 + prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0 + + disk_io = psutil.disk_io_counters(False) + read_bytes = disk_io.read_bytes + write_bytes = disk_io.write_bytes + + rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5) + wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5) + + self.host_stats['disk_read_rate'].append(rd_rate) + self.host_stats['disk_write_rate'].append(wr_rate) + self.host_stats['disk_read_bytes'].append(read_bytes) + self.host_stats['disk_write_bytes'].append(write_bytes) + + def _get_host_network_io_rate(self, seconds): + net_recv_bytes = self.host_stats['net_recv_bytes'] + net_sent_bytes = self.host_stats['net_sent_bytes'] + prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0 + prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0 + + net_ios = None + if hasattr(psutil, 'net_io_counters'): + net_ios = psutil.net_io_counters(True) + elif hasattr(psutil, 'network_io_counters'): + net_ios = psutil.network_io_counters(True) + + recv_bytes = 0 + sent_bytes = 0 + for key in set(self.nics() + + self.wlans()) & set(net_ios.iterkeys()): + recv_bytes = recv_bytes + net_ios[key].bytes_recv + sent_bytes = sent_bytes + net_ios[key].bytes_sent + + rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5) + tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5) + + self.host_stats['net_recv_rate'].append(rx_rate) + self.host_stats['net_sent_rate'].append(tx_rate) + self.host_stats['net_recv_bytes'].append(recv_bytes) + self.host_stats['net_sent_bytes'].append(sent_bytes) + + def wlans(self): + WLAN_PATH = '/sys/class/net/*/wireless' + return [b.split('/')[-2] for b in glob.glob(WLAN_PATH)] + + # FIXME if we do not want to list usb nic + def nics(self): + NIC_PATH = '/sys/class/net/*/device' + return list(set([b.split('/')[-2] for b in glob.glob(NIC_PATH)]) - + set(self.wlans())) + + +class HostStatsHistoryModel(object): + def __init__(self, **kargs): + self.history = HostStatsModel(**kargs) + + def lookup(self, *name): + return {'cpu_utilization': self.history.host_stats['cpu_utilization'], + 'memory': self.history.host_stats['memory'], + 'disk_read_rate': self.history.host_stats['disk_read_rate'], + 'disk_write_rate': self.history.host_stats['disk_write_rate'], + 'net_recv_rate': self.history.host_stats['net_recv_rate'], + 'net_sent_rate': self.history.host_stats['net_sent_rate']} + + +class CapabilitiesModel(object): + __metaclass__ = Singleton + + def __init__(self, **kargs): + pass + + def lookup(self, *ident): + report_tool = DebugReportsModel.get_system_report_tool() + try: + SoftwareUpdate() + except Exception: + update_tool = False + else: + update_tool = True + + try: + repo = Repositories() + except Exception: + repo_mngt_tool = None + else: + repo_mngt_tool = repo._pkg_mnger.TYPE + + return {'system_report_tool': bool(report_tool), + 'update_tool': update_tool, + 'repo_mngt_tool': repo_mngt_tool, + 'federation': kconfig.get("server", "federation") + } + + +class PartitionsModel(object): + def __init__(self, **kargs): + pass + + def get_list(self): + result = disks.get_partitions_names() + return result + + +class PartitionModel(object): + def __init__(self, **kargs): + pass + + def lookup(self, name): + return disks.get_partition_details(name) + + +class PackagesUpdateModel(object): + def __init__(self, **kargs): + try: + self.host_swupdate = SoftwareUpdate() + except: + self.host_swupdate = None + + def get_list(self): + if self.host_swupdate is None: + raise OperationFailed('GGBPKGUPD0004E') + + return self.host_swupdate.getUpdates() + + +class PackageUpdateModel(object): + def __init__(self, **kargs): + pass + + def lookup(self, name): + try: + swupdate = SoftwareUpdate() + except Exception: + raise OperationFailed('GGBPKGUPD0004E') + + return swupdate.getUpdate(name) + + +class RepositoriesModel(object): + def __init__(self, **kargs): + try: + self.host_repositories = Repositories() + except: + self.host_repositories = None + + def get_list(self): + if self.host_repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return sorted(self.host_repositories.getRepositories()) + + def create(self, params): + if self.host_repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return self.host_repositories.addRepository(params) + + +class RepositoryModel(object): + def __init__(self, **kargs): + try: + self._repositories = Repositories() + except: + self._repositories = None + + def lookup(self, repo_id): + if self._repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return self._repositories.getRepository(repo_id) + + def enable(self, repo_id): + if self._repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return self._repositories.enableRepository(repo_id) + + def disable(self, repo_id): + if self._repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return self._repositories.disableRepository(repo_id) + + def update(self, repo_id, params): + if self._repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return self._repositories.updateRepository(repo_id, params) + + def delete(self, repo_id): + if self._repositories is None: + raise InvalidOperation('GGBREPOS0014E') + + return self._repositories.removeRepository(repo_id) diff --git a/src/wok/plugins/gingerbase/model/model.py b/src/wok/plugins/gingerbase/model/model.py new file mode 100644 index 0000000..1f56cc7 --- /dev/null +++ b/src/wok/plugins/gingerbase/model/model.py @@ -0,0 +1,68 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import inspect +import os + +from wok.basemodel import BaseModel +from wok.objectstore import ObjectStore +from wok.plugins.gingerbase import config +from wok.utils import import_module, listPathModules + + +class Model(BaseModel): + def __init__(self, objstore_loc=None): + + def get_instances(module_name): + instances = [] + module = import_module(module_name) + members = inspect.getmembers(module, inspect.isclass) + for cls_name, instance in members: + if inspect.getmodule(instance) == module and \ + cls_name.endswith('Model'): + instances.append(instance) + + return instances + + if objstore_loc is None: + objstore_loc = config.get_object_store() + + self.objstore = ObjectStore(objstore_loc) + kargs = {'objstore': self.objstore} + + models = [] + # Import task model from Wok + instances = get_instances('wok.model.tasks') + for instance in instances: + models.append(instance(**kargs)) + + # Import all Kimchi plugin models + this = os.path.basename(__file__) + this_mod = os.path.splitext(this)[0] + + for mod_name in listPathModules(os.path.dirname(__file__)): + if mod_name.startswith("_") or mod_name == this_mod: + continue + + instances = get_instances('wok.plugins.gingerbase.model.' + + mod_name) + for instance in instances: + models.append(instance(**kargs)) + + return super(Model, self).__init__(models) diff --git a/src/wok/plugins/kimchi/model/debugreports.py b/src/wok/plugins/kimchi/model/debugreports.py deleted file mode 100644 index dcd1b64..0000000 --- a/src/wok/plugins/kimchi/model/debugreports.py +++ /dev/null @@ -1,213 +0,0 @@ -# -# Project Kimchi -# -# Copyright IBM, Corp. 2014-2015 -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -import fnmatch -import glob -import logging -import os -import shutil -import subprocess -import time - -from wok.exception import InvalidParameter, NotFoundError, OperationFailed -from wok.exception import WokException -from wok.utils import add_task, wok_log -from wok.utils import run_command -from wok.model.tasks import TaskModel - -from wok.plugins.kimchi import config - - -class DebugReportsModel(object): - def __init__(self, **kargs): - self.objstore = kargs['objstore'] - self.task = TaskModel(**kargs) - - def create(self, params): - ident = params.get('name').strip() - # Generate a name with time and millisec precision, if necessary - if ident is None or ident == "": - ident = 'report-' + str(int(time.time() * 1000)) - else: - if ident in self.get_list(): - raise InvalidParameter("KCHDR0008E", {"name": ident}) - taskid = self._gen_debugreport_file(ident) - return self.task.lookup(taskid) - - def get_list(self): - path = config.get_debugreports_path() - file_pattern = os.path.join(path, '*.*') - file_lists = glob.glob(file_pattern) - file_lists = [os.path.split(file)[1] for file in file_lists] - name_lists = [file.split('.', 1)[0] for file in file_lists] - - return name_lists - - def _gen_debugreport_file(self, name): - gen_cmd = self.get_system_report_tool() - - if gen_cmd is not None: - return add_task('/plugins/kimchi/debugreports/%s' % name, gen_cmd, - self.objstore, name) - - raise OperationFailed("KCHDR0002E") - - @staticmethod - def sosreport_generate(cb, name): - def log_error(e): - log = logging.getLogger('Model') - log.warning('Exception in generating debug file: %s', e) - - try: - command = ['sosreport', '--batch', '--name=%s' % name] - output, error, retcode = run_command(command) - - if retcode != 0: - raise OperationFailed("KCHDR0003E", {'name': name, - 'err': retcode}) - - # SOSREPORT might create file in /tmp or /var/tmp - # FIXME: The right way should be passing the tar.xz file directory - # though the parameter '--tmp-dir', but it is failing in Fedora 20 - patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*'] - reports = [] - reportFile = None - for p in patterns: - reports = reports + [f for f in glob.glob(p % name)] - for f in reports: - if not fnmatch.fnmatch(f, '*.md5'): - reportFile = f - break - # Some error in sosreport happened - if reportFile is None: - wok_log.error('Debug report file not found. See sosreport ' - 'output for detail:\n%s', output) - fname = (patterns[0] % name).split('/')[-1] - raise OperationFailed('KCHDR0004E', {'name': fname}) - - md5_report_file = reportFile + '.md5' - report_file_extension = '.' + reportFile.split('.', 1)[1] - path = config.get_debugreports_path() - target = os.path.join(path, name + report_file_extension) - # Moving report - msg = 'Moving debug report file "%s" to "%s"' % (reportFile, - target) - wok_log.info(msg) - shutil.move(reportFile, target) - # Deleting md5 - msg = 'Deleting report md5 file: "%s"' % (md5_report_file) - wok_log.info(msg) - with open(md5_report_file) as f: - md5 = f.read().strip() - wok_log.info('Md5 file content: "%s"', md5) - os.remove(md5_report_file) - cb('OK', True) - return - - except WokException as e: - log_error(e) - raise - - except OSError as e: - log_error(e) - raise - - except Exception, e: - # No need to call cb to update the task status here. - # The task object will catch the exception raised here - # and update the task status there - log_error(e) - raise OperationFailed("KCHDR0005E", {'name': name, 'err': e}) - - @staticmethod - def get_system_report_tool(): - # Please add new possible debug report command here - # and implement the report generating function - # based on the new report command - report_tools = ({'cmd': 'sosreport --help', - 'fn': DebugReportsModel.sosreport_generate},) - - # check if the command can be found by shell one by one - for helper_tool in report_tools: - try: - retcode = subprocess.call(helper_tool['cmd'], shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if retcode == 0: - return helper_tool['fn'] - except Exception, e: - wok_log.info('Exception running command: %s', e) - - return None - - -class DebugReportModel(object): - def __init__(self, **kargs): - pass - - def lookup(self, name): - path = config.get_debugreports_path() - file_pattern = os.path.join(path, name) - file_pattern = file_pattern + '.*' - try: - file_target = glob.glob(file_pattern)[0] - except IndexError: - raise NotFoundError("KCHDR0001E", {'name': name}) - - ctime = os.stat(file_target).st_mtime - ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime)) - file_target = os.path.split(file_target)[-1] - file_target = os.path.join("plugins/kimchi/data/debugreports", - file_target) - return {'uri': file_target, - 'ctime': ctime} - - def update(self, name, params): - path = config.get_debugreports_path() - file_pattern = os.path.join(path, name + '.*') - try: - file_source = glob.glob(file_pattern)[0] - except IndexError: - raise NotFoundError("KCHDR0001E", {'name': name}) - - file_target = file_source.replace(name, params['name']) - if os.path.isfile(file_target): - raise InvalidParameter('KCHDR0008E', {'name': params['name']}) - - shutil.move(file_source, file_target) - wok_log.info('%s renamed to %s' % (file_source, file_target)) - return params['name'] - - def delete(self, name): - path = config.get_debugreports_path() - file_pattern = os.path.join(path, name + '.*') - try: - file_target = glob.glob(file_pattern)[0] - except IndexError: - raise NotFoundError("KCHDR0001E", {'name': name}) - - os.remove(file_target) - - -class DebugReportContentModel(object): - def __init__(self, **kargs): - self._debugreport = DebugReportModel() - - def lookup(self, name): - return self._debugreport.lookup(name) -- 2.1.0