Re: [Kimchi-devel] [PATCH 07/17] Ginger Base : base plugin model files

On 01/09/2015 14:58, chandra(a)linux.vnet.ibm.com wrote: > From: chandrureddy <chandra(a)linux.vnet.ibm.com&gt; > > --- > plugins/gingerbase/model/Makefile.am | 25 ++ > plugins/gingerbase/model/__init__.py | 18 ++ > plugins/gingerbase/model/cpuinfo.py | 133 ++++++++++ > plugins/gingerbase/model/debugreports.py | 213 ++++++++++++++++ > plugins/gingerbase/model/host.py | 410 > +++++++++++++++++++++++++++++++ > plugins/gingerbase/model/model.py | 49 ++++ > plugins/gingerbase/model/tasks.py | 64 +++++ > plugins/kimchi/model/debugreports.py | 213 ---------------- > 8 files changed, 912 insertions(+), 213 deletions(-) > create mode 100644 plugins/gingerbase/model/Makefile.am > create mode 100644 plugins/gingerbase/model/__init__.py > create mode 100644 plugins/gingerbase/model/cpuinfo.py Why did you add it to gingerbase? It is used for Kimchi on different areas.

> create mode 100644 plugins/gingerbase/model/debugreports.py > create mode 100644 plugins/gingerbase/model/host.py > create mode 100644 plugins/gingerbase/model/model.py > create mode 100644 plugins/gingerbase/model/tasks.py Why did you add tasks.py to gingerbase?

> delete mode 100644 plugins/kimchi/model/debugreports.py > > diff --git a/plugins/gingerbase/model/Makefile.am > b/plugins/gingerbase/model/Makefile.am > new file mode 100644 > index 0000000..6218b47 > --- /dev/null > +++ b/plugins/gingerbase/model/Makefile.am > @@ -0,0 +1,25 @@ > +# > +# Kimchi > +# > +# Copyright IBM Corp, 2013 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +model_PYTHON = *.py > + > +modeldir = $(pythondir)/wok/plugins/gingerbase/model > + > +install-data-local: > + $(MKDIR_P) $(DESTDIR)$(modeldir) > diff --git a/plugins/gingerbase/model/__init__.py > b/plugins/gingerbase/model/__init__.py > new file mode 100644 > index 0000000..ca7ede4 > --- /dev/null > +++ b/plugins/gingerbase/model/__init__.py > @@ -0,0 +1,18 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > diff --git a/plugins/gingerbase/model/cpuinfo.py > b/plugins/gingerbase/model/cpuinfo.py > new file mode 100644 > index 0000000..fad8814 > --- /dev/null > +++ b/plugins/gingerbase/model/cpuinfo.py > @@ -0,0 +1,133 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014-2015 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +import platform > +# from xml.etree import ElementTree as ET > + > +from wok.exception import InvalidParameter, InvalidOperation > +from wok.utils import run_command, wok_log > +from ..lscpu import LsCpu > + > + > +ARCH = 'power' if platform.machine().startswith('ppc') else 'x86' > + > + > +# def get_topo_capabilities(connect): > +# """ > +# This helper function exists solely to be overridden for > +# mockmodel tests. Since other modules use getCapabilies(), > +# it can't be overridden directly. > +# """ > +# xml = connect.getCapabilities() > +# capabilities = ET.fromstring(xml) > +# return capabilities.find('host').find('cpu').find('topology') > + > + > +class CPUInfoModel(object): > + """ > + Get information about a CPU for hyperthreading (on x86) > + or SMT (on POWER) for logic when creating templates and VMs. > + """ > + > + def __init__(self, **kargs): > + self.guest_threads_enabled = False > + self.sockets = 0 > + self.cores_present = 0 > + self.cores_available = 0 > + self.cores_per_socket = 0 > + self.threads_per_core = 0 > + self.max_threads = 0 > + self.lscpu = LsCpu() > + > + # self.conn = kargs['conn'] > + # libvirt_topology = None > + # try: > + # connect = self.conn.get() > + # libvirt_topology = get_topo_capabilities(connect) > + # except Exception as e: > + # wok_log.info("Unable to get CPU topology capabilities: > %s" > + # % e.message) > + # return > + # if libvirt_topology is None: > + # wok_log.info("cpu_info topology not supported.") > + # return > + > + if ARCH == 'power': > + # IBM PowerPC > + self.guest_threads_enabled = True > + out, error, rc = run_command(['ppc64_cpu', '--smt']) > + if rc or 'on' in out: > + # SMT has to be disabled for guest to use threads as > CPUs. > + # rc is always zero, whether SMT is off or on. > + self.guest_threads_enabled = False > + out, error, rc = run_command(['ppc64_cpu', > '--cores-present']) > + if not rc: > + self.cores_present = int(out.split()[-1]) > + out, error, rc = run_command(['ppc64_cpu', '--cores-on']) > + if not rc: > + self.cores_available = int(out.split()[-1]) > + out, error, rc = run_command(['ppc64_cpu', > '--threads-per-core']) > + if not rc: > + self.threads_per_core = int(out.split()[-1]) > + self.sockets = self.cores_present/self.threads_per_core > + if self.sockets == 0: > + self.sockets = 1 > + self.cores_per_socket = self.cores_present/self.sockets > + else: > + # Intel or AMD > + self.guest_threads_enabled = True > + # self.sockets = int(libvirt_topology.get('sockets')) > + # self.cores_per_socket = > int(libvirt_topology.get('cores')) > + # self.cores_present = self.cores_per_socket * self.sockets > + # self.cores_available = self.cores_present > + # self.threads_per_core = > int(libvirt_topology.get('threads')) > + self.sockets = int(self.lscpu.get_sockets()) > + self.cores_per_socket = > int(self.lscpu.get_cores_per_socket()) > + self.cores_present = self.cores_per_socket * self.sockets > + self.cores_available = self.cores_present > + self.threads_per_core = self.lscpu.get_threads_per_core() > + > + def lookup(self, ident): > + return { > + 'guest_threads_enabled': self.guest_threads_enabled, > + 'sockets': self.sockets, > + 'cores_per_socket': self.cores_per_socket, > + 'cores_present': self.cores_present, > + 'cores_available': self.cores_available, > + 'threads_per_core': self.threads_per_core, > + } > + > + def check_topology(self, vcpus, topology): > + """ > + param vcpus: should be an integer > + param iso_path: the path of the guest ISO > + param topology: {'sockets': x, 'cores': x, 'threads': x} > + """ > + sockets = topology['sockets'] > + cores = topology['cores'] > + threads = topology['threads'] > + > + if not self.guest_threads_enabled: > + raise InvalidOperation("GGBCPUINF0003E") > + if vcpus != sockets * cores * threads: > + raise InvalidParameter("GGBCPUINF0002E") > + if vcpus > self.cores_available * self.threads_per_core: > + raise InvalidParameter("GGBCPUINF0001E") > + if threads > self.threads_per_core: > + raise InvalidParameter("GGBCPUINF0002E") > diff --git a/plugins/gingerbase/model/debugreports.py > b/plugins/gingerbase/model/debugreports.py > new file mode 100644 > index 0000000..fc91a9c > --- /dev/null > +++ b/plugins/gingerbase/model/debugreports.py > @@ -0,0 +1,213 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014-2015 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +import fnmatch > +import glob > +import logging > +import os > +import shutil > +import subprocess > +import time > + > +from wok.exception import InvalidParameter, NotFoundError, > OperationFailed > +from wok.exception import WokException > +from wok.utils import add_task, wok_log > +from wok.utils import run_command > + > +from .. import config > +from tasks import TaskModel > + > + > +class DebugReportsModel(object): > + def __init__(self, **kargs): > + self.objstore = kargs['objstore'] > + self.task = TaskModel(**kargs) > + > + def create(self, params): > + ident = params.get('name').strip() > + # Generate a name with time and millisec precision, if > necessary > + if ident is None or ident == "": > + ident = 'report-' + str(int(time.time() * 1000)) > + else: > + if ident in self.get_list(): > + raise InvalidParameter("GGBDR0008E", {"name": ident}) > + taskid = self._gen_debugreport_file(ident) > + return self.task.lookup(taskid) > + > + def get_list(self): > + path = config.get_debugreports_path() > + file_pattern = os.path.join(path, '*.*') > + file_lists = glob.glob(file_pattern) > + file_lists = [os.path.split(file)[1] for file in file_lists] > + name_lists = [file.split('.', 1)[0] for file in file_lists] > + > + return name_lists > + > + def _gen_debugreport_file(self, name): > + gen_cmd = self.get_system_report_tool() > + > + if gen_cmd is not None: > + return add_task('/plugins/gingerbase/debugreports/%s' % > name, gen_cmd, > + self.objstore, name) > + > + raise OperationFailed("GGBDR0002E") > + > + @staticmethod > + def sosreport_generate(cb, name): > + def log_error(e): > + log = logging.getLogger('Model') > + log.warning('Exception in generating debug file: %s', e) > + > + try: > + command = ['sosreport', '--batch', '--name=%s' % name] > + output, error, retcode = run_command(command) > + > + if retcode != 0: > + raise OperationFailed("GGBDR0003E", {'name': name, > + 'err': retcode}) > + > + # SOSREPORT might create file in /tmp or /var/tmp > + # FIXME: The right way should be passing the tar.xz file > directory > + # though the parameter '--tmp-dir', but it is failing in > Fedora 20 > + patterns = ['/tmp/sosreport-%s-*', > '/var/tmp/sosreport-%s-*'] > + reports = [] > + reportFile = None > + for p in patterns: > + reports = reports + [f for f in glob.glob(p % name)] > + for f in reports: > + if not fnmatch.fnmatch(f, '*.md5'): > + reportFile = f > + break > + # Some error in sosreport happened > + if reportFile is None: > + wok_log.error('Debug report file not found. See > sosreport ' > + 'output for detail:\n%s', output) > + fname = (patterns[0] % name).split('/')[-1] > + raise OperationFailed('GGBDR0004E', {'name': fname}) > + > + md5_report_file = reportFile + '.md5' > + report_file_extension = '.' + reportFile.split('.', 1)[1] > + path = config.get_debugreports_path() > + target = os.path.join(path, name + report_file_extension) > + # Moving report > + msg = 'Moving debug report file "%s" to "%s"' % > (reportFile, > + target) > + wok_log.info(msg) > + shutil.move(reportFile, target) > + # Deleting md5 > + msg = 'Deleting report md5 file: "%s"' % (md5_report_file) > + wok_log.info(msg) > + with open(md5_report_file) as f: > + md5 = f.read().strip() > + wok_log.info('Md5 file content: "%s"', md5) > + os.remove(md5_report_file) > + cb('OK', True) > + return > + > + except WokException as e: > + log_error(e) > + raise > + > + except OSError as e: > + log_error(e) > + raise > + > + except Exception, e: > + # No need to call cb to update the task status here. > + # The task object will catch the exception raised here > + # and update the task status there > + log_error(e) > + raise OperationFailed("GGBDR0005E", {'name': name, > 'err': e}) > + > + @staticmethod > + def get_system_report_tool(): > + # Please add new possible debug report command here > + # and implement the report generating function > + # based on the new report command > + report_tools = ({'cmd': 'sosreport --help', > + 'fn': DebugReportsModel.sosreport_generate},) > + > + # check if the command can be found by shell one by one > + for helper_tool in report_tools: > + try: > + retcode = subprocess.call(helper_tool['cmd'], > shell=True, > + stdout=subprocess.PIPE, > + stderr=subprocess.PIPE) > + if retcode == 0: > + return helper_tool['fn'] > + except Exception, e: > + wok_log.info('Exception running command: %s', e) > + > + return None > + > + > +class DebugReportModel(object): > + def __init__(self, **kargs): > + pass > + > + def lookup(self, name): > + path = config.get_debugreports_path() > + file_pattern = os.path.join(path, name) > + file_pattern = file_pattern + '.*' > + try: > + file_target = glob.glob(file_pattern)[0] > + except IndexError: > + raise NotFoundError("GGBDR0001E", {'name': name}) > + > + ctime = os.stat(file_target).st_mtime > + ctime = time.strftime("%Y-%m-%d-%H:%M:%S", > time.localtime(ctime)) > + file_target = os.path.split(file_target)[-1] > + file_target = > os.path.join("plugins/gingerbase/data/debugreports", > + file_target) > + return {'uri': file_target, > + 'ctime': ctime} > + > + def update(self, name, params): > + path = config.get_debugreports_path() > + file_pattern = os.path.join(path, name + '.*') > + try: > + file_source = glob.glob(file_pattern)[0] > + except IndexError: > + raise NotFoundError("GGBDR0001E", {'name': name}) > + > + file_target = file_source.replace(name, params['name']) > + if os.path.isfile(file_target): > + raise InvalidParameter('GGBDR0008E', {'name': > params['name']}) > + > + shutil.move(file_source, file_target) > + wok_log.info('%s renamed to %s' % (file_source, file_target)) > + return params['name'] > + > + def delete(self, name): > + path = config.get_debugreports_path() > + file_pattern = os.path.join(path, name + '.*') > + try: > + file_target = glob.glob(file_pattern)[0] > + except IndexError: > + raise NotFoundError("GGBDR0001E", {'name': name}) > + > + os.remove(file_target) > + > + > +class DebugReportContentModel(object): > + def __init__(self, **kargs): > + self._debugreport = DebugReportModel() > + > + def lookup(self, name): > + return self._debugreport.lookup(name) > diff --git a/plugins/gingerbase/model/host.py > b/plugins/gingerbase/model/host.py > new file mode 100644 > index 0000000..6522a27 > --- /dev/null > +++ b/plugins/gingerbase/model/host.py > @@ -0,0 +1,410 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014-2015 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +import os > +import platform > +import psutil > +import time > +from cherrypy.process.plugins import BackgroundTask > +from collections import defaultdict > + > +from wok import netinfo > +from wok.basemodel import Singleton > +from wok.exception import InvalidOperation, InvalidParameter > +from wok.exception import NotFoundError, OperationFailed > +from wok.utils import add_task, wok_log > + > + > +from .. import disks > +from ..repositories import Repositories > +from ..swupdate import SoftwareUpdate > +from debugreports import DebugReportsModel > +from tasks import TaskModel > +from wok.config import config as kconfig > + > + > +HOST_STATS_INTERVAL = 1 > + > + > +class HostModel(object): > + def __init__(self, **kargs): > + # self.conn = kargs['conn'] > + self.objstore = kargs['objstore'] > + self.task = TaskModel(**kargs) > + self.host_info = self._get_host_info() > + > + def _get_ppc_cpu_info(self): > + res = {} > + with open('/proc/cpuinfo') as f: > + for line in f.xreadlines(): > + # Parse CPU, CPU's revision and CPU's clock information > + for key in ['cpu', 'revision', 'clock']: > + if key in line: > + info = line.split(':')[1].strip() > + if key == 'clock': > + value = > float(info.split('MHz')[0].strip()) / 1000 > + else: > + value = info.split('(')[0].strip() > + res[key] = value > + > + # Power machines show, for each cpu/core, a > block with > + # all cpu information. Here we control the > scan of the > + # necessary information (1st block provides > + # everything), skipping the function when > find all > + # information. > + if len(res.keys()) == 3: > + return "%(cpu)s (%(revision)s) @ > %(clock)s GHz\ > + " % res > + > + return "" > + > + def _get_host_info(self): > + res = {} > + if platform.machine().startswith('ppc'): > + res['cpu_model'] = self._get_ppc_cpu_info() > + else: > + with open('/proc/cpuinfo') as f: > + for line in f.xreadlines(): > + if "model name" in line: > + res['cpu_model'] = line.split(':')[1].strip() > + break > + > + res['cpus'] = 0 > + res['memory'] = 0L > + > + # Include IBM PowerKVM name to supported distro names > + _sup_distros = platform._supported_dists + ('ibm_powerkvm',) > + # 'fedora' '17' 'Beefy Miracle' > + distro, version, codename = platform.linux_distribution( > + supported_dists=_sup_distros) > + res['os_distro'] = distro > + res['os_version'] = version > + res['os_codename'] = unicode(codename, "utf-8") > + > + return res > + > + def lookup(self, *name): > + cpus = psutil.NUM_CPUS > + > + # psutil is unstable on how to get the number of > + # cpus, different versions call it differently > + if hasattr(psutil, 'cpu_count'): > + cpus = psutil.cpu_count() > + > + elif hasattr(psutil, '_psplatform'): > + for method_name in ['_get_num_cpus', 'get_num_cpus']: > + > + method = getattr(psutil._psplatform, method_name, None) > + if method is not None: > + cpus = method() > + break > + > + self.host_info['cpus'] = cpus > + self.host_info['memory'] = psutil.phymem_usage().total > + return self.host_info > + > + def swupdate(self, *name): > + try: > + swupdate = SoftwareUpdate() > + except: > + raise OperationFailed('GGBPKGUPD0004E') > + > + pkgs = swupdate.getNumOfUpdates() > + if pkgs == 0: > + raise OperationFailed('GGBPKGUPD0001E') > + > + wok_log.debug('Host is going to be updated.') > + taskid = add_task('/plugins/gingerbase/host/swupdate', > swupdate.doUpdate, > + self.objstore, None) > + return self.task.lookup(taskid) > + > + def shutdown(self, args=None): > + # Check for running vms before shutdown > + # FIXME : Find alternative way to figure out if any vms running > + # running_vms = self._get_vms_list_by_state('running') > + # if len(running_vms) > 0: > + # raise OperationFailed("GGBHOST0001E") > + > + wok_log.info('Host is going to shutdown.') > + os.system('shutdown -h now') > + > + def reboot(self, args=None): > + # Find running VMs > + # FIXME : Find alternative way to figure out if any vms running > + # running_vms = self._get_vms_list_by_state('running') > + # if len(running_vms) > 0: > + # raise OperationFailed("GGBHOST0002E") > + > + wok_log.info('Host is going to reboot.') > + os.system('reboot') > + > + # def _get_vms_list_by_state(self, state): > + # conn = self.conn.get() > + # return [dom.name().decode('utf-8') > + # for dom in conn.listAllDomains(0) > + # if (DOM_STATE_MAP[dom.info()[0]]) == state] > + > + > +class HostStatsModel(object): > + __metaclass__ = Singleton > + > + def __init__(self, **kargs): > + self.host_stats = defaultdict(list) > + self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, > + self._update_host_stats) > + self.host_stats_thread.start() > + > + def lookup(self, *name): > + return {'cpu_utilization': > self.host_stats['cpu_utilization'][-1], > + 'memory': self.host_stats['memory'][-1], > + 'disk_read_rate': > self.host_stats['disk_read_rate'][-1], > + 'disk_write_rate': > self.host_stats['disk_write_rate'][-1], > + 'net_recv_rate': self.host_stats['net_recv_rate'][-1], > + 'net_sent_rate': self.host_stats['net_sent_rate'][-1]} > + > + def _update_host_stats(self): > + preTimeStamp = self.host_stats['timestamp'] > + timestamp = time.time() > + # FIXME when we upgrade psutil, we can get uptime by > psutil.uptime > + # we get uptime by > float(open("/proc/uptime").readline().split()[0]) > + # and calculate the first io_rate after the OS started. > + with open("/proc/uptime") as time_f: > + seconds = (timestamp - preTimeStamp if preTimeStamp else > + float(time_f.readline().split()[0])) > + > + self.host_stats['timestamp'] = timestamp > + self._get_host_disk_io_rate(seconds) > + self._get_host_network_io_rate(seconds) > + > + self._get_percentage_host_cpu_usage() > + self._get_host_memory_stats() > + > + # store only 60 stats (1 min) > + for key, value in self.host_stats.iteritems(): > + if isinstance(value, list): > + if len(value) == 60: > + self.host_stats[key] = value[10:] > + > + def _get_percentage_host_cpu_usage(self): > + # This is cpu usage producer. This producer will calculate > the usage > + # at an interval of HOST_STATS_INTERVAL. > + # The psutil.cpu_percent works as non blocking. > + # psutil.cpu_percent maintains a cpu time sample. > + # It will update the cpu time sample when it is called. > + # So only this producer can call psutil.cpu_percent in kimchi. > + self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None)) > + > + def _get_host_memory_stats(self): > + virt_mem = psutil.virtual_memory() > + # available: > + # the actual amount of available memory that can be given > + # instantly to processes that request more memory in bytes; > this > + # is calculated by summing different memory values > depending on > + # the platform (e.g. free + buffers + cached on Linux) > + memory_stats = {'total': virt_mem.total, > + 'free': virt_mem.free, > + 'cached': virt_mem.cached, > + 'buffers': virt_mem.buffers, > + 'avail': virt_mem.available} > + self.host_stats['memory'].append(memory_stats) > + > + def _get_host_disk_io_rate(self, seconds): > + disk_read_bytes = self.host_stats['disk_read_bytes'] > + disk_write_bytes = self.host_stats['disk_write_bytes'] > + prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0 > + prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes > else 0 > + > + disk_io = psutil.disk_io_counters(False) > + read_bytes = disk_io.read_bytes > + write_bytes = disk_io.write_bytes > + > + rd_rate = int(float(read_bytes - prev_read_bytes) / seconds > + 0.5) > + wr_rate = int(float(write_bytes - prev_write_bytes) / > seconds + 0.5) > + > + self.host_stats['disk_read_rate'].append(rd_rate) > + self.host_stats['disk_write_rate'].append(wr_rate) > + self.host_stats['disk_read_bytes'].append(read_bytes) > + self.host_stats['disk_write_bytes'].append(write_bytes) > + > + def _get_host_network_io_rate(self, seconds): > + net_recv_bytes = self.host_stats['net_recv_bytes'] > + net_sent_bytes = self.host_stats['net_sent_bytes'] > + prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0 > + prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0 > + > + net_ios = psutil.network_io_counters(True) > + recv_bytes = 0 > + sent_bytes = 0 > + for key in set(netinfo.nics() + > + netinfo.wlans()) & set(net_ios.iterkeys()): > + recv_bytes = recv_bytes + net_ios[key].bytes_recv > + sent_bytes = sent_bytes + net_ios[key].bytes_sent > + > + rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds > + 0.5) > + tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds > + 0.5) > + > + self.host_stats['net_recv_rate'].append(rx_rate) > + self.host_stats['net_sent_rate'].append(tx_rate) > + self.host_stats['net_recv_bytes'].append(recv_bytes) > + self.host_stats['net_sent_bytes'].append(sent_bytes) > + > + > +class HostStatsHistoryModel(object): > + def __init__(self, **kargs): > + self.history = HostStatsModel(**kargs) > + > + def lookup(self, *name): > + return {'cpu_utilization': > self.history.host_stats['cpu_utilization'], > + 'memory': self.history.host_stats['memory'], > + 'disk_read_rate': > self.history.host_stats['disk_read_rate'], > + 'disk_write_rate': > self.history.host_stats['disk_write_rate'], > + 'net_recv_rate': > self.history.host_stats['net_recv_rate'], > + 'net_sent_rate': > self.history.host_stats['net_sent_rate']} > + > + > +class CapabilitiesModel(object): > + __metaclass__ = Singleton > + > + def __init__(self, **kargs): > + pass > + > + > + def lookup(self, *ident): > + report_tool = DebugReportsModel.get_system_report_tool() > + try: > + SoftwareUpdate() > + except Exception: > + update_tool = False > + else: > + update_tool = True > + > + try: > + repo = Repositories() > + except Exception: > + repo_mngt_tool = None > + else: > + repo_mngt_tool = repo._pkg_mnger.TYPE > + > + return { > + 'system_report_tool': bool(report_tool), > + 'update_tool': update_tool, > + 'repo_mngt_tool': repo_mngt_tool, > + 'federation': kconfig.get("server", "federation") > + } > + > + > +class PartitionsModel(object): > + def __init__(self, **kargs): > + pass > + > + def get_list(self): > + result = disks.get_partitions_names() > + return result > + > + > +class PartitionModel(object): > + def __init__(self, **kargs): > + pass > + > + def lookup(self, name): > + return disks.get_partition_details(name) > + > +class PackagesUpdateModel(object): > + def __init__(self, **kargs): > + try: > + self.host_swupdate = SoftwareUpdate() > + except: > + self.host_swupdate = None > + > + def get_list(self): > + if self.host_swupdate is None: > + raise OperationFailed('GGBPKGUPD0004E') > + > + return self.host_swupdate.getUpdates() > + > + > +class PackageUpdateModel(object): > + def __init__(self, **kargs): > + pass > + > + def lookup(self, name): > + try: > + swupdate = SoftwareUpdate() > + except Exception: > + raise OperationFailed('GGBPKGUPD0004E') > + > + return swupdate.getUpdate(name) > + > + > +class RepositoriesModel(object): > + def __init__(self, **kargs): > + try: > + self.host_repositories = Repositories() > + except: > + self.host_repositories = None > + > + def get_list(self): > + if self.host_repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return sorted(self.host_repositories.getRepositories()) > + > + def create(self, params): > + if self.host_repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return self.host_repositories.addRepository(params) > + > + > +class RepositoryModel(object): > + def __init__(self, **kargs): > + try: > + self._repositories = Repositories() > + except: > + self._repositories = None > + > + def lookup(self, repo_id): > + if self._repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return self._repositories.getRepository(repo_id) > + > + def enable(self, repo_id): > + if self._repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return self._repositories.enableRepository(repo_id) > + > + def disable(self, repo_id): > + if self._repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return self._repositories.disableRepository(repo_id) > + > + def update(self, repo_id, params): > + if self._repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return self._repositories.updateRepository(repo_id, params) > + > + def delete(self, repo_id): > + if self._repositories is None: > + raise InvalidOperation('GGBREPOS0014E') > + > + return self._repositories.removeRepository(repo_id) > diff --git a/plugins/gingerbase/model/model.py > b/plugins/gingerbase/model/model.py > new file mode 100644 > index 0000000..2da70f4 > --- /dev/null > +++ b/plugins/gingerbase/model/model.py > @@ -0,0 +1,49 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014-2015 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +import inspect > +import os > + > +from wok.basemodel import BaseModel > +from wok.objectstore import ObjectStore > +from wok.utils import import_module, listPathModules > + > + > +class Model(BaseModel): > + def __init__(self, objstore_loc=None): > + > + self.objstore = ObjectStore(objstore_loc) > + kargs = {'objstore': self.objstore} > + > + this = os.path.basename(__file__) > + this_mod = os.path.splitext(this)[0] > + > + models = [] > + for mod_name in listPathModules(os.path.dirname(__file__)): > + if mod_name.startswith("_") or mod_name == this_mod: > + continue > + > + module = import_module('plugins.gingerbase.model.' + > mod_name) > + members = inspect.getmembers(module, inspect.isclass) > + for cls_name, instance in members: > + if inspect.getmodule(instance) == module: > + if cls_name.endswith('Model'): > + models.append(instance(**kargs)) > + > + return super(Model, self).__init__(models) > diff --git a/plugins/gingerbase/model/tasks.py > b/plugins/gingerbase/model/tasks.py > new file mode 100644 > index 0000000..fe23e32 > --- /dev/null > +++ b/plugins/gingerbase/model/tasks.py > @@ -0,0 +1,64 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > + > +import time > + > +from wok.exception import TimeoutExpired > + > + > +class TasksModel(object): > + def __init__(self, **kargs): > + self.objstore = kargs['objstore'] > + > + def get_list(self): > + with self.objstore as session: > + return session.get_list('task') > + > + > +class TaskModel(object): > + def __init__(self, **kargs): > + self.objstore = kargs['objstore'] > + > + def lookup(self, id): > + with self.objstore as session: > + return session.get('task', str(id)) > + > + def wait(self, id, timeout=10): > + """Wait for a task until it stops running (successfully or > due to > + an error). If the Task finishes its execution before > <timeout>, this > + function returns normally; otherwise an exception is raised. > + > + Parameters: > + id -- The Task ID. > + timeout -- The maximum time, in seconds, that this function > should wait > + for the Task. If the Task runs for more than <timeout>, > + "TimeoutExpired" is raised. > + """ > + for i in range(0, timeout): > + with self.objstore as session: > + task = session.get('task', str(id)) > + > + if task['status'] != 'running': > + return > + > + time.sleep(1) > + > + raise TimeoutExpired('GGBASYNC0001E', {'seconds': timeout, > + 'task': > task['target_uri']}) > diff --git a/plugins/kimchi/model/debugreports.py > b/plugins/kimchi/model/debugreports.py > deleted file mode 100644 > index d20eb12..0000000 > --- a/plugins/kimchi/model/debugreports.py > +++ /dev/null > @@ -1,213 +0,0 @@ > -# > -# Project Kimchi > -# > -# Copyright IBM, Corp. 2014-2015 > -# > -# This library is free software; you can redistribute it and/or > -# modify it under the terms of the GNU Lesser General Public > -# License as published by the Free Software Foundation; either > -# version 2.1 of the License, or (at your option) any later version. > -# > -# This library is distributed in the hope that it will be useful, > -# but WITHOUT ANY WARRANTY; without even the implied warranty of > -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > -# Lesser General Public License for more details. > -# > -# You should have received a copy of the GNU Lesser General Public > -# License along with this library; if not, write to the Free Software > -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > - > -import fnmatch > -import glob > -import logging > -import os > -import shutil > -import subprocess > -import time > - > -from wok.exception import InvalidParameter, NotFoundError, > OperationFailed > -from wok.exception import WokException > -from wok.utils import add_task, wok_log > -from wok.utils import run_command > - > -from .. import config > -from tasks import TaskModel > - > - > -class DebugReportsModel(object): > - def __init__(self, **kargs): > - self.objstore = kargs['objstore'] > - self.task = TaskModel(**kargs) > - > - def create(self, params): > - ident = params.get('name').strip() > - # Generate a name with time and millisec precision, if > necessary > - if ident is None or ident == "": > - ident = 'report-' + str(int(time.time() * 1000)) > - else: > - if ident in self.get_list(): > - raise InvalidParameter("KCHDR0008E", {"name": ident}) > - taskid = self._gen_debugreport_file(ident) > - return self.task.lookup(taskid) > - > - def get_list(self): > - path = config.get_debugreports_path() > - file_pattern = os.path.join(path, '*.*') > - file_lists = glob.glob(file_pattern) > - file_lists = [os.path.split(file)[1] for file in file_lists] > - name_lists = [file.split('.', 1)[0] for file in file_lists] > - > - return name_lists > - > - def _gen_debugreport_file(self, name): > - gen_cmd = self.get_system_report_tool() > - > - if gen_cmd is not None: > - return add_task('/plugins/kimchi/debugreports/%s' % > name, gen_cmd, > - self.objstore, name) > - > - raise OperationFailed("KCHDR0002E") > - > - @staticmethod > - def sosreport_generate(cb, name): > - def log_error(e): > - log = logging.getLogger('Model') > - log.warning('Exception in generating debug file: %s', e) > - > - try: > - command = ['sosreport', '--batch', '--name=%s' % name] > - output, error, retcode = run_command(command) > - > - if retcode != 0: > - raise OperationFailed("KCHDR0003E", {'name': name, > - 'err': retcode}) > - > - # SOSREPORT might create file in /tmp or /var/tmp > - # FIXME: The right way should be passing the tar.xz file > directory > - # though the parameter '--tmp-dir', but it is failing in > Fedora 20 > - patterns = ['/tmp/sosreport-%s-*', > '/var/tmp/sosreport-%s-*'] > - reports = [] > - reportFile = None > - for p in patterns: > - reports = reports + [f for f in glob.glob(p % name)] > - for f in reports: > - if not fnmatch.fnmatch(f, '*.md5'): > - reportFile = f > - break > - # Some error in sosreport happened > - if reportFile is None: > - wok_log.error('Debug report file not found. See > sosreport ' > - 'output for detail:\n%s', output) > - fname = (patterns[0] % name).split('/')[-1] > - raise OperationFailed('KCHDR0004E', {'name': fname}) > - > - md5_report_file = reportFile + '.md5' > - report_file_extension = '.' + reportFile.split('.', 1)[1] > - path = config.get_debugreports_path() > - target = os.path.join(path, name + report_file_extension) > - # Moving report > - msg = 'Moving debug report file "%s" to "%s"' % > (reportFile, > - target) > - wok_log.info(msg) > - shutil.move(reportFile, target) > - # Deleting md5 > - msg = 'Deleting report md5 file: "%s"' % (md5_report_file) > - wok_log.info(msg) > - with open(md5_report_file) as f: > - md5 = f.read().strip() > - wok_log.info('Md5 file content: "%s"', md5) > - os.remove(md5_report_file) > - cb('OK', True) > - return > - > - except WokException as e: > - log_error(e) > - raise > - > - except OSError as e: > - log_error(e) > - raise > - > - except Exception, e: > - # No need to call cb to update the task status here. > - # The task object will catch the exception raised here > - # and update the task status there > - log_error(e) > - raise OperationFailed("KCHDR0005E", {'name': name, > 'err': e}) > - > - @staticmethod > - def get_system_report_tool(): > - # Please add new possible debug report command here > - # and implement the report generating function > - # based on the new report command > - report_tools = ({'cmd': 'sosreport --help', > - 'fn': DebugReportsModel.sosreport_generate},) > - > - # check if the command can be found by shell one by one > - for helper_tool in report_tools: > - try: > - retcode = subprocess.call(helper_tool['cmd'], > shell=True, > - stdout=subprocess.PIPE, > - stderr=subprocess.PIPE) > - if retcode == 0: > - return helper_tool['fn'] > - except Exception, e: > - wok_log.info('Exception running command: %s', e) > - > - return None > - > - > -class DebugReportModel(object): > - def __init__(self, **kargs): > - pass > - > - def lookup(self, name): > - path = config.get_debugreports_path() > - file_pattern = os.path.join(path, name) > - file_pattern = file_pattern + '.*' > - try: > - file_target = glob.glob(file_pattern)[0] > - except IndexError: > - raise NotFoundError("KCHDR0001E", {'name': name}) > - > - ctime = os.stat(file_target).st_mtime > - ctime = time.strftime("%Y-%m-%d-%H:%M:%S", > time.localtime(ctime)) > - file_target = os.path.split(file_target)[-1] > - file_target = os.path.join("plugins/kimchi/data/debugreports", > - file_target) > - return {'uri': file_target, > - 'ctime': ctime} > - > - def update(self, name, params): > - path = config.get_debugreports_path() > - file_pattern = os.path.join(path, name + '.*') > - try: > - file_source = glob.glob(file_pattern)[0] > - except IndexError: > - raise NotFoundError("KCHDR0001E", {'name': name}) > - > - file_target = file_source.replace(name, params['name']) > - if os.path.isfile(file_target): > - raise InvalidParameter('KCHDR0008E', {'name': > params['name']}) > - > - shutil.move(file_source, file_target) > - wok_log.info('%s renamed to %s' % (file_source, file_target)) > - return params['name'] > - > - def delete(self, name): > - path = config.get_debugreports_path() > - file_pattern = os.path.join(path, name + '.*') > - try: > - file_target = glob.glob(file_pattern)[0] > - except IndexError: > - raise NotFoundError("KCHDR0001E", {'name': name}) > - > - os.remove(file_target) > - > - > -class DebugReportContentModel(object): > - def __init__(self, **kargs): > - self._debugreport = DebugReportModel() > - > - def lookup(self, name): > - return self._debugreport.lookup(name)