[Kimchi-devel] [PATCH 08/15] V3 Ginger Base : base plugin model files
Daniel Henrique Barboza
dhbarboza82 at gmail.com
Wed Oct 7 13:02:14 UTC 2015
Reviewed-by: Daniel Barboza <dhbarboza82 at gmail.com>
On 10/07/2015 07:43 AM, chandra at linux.vnet.ibm.com wrote:
> From: chandrureddy <chandra at linux.vnet.ibm.com>
>
> ---
> src/wok/plugins/gingerbase/model/Makefile.am | 25 ++
> src/wok/plugins/gingerbase/model/__init__.py | 18 +
> src/wok/plugins/gingerbase/model/cpuinfo.py | 103 ++++++
> src/wok/plugins/gingerbase/model/debugreports.py | 213 +++++++++++
> src/wok/plugins/gingerbase/model/host.py | 427 +++++++++++++++++++++++
> src/wok/plugins/gingerbase/model/model.py | 49 +++
> src/wok/plugins/kimchi/model/debugreports.py | 213 -----------
> 7 files changed, 835 insertions(+), 213 deletions(-)
> create mode 100644 src/wok/plugins/gingerbase/model/Makefile.am
> create mode 100644 src/wok/plugins/gingerbase/model/__init__.py
> create mode 100644 src/wok/plugins/gingerbase/model/cpuinfo.py
> create mode 100644 src/wok/plugins/gingerbase/model/debugreports.py
> create mode 100644 src/wok/plugins/gingerbase/model/host.py
> create mode 100644 src/wok/plugins/gingerbase/model/model.py
> delete mode 100644 src/wok/plugins/kimchi/model/debugreports.py
>
> diff --git a/src/wok/plugins/gingerbase/model/Makefile.am b/src/wok/plugins/gingerbase/model/Makefile.am
> new file mode 100644
> index 0000000..80c4e07
> --- /dev/null
> +++ b/src/wok/plugins/gingerbase/model/Makefile.am
> @@ -0,0 +1,25 @@
> +#
> +# Ginger Base
> +#
> +# Copyright IBM Corp, 2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> +
> +model_PYTHON = *.py
> +
> +modeldir = $(pythondir)/wok/plugins/gingerbase/model
> +
> +install-data-local:
> + $(MKDIR_P) $(DESTDIR)$(modeldir)
> diff --git a/src/wok/plugins/gingerbase/model/__init__.py b/src/wok/plugins/gingerbase/model/__init__.py
> new file mode 100644
> index 0000000..355a76c
> --- /dev/null
> +++ b/src/wok/plugins/gingerbase/model/__init__.py
> @@ -0,0 +1,18 @@
> +#
> +# Project Ginger Base
> +#
> +# Copyright IBM, Corp. 2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> diff --git a/src/wok/plugins/gingerbase/model/cpuinfo.py b/src/wok/plugins/gingerbase/model/cpuinfo.py
> new file mode 100644
> index 0000000..688d58f
> --- /dev/null
> +++ b/src/wok/plugins/gingerbase/model/cpuinfo.py
> @@ -0,0 +1,103 @@
> +#
> +# Project Ginger Base
> +#
> +# Copyright IBM, Corp. 2014-2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> +
> +import platform
> +
> +from wok.exception import InvalidParameter, InvalidOperation
> +from wok.utils import run_command
> +from ..lscpu import LsCpu
> +
> +
> +ARCH = 'power' if platform.machine().startswith('ppc') else 'x86'
> +
> +
> +class CPUInfoModel(object):
> + """
> + Get information about a CPU for hyperthreading (on x86)
> + or SMT (on POWER) for logic when creating templates and VMs.
> + """
> +
> + def __init__(self, **kargs):
> + self.guest_threads_enabled = False
> + self.sockets = 0
> + self.cores_present = 0
> + self.cores_available = 0
> + self.cores_per_socket = 0
> + self.threads_per_core = 0
> + self.max_threads = 0
> + self.lscpu = LsCpu()
> +
> + if ARCH == 'power':
> + # IBM PowerPC
> + self.guest_threads_enabled = True
> + out, error, rc = run_command(['ppc64_cpu', '--smt'])
> + if rc or 'on' in out:
> + # SMT has to be disabled for guest to use threads as CPUs.
> + # rc is always zero, whether SMT is off or on.
> + self.guest_threads_enabled = False
> + out, error, rc = run_command(['ppc64_cpu', '--cores-present'])
> + if not rc:
> + self.cores_present = int(out.split()[-1])
> + out, error, rc = run_command(['ppc64_cpu', '--cores-on'])
> + if not rc:
> + self.cores_available = int(out.split()[-1])
> + out, error, rc = run_command(['ppc64_cpu', '--threads-per-core'])
> + if not rc:
> + self.threads_per_core = int(out.split()[-1])
> + self.sockets = self.cores_present/self.threads_per_core
> + if self.sockets == 0:
> + self.sockets = 1
> + self.cores_per_socket = self.cores_present/self.sockets
> + else:
> + # Intel or AMD
> + self.guest_threads_enabled = True
> + self.sockets = int(self.lscpu.get_sockets())
> + self.cores_per_socket = int(self.lscpu.get_cores_per_socket())
> + self.cores_present = self.cores_per_socket * self.sockets
> + self.cores_available = self.cores_present
> + self.threads_per_core = self.lscpu.get_threads_per_core()
> +
> + def lookup(self, ident):
> + return {
> + 'guest_threads_enabled': self.guest_threads_enabled,
> + 'sockets': self.sockets,
> + 'cores_per_socket': self.cores_per_socket,
> + 'cores_present': self.cores_present,
> + 'cores_available': self.cores_available,
> + 'threads_per_core': self.threads_per_core,
> + }
> +
> + def check_topology(self, vcpus, topology):
> + """
> + param vcpus: should be an integer
> + param iso_path: the path of the guest ISO
> + param topology: {'sockets': x, 'cores': x, 'threads': x}
> + """
> + sockets = topology['sockets']
> + cores = topology['cores']
> + threads = topology['threads']
> +
> + if not self.guest_threads_enabled:
> + raise InvalidOperation("GGBCPUINF0003E")
> + if vcpus != sockets * cores * threads:
> + raise InvalidParameter("GGBCPUINF0002E")
> + if vcpus > self.cores_available * self.threads_per_core:
> + raise InvalidParameter("GGBCPUINF0001E")
> + if threads > self.threads_per_core:
> + raise InvalidParameter("GGBCPUINF0002E")
> diff --git a/src/wok/plugins/gingerbase/model/debugreports.py b/src/wok/plugins/gingerbase/model/debugreports.py
> new file mode 100644
> index 0000000..43ba0cb
> --- /dev/null
> +++ b/src/wok/plugins/gingerbase/model/debugreports.py
> @@ -0,0 +1,213 @@
> +#
> +# Project Ginger Base
> +#
> +# Copyright IBM, Corp. 2014-2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> +
> +import fnmatch
> +import glob
> +import logging
> +import os
> +import shutil
> +import subprocess
> +import time
> +
> +from wok.exception import InvalidParameter, NotFoundError, OperationFailed
> +from wok.exception import WokException
> +from wok.utils import add_task, wok_log
> +from wok.utils import run_command
> +from wok.model.tasks import TaskModel
> +
> +from wok.plugins.kimchi import config
> +
> +
> +class DebugReportsModel(object):
> + def __init__(self, **kargs):
> + self.objstore = kargs['objstore']
> + self.task = TaskModel(**kargs)
> +
> + def create(self, params):
> + ident = params.get('name').strip()
> + # Generate a name with time and millisec precision, if necessary
> + if ident is None or ident == "":
> + ident = 'report-' + str(int(time.time() * 1000))
> + else:
> + if ident in self.get_list():
> + raise InvalidParameter("GGBDR0008E", {"name": ident})
> + taskid = self._gen_debugreport_file(ident)
> + return self.task.lookup(taskid)
> +
> + def get_list(self):
> + path = config.get_debugreports_path()
> + file_pattern = os.path.join(path, '*.*')
> + file_lists = glob.glob(file_pattern)
> + file_lists = [os.path.split(file)[1] for file in file_lists]
> + name_lists = [file.split('.', 1)[0] for file in file_lists]
> +
> + return name_lists
> +
> + def _gen_debugreport_file(self, name):
> + gen_cmd = self.get_system_report_tool()
> +
> + if gen_cmd is not None:
> + return add_task('/plugins/gingerbase/debugreports/%s' % name, gen_cmd,
> + self.objstore, name)
> +
> + raise OperationFailed("GGBDR0002E")
> +
> + @staticmethod
> + def sosreport_generate(cb, name):
> + def log_error(e):
> + log = logging.getLogger('Model')
> + log.warning('Exception in generating debug file: %s', e)
> +
> + try:
> + command = ['sosreport', '--batch', '--name=%s' % name]
> + output, error, retcode = run_command(command)
> +
> + if retcode != 0:
> + raise OperationFailed("GGBDR0003E", {'name': name,
> + 'err': retcode})
> +
> + # SOSREPORT might create file in /tmp or /var/tmp
> + # FIXME: The right way should be passing the tar.xz file directory
> + # though the parameter '--tmp-dir', but it is failing in Fedora 20
> + patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*']
> + reports = []
> + reportFile = None
> + for p in patterns:
> + reports = reports + [f for f in glob.glob(p % name)]
> + for f in reports:
> + if not fnmatch.fnmatch(f, '*.md5'):
> + reportFile = f
> + break
> + # Some error in sosreport happened
> + if reportFile is None:
> + wok_log.error('Debug report file not found. See sosreport '
> + 'output for detail:\n%s', output)
> + fname = (patterns[0] % name).split('/')[-1]
> + raise OperationFailed('GGBDR0004E', {'name': fname})
> +
> + md5_report_file = reportFile + '.md5'
> + report_file_extension = '.' + reportFile.split('.', 1)[1]
> + path = config.get_debugreports_path()
> + target = os.path.join(path, name + report_file_extension)
> + # Moving report
> + msg = 'Moving debug report file "%s" to "%s"' % (reportFile,
> + target)
> + wok_log.info(msg)
> + shutil.move(reportFile, target)
> + # Deleting md5
> + msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
> + wok_log.info(msg)
> + with open(md5_report_file) as f:
> + md5 = f.read().strip()
> + wok_log.info('Md5 file content: "%s"', md5)
> + os.remove(md5_report_file)
> + cb('OK', True)
> + return
> +
> + except WokException as e:
> + log_error(e)
> + raise
> +
> + except OSError as e:
> + log_error(e)
> + raise
> +
> + except Exception, e:
> + # No need to call cb to update the task status here.
> + # The task object will catch the exception raised here
> + # and update the task status there
> + log_error(e)
> + raise OperationFailed("GGBDR0005E", {'name': name, 'err': e})
> +
> + @staticmethod
> + def get_system_report_tool():
> + # Please add new possible debug report command here
> + # and implement the report generating function
> + # based on the new report command
> + report_tools = ({'cmd': 'sosreport --help',
> + 'fn': DebugReportsModel.sosreport_generate},)
> +
> + # check if the command can be found by shell one by one
> + for helper_tool in report_tools:
> + try:
> + retcode = subprocess.call(helper_tool['cmd'], shell=True,
> + stdout=subprocess.PIPE,
> + stderr=subprocess.PIPE)
> + if retcode == 0:
> + return helper_tool['fn']
> + except Exception, e:
> + wok_log.info('Exception running command: %s', e)
> +
> + return None
> +
> +
> +class DebugReportModel(object):
> + def __init__(self, **kargs):
> + pass
> +
> + def lookup(self, name):
> + path = config.get_debugreports_path()
> + file_pattern = os.path.join(path, name)
> + file_pattern = file_pattern + '.*'
> + try:
> + file_target = glob.glob(file_pattern)[0]
> + except IndexError:
> + raise NotFoundError("GGBDR0001E", {'name': name})
> +
> + ctime = os.stat(file_target).st_mtime
> + ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime))
> + file_target = os.path.split(file_target)[-1]
> + file_target = os.path.join("plugins/gingerbase/data/debugreports",
> + file_target)
> + return {'uri': file_target,
> + 'ctime': ctime}
> +
> + def update(self, name, params):
> + path = config.get_debugreports_path()
> + file_pattern = os.path.join(path, name + '.*')
> + try:
> + file_source = glob.glob(file_pattern)[0]
> + except IndexError:
> + raise NotFoundError("GGBDR0001E", {'name': name})
> +
> + file_target = file_source.replace(name, params['name'])
> + if os.path.isfile(file_target):
> + raise InvalidParameter('GGBDR0008E', {'name': params['name']})
> +
> + shutil.move(file_source, file_target)
> + wok_log.info('%s renamed to %s' % (file_source, file_target))
> + return params['name']
> +
> + def delete(self, name):
> + path = config.get_debugreports_path()
> + file_pattern = os.path.join(path, name + '.*')
> + try:
> + file_target = glob.glob(file_pattern)[0]
> + except IndexError:
> + raise NotFoundError("GGBDR0001E", {'name': name})
> +
> + os.remove(file_target)
> +
> +
> +class DebugReportContentModel(object):
> + def __init__(self, **kargs):
> + self._debugreport = DebugReportModel()
> +
> + def lookup(self, name):
> + return self._debugreport.lookup(name)
> diff --git a/src/wok/plugins/gingerbase/model/host.py b/src/wok/plugins/gingerbase/model/host.py
> new file mode 100644
> index 0000000..501243e
> --- /dev/null
> +++ b/src/wok/plugins/gingerbase/model/host.py
> @@ -0,0 +1,427 @@
> +#
> +# Project Ginger Base
> +#
> +# Copyright IBM, Corp. 2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> +
> +import os
> +import platform
> +import psutil
> +import time
> +from cherrypy.process.plugins import BackgroundTask
> +from collections import defaultdict
> +import glob
> +
> +from wok.basemodel import Singleton
> +from wok.exception import InvalidOperation
> +from wok.exception import OperationFailed
> +from wok.utils import add_task, wok_log
> +
> +
> +from .. import disks
> +from ..repositories import Repositories
> +from ..swupdate import SoftwareUpdate
> +from debugreports import DebugReportsModel
> +from wok.model.tasks import TaskModel
> +from wok.config import config as kconfig
> +
> +
> +HOST_STATS_INTERVAL = 1
> +
> +
> +class HostModel(object):
> + def __init__(self, **kargs):
> + # self.conn = kargs['conn']
> + self.objstore = kargs['objstore']
> + self.task = TaskModel(**kargs)
> + self.host_info = self._get_host_info()
> +
> + def _get_ppc_cpu_info(self):
> + res = {}
> + with open('/proc/cpuinfo') as f:
> + for line in f.xreadlines():
> + # Parse CPU, CPU's revision and CPU's clock information
> + for key in ['cpu', 'revision', 'clock']:
> + if key in line:
> + info = line.split(':')[1].strip()
> + if key == 'clock':
> + value = float(info.split('MHz')[0].strip()) / 1000
> + else:
> + value = info.split('(')[0].strip()
> + res[key] = value
> +
> + # Power machines show, for each cpu/core, a block with
> + # all cpu information. Here we control the scan of the
> + # necessary information (1st block provides
> + # everything), skipping the function when find all
> + # information.
> + if len(res.keys()) == 3:
> + return "%(cpu)s (%(revision)s) @ %(clock)s GHz\
> + " % res
> +
> + return ""
> +
> + def _get_host_info(self):
> + res = {}
> + if platform.machine().startswith('ppc'):
> + res['cpu_model'] = self._get_ppc_cpu_info()
> + else:
> + with open('/proc/cpuinfo') as f:
> + for line in f.xreadlines():
> + if "model name" in line:
> + res['cpu_model'] = line.split(':')[1].strip()
> + break
> +
> + res['cpus'] = 0
> + res['memory'] = 0L
> +
> + # Include IBM PowerKVM name to supported distro names
> + _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
> + # 'fedora' '17' 'Beefy Miracle'
> + distro, version, codename = platform.linux_distribution(
> + supported_dists=_sup_distros)
> + res['os_distro'] = distro
> + res['os_version'] = version
> + res['os_codename'] = unicode(codename, "utf-8")
> +
> + return res
> +
> + def lookup(self, *name):
> + cpus = psutil.NUM_CPUS
> +
> + # psutil is unstable on how to get the number of
> + # cpus, different versions call it differently
> + if hasattr(psutil, 'cpu_count'):
> + cpus = psutil.cpu_count()
> +
> + elif hasattr(psutil, 'NUM_CPUS'):
> + cpus = psutil.NUM_CPUS
> +
> + elif hasattr(psutil, '_psplatform'):
> + for method_name in ['_get_num_cpus', 'get_num_cpus']:
> +
> + method = getattr(psutil._psplatform, method_name, None)
> + if method is not None:
> + cpus = method()
> + break
> +
> + self.host_info['cpus'] = cpus
> + if hasattr(psutil, 'phymem_usage'):
> + self.host_info['memory'] = psutil.phymem_usage().total
> + elif hasattr(psutil, 'virtual_memory'):
> + self.host_info['memory'] = psutil.virtual_memory().total
> + return self.host_info
> +
> + def swupdate(self, *name):
> + try:
> + swupdate = SoftwareUpdate()
> + except:
> + raise OperationFailed('GGBPKGUPD0004E')
> +
> + pkgs = swupdate.getNumOfUpdates()
> + if pkgs == 0:
> + raise OperationFailed('GGBPKGUPD0001E')
> +
> + wok_log.debug('Host is going to be updated.')
> + taskid = add_task('/plugins/gingerbase/host/swupdate', swupdate.doUpdate,
> + self.objstore, None)
> + return self.task.lookup(taskid)
> +
> + def shutdown(self, args=None):
> + # Check for running vms before shutdown
> + # FIXME : Find alternative way to figure out if any vms running
> + # running_vms = self._get_vms_list_by_state('running')
> + # if len(running_vms) > 0:
> + # raise OperationFailed("GGBHOST0001E")
> +
> + wok_log.info('Host is going to shutdown.')
> + os.system('shutdown -h now')
> +
> + def reboot(self, args=None):
> + # Find running VMs
> + # FIXME : Find alternative way to figure out if any vms running
> + # running_vms = self._get_vms_list_by_state('running')
> + # if len(running_vms) > 0:
> + # raise OperationFailed("GGBHOST0002E")
> +
> + wok_log.info('Host is going to reboot.')
> + os.system('reboot')
> +
> + # def _get_vms_list_by_state(self, state):
> + # conn = self.conn.get()
> + # return [dom.name().decode('utf-8')
> + # for dom in conn.listAllDomains(0)
> + # if (DOM_STATE_MAP[dom.info()[0]]) == state]
> +
> +
> +class HostStatsModel(object):
> + __metaclass__ = Singleton
> +
> + def __init__(self, **kargs):
> + self.host_stats = defaultdict(list)
> + self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL,
> + self._update_host_stats)
> + self.host_stats_thread.start()
> +
> + def lookup(self, *name):
> + return {'cpu_utilization': self.host_stats['cpu_utilization'][-1],
> + 'memory': self.host_stats['memory'][-1],
> + 'disk_read_rate': self.host_stats['disk_read_rate'][-1],
> + 'disk_write_rate': self.host_stats['disk_write_rate'][-1],
> + 'net_recv_rate': self.host_stats['net_recv_rate'][-1],
> + 'net_sent_rate': self.host_stats['net_sent_rate'][-1]}
> +
> + def _update_host_stats(self):
> + preTimeStamp = self.host_stats['timestamp']
> + timestamp = time.time()
> + # FIXME when we upgrade psutil, we can get uptime by psutil.uptime
> + # we get uptime by float(open("/proc/uptime").readline().split()[0])
> + # and calculate the first io_rate after the OS started.
> + with open("/proc/uptime") as time_f:
> + seconds = (timestamp - preTimeStamp if preTimeStamp else
> + float(time_f.readline().split()[0]))
> +
> + self.host_stats['timestamp'] = timestamp
> + self._get_host_disk_io_rate(seconds)
> + self._get_host_network_io_rate(seconds)
> +
> + self._get_percentage_host_cpu_usage()
> + self._get_host_memory_stats()
> +
> + # store only 60 stats (1 min)
> + for key, value in self.host_stats.iteritems():
> + if isinstance(value, list):
> + if len(value) == 60:
> + self.host_stats[key] = value[10:]
> +
> + def _get_percentage_host_cpu_usage(self):
> + # This is cpu usage producer. This producer will calculate the usage
> + # at an interval of HOST_STATS_INTERVAL.
> + # The psutil.cpu_percent works as non blocking.
> + # psutil.cpu_percent maintains a cpu time sample.
> + # It will update the cpu time sample when it is called.
> + # So only this producer can call psutil.cpu_percent in kimchi.
> + self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None))
> +
> + def _get_host_memory_stats(self):
> + virt_mem = psutil.virtual_memory()
> + # available:
> + # the actual amount of available memory that can be given
> + # instantly to processes that request more memory in bytes; this
> + # is calculated by summing different memory values depending on
> + # the platform (e.g. free + buffers + cached on Linux)
> + memory_stats = {'total': virt_mem.total,
> + 'free': virt_mem.free,
> + 'cached': virt_mem.cached,
> + 'buffers': virt_mem.buffers,
> + 'avail': virt_mem.available}
> + self.host_stats['memory'].append(memory_stats)
> +
> + def _get_host_disk_io_rate(self, seconds):
> + disk_read_bytes = self.host_stats['disk_read_bytes']
> + disk_write_bytes = self.host_stats['disk_write_bytes']
> + prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0
> + prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0
> +
> + disk_io = psutil.disk_io_counters(False)
> + read_bytes = disk_io.read_bytes
> + write_bytes = disk_io.write_bytes
> +
> + rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5)
> + wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5)
> +
> + self.host_stats['disk_read_rate'].append(rd_rate)
> + self.host_stats['disk_write_rate'].append(wr_rate)
> + self.host_stats['disk_read_bytes'].append(read_bytes)
> + self.host_stats['disk_write_bytes'].append(write_bytes)
> +
> + def _get_host_network_io_rate(self, seconds):
> + net_recv_bytes = self.host_stats['net_recv_bytes']
> + net_sent_bytes = self.host_stats['net_sent_bytes']
> + prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0
> + prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0
> +
> + net_ios = psutil.network_io_counters(True)
> + recv_bytes = 0
> + sent_bytes = 0
> + for key in set(self.nics() +
> + self.wlans()) & set(net_ios.iterkeys()):
> + recv_bytes = recv_bytes + net_ios[key].bytes_recv
> + sent_bytes = sent_bytes + net_ios[key].bytes_sent
> +
> + rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5)
> + tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5)
> +
> + self.host_stats['net_recv_rate'].append(rx_rate)
> + self.host_stats['net_sent_rate'].append(tx_rate)
> + self.host_stats['net_recv_bytes'].append(recv_bytes)
> + self.host_stats['net_sent_bytes'].append(sent_bytes)
> +
> + def wlans(self):
> + WLAN_PATH = '/sys/class/net/*/wireless'
> + return [b.split('/')[-2] for b in glob.glob(WLAN_PATH)]
> +
> +
> + # FIXME if we do not want to list usb nic
> + def nics(self):
> + NIC_PATH = '/sys/class/net/*/device'
> + return list(set([b.split('/')[-2] for b in glob.glob(NIC_PATH)]) -
> + set(self.wlans()))
> +
> +
> +class HostStatsHistoryModel(object):
> + def __init__(self, **kargs):
> + self.history = HostStatsModel(**kargs)
> +
> + def lookup(self, *name):
> + return {'cpu_utilization': self.history.host_stats['cpu_utilization'],
> + 'memory': self.history.host_stats['memory'],
> + 'disk_read_rate': self.history.host_stats['disk_read_rate'],
> + 'disk_write_rate': self.history.host_stats['disk_write_rate'],
> + 'net_recv_rate': self.history.host_stats['net_recv_rate'],
> + 'net_sent_rate': self.history.host_stats['net_sent_rate']}
> +
> +
> +class CapabilitiesModel(object):
> + __metaclass__ = Singleton
> +
> + def __init__(self, **kargs):
> + pass
> +
> +
> + def lookup(self, *ident):
> + report_tool = DebugReportsModel.get_system_report_tool()
> + try:
> + SoftwareUpdate()
> + except Exception:
> + update_tool = False
> + else:
> + update_tool = True
> +
> + try:
> + repo = Repositories()
> + except Exception:
> + repo_mngt_tool = None
> + else:
> + repo_mngt_tool = repo._pkg_mnger.TYPE
> +
> + return {
> + 'system_report_tool': bool(report_tool),
> + 'update_tool': update_tool,
> + 'repo_mngt_tool': repo_mngt_tool,
> + 'federation': kconfig.get("server", "federation")
> + }
> +
> +
> +class PartitionsModel(object):
> + def __init__(self, **kargs):
> + pass
> +
> + def get_list(self):
> + result = disks.get_partitions_names()
> + return result
> +
> +
> +class PartitionModel(object):
> + def __init__(self, **kargs):
> + pass
> +
> + def lookup(self, name):
> + return disks.get_partition_details(name)
> +
> +class PackagesUpdateModel(object):
> + def __init__(self, **kargs):
> + try:
> + self.host_swupdate = SoftwareUpdate()
> + except:
> + self.host_swupdate = None
> +
> + def get_list(self):
> + if self.host_swupdate is None:
> + raise OperationFailed('GGBPKGUPD0004E')
> +
> + return self.host_swupdate.getUpdates()
> +
> +
> +class PackageUpdateModel(object):
> + def __init__(self, **kargs):
> + pass
> +
> + def lookup(self, name):
> + try:
> + swupdate = SoftwareUpdate()
> + except Exception:
> + raise OperationFailed('GGBPKGUPD0004E')
> +
> + return swupdate.getUpdate(name)
> +
> +
> +class RepositoriesModel(object):
> + def __init__(self, **kargs):
> + try:
> + self.host_repositories = Repositories()
> + except:
> + self.host_repositories = None
> +
> + def get_list(self):
> + if self.host_repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return sorted(self.host_repositories.getRepositories())
> +
> + def create(self, params):
> + if self.host_repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return self.host_repositories.addRepository(params)
> +
> +
> +class RepositoryModel(object):
> + def __init__(self, **kargs):
> + try:
> + self._repositories = Repositories()
> + except:
> + self._repositories = None
> +
> + def lookup(self, repo_id):
> + if self._repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return self._repositories.getRepository(repo_id)
> +
> + def enable(self, repo_id):
> + if self._repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return self._repositories.enableRepository(repo_id)
> +
> + def disable(self, repo_id):
> + if self._repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return self._repositories.disableRepository(repo_id)
> +
> + def update(self, repo_id, params):
> + if self._repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return self._repositories.updateRepository(repo_id, params)
> +
> + def delete(self, repo_id):
> + if self._repositories is None:
> + raise InvalidOperation('GGBREPOS0014E')
> +
> + return self._repositories.removeRepository(repo_id)
> diff --git a/src/wok/plugins/gingerbase/model/model.py b/src/wok/plugins/gingerbase/model/model.py
> new file mode 100644
> index 0000000..90331f5
> --- /dev/null
> +++ b/src/wok/plugins/gingerbase/model/model.py
> @@ -0,0 +1,49 @@
> +#
> +# Project Ginger Base
> +#
> +# Copyright IBM, Corp. 2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> +
> +import inspect
> +import os
> +
> +from wok.basemodel import BaseModel
> +from wok.objectstore import ObjectStore
> +from wok.utils import import_module, listPathModules
> +
> +
> +class Model(BaseModel):
> + def __init__(self, objstore_loc=None):
> +
> + self.objstore = ObjectStore(objstore_loc)
> + kargs = {'objstore': self.objstore}
> +
> + this = os.path.basename(__file__)
> + this_mod = os.path.splitext(this)[0]
> +
> + models = []
> + for mod_name in listPathModules(os.path.dirname(__file__)):
> + if mod_name.startswith("_") or mod_name == this_mod:
> + continue
> +
> + module = import_module('plugins.gingerbase.model.' + mod_name)
> + members = inspect.getmembers(module, inspect.isclass)
> + for cls_name, instance in members:
> + if inspect.getmodule(instance) == module:
> + if cls_name.endswith('Model'):
> + models.append(instance(**kargs))
> +
> + return super(Model, self).__init__(models)
> diff --git a/src/wok/plugins/kimchi/model/debugreports.py b/src/wok/plugins/kimchi/model/debugreports.py
> deleted file mode 100644
> index dcd1b64..0000000
> --- a/src/wok/plugins/kimchi/model/debugreports.py
> +++ /dev/null
> @@ -1,213 +0,0 @@
> -#
> -# Project Kimchi
> -#
> -# Copyright IBM, Corp. 2014-2015
> -#
> -# This library is free software; you can redistribute it and/or
> -# modify it under the terms of the GNU Lesser General Public
> -# License as published by the Free Software Foundation; either
> -# version 2.1 of the License, or (at your option) any later version.
> -#
> -# This library is distributed in the hope that it will be useful,
> -# but WITHOUT ANY WARRANTY; without even the implied warranty of
> -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> -# Lesser General Public License for more details.
> -#
> -# You should have received a copy of the GNU Lesser General Public
> -# License along with this library; if not, write to the Free Software
> -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> -
> -import fnmatch
> -import glob
> -import logging
> -import os
> -import shutil
> -import subprocess
> -import time
> -
> -from wok.exception import InvalidParameter, NotFoundError, OperationFailed
> -from wok.exception import WokException
> -from wok.utils import add_task, wok_log
> -from wok.utils import run_command
> -from wok.model.tasks import TaskModel
> -
> -from wok.plugins.kimchi import config
> -
> -
> -class DebugReportsModel(object):
> - def __init__(self, **kargs):
> - self.objstore = kargs['objstore']
> - self.task = TaskModel(**kargs)
> -
> - def create(self, params):
> - ident = params.get('name').strip()
> - # Generate a name with time and millisec precision, if necessary
> - if ident is None or ident == "":
> - ident = 'report-' + str(int(time.time() * 1000))
> - else:
> - if ident in self.get_list():
> - raise InvalidParameter("KCHDR0008E", {"name": ident})
> - taskid = self._gen_debugreport_file(ident)
> - return self.task.lookup(taskid)
> -
> - def get_list(self):
> - path = config.get_debugreports_path()
> - file_pattern = os.path.join(path, '*.*')
> - file_lists = glob.glob(file_pattern)
> - file_lists = [os.path.split(file)[1] for file in file_lists]
> - name_lists = [file.split('.', 1)[0] for file in file_lists]
> -
> - return name_lists
> -
> - def _gen_debugreport_file(self, name):
> - gen_cmd = self.get_system_report_tool()
> -
> - if gen_cmd is not None:
> - return add_task('/plugins/kimchi/debugreports/%s' % name, gen_cmd,
> - self.objstore, name)
> -
> - raise OperationFailed("KCHDR0002E")
> -
> - @staticmethod
> - def sosreport_generate(cb, name):
> - def log_error(e):
> - log = logging.getLogger('Model')
> - log.warning('Exception in generating debug file: %s', e)
> -
> - try:
> - command = ['sosreport', '--batch', '--name=%s' % name]
> - output, error, retcode = run_command(command)
> -
> - if retcode != 0:
> - raise OperationFailed("KCHDR0003E", {'name': name,
> - 'err': retcode})
> -
> - # SOSREPORT might create file in /tmp or /var/tmp
> - # FIXME: The right way should be passing the tar.xz file directory
> - # though the parameter '--tmp-dir', but it is failing in Fedora 20
> - patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*']
> - reports = []
> - reportFile = None
> - for p in patterns:
> - reports = reports + [f for f in glob.glob(p % name)]
> - for f in reports:
> - if not fnmatch.fnmatch(f, '*.md5'):
> - reportFile = f
> - break
> - # Some error in sosreport happened
> - if reportFile is None:
> - wok_log.error('Debug report file not found. See sosreport '
> - 'output for detail:\n%s', output)
> - fname = (patterns[0] % name).split('/')[-1]
> - raise OperationFailed('KCHDR0004E', {'name': fname})
> -
> - md5_report_file = reportFile + '.md5'
> - report_file_extension = '.' + reportFile.split('.', 1)[1]
> - path = config.get_debugreports_path()
> - target = os.path.join(path, name + report_file_extension)
> - # Moving report
> - msg = 'Moving debug report file "%s" to "%s"' % (reportFile,
> - target)
> - wok_log.info(msg)
> - shutil.move(reportFile, target)
> - # Deleting md5
> - msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
> - wok_log.info(msg)
> - with open(md5_report_file) as f:
> - md5 = f.read().strip()
> - wok_log.info('Md5 file content: "%s"', md5)
> - os.remove(md5_report_file)
> - cb('OK', True)
> - return
> -
> - except WokException as e:
> - log_error(e)
> - raise
> -
> - except OSError as e:
> - log_error(e)
> - raise
> -
> - except Exception, e:
> - # No need to call cb to update the task status here.
> - # The task object will catch the exception raised here
> - # and update the task status there
> - log_error(e)
> - raise OperationFailed("KCHDR0005E", {'name': name, 'err': e})
> -
> - @staticmethod
> - def get_system_report_tool():
> - # Please add new possible debug report command here
> - # and implement the report generating function
> - # based on the new report command
> - report_tools = ({'cmd': 'sosreport --help',
> - 'fn': DebugReportsModel.sosreport_generate},)
> -
> - # check if the command can be found by shell one by one
> - for helper_tool in report_tools:
> - try:
> - retcode = subprocess.call(helper_tool['cmd'], shell=True,
> - stdout=subprocess.PIPE,
> - stderr=subprocess.PIPE)
> - if retcode == 0:
> - return helper_tool['fn']
> - except Exception, e:
> - wok_log.info('Exception running command: %s', e)
> -
> - return None
> -
> -
> -class DebugReportModel(object):
> - def __init__(self, **kargs):
> - pass
> -
> - def lookup(self, name):
> - path = config.get_debugreports_path()
> - file_pattern = os.path.join(path, name)
> - file_pattern = file_pattern + '.*'
> - try:
> - file_target = glob.glob(file_pattern)[0]
> - except IndexError:
> - raise NotFoundError("KCHDR0001E", {'name': name})
> -
> - ctime = os.stat(file_target).st_mtime
> - ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime))
> - file_target = os.path.split(file_target)[-1]
> - file_target = os.path.join("plugins/kimchi/data/debugreports",
> - file_target)
> - return {'uri': file_target,
> - 'ctime': ctime}
> -
> - def update(self, name, params):
> - path = config.get_debugreports_path()
> - file_pattern = os.path.join(path, name + '.*')
> - try:
> - file_source = glob.glob(file_pattern)[0]
> - except IndexError:
> - raise NotFoundError("KCHDR0001E", {'name': name})
> -
> - file_target = file_source.replace(name, params['name'])
> - if os.path.isfile(file_target):
> - raise InvalidParameter('KCHDR0008E', {'name': params['name']})
> -
> - shutil.move(file_source, file_target)
> - wok_log.info('%s renamed to %s' % (file_source, file_target))
> - return params['name']
> -
> - def delete(self, name):
> - path = config.get_debugreports_path()
> - file_pattern = os.path.join(path, name + '.*')
> - try:
> - file_target = glob.glob(file_pattern)[0]
> - except IndexError:
> - raise NotFoundError("KCHDR0001E", {'name': name})
> -
> - os.remove(file_target)
> -
> -
> -class DebugReportContentModel(object):
> - def __init__(self, **kargs):
> - self._debugreport = DebugReportModel()
> -
> - def lookup(self, name):
> - return self._debugreport.lookup(name)
More information about the Kimchi-devel
mailing list