[Kimchi-devel] [PATCH 07/17] Ginger Base : base plugin model files
Chandra Shehkhar Reddy Potula
chandra at linux.vnet.ibm.com
Fri Sep 11 10:12:11 UTC 2015
On 09/10/2015 07:26 PM, Aline Manera wrote:
>
>
> On 01/09/2015 14:58, chandra at linux.vnet.ibm.com wrote:
>> From: chandrureddy <chandra at linux.vnet.ibm.com>
>>
>> ---
>> plugins/gingerbase/model/Makefile.am | 25 ++
>> plugins/gingerbase/model/__init__.py | 18 ++
>> plugins/gingerbase/model/cpuinfo.py | 133 ++++++++++
>> plugins/gingerbase/model/debugreports.py | 213 ++++++++++++++++
>> plugins/gingerbase/model/host.py | 410
>> +++++++++++++++++++++++++++++++
>> plugins/gingerbase/model/model.py | 49 ++++
>> plugins/gingerbase/model/tasks.py | 64 +++++
>> plugins/kimchi/model/debugreports.py | 213 ----------------
>> 8 files changed, 912 insertions(+), 213 deletions(-)
>> create mode 100644 plugins/gingerbase/model/Makefile.am
>> create mode 100644 plugins/gingerbase/model/__init__.py
>
>
>> create mode 100644 plugins/gingerbase/model/cpuinfo.py
>
> Why did you add it to gingerbase? It is used for Kimchi on different
> areas.
I was under the impression that it has to go part of ginger base. When I
was moving this functionality kimchi code does not complain. I am ok to
move it back to kimchi. Let me know.
>
>> create mode 100644 plugins/gingerbase/model/debugreports.py
>> create mode 100644 plugins/gingerbase/model/host.py
>> create mode 100644 plugins/gingerbase/model/model.py
>
>> create mode 100644 plugins/gingerbase/model/tasks.py
>
> Why did you add tasks.py to gingerbase?
I was under the impression that debug reports require it. But never the
less I could remove .
>
>> delete mode 100644 plugins/kimchi/model/debugreports.py
>>
>> diff --git a/plugins/gingerbase/model/Makefile.am
>> b/plugins/gingerbase/model/Makefile.am
>> new file mode 100644
>> index 0000000..6218b47
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/Makefile.am
>> @@ -0,0 +1,25 @@
>> +#
>> +# Kimchi
>> +#
>> +# Copyright IBM Corp, 2013
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> +
>> +model_PYTHON = *.py
>> +
>> +modeldir = $(pythondir)/wok/plugins/gingerbase/model
>> +
>> +install-data-local:
>> + $(MKDIR_P) $(DESTDIR)$(modeldir)
>> diff --git a/plugins/gingerbase/model/__init__.py
>> b/plugins/gingerbase/model/__init__.py
>> new file mode 100644
>> index 0000000..ca7ede4
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/__init__.py
>> @@ -0,0 +1,18 @@
>> +#
>> +# Project Kimchi
>> +#
>> +# Copyright IBM, Corp. 2014
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> diff --git a/plugins/gingerbase/model/cpuinfo.py
>> b/plugins/gingerbase/model/cpuinfo.py
>> new file mode 100644
>> index 0000000..fad8814
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/cpuinfo.py
>> @@ -0,0 +1,133 @@
>> +#
>> +# Project Kimchi
>> +#
>> +# Copyright IBM, Corp. 2014-2015
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> +
>> +import platform
>> +# from xml.etree import ElementTree as ET
>> +
>> +from wok.exception import InvalidParameter, InvalidOperation
>> +from wok.utils import run_command, wok_log
>> +from ..lscpu import LsCpu
>> +
>> +
>> +ARCH = 'power' if platform.machine().startswith('ppc') else 'x86'
>> +
>> +
>> +# def get_topo_capabilities(connect):
>> +# """
>> +# This helper function exists solely to be overridden for
>> +# mockmodel tests. Since other modules use getCapabilies(),
>> +# it can't be overridden directly.
>> +# """
>> +# xml = connect.getCapabilities()
>> +# capabilities = ET.fromstring(xml)
>> +# return capabilities.find('host').find('cpu').find('topology')
>> +
>> +
>> +class CPUInfoModel(object):
>> + """
>> + Get information about a CPU for hyperthreading (on x86)
>> + or SMT (on POWER) for logic when creating templates and VMs.
>> + """
>> +
>> + def __init__(self, **kargs):
>> + self.guest_threads_enabled = False
>> + self.sockets = 0
>> + self.cores_present = 0
>> + self.cores_available = 0
>> + self.cores_per_socket = 0
>> + self.threads_per_core = 0
>> + self.max_threads = 0
>> + self.lscpu = LsCpu()
>> +
>> + # self.conn = kargs['conn']
>> + # libvirt_topology = None
>> + # try:
>> + # connect = self.conn.get()
>> + # libvirt_topology = get_topo_capabilities(connect)
>> + # except Exception as e:
>> + # wok_log.info("Unable to get CPU topology capabilities:
>> %s"
>> + # % e.message)
>> + # return
>> + # if libvirt_topology is None:
>> + # wok_log.info("cpu_info topology not supported.")
>> + # return
>> +
>> + if ARCH == 'power':
>> + # IBM PowerPC
>> + self.guest_threads_enabled = True
>> + out, error, rc = run_command(['ppc64_cpu', '--smt'])
>> + if rc or 'on' in out:
>> + # SMT has to be disabled for guest to use threads as
>> CPUs.
>> + # rc is always zero, whether SMT is off or on.
>> + self.guest_threads_enabled = False
>> + out, error, rc = run_command(['ppc64_cpu',
>> '--cores-present'])
>> + if not rc:
>> + self.cores_present = int(out.split()[-1])
>> + out, error, rc = run_command(['ppc64_cpu', '--cores-on'])
>> + if not rc:
>> + self.cores_available = int(out.split()[-1])
>> + out, error, rc = run_command(['ppc64_cpu',
>> '--threads-per-core'])
>> + if not rc:
>> + self.threads_per_core = int(out.split()[-1])
>> + self.sockets = self.cores_present/self.threads_per_core
>> + if self.sockets == 0:
>> + self.sockets = 1
>> + self.cores_per_socket = self.cores_present/self.sockets
>> + else:
>> + # Intel or AMD
>> + self.guest_threads_enabled = True
>> + # self.sockets = int(libvirt_topology.get('sockets'))
>> + # self.cores_per_socket =
>> int(libvirt_topology.get('cores'))
>> + # self.cores_present = self.cores_per_socket * self.sockets
>> + # self.cores_available = self.cores_present
>> + # self.threads_per_core =
>> int(libvirt_topology.get('threads'))
>> + self.sockets = int(self.lscpu.get_sockets())
>> + self.cores_per_socket =
>> int(self.lscpu.get_cores_per_socket())
>> + self.cores_present = self.cores_per_socket * self.sockets
>> + self.cores_available = self.cores_present
>> + self.threads_per_core = self.lscpu.get_threads_per_core()
>> +
>> + def lookup(self, ident):
>> + return {
>> + 'guest_threads_enabled': self.guest_threads_enabled,
>> + 'sockets': self.sockets,
>> + 'cores_per_socket': self.cores_per_socket,
>> + 'cores_present': self.cores_present,
>> + 'cores_available': self.cores_available,
>> + 'threads_per_core': self.threads_per_core,
>> + }
>> +
>> + def check_topology(self, vcpus, topology):
>> + """
>> + param vcpus: should be an integer
>> + param iso_path: the path of the guest ISO
>> + param topology: {'sockets': x, 'cores': x, 'threads': x}
>> + """
>> + sockets = topology['sockets']
>> + cores = topology['cores']
>> + threads = topology['threads']
>> +
>> + if not self.guest_threads_enabled:
>> + raise InvalidOperation("GGBCPUINF0003E")
>> + if vcpus != sockets * cores * threads:
>> + raise InvalidParameter("GGBCPUINF0002E")
>> + if vcpus > self.cores_available * self.threads_per_core:
>> + raise InvalidParameter("GGBCPUINF0001E")
>> + if threads > self.threads_per_core:
>> + raise InvalidParameter("GGBCPUINF0002E")
>> diff --git a/plugins/gingerbase/model/debugreports.py
>> b/plugins/gingerbase/model/debugreports.py
>> new file mode 100644
>> index 0000000..fc91a9c
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/debugreports.py
>> @@ -0,0 +1,213 @@
>> +#
>> +# Project Kimchi
>> +#
>> +# Copyright IBM, Corp. 2014-2015
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> +
>> +import fnmatch
>> +import glob
>> +import logging
>> +import os
>> +import shutil
>> +import subprocess
>> +import time
>> +
>> +from wok.exception import InvalidParameter, NotFoundError,
>> OperationFailed
>> +from wok.exception import WokException
>> +from wok.utils import add_task, wok_log
>> +from wok.utils import run_command
>> +
>> +from .. import config
>> +from tasks import TaskModel
>> +
>> +
>> +class DebugReportsModel(object):
>> + def __init__(self, **kargs):
>> + self.objstore = kargs['objstore']
>> + self.task = TaskModel(**kargs)
>> +
>> + def create(self, params):
>> + ident = params.get('name').strip()
>> + # Generate a name with time and millisec precision, if
>> necessary
>> + if ident is None or ident == "":
>> + ident = 'report-' + str(int(time.time() * 1000))
>> + else:
>> + if ident in self.get_list():
>> + raise InvalidParameter("GGBDR0008E", {"name": ident})
>> + taskid = self._gen_debugreport_file(ident)
>> + return self.task.lookup(taskid)
>> +
>> + def get_list(self):
>> + path = config.get_debugreports_path()
>> + file_pattern = os.path.join(path, '*.*')
>> + file_lists = glob.glob(file_pattern)
>> + file_lists = [os.path.split(file)[1] for file in file_lists]
>> + name_lists = [file.split('.', 1)[0] for file in file_lists]
>> +
>> + return name_lists
>> +
>> + def _gen_debugreport_file(self, name):
>> + gen_cmd = self.get_system_report_tool()
>> +
>> + if gen_cmd is not None:
>> + return add_task('/plugins/gingerbase/debugreports/%s' %
>> name, gen_cmd,
>> + self.objstore, name)
>> +
>> + raise OperationFailed("GGBDR0002E")
>> +
>> + @staticmethod
>> + def sosreport_generate(cb, name):
>> + def log_error(e):
>> + log = logging.getLogger('Model')
>> + log.warning('Exception in generating debug file: %s', e)
>> +
>> + try:
>> + command = ['sosreport', '--batch', '--name=%s' % name]
>> + output, error, retcode = run_command(command)
>> +
>> + if retcode != 0:
>> + raise OperationFailed("GGBDR0003E", {'name': name,
>> + 'err': retcode})
>> +
>> + # SOSREPORT might create file in /tmp or /var/tmp
>> + # FIXME: The right way should be passing the tar.xz file
>> directory
>> + # though the parameter '--tmp-dir', but it is failing in
>> Fedora 20
>> + patterns = ['/tmp/sosreport-%s-*',
>> '/var/tmp/sosreport-%s-*']
>> + reports = []
>> + reportFile = None
>> + for p in patterns:
>> + reports = reports + [f for f in glob.glob(p % name)]
>> + for f in reports:
>> + if not fnmatch.fnmatch(f, '*.md5'):
>> + reportFile = f
>> + break
>> + # Some error in sosreport happened
>> + if reportFile is None:
>> + wok_log.error('Debug report file not found. See
>> sosreport '
>> + 'output for detail:\n%s', output)
>> + fname = (patterns[0] % name).split('/')[-1]
>> + raise OperationFailed('GGBDR0004E', {'name': fname})
>> +
>> + md5_report_file = reportFile + '.md5'
>> + report_file_extension = '.' + reportFile.split('.', 1)[1]
>> + path = config.get_debugreports_path()
>> + target = os.path.join(path, name + report_file_extension)
>> + # Moving report
>> + msg = 'Moving debug report file "%s" to "%s"' %
>> (reportFile,
>> + target)
>> + wok_log.info(msg)
>> + shutil.move(reportFile, target)
>> + # Deleting md5
>> + msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
>> + wok_log.info(msg)
>> + with open(md5_report_file) as f:
>> + md5 = f.read().strip()
>> + wok_log.info('Md5 file content: "%s"', md5)
>> + os.remove(md5_report_file)
>> + cb('OK', True)
>> + return
>> +
>> + except WokException as e:
>> + log_error(e)
>> + raise
>> +
>> + except OSError as e:
>> + log_error(e)
>> + raise
>> +
>> + except Exception, e:
>> + # No need to call cb to update the task status here.
>> + # The task object will catch the exception raised here
>> + # and update the task status there
>> + log_error(e)
>> + raise OperationFailed("GGBDR0005E", {'name': name,
>> 'err': e})
>> +
>> + @staticmethod
>> + def get_system_report_tool():
>> + # Please add new possible debug report command here
>> + # and implement the report generating function
>> + # based on the new report command
>> + report_tools = ({'cmd': 'sosreport --help',
>> + 'fn': DebugReportsModel.sosreport_generate},)
>> +
>> + # check if the command can be found by shell one by one
>> + for helper_tool in report_tools:
>> + try:
>> + retcode = subprocess.call(helper_tool['cmd'],
>> shell=True,
>> + stdout=subprocess.PIPE,
>> + stderr=subprocess.PIPE)
>> + if retcode == 0:
>> + return helper_tool['fn']
>> + except Exception, e:
>> + wok_log.info('Exception running command: %s', e)
>> +
>> + return None
>> +
>> +
>> +class DebugReportModel(object):
>> + def __init__(self, **kargs):
>> + pass
>> +
>> + def lookup(self, name):
>> + path = config.get_debugreports_path()
>> + file_pattern = os.path.join(path, name)
>> + file_pattern = file_pattern + '.*'
>> + try:
>> + file_target = glob.glob(file_pattern)[0]
>> + except IndexError:
>> + raise NotFoundError("GGBDR0001E", {'name': name})
>> +
>> + ctime = os.stat(file_target).st_mtime
>> + ctime = time.strftime("%Y-%m-%d-%H:%M:%S",
>> time.localtime(ctime))
>> + file_target = os.path.split(file_target)[-1]
>> + file_target =
>> os.path.join("plugins/gingerbase/data/debugreports",
>> + file_target)
>> + return {'uri': file_target,
>> + 'ctime': ctime}
>> +
>> + def update(self, name, params):
>> + path = config.get_debugreports_path()
>> + file_pattern = os.path.join(path, name + '.*')
>> + try:
>> + file_source = glob.glob(file_pattern)[0]
>> + except IndexError:
>> + raise NotFoundError("GGBDR0001E", {'name': name})
>> +
>> + file_target = file_source.replace(name, params['name'])
>> + if os.path.isfile(file_target):
>> + raise InvalidParameter('GGBDR0008E', {'name':
>> params['name']})
>> +
>> + shutil.move(file_source, file_target)
>> + wok_log.info('%s renamed to %s' % (file_source, file_target))
>> + return params['name']
>> +
>> + def delete(self, name):
>> + path = config.get_debugreports_path()
>> + file_pattern = os.path.join(path, name + '.*')
>> + try:
>> + file_target = glob.glob(file_pattern)[0]
>> + except IndexError:
>> + raise NotFoundError("GGBDR0001E", {'name': name})
>> +
>> + os.remove(file_target)
>> +
>> +
>> +class DebugReportContentModel(object):
>> + def __init__(self, **kargs):
>> + self._debugreport = DebugReportModel()
>> +
>> + def lookup(self, name):
>> + return self._debugreport.lookup(name)
>> diff --git a/plugins/gingerbase/model/host.py
>> b/plugins/gingerbase/model/host.py
>> new file mode 100644
>> index 0000000..6522a27
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/host.py
>> @@ -0,0 +1,410 @@
>> +#
>> +# Project Kimchi
>> +#
>> +# Copyright IBM, Corp. 2014-2015
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> +
>> +import os
>> +import platform
>> +import psutil
>> +import time
>> +from cherrypy.process.plugins import BackgroundTask
>> +from collections import defaultdict
>> +
>> +from wok import netinfo
>> +from wok.basemodel import Singleton
>> +from wok.exception import InvalidOperation, InvalidParameter
>> +from wok.exception import NotFoundError, OperationFailed
>> +from wok.utils import add_task, wok_log
>> +
>> +
>> +from .. import disks
>> +from ..repositories import Repositories
>> +from ..swupdate import SoftwareUpdate
>> +from debugreports import DebugReportsModel
>> +from tasks import TaskModel
>> +from wok.config import config as kconfig
>> +
>> +
>> +HOST_STATS_INTERVAL = 1
>> +
>> +
>> +class HostModel(object):
>> + def __init__(self, **kargs):
>> + # self.conn = kargs['conn']
>> + self.objstore = kargs['objstore']
>> + self.task = TaskModel(**kargs)
>> + self.host_info = self._get_host_info()
>> +
>> + def _get_ppc_cpu_info(self):
>> + res = {}
>> + with open('/proc/cpuinfo') as f:
>> + for line in f.xreadlines():
>> + # Parse CPU, CPU's revision and CPU's clock information
>> + for key in ['cpu', 'revision', 'clock']:
>> + if key in line:
>> + info = line.split(':')[1].strip()
>> + if key == 'clock':
>> + value =
>> float(info.split('MHz')[0].strip()) / 1000
>> + else:
>> + value = info.split('(')[0].strip()
>> + res[key] = value
>> +
>> + # Power machines show, for each cpu/core, a
>> block with
>> + # all cpu information. Here we control the
>> scan of the
>> + # necessary information (1st block provides
>> + # everything), skipping the function when
>> find all
>> + # information.
>> + if len(res.keys()) == 3:
>> + return "%(cpu)s (%(revision)s) @
>> %(clock)s GHz\
>> + " % res
>> +
>> + return ""
>> +
>> + def _get_host_info(self):
>> + res = {}
>> + if platform.machine().startswith('ppc'):
>> + res['cpu_model'] = self._get_ppc_cpu_info()
>> + else:
>> + with open('/proc/cpuinfo') as f:
>> + for line in f.xreadlines():
>> + if "model name" in line:
>> + res['cpu_model'] = line.split(':')[1].strip()
>> + break
>> +
>> + res['cpus'] = 0
>> + res['memory'] = 0L
>> +
>> + # Include IBM PowerKVM name to supported distro names
>> + _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
>> + # 'fedora' '17' 'Beefy Miracle'
>> + distro, version, codename = platform.linux_distribution(
>> + supported_dists=_sup_distros)
>> + res['os_distro'] = distro
>> + res['os_version'] = version
>> + res['os_codename'] = unicode(codename, "utf-8")
>> +
>> + return res
>> +
>> + def lookup(self, *name):
>> + cpus = psutil.NUM_CPUS
>> +
>> + # psutil is unstable on how to get the number of
>> + # cpus, different versions call it differently
>> + if hasattr(psutil, 'cpu_count'):
>> + cpus = psutil.cpu_count()
>> +
>> + elif hasattr(psutil, '_psplatform'):
>> + for method_name in ['_get_num_cpus', 'get_num_cpus']:
>> +
>> + method = getattr(psutil._psplatform, method_name, None)
>> + if method is not None:
>> + cpus = method()
>> + break
>> +
>> + self.host_info['cpus'] = cpus
>> + self.host_info['memory'] = psutil.phymem_usage().total
>> + return self.host_info
>> +
>> + def swupdate(self, *name):
>> + try:
>> + swupdate = SoftwareUpdate()
>> + except:
>> + raise OperationFailed('GGBPKGUPD0004E')
>> +
>> + pkgs = swupdate.getNumOfUpdates()
>> + if pkgs == 0:
>> + raise OperationFailed('GGBPKGUPD0001E')
>> +
>> + wok_log.debug('Host is going to be updated.')
>> + taskid = add_task('/plugins/gingerbase/host/swupdate',
>> swupdate.doUpdate,
>> + self.objstore, None)
>> + return self.task.lookup(taskid)
>> +
>> + def shutdown(self, args=None):
>> + # Check for running vms before shutdown
>> + # FIXME : Find alternative way to figure out if any vms running
>> + # running_vms = self._get_vms_list_by_state('running')
>> + # if len(running_vms) > 0:
>> + # raise OperationFailed("GGBHOST0001E")
>> +
>> + wok_log.info('Host is going to shutdown.')
>> + os.system('shutdown -h now')
>> +
>> + def reboot(self, args=None):
>> + # Find running VMs
>> + # FIXME : Find alternative way to figure out if any vms running
>> + # running_vms = self._get_vms_list_by_state('running')
>> + # if len(running_vms) > 0:
>> + # raise OperationFailed("GGBHOST0002E")
>> +
>> + wok_log.info('Host is going to reboot.')
>> + os.system('reboot')
>> +
>> + # def _get_vms_list_by_state(self, state):
>> + # conn = self.conn.get()
>> + # return [dom.name().decode('utf-8')
>> + # for dom in conn.listAllDomains(0)
>> + # if (DOM_STATE_MAP[dom.info()[0]]) == state]
>> +
>> +
>> +class HostStatsModel(object):
>> + __metaclass__ = Singleton
>> +
>> + def __init__(self, **kargs):
>> + self.host_stats = defaultdict(list)
>> + self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL,
>> + self._update_host_stats)
>> + self.host_stats_thread.start()
>> +
>> + def lookup(self, *name):
>> + return {'cpu_utilization':
>> self.host_stats['cpu_utilization'][-1],
>> + 'memory': self.host_stats['memory'][-1],
>> + 'disk_read_rate':
>> self.host_stats['disk_read_rate'][-1],
>> + 'disk_write_rate':
>> self.host_stats['disk_write_rate'][-1],
>> + 'net_recv_rate': self.host_stats['net_recv_rate'][-1],
>> + 'net_sent_rate': self.host_stats['net_sent_rate'][-1]}
>> +
>> + def _update_host_stats(self):
>> + preTimeStamp = self.host_stats['timestamp']
>> + timestamp = time.time()
>> + # FIXME when we upgrade psutil, we can get uptime by
>> psutil.uptime
>> + # we get uptime by
>> float(open("/proc/uptime").readline().split()[0])
>> + # and calculate the first io_rate after the OS started.
>> + with open("/proc/uptime") as time_f:
>> + seconds = (timestamp - preTimeStamp if preTimeStamp else
>> + float(time_f.readline().split()[0]))
>> +
>> + self.host_stats['timestamp'] = timestamp
>> + self._get_host_disk_io_rate(seconds)
>> + self._get_host_network_io_rate(seconds)
>> +
>> + self._get_percentage_host_cpu_usage()
>> + self._get_host_memory_stats()
>> +
>> + # store only 60 stats (1 min)
>> + for key, value in self.host_stats.iteritems():
>> + if isinstance(value, list):
>> + if len(value) == 60:
>> + self.host_stats[key] = value[10:]
>> +
>> + def _get_percentage_host_cpu_usage(self):
>> + # This is cpu usage producer. This producer will calculate
>> the usage
>> + # at an interval of HOST_STATS_INTERVAL.
>> + # The psutil.cpu_percent works as non blocking.
>> + # psutil.cpu_percent maintains a cpu time sample.
>> + # It will update the cpu time sample when it is called.
>> + # So only this producer can call psutil.cpu_percent in kimchi.
>> + self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None))
>> +
>> + def _get_host_memory_stats(self):
>> + virt_mem = psutil.virtual_memory()
>> + # available:
>> + # the actual amount of available memory that can be given
>> + # instantly to processes that request more memory in bytes;
>> this
>> + # is calculated by summing different memory values
>> depending on
>> + # the platform (e.g. free + buffers + cached on Linux)
>> + memory_stats = {'total': virt_mem.total,
>> + 'free': virt_mem.free,
>> + 'cached': virt_mem.cached,
>> + 'buffers': virt_mem.buffers,
>> + 'avail': virt_mem.available}
>> + self.host_stats['memory'].append(memory_stats)
>> +
>> + def _get_host_disk_io_rate(self, seconds):
>> + disk_read_bytes = self.host_stats['disk_read_bytes']
>> + disk_write_bytes = self.host_stats['disk_write_bytes']
>> + prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0
>> + prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes
>> else 0
>> +
>> + disk_io = psutil.disk_io_counters(False)
>> + read_bytes = disk_io.read_bytes
>> + write_bytes = disk_io.write_bytes
>> +
>> + rd_rate = int(float(read_bytes - prev_read_bytes) / seconds
>> + 0.5)
>> + wr_rate = int(float(write_bytes - prev_write_bytes) /
>> seconds + 0.5)
>> +
>> + self.host_stats['disk_read_rate'].append(rd_rate)
>> + self.host_stats['disk_write_rate'].append(wr_rate)
>> + self.host_stats['disk_read_bytes'].append(read_bytes)
>> + self.host_stats['disk_write_bytes'].append(write_bytes)
>> +
>> + def _get_host_network_io_rate(self, seconds):
>> + net_recv_bytes = self.host_stats['net_recv_bytes']
>> + net_sent_bytes = self.host_stats['net_sent_bytes']
>> + prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0
>> + prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0
>> +
>> + net_ios = psutil.network_io_counters(True)
>> + recv_bytes = 0
>> + sent_bytes = 0
>> + for key in set(netinfo.nics() +
>> + netinfo.wlans()) & set(net_ios.iterkeys()):
>> + recv_bytes = recv_bytes + net_ios[key].bytes_recv
>> + sent_bytes = sent_bytes + net_ios[key].bytes_sent
>> +
>> + rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds
>> + 0.5)
>> + tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds
>> + 0.5)
>> +
>> + self.host_stats['net_recv_rate'].append(rx_rate)
>> + self.host_stats['net_sent_rate'].append(tx_rate)
>> + self.host_stats['net_recv_bytes'].append(recv_bytes)
>> + self.host_stats['net_sent_bytes'].append(sent_bytes)
>> +
>> +
>> +class HostStatsHistoryModel(object):
>> + def __init__(self, **kargs):
>> + self.history = HostStatsModel(**kargs)
>> +
>> + def lookup(self, *name):
>> + return {'cpu_utilization':
>> self.history.host_stats['cpu_utilization'],
>> + 'memory': self.history.host_stats['memory'],
>> + 'disk_read_rate':
>> self.history.host_stats['disk_read_rate'],
>> + 'disk_write_rate':
>> self.history.host_stats['disk_write_rate'],
>> + 'net_recv_rate':
>> self.history.host_stats['net_recv_rate'],
>> + 'net_sent_rate':
>> self.history.host_stats['net_sent_rate']}
>> +
>> +
>> +class CapabilitiesModel(object):
>> + __metaclass__ = Singleton
>> +
>> + def __init__(self, **kargs):
>> + pass
>> +
>> +
>> + def lookup(self, *ident):
>> + report_tool = DebugReportsModel.get_system_report_tool()
>> + try:
>> + SoftwareUpdate()
>> + except Exception:
>> + update_tool = False
>> + else:
>> + update_tool = True
>> +
>> + try:
>> + repo = Repositories()
>> + except Exception:
>> + repo_mngt_tool = None
>> + else:
>> + repo_mngt_tool = repo._pkg_mnger.TYPE
>> +
>> + return {
>> + 'system_report_tool': bool(report_tool),
>> + 'update_tool': update_tool,
>> + 'repo_mngt_tool': repo_mngt_tool,
>> + 'federation': kconfig.get("server", "federation")
>> + }
>> +
>> +
>> +class PartitionsModel(object):
>> + def __init__(self, **kargs):
>> + pass
>> +
>> + def get_list(self):
>> + result = disks.get_partitions_names()
>> + return result
>> +
>> +
>> +class PartitionModel(object):
>> + def __init__(self, **kargs):
>> + pass
>> +
>> + def lookup(self, name):
>> + return disks.get_partition_details(name)
>> +
>> +class PackagesUpdateModel(object):
>> + def __init__(self, **kargs):
>> + try:
>> + self.host_swupdate = SoftwareUpdate()
>> + except:
>> + self.host_swupdate = None
>> +
>> + def get_list(self):
>> + if self.host_swupdate is None:
>> + raise OperationFailed('GGBPKGUPD0004E')
>> +
>> + return self.host_swupdate.getUpdates()
>> +
>> +
>> +class PackageUpdateModel(object):
>> + def __init__(self, **kargs):
>> + pass
>> +
>> + def lookup(self, name):
>> + try:
>> + swupdate = SoftwareUpdate()
>> + except Exception:
>> + raise OperationFailed('GGBPKGUPD0004E')
>> +
>> + return swupdate.getUpdate(name)
>> +
>> +
>> +class RepositoriesModel(object):
>> + def __init__(self, **kargs):
>> + try:
>> + self.host_repositories = Repositories()
>> + except:
>> + self.host_repositories = None
>> +
>> + def get_list(self):
>> + if self.host_repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return sorted(self.host_repositories.getRepositories())
>> +
>> + def create(self, params):
>> + if self.host_repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return self.host_repositories.addRepository(params)
>> +
>> +
>> +class RepositoryModel(object):
>> + def __init__(self, **kargs):
>> + try:
>> + self._repositories = Repositories()
>> + except:
>> + self._repositories = None
>> +
>> + def lookup(self, repo_id):
>> + if self._repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return self._repositories.getRepository(repo_id)
>> +
>> + def enable(self, repo_id):
>> + if self._repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return self._repositories.enableRepository(repo_id)
>> +
>> + def disable(self, repo_id):
>> + if self._repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return self._repositories.disableRepository(repo_id)
>> +
>> + def update(self, repo_id, params):
>> + if self._repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return self._repositories.updateRepository(repo_id, params)
>> +
>> + def delete(self, repo_id):
>> + if self._repositories is None:
>> + raise InvalidOperation('GGBREPOS0014E')
>> +
>> + return self._repositories.removeRepository(repo_id)
>> diff --git a/plugins/gingerbase/model/model.py
>> b/plugins/gingerbase/model/model.py
>> new file mode 100644
>> index 0000000..2da70f4
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/model.py
>> @@ -0,0 +1,49 @@
>> +#
>> +# Project Kimchi
>> +#
>> +# Copyright IBM, Corp. 2014-2015
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> +
>> +import inspect
>> +import os
>> +
>> +from wok.basemodel import BaseModel
>> +from wok.objectstore import ObjectStore
>> +from wok.utils import import_module, listPathModules
>> +
>> +
>> +class Model(BaseModel):
>> + def __init__(self, objstore_loc=None):
>> +
>> + self.objstore = ObjectStore(objstore_loc)
>> + kargs = {'objstore': self.objstore}
>> +
>> + this = os.path.basename(__file__)
>> + this_mod = os.path.splitext(this)[0]
>> +
>> + models = []
>> + for mod_name in listPathModules(os.path.dirname(__file__)):
>> + if mod_name.startswith("_") or mod_name == this_mod:
>> + continue
>> +
>> + module = import_module('plugins.gingerbase.model.' +
>> mod_name)
>> + members = inspect.getmembers(module, inspect.isclass)
>> + for cls_name, instance in members:
>> + if inspect.getmodule(instance) == module:
>> + if cls_name.endswith('Model'):
>> + models.append(instance(**kargs))
>> +
>> + return super(Model, self).__init__(models)
>> diff --git a/plugins/gingerbase/model/tasks.py
>> b/plugins/gingerbase/model/tasks.py
>> new file mode 100644
>> index 0000000..fe23e32
>> --- /dev/null
>> +++ b/plugins/gingerbase/model/tasks.py
>> @@ -0,0 +1,64 @@
>> +#
>> +# Project Kimchi
>> +#
>> +# Copyright IBM, Corp. 2014
>> +#
>> +# This library is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU Lesser General Public
>> +# License as published by the Free Software Foundation; either
>> +# version 2.1 of the License, or (at your option) any later version.
>> +#
>> +# This library is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# Lesser General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU Lesser General Public
>> +# License along with this library; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> +
>> +
>> +import time
>> +
>> +from wok.exception import TimeoutExpired
>> +
>> +
>> +class TasksModel(object):
>> + def __init__(self, **kargs):
>> + self.objstore = kargs['objstore']
>> +
>> + def get_list(self):
>> + with self.objstore as session:
>> + return session.get_list('task')
>> +
>> +
>> +class TaskModel(object):
>> + def __init__(self, **kargs):
>> + self.objstore = kargs['objstore']
>> +
>> + def lookup(self, id):
>> + with self.objstore as session:
>> + return session.get('task', str(id))
>> +
>> + def wait(self, id, timeout=10):
>> + """Wait for a task until it stops running (successfully or
>> due to
>> + an error). If the Task finishes its execution before
>> <timeout>, this
>> + function returns normally; otherwise an exception is raised.
>> +
>> + Parameters:
>> + id -- The Task ID.
>> + timeout -- The maximum time, in seconds, that this function
>> should wait
>> + for the Task. If the Task runs for more than <timeout>,
>> + "TimeoutExpired" is raised.
>> + """
>> + for i in range(0, timeout):
>> + with self.objstore as session:
>> + task = session.get('task', str(id))
>> +
>> + if task['status'] != 'running':
>> + return
>> +
>> + time.sleep(1)
>> +
>> + raise TimeoutExpired('GGBASYNC0001E', {'seconds': timeout,
>> + 'task':
>> task['target_uri']})
>> diff --git a/plugins/kimchi/model/debugreports.py
>> b/plugins/kimchi/model/debugreports.py
>> deleted file mode 100644
>> index d20eb12..0000000
>> --- a/plugins/kimchi/model/debugreports.py
>> +++ /dev/null
>> @@ -1,213 +0,0 @@
>> -#
>> -# Project Kimchi
>> -#
>> -# Copyright IBM, Corp. 2014-2015
>> -#
>> -# This library is free software; you can redistribute it and/or
>> -# modify it under the terms of the GNU Lesser General Public
>> -# License as published by the Free Software Foundation; either
>> -# version 2.1 of the License, or (at your option) any later version.
>> -#
>> -# This library is distributed in the hope that it will be useful,
>> -# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> -# Lesser General Public License for more details.
>> -#
>> -# You should have received a copy of the GNU Lesser General Public
>> -# License along with this library; if not, write to the Free Software
>> -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> -
>> -import fnmatch
>> -import glob
>> -import logging
>> -import os
>> -import shutil
>> -import subprocess
>> -import time
>> -
>> -from wok.exception import InvalidParameter, NotFoundError,
>> OperationFailed
>> -from wok.exception import WokException
>> -from wok.utils import add_task, wok_log
>> -from wok.utils import run_command
>> -
>> -from .. import config
>> -from tasks import TaskModel
>> -
>> -
>> -class DebugReportsModel(object):
>> - def __init__(self, **kargs):
>> - self.objstore = kargs['objstore']
>> - self.task = TaskModel(**kargs)
>> -
>> - def create(self, params):
>> - ident = params.get('name').strip()
>> - # Generate a name with time and millisec precision, if
>> necessary
>> - if ident is None or ident == "":
>> - ident = 'report-' + str(int(time.time() * 1000))
>> - else:
>> - if ident in self.get_list():
>> - raise InvalidParameter("KCHDR0008E", {"name": ident})
>> - taskid = self._gen_debugreport_file(ident)
>> - return self.task.lookup(taskid)
>> -
>> - def get_list(self):
>> - path = config.get_debugreports_path()
>> - file_pattern = os.path.join(path, '*.*')
>> - file_lists = glob.glob(file_pattern)
>> - file_lists = [os.path.split(file)[1] for file in file_lists]
>> - name_lists = [file.split('.', 1)[0] for file in file_lists]
>> -
>> - return name_lists
>> -
>> - def _gen_debugreport_file(self, name):
>> - gen_cmd = self.get_system_report_tool()
>> -
>> - if gen_cmd is not None:
>> - return add_task('/plugins/kimchi/debugreports/%s' %
>> name, gen_cmd,
>> - self.objstore, name)
>> -
>> - raise OperationFailed("KCHDR0002E")
>> -
>> - @staticmethod
>> - def sosreport_generate(cb, name):
>> - def log_error(e):
>> - log = logging.getLogger('Model')
>> - log.warning('Exception in generating debug file: %s', e)
>> -
>> - try:
>> - command = ['sosreport', '--batch', '--name=%s' % name]
>> - output, error, retcode = run_command(command)
>> -
>> - if retcode != 0:
>> - raise OperationFailed("KCHDR0003E", {'name': name,
>> - 'err': retcode})
>> -
>> - # SOSREPORT might create file in /tmp or /var/tmp
>> - # FIXME: The right way should be passing the tar.xz file
>> directory
>> - # though the parameter '--tmp-dir', but it is failing in
>> Fedora 20
>> - patterns = ['/tmp/sosreport-%s-*',
>> '/var/tmp/sosreport-%s-*']
>> - reports = []
>> - reportFile = None
>> - for p in patterns:
>> - reports = reports + [f for f in glob.glob(p % name)]
>> - for f in reports:
>> - if not fnmatch.fnmatch(f, '*.md5'):
>> - reportFile = f
>> - break
>> - # Some error in sosreport happened
>> - if reportFile is None:
>> - wok_log.error('Debug report file not found. See
>> sosreport '
>> - 'output for detail:\n%s', output)
>> - fname = (patterns[0] % name).split('/')[-1]
>> - raise OperationFailed('KCHDR0004E', {'name': fname})
>> -
>> - md5_report_file = reportFile + '.md5'
>> - report_file_extension = '.' + reportFile.split('.', 1)[1]
>> - path = config.get_debugreports_path()
>> - target = os.path.join(path, name + report_file_extension)
>> - # Moving report
>> - msg = 'Moving debug report file "%s" to "%s"' %
>> (reportFile,
>> - target)
>> - wok_log.info(msg)
>> - shutil.move(reportFile, target)
>> - # Deleting md5
>> - msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
>> - wok_log.info(msg)
>> - with open(md5_report_file) as f:
>> - md5 = f.read().strip()
>> - wok_log.info('Md5 file content: "%s"', md5)
>> - os.remove(md5_report_file)
>> - cb('OK', True)
>> - return
>> -
>> - except WokException as e:
>> - log_error(e)
>> - raise
>> -
>> - except OSError as e:
>> - log_error(e)
>> - raise
>> -
>> - except Exception, e:
>> - # No need to call cb to update the task status here.
>> - # The task object will catch the exception raised here
>> - # and update the task status there
>> - log_error(e)
>> - raise OperationFailed("KCHDR0005E", {'name': name,
>> 'err': e})
>> -
>> - @staticmethod
>> - def get_system_report_tool():
>> - # Please add new possible debug report command here
>> - # and implement the report generating function
>> - # based on the new report command
>> - report_tools = ({'cmd': 'sosreport --help',
>> - 'fn': DebugReportsModel.sosreport_generate},)
>> -
>> - # check if the command can be found by shell one by one
>> - for helper_tool in report_tools:
>> - try:
>> - retcode = subprocess.call(helper_tool['cmd'],
>> shell=True,
>> - stdout=subprocess.PIPE,
>> - stderr=subprocess.PIPE)
>> - if retcode == 0:
>> - return helper_tool['fn']
>> - except Exception, e:
>> - wok_log.info('Exception running command: %s', e)
>> -
>> - return None
>> -
>> -
>> -class DebugReportModel(object):
>> - def __init__(self, **kargs):
>> - pass
>> -
>> - def lookup(self, name):
>> - path = config.get_debugreports_path()
>> - file_pattern = os.path.join(path, name)
>> - file_pattern = file_pattern + '.*'
>> - try:
>> - file_target = glob.glob(file_pattern)[0]
>> - except IndexError:
>> - raise NotFoundError("KCHDR0001E", {'name': name})
>> -
>> - ctime = os.stat(file_target).st_mtime
>> - ctime = time.strftime("%Y-%m-%d-%H:%M:%S",
>> time.localtime(ctime))
>> - file_target = os.path.split(file_target)[-1]
>> - file_target = os.path.join("plugins/kimchi/data/debugreports",
>> - file_target)
>> - return {'uri': file_target,
>> - 'ctime': ctime}
>> -
>> - def update(self, name, params):
>> - path = config.get_debugreports_path()
>> - file_pattern = os.path.join(path, name + '.*')
>> - try:
>> - file_source = glob.glob(file_pattern)[0]
>> - except IndexError:
>> - raise NotFoundError("KCHDR0001E", {'name': name})
>> -
>> - file_target = file_source.replace(name, params['name'])
>> - if os.path.isfile(file_target):
>> - raise InvalidParameter('KCHDR0008E', {'name':
>> params['name']})
>> -
>> - shutil.move(file_source, file_target)
>> - wok_log.info('%s renamed to %s' % (file_source, file_target))
>> - return params['name']
>> -
>> - def delete(self, name):
>> - path = config.get_debugreports_path()
>> - file_pattern = os.path.join(path, name + '.*')
>> - try:
>> - file_target = glob.glob(file_pattern)[0]
>> - except IndexError:
>> - raise NotFoundError("KCHDR0001E", {'name': name})
>> -
>> - os.remove(file_target)
>> -
>> -
>> -class DebugReportContentModel(object):
>> - def __init__(self, **kargs):
>> - self._debugreport = DebugReportModel()
>> -
>> - def lookup(self, name):
>> - return self._debugreport.lookup(name)
>
>
More information about the Kimchi-devel
mailing list