[Kimchi-devel] [PATCH 07/17] Ginger Base : base plugin model files

Aline Manera alinefm at linux.vnet.ibm.com
Thu Sep 10 13:56:40 UTC 2015



On 01/09/2015 14:58, chandra at linux.vnet.ibm.com wrote:
> From: chandrureddy <chandra at linux.vnet.ibm.com>
>
> ---
>   plugins/gingerbase/model/Makefile.am     |  25 ++
>   plugins/gingerbase/model/__init__.py     |  18 ++
>   plugins/gingerbase/model/cpuinfo.py      | 133 ++++++++++
>   plugins/gingerbase/model/debugreports.py | 213 ++++++++++++++++
>   plugins/gingerbase/model/host.py         | 410 +++++++++++++++++++++++++++++++
>   plugins/gingerbase/model/model.py        |  49 ++++
>   plugins/gingerbase/model/tasks.py        |  64 +++++
>   plugins/kimchi/model/debugreports.py     | 213 ----------------
>   8 files changed, 912 insertions(+), 213 deletions(-)
>   create mode 100644 plugins/gingerbase/model/Makefile.am
>   create mode 100644 plugins/gingerbase/model/__init__.py


>   create mode 100644 plugins/gingerbase/model/cpuinfo.py

Why did you add it to gingerbase? It is used for Kimchi on different areas.

>   create mode 100644 plugins/gingerbase/model/debugreports.py
>   create mode 100644 plugins/gingerbase/model/host.py
>   create mode 100644 plugins/gingerbase/model/model.py

>   create mode 100644 plugins/gingerbase/model/tasks.py

Why did you add tasks.py to gingerbase?

>   delete mode 100644 plugins/kimchi/model/debugreports.py
>
> diff --git a/plugins/gingerbase/model/Makefile.am b/plugins/gingerbase/model/Makefile.am
> new file mode 100644
> index 0000000..6218b47
> --- /dev/null
> +++ b/plugins/gingerbase/model/Makefile.am
> @@ -0,0 +1,25 @@
> +#
> +# Kimchi
> +#
> +# Copyright IBM Corp, 2013
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
> +
> +model_PYTHON = *.py
> +
> +modeldir = $(pythondir)/wok/plugins/gingerbase/model
> +
> +install-data-local:
> +	$(MKDIR_P) $(DESTDIR)$(modeldir)
> diff --git a/plugins/gingerbase/model/__init__.py b/plugins/gingerbase/model/__init__.py
> new file mode 100644
> index 0000000..ca7ede4
> --- /dev/null
> +++ b/plugins/gingerbase/model/__init__.py
> @@ -0,0 +1,18 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> diff --git a/plugins/gingerbase/model/cpuinfo.py b/plugins/gingerbase/model/cpuinfo.py
> new file mode 100644
> index 0000000..fad8814
> --- /dev/null
> +++ b/plugins/gingerbase/model/cpuinfo.py
> @@ -0,0 +1,133 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014-2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> +
> +import platform
> +# from xml.etree import ElementTree as ET
> +
> +from wok.exception import InvalidParameter, InvalidOperation
> +from wok.utils import run_command, wok_log
> +from ..lscpu import LsCpu
> +
> +
> +ARCH = 'power' if platform.machine().startswith('ppc') else 'x86'
> +
> +
> +# def get_topo_capabilities(connect):
> +#     """
> +#     This helper function exists solely to be overridden for
> +#     mockmodel tests. Since other modules use getCapabilies(),
> +#     it can't be overridden directly.
> +#     """
> +#     xml = connect.getCapabilities()
> +#     capabilities = ET.fromstring(xml)
> +#     return capabilities.find('host').find('cpu').find('topology')
> +
> +
> +class CPUInfoModel(object):
> +    """
> +    Get information about a CPU for hyperthreading (on x86)
> +    or SMT (on POWER) for logic when creating templates and VMs.
> +    """
> +
> +    def __init__(self, **kargs):
> +        self.guest_threads_enabled = False
> +        self.sockets = 0
> +        self.cores_present = 0
> +        self.cores_available = 0
> +        self.cores_per_socket = 0
> +        self.threads_per_core = 0
> +        self.max_threads = 0
> +        self.lscpu = LsCpu()
> +
> +        # self.conn = kargs['conn']
> +        # libvirt_topology = None
> +        # try:
> +        #     connect = self.conn.get()
> +        #     libvirt_topology = get_topo_capabilities(connect)
> +        # except Exception as e:
> +        #     wok_log.info("Unable to get CPU topology capabilities: %s"
> +        #                     % e.message)
> +        #     return
> +        # if libvirt_topology is None:
> +        #     wok_log.info("cpu_info topology not supported.")
> +        #     return
> +
> +        if ARCH == 'power':
> +            # IBM PowerPC
> +            self.guest_threads_enabled = True
> +            out, error, rc = run_command(['ppc64_cpu', '--smt'])
> +            if rc or 'on' in out:
> +                # SMT has to be disabled for guest to use threads as CPUs.
> +                # rc is always zero, whether SMT is off or on.
> +                self.guest_threads_enabled = False
> +            out, error, rc = run_command(['ppc64_cpu', '--cores-present'])
> +            if not rc:
> +                self.cores_present = int(out.split()[-1])
> +            out, error, rc = run_command(['ppc64_cpu', '--cores-on'])
> +            if not rc:
> +                self.cores_available = int(out.split()[-1])
> +            out, error, rc = run_command(['ppc64_cpu', '--threads-per-core'])
> +            if not rc:
> +                self.threads_per_core = int(out.split()[-1])
> +            self.sockets = self.cores_present/self.threads_per_core
> +            if self.sockets == 0:
> +                self.sockets = 1
> +            self.cores_per_socket = self.cores_present/self.sockets
> +        else:
> +            # Intel or AMD
> +            self.guest_threads_enabled = True
> +            # self.sockets = int(libvirt_topology.get('sockets'))
> +            # self.cores_per_socket = int(libvirt_topology.get('cores'))
> +            # self.cores_present = self.cores_per_socket * self.sockets
> +            # self.cores_available = self.cores_present
> +            # self.threads_per_core = int(libvirt_topology.get('threads'))
> +            self.sockets = int(self.lscpu.get_sockets())
> +            self.cores_per_socket = int(self.lscpu.get_cores_per_socket())
> +            self.cores_present = self.cores_per_socket * self.sockets
> +            self.cores_available = self.cores_present
> +            self.threads_per_core = self.lscpu.get_threads_per_core()
> +
> +    def lookup(self, ident):
> +        return {
> +            'guest_threads_enabled': self.guest_threads_enabled,
> +            'sockets': self.sockets,
> +            'cores_per_socket': self.cores_per_socket,
> +            'cores_present': self.cores_present,
> +            'cores_available': self.cores_available,
> +            'threads_per_core': self.threads_per_core,
> +            }
> +
> +    def check_topology(self, vcpus, topology):
> +        """
> +            param vcpus: should be an integer
> +            param iso_path: the path of the guest ISO
> +            param topology: {'sockets': x, 'cores': x, 'threads': x}
> +        """
> +        sockets = topology['sockets']
> +        cores = topology['cores']
> +        threads = topology['threads']
> +
> +        if not self.guest_threads_enabled:
> +            raise InvalidOperation("GGBCPUINF0003E")
> +        if vcpus != sockets * cores * threads:
> +            raise InvalidParameter("GGBCPUINF0002E")
> +        if vcpus > self.cores_available * self.threads_per_core:
> +            raise InvalidParameter("GGBCPUINF0001E")
> +        if threads > self.threads_per_core:
> +            raise InvalidParameter("GGBCPUINF0002E")
> diff --git a/plugins/gingerbase/model/debugreports.py b/plugins/gingerbase/model/debugreports.py
> new file mode 100644
> index 0000000..fc91a9c
> --- /dev/null
> +++ b/plugins/gingerbase/model/debugreports.py
> @@ -0,0 +1,213 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014-2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> +
> +import fnmatch
> +import glob
> +import logging
> +import os
> +import shutil
> +import subprocess
> +import time
> +
> +from wok.exception import InvalidParameter, NotFoundError, OperationFailed
> +from wok.exception import WokException
> +from wok.utils import add_task, wok_log
> +from wok.utils import run_command
> +
> +from .. import config
> +from tasks import TaskModel
> +
> +
> +class DebugReportsModel(object):
> +    def __init__(self, **kargs):
> +        self.objstore = kargs['objstore']
> +        self.task = TaskModel(**kargs)
> +
> +    def create(self, params):
> +        ident = params.get('name').strip()
> +        # Generate a name with time and millisec precision, if necessary
> +        if ident is None or ident == "":
> +            ident = 'report-' + str(int(time.time() * 1000))
> +        else:
> +            if ident in self.get_list():
> +                raise InvalidParameter("GGBDR0008E", {"name": ident})
> +        taskid = self._gen_debugreport_file(ident)
> +        return self.task.lookup(taskid)
> +
> +    def get_list(self):
> +        path = config.get_debugreports_path()
> +        file_pattern = os.path.join(path, '*.*')
> +        file_lists = glob.glob(file_pattern)
> +        file_lists = [os.path.split(file)[1] for file in file_lists]
> +        name_lists = [file.split('.', 1)[0] for file in file_lists]
> +
> +        return name_lists
> +
> +    def _gen_debugreport_file(self, name):
> +        gen_cmd = self.get_system_report_tool()
> +
> +        if gen_cmd is not None:
> +            return add_task('/plugins/gingerbase/debugreports/%s' % name, gen_cmd,
> +                            self.objstore, name)
> +
> +        raise OperationFailed("GGBDR0002E")
> +
> +    @staticmethod
> +    def sosreport_generate(cb, name):
> +        def log_error(e):
> +            log = logging.getLogger('Model')
> +            log.warning('Exception in generating debug file: %s', e)
> +
> +        try:
> +            command = ['sosreport', '--batch', '--name=%s' % name]
> +            output, error, retcode = run_command(command)
> +
> +            if retcode != 0:
> +                raise OperationFailed("GGBDR0003E", {'name': name,
> +                                                     'err': retcode})
> +
> +            # SOSREPORT might create file in /tmp or /var/tmp
> +            # FIXME: The right way should be passing the tar.xz file directory
> +            # though the parameter '--tmp-dir', but it is failing in Fedora 20
> +            patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*']
> +            reports = []
> +            reportFile = None
> +            for p in patterns:
> +                reports = reports + [f for f in glob.glob(p % name)]
> +            for f in reports:
> +                if not fnmatch.fnmatch(f, '*.md5'):
> +                    reportFile = f
> +                    break
> +            # Some error in sosreport happened
> +            if reportFile is None:
> +                wok_log.error('Debug report file not found. See sosreport '
> +                                 'output for detail:\n%s', output)
> +                fname = (patterns[0] % name).split('/')[-1]
> +                raise OperationFailed('GGBDR0004E', {'name': fname})
> +
> +            md5_report_file = reportFile + '.md5'
> +            report_file_extension = '.' + reportFile.split('.', 1)[1]
> +            path = config.get_debugreports_path()
> +            target = os.path.join(path, name + report_file_extension)
> +            # Moving report
> +            msg = 'Moving debug report file "%s" to "%s"' % (reportFile,
> +                                                             target)
> +            wok_log.info(msg)
> +            shutil.move(reportFile, target)
> +            # Deleting md5
> +            msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
> +            wok_log.info(msg)
> +            with open(md5_report_file) as f:
> +                md5 = f.read().strip()
> +                wok_log.info('Md5 file content: "%s"', md5)
> +            os.remove(md5_report_file)
> +            cb('OK', True)
> +            return
> +
> +        except WokException as e:
> +            log_error(e)
> +            raise
> +
> +        except OSError as e:
> +            log_error(e)
> +            raise
> +
> +        except Exception, e:
> +            # No need to call cb to update the task status here.
> +            # The task object will catch the exception raised here
> +            # and update the task status there
> +            log_error(e)
> +            raise OperationFailed("GGBDR0005E", {'name': name, 'err': e})
> +
> +    @staticmethod
> +    def get_system_report_tool():
> +        # Please add new possible debug report command here
> +        # and implement the report generating function
> +        # based on the new report command
> +        report_tools = ({'cmd': 'sosreport --help',
> +                         'fn': DebugReportsModel.sosreport_generate},)
> +
> +        # check if the command can be found by shell one by one
> +        for helper_tool in report_tools:
> +            try:
> +                retcode = subprocess.call(helper_tool['cmd'], shell=True,
> +                                          stdout=subprocess.PIPE,
> +                                          stderr=subprocess.PIPE)
> +                if retcode == 0:
> +                    return helper_tool['fn']
> +            except Exception, e:
> +                wok_log.info('Exception running command: %s', e)
> +
> +        return None
> +
> +
> +class DebugReportModel(object):
> +    def __init__(self, **kargs):
> +        pass
> +
> +    def lookup(self, name):
> +        path = config.get_debugreports_path()
> +        file_pattern = os.path.join(path, name)
> +        file_pattern = file_pattern + '.*'
> +        try:
> +            file_target = glob.glob(file_pattern)[0]
> +        except IndexError:
> +            raise NotFoundError("GGBDR0001E", {'name': name})
> +
> +        ctime = os.stat(file_target).st_mtime
> +        ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime))
> +        file_target = os.path.split(file_target)[-1]
> +        file_target = os.path.join("plugins/gingerbase/data/debugreports",
> +                                   file_target)
> +        return {'uri': file_target,
> +                'ctime': ctime}
> +
> +    def update(self, name, params):
> +        path = config.get_debugreports_path()
> +        file_pattern = os.path.join(path, name + '.*')
> +        try:
> +            file_source = glob.glob(file_pattern)[0]
> +        except IndexError:
> +            raise NotFoundError("GGBDR0001E", {'name': name})
> +
> +        file_target = file_source.replace(name, params['name'])
> +        if os.path.isfile(file_target):
> +            raise InvalidParameter('GGBDR0008E', {'name': params['name']})
> +
> +        shutil.move(file_source, file_target)
> +        wok_log.info('%s renamed to %s' % (file_source, file_target))
> +        return params['name']
> +
> +    def delete(self, name):
> +        path = config.get_debugreports_path()
> +        file_pattern = os.path.join(path, name + '.*')
> +        try:
> +            file_target = glob.glob(file_pattern)[0]
> +        except IndexError:
> +            raise NotFoundError("GGBDR0001E", {'name': name})
> +
> +        os.remove(file_target)
> +
> +
> +class DebugReportContentModel(object):
> +    def __init__(self, **kargs):
> +        self._debugreport = DebugReportModel()
> +
> +    def lookup(self, name):
> +        return self._debugreport.lookup(name)
> diff --git a/plugins/gingerbase/model/host.py b/plugins/gingerbase/model/host.py
> new file mode 100644
> index 0000000..6522a27
> --- /dev/null
> +++ b/plugins/gingerbase/model/host.py
> @@ -0,0 +1,410 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014-2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> +
> +import os
> +import platform
> +import psutil
> +import time
> +from cherrypy.process.plugins import BackgroundTask
> +from collections import defaultdict
> +
> +from wok import netinfo
> +from wok.basemodel import Singleton
> +from wok.exception import InvalidOperation, InvalidParameter
> +from wok.exception import NotFoundError, OperationFailed
> +from wok.utils import add_task, wok_log
> +
> +
> +from .. import disks
> +from ..repositories import Repositories
> +from ..swupdate import SoftwareUpdate
> +from debugreports import DebugReportsModel
> +from tasks import TaskModel
> +from wok.config import config as kconfig
> +
> +
> +HOST_STATS_INTERVAL = 1
> +
> +
> +class HostModel(object):
> +    def __init__(self, **kargs):
> +        # self.conn = kargs['conn']
> +        self.objstore = kargs['objstore']
> +        self.task = TaskModel(**kargs)
> +        self.host_info = self._get_host_info()
> +
> +    def _get_ppc_cpu_info(self):
> +        res = {}
> +        with open('/proc/cpuinfo') as f:
> +            for line in f.xreadlines():
> +                # Parse CPU, CPU's revision and CPU's clock information
> +                for key in ['cpu', 'revision', 'clock']:
> +                    if key in line:
> +                        info = line.split(':')[1].strip()
> +                        if key == 'clock':
> +                            value = float(info.split('MHz')[0].strip()) / 1000
> +                        else:
> +                            value = info.split('(')[0].strip()
> +                        res[key] = value
> +
> +                        # Power machines show, for each cpu/core, a block with
> +                        # all cpu information. Here we control the scan of the
> +                        # necessary information (1st block provides
> +                        # everything), skipping the function when find all
> +                        # information.
> +                        if len(res.keys()) == 3:
> +                            return "%(cpu)s (%(revision)s) @ %(clock)s GHz\
> +                                    " % res
> +
> +        return ""
> +
> +    def _get_host_info(self):
> +        res = {}
> +        if platform.machine().startswith('ppc'):
> +            res['cpu_model'] = self._get_ppc_cpu_info()
> +        else:
> +            with open('/proc/cpuinfo') as f:
> +                for line in f.xreadlines():
> +                    if "model name" in line:
> +                        res['cpu_model'] = line.split(':')[1].strip()
> +                        break
> +
> +        res['cpus'] = 0
> +        res['memory'] = 0L
> +
> +        # Include IBM PowerKVM name to supported distro names
> +        _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
> +        # 'fedora' '17' 'Beefy Miracle'
> +        distro, version, codename = platform.linux_distribution(
> +            supported_dists=_sup_distros)
> +        res['os_distro'] = distro
> +        res['os_version'] = version
> +        res['os_codename'] = unicode(codename, "utf-8")
> +
> +        return res
> +
> +    def lookup(self, *name):
> +        cpus = psutil.NUM_CPUS
> +
> +        # psutil is unstable on how to get the number of
> +        # cpus, different versions call it differently
> +        if hasattr(psutil, 'cpu_count'):
> +            cpus = psutil.cpu_count()
> +
> +        elif hasattr(psutil, '_psplatform'):
> +            for method_name in ['_get_num_cpus', 'get_num_cpus']:
> +
> +                method = getattr(psutil._psplatform, method_name, None)
> +                if method is not None:
> +                    cpus = method()
> +                    break
> +
> +        self.host_info['cpus'] = cpus
> +        self.host_info['memory'] = psutil.phymem_usage().total
> +        return self.host_info
> +
> +    def swupdate(self, *name):
> +        try:
> +            swupdate = SoftwareUpdate()
> +        except:
> +            raise OperationFailed('GGBPKGUPD0004E')
> +
> +        pkgs = swupdate.getNumOfUpdates()
> +        if pkgs == 0:
> +            raise OperationFailed('GGBPKGUPD0001E')
> +
> +        wok_log.debug('Host is going to be updated.')
> +        taskid = add_task('/plugins/gingerbase/host/swupdate', swupdate.doUpdate,
> +                          self.objstore, None)
> +        return self.task.lookup(taskid)
> +
> +    def shutdown(self, args=None):
> +        # Check for running vms before shutdown
> +        # FIXME : Find alternative way to figure out if any vms running
> +        # running_vms = self._get_vms_list_by_state('running')
> +        # if len(running_vms) > 0:
> +        #     raise OperationFailed("GGBHOST0001E")
> +
> +        wok_log.info('Host is going to shutdown.')
> +        os.system('shutdown -h now')
> +
> +    def reboot(self, args=None):
> +        # Find running VMs
> +        # FIXME : Find alternative way to figure out if any vms running
> +        # running_vms = self._get_vms_list_by_state('running')
> +        # if len(running_vms) > 0:
> +        #     raise OperationFailed("GGBHOST0002E")
> +
> +        wok_log.info('Host is going to reboot.')
> +        os.system('reboot')
> +
> +    # def _get_vms_list_by_state(self, state):
> +    #     conn = self.conn.get()
> +    #     return [dom.name().decode('utf-8')
> +    #             for dom in conn.listAllDomains(0)
> +    #             if (DOM_STATE_MAP[dom.info()[0]]) == state]
> +
> +
> +class HostStatsModel(object):
> +    __metaclass__ = Singleton
> +
> +    def __init__(self, **kargs):
> +        self.host_stats = defaultdict(list)
> +        self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL,
> +                                                self._update_host_stats)
> +        self.host_stats_thread.start()
> +
> +    def lookup(self, *name):
> +        return {'cpu_utilization': self.host_stats['cpu_utilization'][-1],
> +                'memory': self.host_stats['memory'][-1],
> +                'disk_read_rate': self.host_stats['disk_read_rate'][-1],
> +                'disk_write_rate': self.host_stats['disk_write_rate'][-1],
> +                'net_recv_rate': self.host_stats['net_recv_rate'][-1],
> +                'net_sent_rate': self.host_stats['net_sent_rate'][-1]}
> +
> +    def _update_host_stats(self):
> +        preTimeStamp = self.host_stats['timestamp']
> +        timestamp = time.time()
> +        # FIXME when we upgrade psutil, we can get uptime by psutil.uptime
> +        # we get uptime by float(open("/proc/uptime").readline().split()[0])
> +        # and calculate the first io_rate after the OS started.
> +        with open("/proc/uptime") as time_f:
> +            seconds = (timestamp - preTimeStamp if preTimeStamp else
> +                       float(time_f.readline().split()[0]))
> +
> +        self.host_stats['timestamp'] = timestamp
> +        self._get_host_disk_io_rate(seconds)
> +        self._get_host_network_io_rate(seconds)
> +
> +        self._get_percentage_host_cpu_usage()
> +        self._get_host_memory_stats()
> +
> +        # store only 60 stats (1 min)
> +        for key, value in self.host_stats.iteritems():
> +            if isinstance(value, list):
> +                if len(value) == 60:
> +                    self.host_stats[key] = value[10:]
> +
> +    def _get_percentage_host_cpu_usage(self):
> +        # This is cpu usage producer. This producer will calculate the usage
> +        # at an interval of HOST_STATS_INTERVAL.
> +        # The psutil.cpu_percent works as non blocking.
> +        # psutil.cpu_percent maintains a cpu time sample.
> +        # It will update the cpu time sample when it is called.
> +        # So only this producer can call psutil.cpu_percent in kimchi.
> +        self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None))
> +
> +    def _get_host_memory_stats(self):
> +        virt_mem = psutil.virtual_memory()
> +        # available:
> +        #  the actual amount of available memory that can be given
> +        #  instantly to processes that request more memory in bytes; this
> +        #  is calculated by summing different memory values depending on
> +        #  the platform (e.g. free + buffers + cached on Linux)
> +        memory_stats = {'total': virt_mem.total,
> +                        'free': virt_mem.free,
> +                        'cached': virt_mem.cached,
> +                        'buffers': virt_mem.buffers,
> +                        'avail': virt_mem.available}
> +        self.host_stats['memory'].append(memory_stats)
> +
> +    def _get_host_disk_io_rate(self, seconds):
> +        disk_read_bytes = self.host_stats['disk_read_bytes']
> +        disk_write_bytes = self.host_stats['disk_write_bytes']
> +        prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0
> +        prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0
> +
> +        disk_io = psutil.disk_io_counters(False)
> +        read_bytes = disk_io.read_bytes
> +        write_bytes = disk_io.write_bytes
> +
> +        rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5)
> +        wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5)
> +
> +        self.host_stats['disk_read_rate'].append(rd_rate)
> +        self.host_stats['disk_write_rate'].append(wr_rate)
> +        self.host_stats['disk_read_bytes'].append(read_bytes)
> +        self.host_stats['disk_write_bytes'].append(write_bytes)
> +
> +    def _get_host_network_io_rate(self, seconds):
> +        net_recv_bytes = self.host_stats['net_recv_bytes']
> +        net_sent_bytes = self.host_stats['net_sent_bytes']
> +        prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0
> +        prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0
> +
> +        net_ios = psutil.network_io_counters(True)
> +        recv_bytes = 0
> +        sent_bytes = 0
> +        for key in set(netinfo.nics() +
> +                       netinfo.wlans()) & set(net_ios.iterkeys()):
> +            recv_bytes = recv_bytes + net_ios[key].bytes_recv
> +            sent_bytes = sent_bytes + net_ios[key].bytes_sent
> +
> +        rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5)
> +        tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5)
> +
> +        self.host_stats['net_recv_rate'].append(rx_rate)
> +        self.host_stats['net_sent_rate'].append(tx_rate)
> +        self.host_stats['net_recv_bytes'].append(recv_bytes)
> +        self.host_stats['net_sent_bytes'].append(sent_bytes)
> +
> +
> +class HostStatsHistoryModel(object):
> +    def __init__(self, **kargs):
> +        self.history = HostStatsModel(**kargs)
> +
> +    def lookup(self, *name):
> +        return {'cpu_utilization': self.history.host_stats['cpu_utilization'],
> +                'memory': self.history.host_stats['memory'],
> +                'disk_read_rate': self.history.host_stats['disk_read_rate'],
> +                'disk_write_rate': self.history.host_stats['disk_write_rate'],
> +                'net_recv_rate': self.history.host_stats['net_recv_rate'],
> +                'net_sent_rate': self.history.host_stats['net_sent_rate']}
> +
> +
> +class CapabilitiesModel(object):
> +    __metaclass__ = Singleton
> +
> +    def __init__(self, **kargs):
> +        pass
> +
> +
> +    def lookup(self, *ident):
> +        report_tool = DebugReportsModel.get_system_report_tool()
> +        try:
> +            SoftwareUpdate()
> +        except Exception:
> +            update_tool = False
> +        else:
> +            update_tool = True
> +
> +        try:
> +            repo = Repositories()
> +        except Exception:
> +            repo_mngt_tool = None
> +        else:
> +            repo_mngt_tool = repo._pkg_mnger.TYPE
> +
> +        return {
> +                'system_report_tool': bool(report_tool),
> +                'update_tool': update_tool,
> +                'repo_mngt_tool': repo_mngt_tool,
> +                'federation': kconfig.get("server", "federation")
> +                }
> +
> +
> +class PartitionsModel(object):
> +    def __init__(self, **kargs):
> +        pass
> +
> +    def get_list(self):
> +        result = disks.get_partitions_names()
> +        return result
> +
> +
> +class PartitionModel(object):
> +    def __init__(self, **kargs):
> +        pass
> +
> +    def lookup(self, name):
> +        return disks.get_partition_details(name)
> +
> +class PackagesUpdateModel(object):
> +    def __init__(self, **kargs):
> +        try:
> +            self.host_swupdate = SoftwareUpdate()
> +        except:
> +            self.host_swupdate = None
> +
> +    def get_list(self):
> +        if self.host_swupdate is None:
> +            raise OperationFailed('GGBPKGUPD0004E')
> +
> +        return self.host_swupdate.getUpdates()
> +
> +
> +class PackageUpdateModel(object):
> +    def __init__(self, **kargs):
> +        pass
> +
> +    def lookup(self, name):
> +        try:
> +            swupdate = SoftwareUpdate()
> +        except Exception:
> +            raise OperationFailed('GGBPKGUPD0004E')
> +
> +        return swupdate.getUpdate(name)
> +
> +
> +class RepositoriesModel(object):
> +    def __init__(self, **kargs):
> +        try:
> +            self.host_repositories = Repositories()
> +        except:
> +            self.host_repositories = None
> +
> +    def get_list(self):
> +        if self.host_repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return sorted(self.host_repositories.getRepositories())
> +
> +    def create(self, params):
> +        if self.host_repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return self.host_repositories.addRepository(params)
> +
> +
> +class RepositoryModel(object):
> +    def __init__(self, **kargs):
> +        try:
> +            self._repositories = Repositories()
> +        except:
> +            self._repositories = None
> +
> +    def lookup(self, repo_id):
> +        if self._repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return self._repositories.getRepository(repo_id)
> +
> +    def enable(self, repo_id):
> +        if self._repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return self._repositories.enableRepository(repo_id)
> +
> +    def disable(self, repo_id):
> +        if self._repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return self._repositories.disableRepository(repo_id)
> +
> +    def update(self, repo_id, params):
> +        if self._repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return self._repositories.updateRepository(repo_id, params)
> +
> +    def delete(self, repo_id):
> +        if self._repositories is None:
> +            raise InvalidOperation('GGBREPOS0014E')
> +
> +        return self._repositories.removeRepository(repo_id)
> diff --git a/plugins/gingerbase/model/model.py b/plugins/gingerbase/model/model.py
> new file mode 100644
> index 0000000..2da70f4
> --- /dev/null
> +++ b/plugins/gingerbase/model/model.py
> @@ -0,0 +1,49 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014-2015
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> +
> +import inspect
> +import os
> +
> +from wok.basemodel import BaseModel
> +from wok.objectstore import ObjectStore
> +from wok.utils import import_module, listPathModules
> +
> +
> +class Model(BaseModel):
> +    def __init__(self, objstore_loc=None):
> +
> +        self.objstore = ObjectStore(objstore_loc)
> +        kargs = {'objstore': self.objstore}
> +
> +        this = os.path.basename(__file__)
> +        this_mod = os.path.splitext(this)[0]
> +
> +        models = []
> +        for mod_name in listPathModules(os.path.dirname(__file__)):
> +            if mod_name.startswith("_") or mod_name == this_mod:
> +                continue
> +
> +            module = import_module('plugins.gingerbase.model.' + mod_name)
> +            members = inspect.getmembers(module, inspect.isclass)
> +            for cls_name, instance in members:
> +                if inspect.getmodule(instance) == module:
> +                    if cls_name.endswith('Model'):
> +                        models.append(instance(**kargs))
> +
> +        return super(Model, self).__init__(models)
> diff --git a/plugins/gingerbase/model/tasks.py b/plugins/gingerbase/model/tasks.py
> new file mode 100644
> index 0000000..fe23e32
> --- /dev/null
> +++ b/plugins/gingerbase/model/tasks.py
> @@ -0,0 +1,64 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> +
> +
> +import time
> +
> +from wok.exception import TimeoutExpired
> +
> +
> +class TasksModel(object):
> +    def __init__(self, **kargs):
> +        self.objstore = kargs['objstore']
> +
> +    def get_list(self):
> +        with self.objstore as session:
> +            return session.get_list('task')
> +
> +
> +class TaskModel(object):
> +    def __init__(self, **kargs):
> +        self.objstore = kargs['objstore']
> +
> +    def lookup(self, id):
> +        with self.objstore as session:
> +            return session.get('task', str(id))
> +
> +    def wait(self, id, timeout=10):
> +        """Wait for a task until it stops running (successfully or due to
> +        an error). If the Task finishes its execution before <timeout>, this
> +        function returns normally; otherwise an exception is raised.
> +
> +        Parameters:
> +        id -- The Task ID.
> +        timeout -- The maximum time, in seconds, that this function should wait
> +            for the Task. If the Task runs for more than <timeout>,
> +            "TimeoutExpired" is raised.
> +        """
> +        for i in range(0, timeout):
> +            with self.objstore as session:
> +                task = session.get('task', str(id))
> +
> +            if task['status'] != 'running':
> +                return
> +
> +            time.sleep(1)
> +
> +        raise TimeoutExpired('GGBASYNC0001E', {'seconds': timeout,
> +                                               'task': task['target_uri']})
> diff --git a/plugins/kimchi/model/debugreports.py b/plugins/kimchi/model/debugreports.py
> deleted file mode 100644
> index d20eb12..0000000
> --- a/plugins/kimchi/model/debugreports.py
> +++ /dev/null
> @@ -1,213 +0,0 @@
> -#
> -# Project Kimchi
> -#
> -# Copyright IBM, Corp. 2014-2015
> -#
> -# This library is free software; you can redistribute it and/or
> -# modify it under the terms of the GNU Lesser General Public
> -# License as published by the Free Software Foundation; either
> -# version 2.1 of the License, or (at your option) any later version.
> -#
> -# This library is distributed in the hope that it will be useful,
> -# but WITHOUT ANY WARRANTY; without even the implied warranty of
> -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> -# Lesser General Public License for more details.
> -#
> -# You should have received a copy of the GNU Lesser General Public
> -# License along with this library; if not, write to the Free Software
> -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> -
> -import fnmatch
> -import glob
> -import logging
> -import os
> -import shutil
> -import subprocess
> -import time
> -
> -from wok.exception import InvalidParameter, NotFoundError, OperationFailed
> -from wok.exception import WokException
> -from wok.utils import add_task, wok_log
> -from wok.utils import run_command
> -
> -from .. import config
> -from tasks import TaskModel
> -
> -
> -class DebugReportsModel(object):
> -    def __init__(self, **kargs):
> -        self.objstore = kargs['objstore']
> -        self.task = TaskModel(**kargs)
> -
> -    def create(self, params):
> -        ident = params.get('name').strip()
> -        # Generate a name with time and millisec precision, if necessary
> -        if ident is None or ident == "":
> -            ident = 'report-' + str(int(time.time() * 1000))
> -        else:
> -            if ident in self.get_list():
> -                raise InvalidParameter("KCHDR0008E", {"name": ident})
> -        taskid = self._gen_debugreport_file(ident)
> -        return self.task.lookup(taskid)
> -
> -    def get_list(self):
> -        path = config.get_debugreports_path()
> -        file_pattern = os.path.join(path, '*.*')
> -        file_lists = glob.glob(file_pattern)
> -        file_lists = [os.path.split(file)[1] for file in file_lists]
> -        name_lists = [file.split('.', 1)[0] for file in file_lists]
> -
> -        return name_lists
> -
> -    def _gen_debugreport_file(self, name):
> -        gen_cmd = self.get_system_report_tool()
> -
> -        if gen_cmd is not None:
> -            return add_task('/plugins/kimchi/debugreports/%s' % name, gen_cmd,
> -                            self.objstore, name)
> -
> -        raise OperationFailed("KCHDR0002E")
> -
> -    @staticmethod
> -    def sosreport_generate(cb, name):
> -        def log_error(e):
> -            log = logging.getLogger('Model')
> -            log.warning('Exception in generating debug file: %s', e)
> -
> -        try:
> -            command = ['sosreport', '--batch', '--name=%s' % name]
> -            output, error, retcode = run_command(command)
> -
> -            if retcode != 0:
> -                raise OperationFailed("KCHDR0003E", {'name': name,
> -                                                     'err': retcode})
> -
> -            # SOSREPORT might create file in /tmp or /var/tmp
> -            # FIXME: The right way should be passing the tar.xz file directory
> -            # though the parameter '--tmp-dir', but it is failing in Fedora 20
> -            patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*']
> -            reports = []
> -            reportFile = None
> -            for p in patterns:
> -                reports = reports + [f for f in glob.glob(p % name)]
> -            for f in reports:
> -                if not fnmatch.fnmatch(f, '*.md5'):
> -                    reportFile = f
> -                    break
> -            # Some error in sosreport happened
> -            if reportFile is None:
> -                wok_log.error('Debug report file not found. See sosreport '
> -                                 'output for detail:\n%s', output)
> -                fname = (patterns[0] % name).split('/')[-1]
> -                raise OperationFailed('KCHDR0004E', {'name': fname})
> -
> -            md5_report_file = reportFile + '.md5'
> -            report_file_extension = '.' + reportFile.split('.', 1)[1]
> -            path = config.get_debugreports_path()
> -            target = os.path.join(path, name + report_file_extension)
> -            # Moving report
> -            msg = 'Moving debug report file "%s" to "%s"' % (reportFile,
> -                                                             target)
> -            wok_log.info(msg)
> -            shutil.move(reportFile, target)
> -            # Deleting md5
> -            msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
> -            wok_log.info(msg)
> -            with open(md5_report_file) as f:
> -                md5 = f.read().strip()
> -                wok_log.info('Md5 file content: "%s"', md5)
> -            os.remove(md5_report_file)
> -            cb('OK', True)
> -            return
> -
> -        except WokException as e:
> -            log_error(e)
> -            raise
> -
> -        except OSError as e:
> -            log_error(e)
> -            raise
> -
> -        except Exception, e:
> -            # No need to call cb to update the task status here.
> -            # The task object will catch the exception raised here
> -            # and update the task status there
> -            log_error(e)
> -            raise OperationFailed("KCHDR0005E", {'name': name, 'err': e})
> -
> -    @staticmethod
> -    def get_system_report_tool():
> -        # Please add new possible debug report command here
> -        # and implement the report generating function
> -        # based on the new report command
> -        report_tools = ({'cmd': 'sosreport --help',
> -                         'fn': DebugReportsModel.sosreport_generate},)
> -
> -        # check if the command can be found by shell one by one
> -        for helper_tool in report_tools:
> -            try:
> -                retcode = subprocess.call(helper_tool['cmd'], shell=True,
> -                                          stdout=subprocess.PIPE,
> -                                          stderr=subprocess.PIPE)
> -                if retcode == 0:
> -                    return helper_tool['fn']
> -            except Exception, e:
> -                wok_log.info('Exception running command: %s', e)
> -
> -        return None
> -
> -
> -class DebugReportModel(object):
> -    def __init__(self, **kargs):
> -        pass
> -
> -    def lookup(self, name):
> -        path = config.get_debugreports_path()
> -        file_pattern = os.path.join(path, name)
> -        file_pattern = file_pattern + '.*'
> -        try:
> -            file_target = glob.glob(file_pattern)[0]
> -        except IndexError:
> -            raise NotFoundError("KCHDR0001E", {'name': name})
> -
> -        ctime = os.stat(file_target).st_mtime
> -        ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime))
> -        file_target = os.path.split(file_target)[-1]
> -        file_target = os.path.join("plugins/kimchi/data/debugreports",
> -                                   file_target)
> -        return {'uri': file_target,
> -                'ctime': ctime}
> -
> -    def update(self, name, params):
> -        path = config.get_debugreports_path()
> -        file_pattern = os.path.join(path, name + '.*')
> -        try:
> -            file_source = glob.glob(file_pattern)[0]
> -        except IndexError:
> -            raise NotFoundError("KCHDR0001E", {'name': name})
> -
> -        file_target = file_source.replace(name, params['name'])
> -        if os.path.isfile(file_target):
> -            raise InvalidParameter('KCHDR0008E', {'name': params['name']})
> -
> -        shutil.move(file_source, file_target)
> -        wok_log.info('%s renamed to %s' % (file_source, file_target))
> -        return params['name']
> -
> -    def delete(self, name):
> -        path = config.get_debugreports_path()
> -        file_pattern = os.path.join(path, name + '.*')
> -        try:
> -            file_target = glob.glob(file_pattern)[0]
> -        except IndexError:
> -            raise NotFoundError("KCHDR0001E", {'name': name})
> -
> -        os.remove(file_target)
> -
> -
> -class DebugReportContentModel(object):
> -    def __init__(self, **kargs):
> -        self._debugreport = DebugReportModel()
> -
> -    def lookup(self, name):
> -        return self._debugreport.lookup(name)




More information about the Kimchi-devel mailing list