[Kimchi-devel] [PATCH 10/17] V7 Ginger Base : base plugin model files

chandra at linux.vnet.ibm.com chandra at linux.vnet.ibm.com
Wed Oct 21 11:10:55 UTC 2015


From: chandrureddy <chandra at linux.vnet.ibm.com>

---
 src/wok/plugins/gingerbase/model/Makefile.am     |  25 ++
 src/wok/plugins/gingerbase/model/__init__.py     |  18 +
 src/wok/plugins/gingerbase/model/config.py       |  83 +++++
 src/wok/plugins/gingerbase/model/cpuinfo.py      | 105 ++++++
 src/wok/plugins/gingerbase/model/debugreports.py | 215 +++++++++++
 src/wok/plugins/gingerbase/model/host.py         | 449 +++++++++++++++++++++++
 src/wok/plugins/gingerbase/model/model.py        |  68 ++++
 src/wok/plugins/kimchi/model/debugreports.py     | 213 -----------
 8 files changed, 963 insertions(+), 213 deletions(-)
 create mode 100644 src/wok/plugins/gingerbase/model/Makefile.am
 create mode 100644 src/wok/plugins/gingerbase/model/__init__.py
 create mode 100644 src/wok/plugins/gingerbase/model/config.py
 create mode 100644 src/wok/plugins/gingerbase/model/cpuinfo.py
 create mode 100644 src/wok/plugins/gingerbase/model/debugreports.py
 create mode 100644 src/wok/plugins/gingerbase/model/host.py
 create mode 100644 src/wok/plugins/gingerbase/model/model.py
 delete mode 100644 src/wok/plugins/kimchi/model/debugreports.py

diff --git a/src/wok/plugins/gingerbase/model/Makefile.am b/src/wok/plugins/gingerbase/model/Makefile.am
new file mode 100644
index 0000000..80c4e07
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/Makefile.am
@@ -0,0 +1,25 @@
+#
+# Ginger Base
+#
+# Copyright IBM Corp, 2015
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+model_PYTHON = *.py
+
+modeldir = $(pythondir)/wok/plugins/gingerbase/model
+
+install-data-local:
+	$(MKDIR_P) $(DESTDIR)$(modeldir)
diff --git a/src/wok/plugins/gingerbase/model/__init__.py b/src/wok/plugins/gingerbase/model/__init__.py
new file mode 100644
index 0000000..355a76c
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/__init__.py
@@ -0,0 +1,18 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2015
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
diff --git a/src/wok/plugins/gingerbase/model/config.py b/src/wok/plugins/gingerbase/model/config.py
new file mode 100644
index 0000000..8aed61f
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/config.py
@@ -0,0 +1,83 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2014-2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+import cherrypy
+
+from wok.basemodel import Singleton
+from wok.config import config as kconfig
+from wok.config import get_version
+from wok.utils import wok_log
+
+from ..repositories import Repositories
+from ..swupdate import SoftwareUpdate
+from debugreports import DebugReportsModel
+
+
+class ConfigModel(object):
+    def __init__(self, **kargs):
+        pass
+
+    def lookup(self, name):
+        proxy_port = kconfig.get('display', 'display_proxy_port')
+        return {'display_proxy_port': proxy_port,
+                'version': get_version()}
+
+
+class CapabilitiesModel(object):
+    __metaclass__ = Singleton
+
+    def __init__(self, **kargs):
+        # Subscribe function to set host capabilities to be run when cherrypy
+        # server is up
+        # It is needed because some features tests depends on the server
+        cherrypy.engine.subscribe('start', self._set_capabilities)
+
+        # Subscribe function to clean any Kimchi leftovers
+        cherrypy.engine.subscribe('stop', self._clean_leftovers)
+
+    def _clean_leftovers(self):
+        pass
+
+    def _set_capabilities(self):
+        wok_log.info("*** Running feature tests ***")
+        wok_log.info("*** Feature tests completed ***")
+    _set_capabilities.priority = 90
+
+    def lookup(self, *ident):
+        report_tool = DebugReportsModel.get_system_report_tool()
+        try:
+            SoftwareUpdate()
+        except Exception:
+            update_tool = False
+        else:
+            update_tool = True
+
+        try:
+            repo = Repositories()
+        except Exception:
+            repo_mngt_tool = None
+        else:
+            repo_mngt_tool = repo._pkg_mnger.TYPE
+
+        return {'system_report_tool': bool(report_tool),
+                'update_tool': update_tool,
+                'repo_mngt_tool': repo_mngt_tool,
+                }
diff --git a/src/wok/plugins/gingerbase/model/cpuinfo.py b/src/wok/plugins/gingerbase/model/cpuinfo.py
new file mode 100644
index 0000000..d28d4e1
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/cpuinfo.py
@@ -0,0 +1,105 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2014-2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+import platform
+
+from wok.exception import InvalidParameter, InvalidOperation
+from wok.utils import run_command
+from wok.plugins.gingerbase.lscpu import LsCpu
+
+
+ARCH = 'power' if platform.machine().startswith('ppc') else 'x86'
+
+
+class CPUInfoModel(object):
+    """
+    Get information about a CPU for hyperthreading (on x86)
+    or SMT (on POWER) for logic when creating templates and VMs.
+    """
+
+    def __init__(self, **kargs):
+        self.guest_threads_enabled = False
+        self.sockets = 0
+        self.cores_present = 0
+        self.cores_available = 0
+        self.cores_per_socket = 0
+        self.threads_per_core = 0
+        self.max_threads = 0
+        self.lscpu = LsCpu()
+
+        if ARCH == 'power':
+            # IBM PowerPC
+            self.guest_threads_enabled = True
+            out, error, rc = run_command(['ppc64_cpu', '--smt'])
+            if rc or 'on' in out:
+                # SMT has to be disabled for guest to use threads as CPUs.
+                # rc is always zero, whether SMT is off or on.
+                self.guest_threads_enabled = False
+            out, error, rc = run_command(['ppc64_cpu', '--cores-present'])
+            if not rc:
+                self.cores_present = int(out.split()[-1])
+            out, error, rc = run_command(['ppc64_cpu', '--cores-on'])
+            if not rc:
+                self.cores_available = int(out.split()[-1])
+            out, error, rc = run_command(['ppc64_cpu', '--threads-per-core'])
+            if not rc:
+                self.threads_per_core = int(out.split()[-1])
+            self.sockets = self.cores_present/self.threads_per_core
+            if self.sockets == 0:
+                self.sockets = 1
+            self.cores_per_socket = self.cores_present/self.sockets
+        else:
+            # Intel or AMD
+            self.guest_threads_enabled = True
+            self.sockets = int(self.lscpu.get_sockets())
+            self.cores_per_socket = int(self.lscpu.get_cores_per_socket())
+            self.cores_present = self.cores_per_socket * self.sockets
+            self.cores_available = self.cores_present
+            self.threads_per_core = self.lscpu.get_threads_per_core()
+
+    def lookup(self, ident):
+        return {
+            'guest_threads_enabled': self.guest_threads_enabled,
+            'sockets': self.sockets,
+            'cores_per_socket': self.cores_per_socket,
+            'cores_present': self.cores_present,
+            'cores_available': self.cores_available,
+            'threads_per_core': self.threads_per_core,
+            }
+
+    def check_topology(self, vcpus, topology):
+        """
+            param vcpus: should be an integer
+            param iso_path: the path of the guest ISO
+            param topology: {'sockets': x, 'cores': x, 'threads': x}
+        """
+        sockets = topology['sockets']
+        cores = topology['cores']
+        threads = topology['threads']
+
+        if not self.guest_threads_enabled:
+            raise InvalidOperation("GGBCPUINF0003E")
+        if vcpus != sockets * cores * threads:
+            raise InvalidParameter("GGBCPUINF0002E")
+        if vcpus > self.cores_available * self.threads_per_core:
+            raise InvalidParameter("GGBCPUINF0001E")
+        if threads > self.threads_per_core:
+            raise InvalidParameter("GGBCPUINF0002E")
diff --git a/src/wok/plugins/gingerbase/model/debugreports.py b/src/wok/plugins/gingerbase/model/debugreports.py
new file mode 100644
index 0000000..94ab7fe
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/debugreports.py
@@ -0,0 +1,215 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2014-2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+import fnmatch
+import glob
+import logging
+import os
+import shutil
+import subprocess
+import time
+
+from wok.exception import InvalidParameter, NotFoundError, OperationFailed
+from wok.exception import WokException
+from wok.utils import add_task, wok_log
+from wok.utils import run_command
+from wok.model.tasks import TaskModel
+
+from wok.plugins.gingerbase import config
+
+
+class DebugReportsModel(object):
+    def __init__(self, **kargs):
+        self.objstore = kargs['objstore']
+        self.task = TaskModel(**kargs)
+
+    def create(self, params):
+        ident = params.get('name').strip()
+        # Generate a name with time and millisec precision, if necessary
+        if ident is None or ident == "":
+            ident = 'report-' + str(int(time.time() * 1000))
+        else:
+            if ident in self.get_list():
+                raise InvalidParameter("GGBDR0008E", {"name": ident})
+        taskid = self._gen_debugreport_file(ident)
+        return self.task.lookup(taskid)
+
+    def get_list(self):
+        path = config.get_debugreports_path()
+        file_pattern = os.path.join(path, '*.*')
+        file_lists = glob.glob(file_pattern)
+        file_lists = [os.path.split(file)[1] for file in file_lists]
+        name_lists = [file.split('.', 1)[0] for file in file_lists]
+
+        return name_lists
+
+    def _gen_debugreport_file(self, name):
+        gen_cmd = self.get_system_report_tool()
+
+        if gen_cmd is not None:
+            return add_task('/plugins/gingerbase/debugreports/%s' % name,
+                            gen_cmd, self.objstore, name)
+
+        raise OperationFailed("GGBDR0002E")
+
+    @staticmethod
+    def sosreport_generate(cb, name):
+        def log_error(e):
+            log = logging.getLogger('Model')
+            log.warning('Exception in generating debug file: %s', e)
+
+        try:
+            command = ['sosreport', '--batch', '--name=%s' % name]
+            output, error, retcode = run_command(command)
+
+            if retcode != 0:
+                raise OperationFailed("GGBDR0003E", {'name': name,
+                                                     'err': retcode})
+
+            # SOSREPORT might create file in /tmp or /var/tmp
+            # FIXME: The right way should be passing the tar.xz file directory
+            # though the parameter '--tmp-dir', but it is failing in Fedora 20
+            patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*']
+            reports = []
+            reportFile = None
+            for p in patterns:
+                reports = reports + [f for f in glob.glob(p % name)]
+            for f in reports:
+                if not fnmatch.fnmatch(f, '*.md5'):
+                    reportFile = f
+                    break
+            # Some error in sosreport happened
+            if reportFile is None:
+                wok_log.error('Debug report file not found. See sosreport '
+                              'output for detail:\n%s', output)
+                fname = (patterns[0] % name).split('/')[-1]
+                raise OperationFailed('GGBDR0004E', {'name': fname})
+
+            md5_report_file = reportFile + '.md5'
+            report_file_extension = '.' + reportFile.split('.', 1)[1]
+            path = config.get_debugreports_path()
+            target = os.path.join(path, name + report_file_extension)
+            # Moving report
+            msg = 'Moving debug report file "%s" to "%s"' % (reportFile,
+                                                             target)
+            wok_log.info(msg)
+            shutil.move(reportFile, target)
+            # Deleting md5
+            msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
+            wok_log.info(msg)
+            with open(md5_report_file) as f:
+                md5 = f.read().strip()
+                wok_log.info('Md5 file content: "%s"', md5)
+            os.remove(md5_report_file)
+            cb('OK', True)
+            return
+
+        except WokException as e:
+            log_error(e)
+            raise
+
+        except OSError as e:
+            log_error(e)
+            raise
+
+        except Exception, e:
+            # No need to call cb to update the task status here.
+            # The task object will catch the exception raised here
+            # and update the task status there
+            log_error(e)
+            raise OperationFailed("GGBDR0005E", {'name': name, 'err': e})
+
+    @staticmethod
+    def get_system_report_tool():
+        # Please add new possible debug report command here
+        # and implement the report generating function
+        # based on the new report command
+        report_tools = ({'cmd': 'sosreport --help',
+                         'fn': DebugReportsModel.sosreport_generate},)
+
+        # check if the command can be found by shell one by one
+        for helper_tool in report_tools:
+            try:
+                retcode = subprocess.call(helper_tool['cmd'], shell=True,
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.PIPE)
+                if retcode == 0:
+                    return helper_tool['fn']
+            except Exception, e:
+                wok_log.info('Exception running command: %s', e)
+
+        return None
+
+
+class DebugReportModel(object):
+    def __init__(self, **kargs):
+        pass
+
+    def lookup(self, name):
+        path = config.get_debugreports_path()
+        file_pattern = os.path.join(path, name)
+        file_pattern = file_pattern + '.*'
+        try:
+            file_target = glob.glob(file_pattern)[0]
+        except IndexError:
+            raise NotFoundError("GGBDR0001E", {'name': name})
+
+        ctime = os.stat(file_target).st_mtime
+        ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime))
+        file_target = os.path.split(file_target)[-1]
+        file_target = os.path.join("plugins/gingerbase/data/debugreports",
+                                   file_target)
+        return {'uri': file_target,
+                'ctime': ctime}
+
+    def update(self, name, params):
+        path = config.get_debugreports_path()
+        file_pattern = os.path.join(path, name + '.*')
+        try:
+            file_source = glob.glob(file_pattern)[0]
+        except IndexError:
+            raise NotFoundError("GGBDR0001E", {'name': name})
+
+        file_target = file_source.replace(name, params['name'])
+        if os.path.isfile(file_target):
+            raise InvalidParameter('GGBDR0008E', {'name': params['name']})
+
+        shutil.move(file_source, file_target)
+        wok_log.info('%s renamed to %s' % (file_source, file_target))
+        return params['name']
+
+    def delete(self, name):
+        path = config.get_debugreports_path()
+        file_pattern = os.path.join(path, name + '.*')
+        try:
+            file_target = glob.glob(file_pattern)[0]
+        except IndexError:
+            raise NotFoundError("GGBDR0001E", {'name': name})
+
+        os.remove(file_target)
+
+
+class DebugReportContentModel(object):
+    def __init__(self, **kargs):
+        self._debugreport = DebugReportModel()
+
+    def lookup(self, name):
+        return self._debugreport.lookup(name)
diff --git a/src/wok/plugins/gingerbase/model/host.py b/src/wok/plugins/gingerbase/model/host.py
new file mode 100644
index 0000000..062cf24
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/host.py
@@ -0,0 +1,449 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+import os
+import platform
+import psutil
+import time
+from cherrypy.process.plugins import BackgroundTask
+from collections import defaultdict
+import glob
+
+from wok.basemodel import Singleton
+from wok.exception import InvalidOperation
+from wok.exception import OperationFailed
+from wok.utils import add_task, wok_log
+from wok.model.tasks import TaskModel
+from wok.config import config as kconfig
+
+
+from wok.plugins.gingerbase import disks
+from wok.plugins.gingerbase.repositories import Repositories
+from wok.plugins.gingerbase.swupdate import SoftwareUpdate
+from wok.plugins.gingerbase.model.debugreports import DebugReportsModel
+
+
+HOST_STATS_INTERVAL = 1
+
+
+class HostModel(object):
+    def __init__(self, **kargs):
+        # self.conn = kargs['conn']
+        self.objstore = kargs['objstore']
+        self.task = TaskModel(**kargs)
+        self.host_info = self._get_host_info()
+
+    def _get_ppc_cpu_info(self):
+        res = {}
+        with open('/proc/cpuinfo') as f:
+            for line in f.xreadlines():
+                # Parse CPU, CPU's revision and CPU's clock information
+                for key in ['cpu', 'revision', 'clock']:
+                    if key in line:
+                        info = line.split(':')[1].strip()
+                        if key == 'clock':
+                            value = float(info.split('MHz')[0].strip()) / 1000
+                        else:
+                            value = info.split('(')[0].strip()
+                        res[key] = value
+
+                        # Power machines show, for each cpu/core, a block with
+                        # all cpu information. Here we control the scan of the
+                        # necessary information (1st block provides
+                        # everything), skipping the function when find all
+                        # information.
+                        if len(res.keys()) == 3:
+                            return "%(cpu)s (%(revision)s) @ %(clock)s GHz\
+                                    " % res
+
+        return ""
+
+    def _get_host_info(self):
+        res = {}
+        if platform.machine().startswith('ppc'):
+            res['cpu_model'] = self._get_ppc_cpu_info()
+        else:
+            with open('/proc/cpuinfo') as f:
+                for line in f.xreadlines():
+                    if "model name" in line:
+                        res['cpu_model'] = line.split(':')[1].strip()
+                        break
+
+        res['cpus'] = 0
+        res['memory'] = 0L
+
+        # Include IBM PowerKVM name to supported distro names
+        _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
+        # 'fedora' '17' 'Beefy Miracle'
+        distro, version, codename = platform.linux_distribution(
+            supported_dists=_sup_distros)
+        res['os_distro'] = distro
+        res['os_version'] = version
+        res['os_codename'] = unicode(codename, "utf-8")
+
+        return res
+
+    def lookup(self, *name):
+        cpus = psutil.NUM_CPUS
+
+        # psutil is unstable on how to get the number of
+        # cpus, different versions call it differently
+        if hasattr(psutil, 'cpu_count'):
+            cpus = psutil.cpu_count()
+
+        elif hasattr(psutil, 'NUM_CPUS'):
+            cpus = psutil.NUM_CPUS
+
+        elif hasattr(psutil, '_psplatform'):
+            for method_name in ['_get_num_cpus', 'get_num_cpus']:
+
+                method = getattr(psutil._psplatform, method_name, None)
+                if method is not None:
+                    cpus = method()
+                    break
+
+        self.host_info['cpus'] = cpus
+        if hasattr(psutil, 'phymem_usage'):
+            self.host_info['memory'] = psutil.phymem_usage().total
+        elif hasattr(psutil, 'virtual_memory'):
+            self.host_info['memory'] = psutil.virtual_memory().total
+        return self.host_info
+
+    def swupdate(self, *name):
+        try:
+            swupdate = SoftwareUpdate()
+        except:
+            raise OperationFailed('GGBPKGUPD0004E')
+
+        pkgs = swupdate.getNumOfUpdates()
+        if pkgs == 0:
+            raise OperationFailed('GGBPKGUPD0001E')
+
+        wok_log.debug('Host is going to be updated.')
+        taskid = add_task('/plugins/gingerbase/host/swupdate',
+                          swupdate.doUpdate,
+                          self.objstore, None)
+        return self.task.lookup(taskid)
+
+    def shutdown(self, args=None):
+        # Check for running vms before shutdown
+        # FIXME : Find alternative way to figure out if any vms running
+        # running_vms = self._get_vms_list_by_state('running')
+        # if len(running_vms) > 0:
+        #     raise OperationFailed("GGBHOST0001E")
+
+        wok_log.info('Host is going to shutdown.')
+        os.system('shutdown -h now')
+
+    def reboot(self, args=None):
+        # Find running VMs
+        # FIXME : Find alternative way to figure out if any vms running
+        # running_vms = self._get_vms_list_by_state('running')
+        # if len(running_vms) > 0:
+        #     raise OperationFailed("GGBHOST0002E")
+
+        wok_log.info('Host is going to reboot.')
+        os.system('reboot')
+
+    # def _get_vms_list_by_state(self, state):
+    #     conn = self.conn.get()
+    #     return [dom.name().decode('utf-8')
+    #             for dom in conn.listAllDomains(0)
+    #             if (DOM_STATE_MAP[dom.info()[0]]) == state]
+
+
+class SoftwareUpdateProgressModel(object):
+    def __init__(self, **kargs):
+        self.task = TaskModel(**kargs)
+        self.objstore = kargs['objstore']
+
+    def lookup(self, *name):
+        try:
+            swupdate = SoftwareUpdate()
+        except:
+            raise OperationFailed('GGBPKGUPD0004E')
+
+        taskid = add_task('/plugins/kimchi/host/swupdateprogress',
+                          swupdate.tailUpdateLogs, self.objstore, None)
+        return self.task.lookup(taskid)
+
+
+class HostStatsModel(object):
+    __metaclass__ = Singleton
+
+    def __init__(self, **kargs):
+        self.host_stats = defaultdict(list)
+        self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL,
+                                                self._update_host_stats)
+        self.host_stats_thread.start()
+
+    def lookup(self, *name):
+        return {'cpu_utilization': self.host_stats['cpu_utilization'][-1],
+                'memory': self.host_stats['memory'][-1],
+                'disk_read_rate': self.host_stats['disk_read_rate'][-1],
+                'disk_write_rate': self.host_stats['disk_write_rate'][-1],
+                'net_recv_rate': self.host_stats['net_recv_rate'][-1],
+                'net_sent_rate': self.host_stats['net_sent_rate'][-1]}
+
+    def _update_host_stats(self):
+        preTimeStamp = self.host_stats['timestamp']
+        timestamp = time.time()
+        # FIXME when we upgrade psutil, we can get uptime by psutil.uptime
+        # we get uptime by float(open("/proc/uptime").readline().split()[0])
+        # and calculate the first io_rate after the OS started.
+        with open("/proc/uptime") as time_f:
+            seconds = (timestamp - preTimeStamp if preTimeStamp else
+                       float(time_f.readline().split()[0]))
+
+        self.host_stats['timestamp'] = timestamp
+        self._get_host_disk_io_rate(seconds)
+        self._get_host_network_io_rate(seconds)
+
+        self._get_percentage_host_cpu_usage()
+        self._get_host_memory_stats()
+
+        # store only 60 stats (1 min)
+        for key, value in self.host_stats.iteritems():
+            if isinstance(value, list):
+                if len(value) == 60:
+                    self.host_stats[key] = value[10:]
+
+    def _get_percentage_host_cpu_usage(self):
+        # This is cpu usage producer. This producer will calculate the usage
+        # at an interval of HOST_STATS_INTERVAL.
+        # The psutil.cpu_percent works as non blocking.
+        # psutil.cpu_percent maintains a cpu time sample.
+        # It will update the cpu time sample when it is called.
+        # So only this producer can call psutil.cpu_percent in kimchi.
+        self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None))
+
+    def _get_host_memory_stats(self):
+        virt_mem = psutil.virtual_memory()
+        # available:
+        #  the actual amount of available memory that can be given
+        #  instantly to processes that request more memory in bytes; this
+        #  is calculated by summing different memory values depending on
+        #  the platform (e.g. free + buffers + cached on Linux)
+        memory_stats = {'total': virt_mem.total,
+                        'free': virt_mem.free,
+                        'cached': virt_mem.cached,
+                        'buffers': virt_mem.buffers,
+                        'avail': virt_mem.available}
+        self.host_stats['memory'].append(memory_stats)
+
+    def _get_host_disk_io_rate(self, seconds):
+        disk_read_bytes = self.host_stats['disk_read_bytes']
+        disk_write_bytes = self.host_stats['disk_write_bytes']
+        prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0
+        prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0
+
+        disk_io = psutil.disk_io_counters(False)
+        read_bytes = disk_io.read_bytes
+        write_bytes = disk_io.write_bytes
+
+        rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5)
+        wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5)
+
+        self.host_stats['disk_read_rate'].append(rd_rate)
+        self.host_stats['disk_write_rate'].append(wr_rate)
+        self.host_stats['disk_read_bytes'].append(read_bytes)
+        self.host_stats['disk_write_bytes'].append(write_bytes)
+
+    def _get_host_network_io_rate(self, seconds):
+        net_recv_bytes = self.host_stats['net_recv_bytes']
+        net_sent_bytes = self.host_stats['net_sent_bytes']
+        prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0
+        prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0
+
+        net_ios = None
+        if hasattr(psutil, 'net_io_counters'):
+            net_ios = psutil.net_io_counters(True)
+        elif hasattr(psutil, 'network_io_counters'):
+            net_ios = psutil.network_io_counters(True)
+
+        recv_bytes = 0
+        sent_bytes = 0
+        for key in set(self.nics() +
+                       self.wlans()) & set(net_ios.iterkeys()):
+            recv_bytes = recv_bytes + net_ios[key].bytes_recv
+            sent_bytes = sent_bytes + net_ios[key].bytes_sent
+
+        rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5)
+        tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5)
+
+        self.host_stats['net_recv_rate'].append(rx_rate)
+        self.host_stats['net_sent_rate'].append(tx_rate)
+        self.host_stats['net_recv_bytes'].append(recv_bytes)
+        self.host_stats['net_sent_bytes'].append(sent_bytes)
+
+    def wlans(self):
+        WLAN_PATH = '/sys/class/net/*/wireless'
+        return [b.split('/')[-2] for b in glob.glob(WLAN_PATH)]
+
+    # FIXME if we do not want to list usb nic
+    def nics(self):
+        NIC_PATH = '/sys/class/net/*/device'
+        return list(set([b.split('/')[-2] for b in glob.glob(NIC_PATH)]) -
+                    set(self.wlans()))
+
+
+class HostStatsHistoryModel(object):
+    def __init__(self, **kargs):
+        self.history = HostStatsModel(**kargs)
+
+    def lookup(self, *name):
+        return {'cpu_utilization': self.history.host_stats['cpu_utilization'],
+                'memory': self.history.host_stats['memory'],
+                'disk_read_rate': self.history.host_stats['disk_read_rate'],
+                'disk_write_rate': self.history.host_stats['disk_write_rate'],
+                'net_recv_rate': self.history.host_stats['net_recv_rate'],
+                'net_sent_rate': self.history.host_stats['net_sent_rate']}
+
+
+class CapabilitiesModel(object):
+    __metaclass__ = Singleton
+
+    def __init__(self, **kargs):
+        pass
+
+    def lookup(self, *ident):
+        report_tool = DebugReportsModel.get_system_report_tool()
+        try:
+            SoftwareUpdate()
+        except Exception:
+            update_tool = False
+        else:
+            update_tool = True
+
+        try:
+            repo = Repositories()
+        except Exception:
+            repo_mngt_tool = None
+        else:
+            repo_mngt_tool = repo._pkg_mnger.TYPE
+
+        return {'system_report_tool': bool(report_tool),
+                'update_tool': update_tool,
+                'repo_mngt_tool': repo_mngt_tool,
+                'federation': kconfig.get("server", "federation")
+                }
+
+
+class PartitionsModel(object):
+    def __init__(self, **kargs):
+        pass
+
+    def get_list(self):
+        result = disks.get_partitions_names()
+        return result
+
+
+class PartitionModel(object):
+    def __init__(self, **kargs):
+        pass
+
+    def lookup(self, name):
+        return disks.get_partition_details(name)
+
+
+class PackagesUpdateModel(object):
+    def __init__(self, **kargs):
+        try:
+            self.host_swupdate = SoftwareUpdate()
+        except:
+            self.host_swupdate = None
+
+    def get_list(self):
+        if self.host_swupdate is None:
+            raise OperationFailed('GGBPKGUPD0004E')
+
+        return self.host_swupdate.getUpdates()
+
+
+class PackageUpdateModel(object):
+    def __init__(self, **kargs):
+        pass
+
+    def lookup(self, name):
+        try:
+            swupdate = SoftwareUpdate()
+        except Exception:
+            raise OperationFailed('GGBPKGUPD0004E')
+
+        return swupdate.getUpdate(name)
+
+
+class RepositoriesModel(object):
+    def __init__(self, **kargs):
+        try:
+            self.host_repositories = Repositories()
+        except:
+            self.host_repositories = None
+
+    def get_list(self):
+        if self.host_repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return sorted(self.host_repositories.getRepositories())
+
+    def create(self, params):
+        if self.host_repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return self.host_repositories.addRepository(params)
+
+
+class RepositoryModel(object):
+    def __init__(self, **kargs):
+        try:
+            self._repositories = Repositories()
+        except:
+            self._repositories = None
+
+    def lookup(self, repo_id):
+        if self._repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return self._repositories.getRepository(repo_id)
+
+    def enable(self, repo_id):
+        if self._repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return self._repositories.enableRepository(repo_id)
+
+    def disable(self, repo_id):
+        if self._repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return self._repositories.disableRepository(repo_id)
+
+    def update(self, repo_id, params):
+        if self._repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return self._repositories.updateRepository(repo_id, params)
+
+    def delete(self, repo_id):
+        if self._repositories is None:
+            raise InvalidOperation('GGBREPOS0014E')
+
+        return self._repositories.removeRepository(repo_id)
diff --git a/src/wok/plugins/gingerbase/model/model.py b/src/wok/plugins/gingerbase/model/model.py
new file mode 100644
index 0000000..1f56cc7
--- /dev/null
+++ b/src/wok/plugins/gingerbase/model/model.py
@@ -0,0 +1,68 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2015
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+import inspect
+import os
+
+from wok.basemodel import BaseModel
+from wok.objectstore import ObjectStore
+from wok.plugins.gingerbase import config
+from wok.utils import import_module, listPathModules
+
+
+class Model(BaseModel):
+    def __init__(self, objstore_loc=None):
+
+        def get_instances(module_name):
+            instances = []
+            module = import_module(module_name)
+            members = inspect.getmembers(module, inspect.isclass)
+            for cls_name, instance in members:
+                if inspect.getmodule(instance) == module and \
+                   cls_name.endswith('Model'):
+                    instances.append(instance)
+
+            return instances
+
+        if objstore_loc is None:
+            objstore_loc = config.get_object_store()
+
+        self.objstore = ObjectStore(objstore_loc)
+        kargs = {'objstore': self.objstore}
+
+        models = []
+        # Import task model from Wok
+        instances = get_instances('wok.model.tasks')
+        for instance in instances:
+            models.append(instance(**kargs))
+
+        # Import all Kimchi plugin models
+        this = os.path.basename(__file__)
+        this_mod = os.path.splitext(this)[0]
+
+        for mod_name in listPathModules(os.path.dirname(__file__)):
+            if mod_name.startswith("_") or mod_name == this_mod:
+                continue
+
+            instances = get_instances('wok.plugins.gingerbase.model.'
+                                      + mod_name)
+            for instance in instances:
+                models.append(instance(**kargs))
+
+        return super(Model, self).__init__(models)
diff --git a/src/wok/plugins/kimchi/model/debugreports.py b/src/wok/plugins/kimchi/model/debugreports.py
deleted file mode 100644
index dcd1b64..0000000
--- a/src/wok/plugins/kimchi/model/debugreports.py
+++ /dev/null
@@ -1,213 +0,0 @@
-#
-# Project Kimchi
-#
-# Copyright IBM, Corp. 2014-2015
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-
-import fnmatch
-import glob
-import logging
-import os
-import shutil
-import subprocess
-import time
-
-from wok.exception import InvalidParameter, NotFoundError, OperationFailed
-from wok.exception import WokException
-from wok.utils import add_task, wok_log
-from wok.utils import run_command
-from wok.model.tasks import TaskModel
-
-from wok.plugins.kimchi import config
-
-
-class DebugReportsModel(object):
-    def __init__(self, **kargs):
-        self.objstore = kargs['objstore']
-        self.task = TaskModel(**kargs)
-
-    def create(self, params):
-        ident = params.get('name').strip()
-        # Generate a name with time and millisec precision, if necessary
-        if ident is None or ident == "":
-            ident = 'report-' + str(int(time.time() * 1000))
-        else:
-            if ident in self.get_list():
-                raise InvalidParameter("KCHDR0008E", {"name": ident})
-        taskid = self._gen_debugreport_file(ident)
-        return self.task.lookup(taskid)
-
-    def get_list(self):
-        path = config.get_debugreports_path()
-        file_pattern = os.path.join(path, '*.*')
-        file_lists = glob.glob(file_pattern)
-        file_lists = [os.path.split(file)[1] for file in file_lists]
-        name_lists = [file.split('.', 1)[0] for file in file_lists]
-
-        return name_lists
-
-    def _gen_debugreport_file(self, name):
-        gen_cmd = self.get_system_report_tool()
-
-        if gen_cmd is not None:
-            return add_task('/plugins/kimchi/debugreports/%s' % name, gen_cmd,
-                            self.objstore, name)
-
-        raise OperationFailed("KCHDR0002E")
-
-    @staticmethod
-    def sosreport_generate(cb, name):
-        def log_error(e):
-            log = logging.getLogger('Model')
-            log.warning('Exception in generating debug file: %s', e)
-
-        try:
-            command = ['sosreport', '--batch', '--name=%s' % name]
-            output, error, retcode = run_command(command)
-
-            if retcode != 0:
-                raise OperationFailed("KCHDR0003E", {'name': name,
-                                                     'err': retcode})
-
-            # SOSREPORT might create file in /tmp or /var/tmp
-            # FIXME: The right way should be passing the tar.xz file directory
-            # though the parameter '--tmp-dir', but it is failing in Fedora 20
-            patterns = ['/tmp/sosreport-%s-*', '/var/tmp/sosreport-%s-*']
-            reports = []
-            reportFile = None
-            for p in patterns:
-                reports = reports + [f for f in glob.glob(p % name)]
-            for f in reports:
-                if not fnmatch.fnmatch(f, '*.md5'):
-                    reportFile = f
-                    break
-            # Some error in sosreport happened
-            if reportFile is None:
-                wok_log.error('Debug report file not found. See sosreport '
-                              'output for detail:\n%s', output)
-                fname = (patterns[0] % name).split('/')[-1]
-                raise OperationFailed('KCHDR0004E', {'name': fname})
-
-            md5_report_file = reportFile + '.md5'
-            report_file_extension = '.' + reportFile.split('.', 1)[1]
-            path = config.get_debugreports_path()
-            target = os.path.join(path, name + report_file_extension)
-            # Moving report
-            msg = 'Moving debug report file "%s" to "%s"' % (reportFile,
-                                                             target)
-            wok_log.info(msg)
-            shutil.move(reportFile, target)
-            # Deleting md5
-            msg = 'Deleting report md5 file: "%s"' % (md5_report_file)
-            wok_log.info(msg)
-            with open(md5_report_file) as f:
-                md5 = f.read().strip()
-                wok_log.info('Md5 file content: "%s"', md5)
-            os.remove(md5_report_file)
-            cb('OK', True)
-            return
-
-        except WokException as e:
-            log_error(e)
-            raise
-
-        except OSError as e:
-            log_error(e)
-            raise
-
-        except Exception, e:
-            # No need to call cb to update the task status here.
-            # The task object will catch the exception raised here
-            # and update the task status there
-            log_error(e)
-            raise OperationFailed("KCHDR0005E", {'name': name, 'err': e})
-
-    @staticmethod
-    def get_system_report_tool():
-        # Please add new possible debug report command here
-        # and implement the report generating function
-        # based on the new report command
-        report_tools = ({'cmd': 'sosreport --help',
-                         'fn': DebugReportsModel.sosreport_generate},)
-
-        # check if the command can be found by shell one by one
-        for helper_tool in report_tools:
-            try:
-                retcode = subprocess.call(helper_tool['cmd'], shell=True,
-                                          stdout=subprocess.PIPE,
-                                          stderr=subprocess.PIPE)
-                if retcode == 0:
-                    return helper_tool['fn']
-            except Exception, e:
-                wok_log.info('Exception running command: %s', e)
-
-        return None
-
-
-class DebugReportModel(object):
-    def __init__(self, **kargs):
-        pass
-
-    def lookup(self, name):
-        path = config.get_debugreports_path()
-        file_pattern = os.path.join(path, name)
-        file_pattern = file_pattern + '.*'
-        try:
-            file_target = glob.glob(file_pattern)[0]
-        except IndexError:
-            raise NotFoundError("KCHDR0001E", {'name': name})
-
-        ctime = os.stat(file_target).st_mtime
-        ctime = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime(ctime))
-        file_target = os.path.split(file_target)[-1]
-        file_target = os.path.join("plugins/kimchi/data/debugreports",
-                                   file_target)
-        return {'uri': file_target,
-                'ctime': ctime}
-
-    def update(self, name, params):
-        path = config.get_debugreports_path()
-        file_pattern = os.path.join(path, name + '.*')
-        try:
-            file_source = glob.glob(file_pattern)[0]
-        except IndexError:
-            raise NotFoundError("KCHDR0001E", {'name': name})
-
-        file_target = file_source.replace(name, params['name'])
-        if os.path.isfile(file_target):
-            raise InvalidParameter('KCHDR0008E', {'name': params['name']})
-
-        shutil.move(file_source, file_target)
-        wok_log.info('%s renamed to %s' % (file_source, file_target))
-        return params['name']
-
-    def delete(self, name):
-        path = config.get_debugreports_path()
-        file_pattern = os.path.join(path, name + '.*')
-        try:
-            file_target = glob.glob(file_pattern)[0]
-        except IndexError:
-            raise NotFoundError("KCHDR0001E", {'name': name})
-
-        os.remove(file_target)
-
-
-class DebugReportContentModel(object):
-    def __init__(self, **kargs):
-        self._debugreport = DebugReportModel()
-
-    def lookup(self, name):
-        return self._debugreport.lookup(name)
-- 
2.1.0




More information about the Kimchi-devel mailing list