[Kimchi-devel] [PATCH V2] [Ginger-Base 1/1] Issue 728 - Processor Info in s390 architecture

From: Suresh Babu Angadi <sureshab(a)in.ibm.com&gt; Fix for issue 728: processor info displays blank for system z this patch set also adds additional capability: retrieving architecture and host name (for all architecture) split CPUs to show online and offline cpus also includes shared and dedicated cpus for s390x split memory to show online and offline additional virtualization details(for s390x): virtualization will have hypervisor details and lpar details Signed-off-by: Suresh Babu Angadi <sureshab(a)in.ibm.com&gt; --- src/wok/plugins/gingerbase/docs/API.md | 18 +- src/wok/plugins/gingerbase/i18n.py | 1 + src/wok/plugins/gingerbase/lscpu.py | 60 ++++++ src/wok/plugins/gingerbase/model/host.py | 259 ++++++++++++++++++++++---- src/wok/plugins/gingerbase/tests/test_host.py | 12 +- 5 files changed, 307 insertions(+), 43 deletions(-) diff --git a/src/wok/plugins/gingerbase/docs/API.md b/src/wok/plugins/gingerbase/docs/API.md index 8f37ddc..30b0c8f 100644 --- a/src/wok/plugins/gingerbase/docs/API.md +++ b/src/wok/plugins/gingerbase/docs/API.md @@ -95,13 +95,25 @@ Contains information of host. **Methods:** * **GET**: Retrieve host static information - * memory: Total size of host physical memory - The unit is Bytes + * memory: Memory details of the host. The unit is Bytes + * online: Total size of physical memory currently available to host + * offline: Total offline memory(for s390x), zero for other architectures. * cpu_model: The model name of host CPU - * cpus: The number of online CPUs available on host + * cpus: The host cpus information + * online: Number of online CPUs + * offline: Number of offline CPUs + * shared *(s390x only)*: Number of shared CPUs + * dedicated *(s390x only)*: Number of dedicated CPUs * os_distro: The OS distribution that runs on host * os_version: The version of OS distribution * os_codename: The code name of OS distribution + * host: Host name of the network node + * architecture: Architecture of the host + * virtualization *(s390x only)*: This is a json with following keys + * hypervisor: Hypervisor name of the host + * hypervisor_vendor: Hypervisor Vendor name + * lpar_number: LPAR Number of host + * lpar_name: Name of host LPAR * **POST**: *See Host Actions* diff --git a/src/wok/plugins/gingerbase/i18n.py b/src/wok/plugins/gingerbase/i18n.py index af75c70..a0a8c36 100644 --- a/src/wok/plugins/gingerbase/i18n.py +++ b/src/wok/plugins/gingerbase/i18n.py @@ -88,5 +88,6 @@ messages = { "GGBCPUINF0005E": _("This host (or current configuration) does not provide Socket(s) information."), "GGBCPUINF0006E": _("This host (or current configuration) does not provide Core(s) per socket information."), "GGBCPUINF0007E": _("This host (or current configuration) does not provide Thread(s) per core information."), + "GGBCPUINF0008E": _("This host (or current configuration) does not provide CPU(s) information."), } diff --git a/src/wok/plugins/gingerbase/lscpu.py b/src/wok/plugins/gingerbase/lscpu.py index 0fc1e72..4686049 100644 --- a/src/wok/plugins/gingerbase/lscpu.py +++ b/src/wok/plugins/gingerbase/lscpu.py @@ -17,10 +17,13 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging +import platform from wok.utils import run_command from wok.exception import NotFoundError +ARCH = platform.machine() + class LsCpu(object): """ @@ -64,6 +67,28 @@ class LsCpu(object): # L2 cache: 256K # L3 cache: 3072K # NUMA node0 CPU(s): 0-3 + # + # Output of lscpu in s390x is expected to be + # Architecture: s390x + # CPU op-mode(s): 32-bit, 64-bit + # Byte Order: Big Endian + # CPU(s): 4 + # On-line CPU(s) list: 0,1 + # Off-line CPU(s) list: 2,3 + # Thread(s) per core: 1 + # Core(s) per socket: 6 + # Socket(s) per book: 6 + # Book(s): 4 + # Vendor ID: IBM/S390 + # BogoMIPS: 18115.00 + # Hypervisor: PR/SM + # Hypervisor vendor: IBM + # Virtualization type: full + # Dispatching mode: horizontal + # L1d cache: 96K + # L1i cache: 64K + # L2d cache: 1024K + # L2i cache: 1024K if not rc and (not out.isspace()): lscpuout = out.split('\n') @@ -85,6 +110,8 @@ class LsCpu(object): """ try: sockets = "Socket(s)" + if ARCH.startswith('s390x'): + sockets = "Socket(s) per book" if len(self.lsCpuInfo) > 0 and sockets in self.lsCpuInfo.keys(): return int(self.lsCpuInfo[sockets]) else: @@ -124,3 +151,36 @@ class LsCpu(object): except IndexError, e: self.log_error(e) raise NotFoundError("GGBCPUINF0007E") + + def get_total_cpus(self): + """ + + :return: + """ + total_cpus = 'CPU(s)' + if len(self.lsCpuInfo) > 0 and total_cpus in self.lsCpuInfo.keys(): + return int(self.lsCpuInfo[total_cpus]) + else: + self.log_error("Failed to fetch total cpus count in lscpu output") + raise NotFoundError("GGBCPUINF0008E") + + def get_hypervisor(self): + """ + method to get hypervisor name if present in lscpu o/p + :return: Hypervisor Name + """ + hypervisor = 'Hypervisor' + if len(self.lsCpuInfo) > 0 and hypervisor in self.lsCpuInfo.keys(): + return self.lsCpuInfo[hypervisor] + return None + + def get_hypervisor_vendor(self): + """ + method to get hypervisor vendor if present in lscpu o/p + :return: Hypervisor Vendor + """ + hypervisor_vendor = 'Hypervisor vendor' + if len(self.lsCpuInfo) > 0 and hypervisor_vendor in \ + self.lsCpuInfo.keys(): + return self.lsCpuInfo[hypervisor_vendor] + return None diff --git a/src/wok/plugins/gingerbase/model/host.py b/src/wok/plugins/gingerbase/model/host.py index a58bada..fd4db29 100644 --- a/src/wok/plugins/gingerbase/model/host.py +++ b/src/wok/plugins/gingerbase/model/host.py @@ -22,6 +22,7 @@ import os import platform import psutil +import re import time from cherrypy.process.plugins import BackgroundTask from collections import defaultdict @@ -31,9 +32,10 @@ from wok.basemodel import Singleton from wok.config import config as kconfig from wok.exception import InvalidOperation from wok.exception import OperationFailed -from wok.utils import add_task, wok_log +from wok.utils import add_task, run_command, wok_log from wok.model.tasks import TaskModel +from wok.plugins.gingerbase.lscpu import LsCpu from wok.plugins.gingerbase.model.debugreports import DebugReportsModel from wok.plugins.gingerbase.repositories import Repositories from wok.plugins.gingerbase.swupdate import SoftwareUpdate @@ -48,17 +50,29 @@ DOM_STATE_MAP = {0: 'nostate', 6: 'crashed', 7: 'pmsuspended'} +ARCH = platform.machine() +PROC_CPUINFO = '/proc/cpuinfo' +PROC_SYSINFO = '/proc/sysinfo' +LSMEM = 'lsmem' +CPUS_DEDICATED = 'cpus_dedicated' +CPUS_SHARED = 'cpus_shared' +LPAR_NAME = 'lpar_name' +LPAR_NUMBER = 'lpar_number' + class HostModel(object): def __init__(self, **kargs): # self.conn = kargs['conn'] self.objstore = kargs['objstore'] self.task = TaskModel(**kargs) - self.host_info = self._get_host_info() + self.lscpu = LsCpu() - def _get_ppc_cpu_info(self): + def _get_ppc_cpu_model(self): + """ + method to get cpu_model for ppc architecture + """ res = {} - with open('/proc/cpuinfo') as f: + with open(PROC_CPUINFO) as f: for line in f.xreadlines(): # Parse CPU, CPU's revision and CPU's clock information for key in ['cpu', 'revision', 'clock']: @@ -81,56 +95,227 @@ class HostModel(object): return "" - def _get_host_info(self): - res = {} - if platform.machine().startswith('ppc'): - res['cpu_model'] = self._get_ppc_cpu_info() - else: - with open('/proc/cpuinfo') as f: + def _get_x86_cpu_model(self): + """ + method to get cpu_model for x86 architecture + """ + try: + with open(PROC_CPUINFO) as f: for line in f.xreadlines(): if "model name" in line: - res['cpu_model'] = line.split(':')[1].strip() + return line.split(':')[1].strip() break + except Exception as e: + wok_log.error("Failed to retrive cpu_model for " + "%s. Error: %s", ARCH, e.__str__()) + return "" - res['cpus'] = 0 - res['memory'] = 0L - - # Include IBM PowerKVM name to supported distro names - _sup_distros = platform._supported_dists + ('ibm_powerkvm',) - # 'fedora' '17' 'Beefy Miracle' - distro, version, codename = platform.linux_distribution( - supported_dists=_sup_distros) - res['os_distro'] = distro - res['os_version'] = version - res['os_codename'] = unicode(codename, "utf-8") - - return res - - def lookup(self, *name): - cpus = psutil.NUM_CPUS - + def _get_s390x_host_info(self): + """ + method to get additional host details + specific to s390x architecture + :return: dictionary + """ + host_info = {} + host_info['cpus'] = self._get_cpus() + host_info['cpus']['dedicated'] = 0 + host_info['cpus']['shared'] = 0 + host_info['cpu_model'] = "" + host_info['virtualization'] = {} + s390x_sysinfo = self._get_s390x_sysinfo() + if 'manufacturer' in s390x_sysinfo.keys(): + host_info['cpu_model'] = s390x_sysinfo['manufacturer'] + if 'type' in s390x_sysinfo.keys(): + host_info['cpu_model'] = \ + host_info['cpu_model'] + "/" + s390x_sysinfo['type'] + if 'model' in s390x_sysinfo.keys(): + host_info['cpu_model'] = \ + host_info['cpu_model'] + "/" + s390x_sysinfo['model'] + if CPUS_DEDICATED in s390x_sysinfo.keys(): + host_info['cpus']['dedicated'] = s390x_sysinfo[CPUS_DEDICATED] + if CPUS_SHARED in s390x_sysinfo.keys(): + host_info['cpus']['shared'] = s390x_sysinfo[CPUS_SHARED] + host_info['virtualization']['hypervisor'] = \ + self.lscpu.get_hypervisor() + host_info['virtualization']['hypervisor_vendor'] = \ + self.lscpu.get_hypervisor_vendor() + host_info['virtualization'][LPAR_NAME] = '' + host_info['virtualization'][LPAR_NUMBER] = '' + if LPAR_NAME in s390x_sysinfo.keys(): + host_info['virtualization'][LPAR_NAME] = s390x_sysinfo[LPAR_NAME] + if LPAR_NUMBER in s390x_sysinfo.keys(): + host_info['virtualization'][LPAR_NUMBER] = \ + s390x_sysinfo[LPAR_NUMBER] + + return host_info + + def _get_s390x_sysinfo(self): + """ + This method retrieves following system information + for s390 architecture + * manufacturer: Manufacturer of host machine + * type: Type of the host machine + * model:Model of host machine + * LPAR_NUMBER: LPAR Number of host + * LPAR_NAME: Name of host LPAR + * CPUS_DEDICATED: LPAR CPUs Dedicated + * CPUS_SHARED: LPAR CPUs Shared + + :param self: object of the class self + :return: dictionary with following keys - + 'manufacturer', 'type', 'model', CPUS_SHARED, + CPUS_DEDICATED, LPAR_NUMBER, LPAR_NAME + """ + s390x_sysinfo = {} + try: + with open(PROC_SYSINFO) as f: + for line in f.xreadlines(): + if ":" in line and (len(line.split(':')) == 2): + info = line.split(':') + if info[0] == 'Model' and (len(info[1].split()) == 2): + s390x_sysinfo['model'] = \ + info[1].split()[0].strip() +\ + " "+info[1].split()[1].strip() + elif info[0] == 'Manufacturer': + s390x_sysinfo['manufacturer'] = info[1].strip() + elif info[0] == 'Type': + s390x_sysinfo['type'] = info[1].strip() + elif info[0] == 'LPAR Number': + s390x_sysinfo[LPAR_NUMBER] = int(info[1].strip()) + elif info[0] == 'LPAR Name': + s390x_sysinfo[LPAR_NAME] = info[1].strip() + elif info[0] == 'LPAR CPUs Dedicated': + s390x_sysinfo[CPUS_DEDICATED] =\ + int(info[1].strip()) + elif info[0] == 'LPAR CPUs Shared': + s390x_sysinfo[CPUS_SHARED] = int(info[1].strip()) + except Exception as e: + wok_log.error("Failed to retrieve information from %s file. " + "Error: %s", PROC_SYSINFO, e.__str__()) + + return s390x_sysinfo + + def _get_memory(self): + """ + method to retrieve memory information for all architecture + :return: dictionary with keys "online" and "offline" + """ + memory = {} + online_memory = 0 + offline_memory = 0 + if ARCH.startswith('s390x'): + online_mem_pat = r'^Total online memory :\s+(\d+)\s+MB$' + offline_mem_pat = r'^Total offline memory:\s+(\d+)\s+MB$' + out, err, rc = run_command(LSMEM) + # output of lsmem in s390x architecture is expected to be + # Address Range Size (MB) State\ + # Removable Device + # ========================================================\ + # ======================= + # 0x0000000000000000-0x000000000fffffff 256 online\ + # no 0 + # 0x0000000010000000-0x000000002fffffff 512 online\ + # yes 1-2 + # 0x0000000030000000-0x000000007fffffff 1280 online\ + # no 3-7 + # 0x0000000080000000-0x00000000ffffffff 2048 offline\ + # - 8-15 + # + # Memory device size : 256 MB + # Memory block size : 256 MB + # Total online memory : 2048 MB + # Total offline memory: 2048 MB + if not rc: + online_mem =\ + re.search(online_mem_pat, out.strip(), re.M | re.I) + offline_mem =\ + re.search(offline_mem_pat, out.strip(), re.M | re.I) + if online_mem and len(online_mem.groups()) == 1: + online_memory = int(online_mem.group(1)) * 1024 * 1024 + # converting MB to bytes + # lsmem always returns memory in MB + if offline_mem and len(offline_mem.groups()) == 1: + offline_memory = int(offline_mem.group(1)) * 1024 * 1024 + else: + wok_log.error('Failed to retrieve memory information with' + ' command %s. Error: %s' % (LSMEM, err)) + else: + if hasattr(psutil, 'phymem_usage'): + online_memory = psutil.phymem_usage().total + elif hasattr(psutil, 'virtual_memory'): + online_memory = psutil.virtual_memory().total + + memory['online'] = online_memory + memory['offline'] = offline_memory + return memory + + def _get_cpus_(self): + """ + method to retrieve online cpus count and offline cpus + count for all architecture + :return: dictionary with keys "online" and "offline" + """ + cpus = {} + total_cpus = int(self.lscpu.get_total_cpus()) + offline_cpus = 0 + + online_cpus = psutil.NUM_CPUS # psutil is unstable on how to get the number of # cpus, different versions call it differently if hasattr(psutil, 'cpu_count'): - cpus = psutil.cpu_count() + online_cpus = psutil.cpu_count() elif hasattr(psutil, 'NUM_CPUS'): - cpus = psutil.NUM_CPUS + online_cpus = psutil.NUM_CPUS elif hasattr(psutil, '_psplatform'): for method_name in ['_get_num_cpus', 'get_num_cpus']: method = getattr(psutil._psplatform, method_name, None) if method is not None: - cpus = method() + online_cpus = method() break + if total_cpus >= online_cpus: + offline_cpus = total_cpus - online_cpus + cpus['online'] = online_cpus + cpus['offline'] = offline_cpus + return cpus + + def _get_base_info(self): + """ + method to retrieve common host information for all architectures + :return: dictionary with keys 'os_distro', 'os_version', 'os_codename' + 'architecture', 'host', memory + """ + common_info = {} + # Include IBM PowerKVM name to supported distro names + _sup_distros = platform._supported_dists + ('ibm_powerkvm',) + # 'fedora' '17' 'Beefy Miracle' + distro, version, codename = platform.linux_distribution( + supported_dists=_sup_distros) + common_info['os_distro'] = distro + common_info['os_version'] = version + common_info['os_codename'] = unicode(codename, "utf-8") + common_info['architecture'] = ARCH + common_info['host'] = platform.node() + common_info['memory'] = self._get_memory() - self.host_info['cpus'] = cpus - if hasattr(psutil, 'phymem_usage'): - self.host_info['memory'] = psutil.phymem_usage().total - elif hasattr(psutil, 'virtual_memory'): - self.host_info['memory'] = psutil.virtual_memory().total - return self.host_info + return common_info + + def lookup(self, *name): + """ + method to get basic information for host + """ + host_info = self._get_base_info() + if ARCH.startswith('s390x'): + host_info.update(self._get_s390x_host_info()) + elif ARCH.startswith('ppc'): + host_info['cpus'] = self._get_cpus_() + host_info['cpu_model'] = self._get_ppc_cpu_model() + else: + host_info['cpus'] = self._get_cpus_() + host_info['cpu_model'] = self._get_x86_cpu_model() + return host_info def swupdate(self, *name): try: diff --git a/src/wok/plugins/gingerbase/tests/test_host.py b/src/wok/plugins/gingerbase/tests/test_host.py index 5d158bb..a994177 100644 --- a/src/wok/plugins/gingerbase/tests/test_host.py +++ b/src/wok/plugins/gingerbase/tests/test_host.py @@ -68,15 +68,21 @@ class HostTests(unittest.TestCase): def test_hostinfo(self): resp = self.request('/plugins/gingerbase/host').read() info = json.loads(resp) - keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model', - 'memory', 'cpus'] + if platform.machine().startswith('s390x'): + keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model', + 'memory', 'cpus', 'architecture', 'host', 'virtualization'] + else: + keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model', + 'memory', 'cpus', 'architecture', 'host'] + self.assertEquals(psutil.TOTAL_PHYMEM, info['memory']) self.assertEquals(sorted(keys), sorted(info.keys())) distro, version, codename = platform.linux_distribution() self.assertEquals(distro, info['os_distro']) self.assertEquals(version, info['os_version']) self.assertEquals(unicode(codename, "utf-8"), info['os_codename']) - self.assertEquals(psutil.TOTAL_PHYMEM, info['memory']) + self.assertEqual(platform.node(), info['host']) + self.assertEqual(platform.machine(), info['architecture']) def test_hoststats(self): time.sleep(1) -- 2.1.0