[Kimchi-devel] [PATCH V3] [Ginger-Base 1/1] Issue 728 - Processor Info in s390 architecture
Aline Manera
alinefm at linux.vnet.ibm.com
Thu Nov 26 12:03:25 UTC 2015
Reviewed-by: Aline Manera <alinefm at linux.vnet.ibm.com>
On 26/11/2015 09:01, sureshab at linux.vnet.ibm.com wrote:
> From: Suresh Babu Angadi <sureshab at in.ibm.com>
>
> Fix for issue 728: processor info displays blank for system z
> this patch set also adds additional capability:
> retrieving architecture and host name (for all architecture)
> split CPUs to show online and offline cpus also
> includes shared and dedicated cpus for s390x
> split memory to show online and offline
> additional virtualization details(for s390x):
> virtualization will have hypervisor details and lpar details
>
> Signed-off-by: Suresh Babu Angadi <sureshab at in.ibm.com>
> ---
> src/wok/plugins/gingerbase/docs/API.md | 18 +-
> src/wok/plugins/gingerbase/i18n.py | 1 +
> src/wok/plugins/gingerbase/lscpu.py | 60 ++++++
> src/wok/plugins/gingerbase/model/host.py | 259 ++++++++++++++++++++++----
> src/wok/plugins/gingerbase/tests/test_host.py | 12 +-
> 5 files changed, 307 insertions(+), 43 deletions(-)
>
> diff --git a/src/wok/plugins/gingerbase/docs/API.md b/src/wok/plugins/gingerbase/docs/API.md
> index 8f37ddc..30b0c8f 100644
> --- a/src/wok/plugins/gingerbase/docs/API.md
> +++ b/src/wok/plugins/gingerbase/docs/API.md
> @@ -95,13 +95,25 @@ Contains information of host.
> **Methods:**
>
> * **GET**: Retrieve host static information
> - * memory: Total size of host physical memory
> - The unit is Bytes
> + * memory: Memory details of the host. The unit is Bytes
> + * online: Total size of physical memory currently available to host
> + * offline: Total offline memory(for s390x), zero for other architectures.
> * cpu_model: The model name of host CPU
> - * cpus: The number of online CPUs available on host
> + * cpus: The host cpus information
> + * online: Number of online CPUs
> + * offline: Number of offline CPUs
> + * shared *(s390x only)*: Number of shared CPUs
> + * dedicated *(s390x only)*: Number of dedicated CPUs
> * os_distro: The OS distribution that runs on host
> * os_version: The version of OS distribution
> * os_codename: The code name of OS distribution
> + * host: Host name of the network node
> + * architecture: Architecture of the host
> + * virtualization *(s390x only)*: This is a json with following keys
> + * hypervisor: Hypervisor name of the host
> + * hypervisor_vendor: Hypervisor Vendor name
> + * lpar_number: LPAR Number of host
> + * lpar_name: Name of host LPAR
>
> * **POST**: *See Host Actions*
>
> diff --git a/src/wok/plugins/gingerbase/i18n.py b/src/wok/plugins/gingerbase/i18n.py
> index af75c70..a0a8c36 100644
> --- a/src/wok/plugins/gingerbase/i18n.py
> +++ b/src/wok/plugins/gingerbase/i18n.py
> @@ -88,5 +88,6 @@ messages = {
> "GGBCPUINF0005E": _("This host (or current configuration) does not provide Socket(s) information."),
> "GGBCPUINF0006E": _("This host (or current configuration) does not provide Core(s) per socket information."),
> "GGBCPUINF0007E": _("This host (or current configuration) does not provide Thread(s) per core information."),
> + "GGBCPUINF0008E": _("This host (or current configuration) does not provide CPU(s) information."),
>
> }
> diff --git a/src/wok/plugins/gingerbase/lscpu.py b/src/wok/plugins/gingerbase/lscpu.py
> index 0fc1e72..a822a15 100644
> --- a/src/wok/plugins/gingerbase/lscpu.py
> +++ b/src/wok/plugins/gingerbase/lscpu.py
> @@ -17,10 +17,13 @@
> # License along with this library; if not, write to the Free Software
> # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> import logging
> +import platform
>
> from wok.utils import run_command
> from wok.exception import NotFoundError
>
> +ARCH = platform.machine()
> +
>
> class LsCpu(object):
> """
> @@ -64,6 +67,28 @@ class LsCpu(object):
> # L2 cache: 256K
> # L3 cache: 3072K
> # NUMA node0 CPU(s): 0-3
> + #
> + # Output of lscpu in s390x is expected to be
> + # Architecture: s390x
> + # CPU op-mode(s): 32-bit, 64-bit
> + # Byte Order: Big Endian
> + # CPU(s): 4
> + # On-line CPU(s) list: 0,1
> + # Off-line CPU(s) list: 2,3
> + # Thread(s) per core: 1
> + # Core(s) per socket: 6
> + # Socket(s) per book: 6
> + # Book(s): 4
> + # Vendor ID: IBM/S390
> + # BogoMIPS: 18115.00
> + # Hypervisor: PR/SM
> + # Hypervisor vendor: IBM
> + # Virtualization type: full
> + # Dispatching mode: horizontal
> + # L1d cache: 96K
> + # L1i cache: 64K
> + # L2d cache: 1024K
> + # L2i cache: 1024K
>
> if not rc and (not out.isspace()):
> lscpuout = out.split('\n')
> @@ -85,6 +110,8 @@ class LsCpu(object):
> """
> try:
> sockets = "Socket(s)"
> + if ARCH.startswith('s390x'):
> + sockets = "Socket(s) per book"
> if len(self.lsCpuInfo) > 0 and sockets in self.lsCpuInfo.keys():
> return int(self.lsCpuInfo[sockets])
> else:
> @@ -124,3 +151,36 @@ class LsCpu(object):
> except IndexError, e:
> self.log_error(e)
> raise NotFoundError("GGBCPUINF0007E")
> +
> + def get_total_cpus(self):
> + """
> + method to get total cpus retrieved from CPU(s) field of lscpu
> + :return: total cpus
> + """
> + total_cpus = 'CPU(s)'
> + if len(self.lsCpuInfo) > 0 and total_cpus in self.lsCpuInfo.keys():
> + return int(self.lsCpuInfo[total_cpus])
> + else:
> + self.log_error("Failed to fetch total cpus count in lscpu output")
> + raise NotFoundError("GGBCPUINF0008E")
> +
> + def get_hypervisor(self):
> + """
> + method to get hypervisor name if present in lscpu o/p
> + :return: Hypervisor Name
> + """
> + hypervisor = 'Hypervisor'
> + if len(self.lsCpuInfo) > 0 and hypervisor in self.lsCpuInfo.keys():
> + return self.lsCpuInfo[hypervisor]
> + return None
> +
> + def get_hypervisor_vendor(self):
> + """
> + method to get hypervisor vendor if present in lscpu o/p
> + :return: Hypervisor Vendor
> + """
> + hypervisor_vendor = 'Hypervisor vendor'
> + if len(self.lsCpuInfo) > 0 and hypervisor_vendor in \
> + self.lsCpuInfo.keys():
> + return self.lsCpuInfo[hypervisor_vendor]
> + return None
> diff --git a/src/wok/plugins/gingerbase/model/host.py b/src/wok/plugins/gingerbase/model/host.py
> index a58bada..fd4db29 100644
> --- a/src/wok/plugins/gingerbase/model/host.py
> +++ b/src/wok/plugins/gingerbase/model/host.py
> @@ -22,6 +22,7 @@
> import os
> import platform
> import psutil
> +import re
> import time
> from cherrypy.process.plugins import BackgroundTask
> from collections import defaultdict
> @@ -31,9 +32,10 @@ from wok.basemodel import Singleton
> from wok.config import config as kconfig
> from wok.exception import InvalidOperation
> from wok.exception import OperationFailed
> -from wok.utils import add_task, wok_log
> +from wok.utils import add_task, run_command, wok_log
> from wok.model.tasks import TaskModel
>
> +from wok.plugins.gingerbase.lscpu import LsCpu
> from wok.plugins.gingerbase.model.debugreports import DebugReportsModel
> from wok.plugins.gingerbase.repositories import Repositories
> from wok.plugins.gingerbase.swupdate import SoftwareUpdate
> @@ -48,17 +50,29 @@ DOM_STATE_MAP = {0: 'nostate',
> 6: 'crashed',
> 7: 'pmsuspended'}
>
> +ARCH = platform.machine()
> +PROC_CPUINFO = '/proc/cpuinfo'
> +PROC_SYSINFO = '/proc/sysinfo'
> +LSMEM = 'lsmem'
> +CPUS_DEDICATED = 'cpus_dedicated'
> +CPUS_SHARED = 'cpus_shared'
> +LPAR_NAME = 'lpar_name'
> +LPAR_NUMBER = 'lpar_number'
> +
>
> class HostModel(object):
> def __init__(self, **kargs):
> # self.conn = kargs['conn']
> self.objstore = kargs['objstore']
> self.task = TaskModel(**kargs)
> - self.host_info = self._get_host_info()
> + self.lscpu = LsCpu()
>
> - def _get_ppc_cpu_info(self):
> + def _get_ppc_cpu_model(self):
> + """
> + method to get cpu_model for ppc architecture
> + """
> res = {}
> - with open('/proc/cpuinfo') as f:
> + with open(PROC_CPUINFO) as f:
> for line in f.xreadlines():
> # Parse CPU, CPU's revision and CPU's clock information
> for key in ['cpu', 'revision', 'clock']:
> @@ -81,56 +95,227 @@ class HostModel(object):
>
> return ""
>
> - def _get_host_info(self):
> - res = {}
> - if platform.machine().startswith('ppc'):
> - res['cpu_model'] = self._get_ppc_cpu_info()
> - else:
> - with open('/proc/cpuinfo') as f:
> + def _get_x86_cpu_model(self):
> + """
> + method to get cpu_model for x86 architecture
> + """
> + try:
> + with open(PROC_CPUINFO) as f:
> for line in f.xreadlines():
> if "model name" in line:
> - res['cpu_model'] = line.split(':')[1].strip()
> + return line.split(':')[1].strip()
> break
> + except Exception as e:
> + wok_log.error("Failed to retrive cpu_model for "
> + "%s. Error: %s", ARCH, e.__str__())
> + return ""
>
> - res['cpus'] = 0
> - res['memory'] = 0L
> -
> - # Include IBM PowerKVM name to supported distro names
> - _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
> - # 'fedora' '17' 'Beefy Miracle'
> - distro, version, codename = platform.linux_distribution(
> - supported_dists=_sup_distros)
> - res['os_distro'] = distro
> - res['os_version'] = version
> - res['os_codename'] = unicode(codename, "utf-8")
> -
> - return res
> -
> - def lookup(self, *name):
> - cpus = psutil.NUM_CPUS
> -
> + def _get_s390x_host_info(self):
> + """
> + method to get additional host details
> + specific to s390x architecture
> + :return: dictionary
> + """
> + host_info = {}
> + host_info['cpus'] = self._get_cpus()
> + host_info['cpus']['dedicated'] = 0
> + host_info['cpus']['shared'] = 0
> + host_info['cpu_model'] = ""
> + host_info['virtualization'] = {}
> + s390x_sysinfo = self._get_s390x_sysinfo()
> + if 'manufacturer' in s390x_sysinfo.keys():
> + host_info['cpu_model'] = s390x_sysinfo['manufacturer']
> + if 'type' in s390x_sysinfo.keys():
> + host_info['cpu_model'] = \
> + host_info['cpu_model'] + "/" + s390x_sysinfo['type']
> + if 'model' in s390x_sysinfo.keys():
> + host_info['cpu_model'] = \
> + host_info['cpu_model'] + "/" + s390x_sysinfo['model']
> + if CPUS_DEDICATED in s390x_sysinfo.keys():
> + host_info['cpus']['dedicated'] = s390x_sysinfo[CPUS_DEDICATED]
> + if CPUS_SHARED in s390x_sysinfo.keys():
> + host_info['cpus']['shared'] = s390x_sysinfo[CPUS_SHARED]
> + host_info['virtualization']['hypervisor'] = \
> + self.lscpu.get_hypervisor()
> + host_info['virtualization']['hypervisor_vendor'] = \
> + self.lscpu.get_hypervisor_vendor()
> + host_info['virtualization'][LPAR_NAME] = ''
> + host_info['virtualization'][LPAR_NUMBER] = ''
> + if LPAR_NAME in s390x_sysinfo.keys():
> + host_info['virtualization'][LPAR_NAME] = s390x_sysinfo[LPAR_NAME]
> + if LPAR_NUMBER in s390x_sysinfo.keys():
> + host_info['virtualization'][LPAR_NUMBER] = \
> + s390x_sysinfo[LPAR_NUMBER]
> +
> + return host_info
> +
> + def _get_s390x_sysinfo(self):
> + """
> + This method retrieves following system information
> + for s390 architecture
> + * manufacturer: Manufacturer of host machine
> + * type: Type of the host machine
> + * model:Model of host machine
> + * LPAR_NUMBER: LPAR Number of host
> + * LPAR_NAME: Name of host LPAR
> + * CPUS_DEDICATED: LPAR CPUs Dedicated
> + * CPUS_SHARED: LPAR CPUs Shared
> +
> + :param self: object of the class self
> + :return: dictionary with following keys -
> + 'manufacturer', 'type', 'model', CPUS_SHARED,
> + CPUS_DEDICATED, LPAR_NUMBER, LPAR_NAME
> + """
> + s390x_sysinfo = {}
> + try:
> + with open(PROC_SYSINFO) as f:
> + for line in f.xreadlines():
> + if ":" in line and (len(line.split(':')) == 2):
> + info = line.split(':')
> + if info[0] == 'Model' and (len(info[1].split()) == 2):
> + s390x_sysinfo['model'] = \
> + info[1].split()[0].strip() +\
> + " "+info[1].split()[1].strip()
> + elif info[0] == 'Manufacturer':
> + s390x_sysinfo['manufacturer'] = info[1].strip()
> + elif info[0] == 'Type':
> + s390x_sysinfo['type'] = info[1].strip()
> + elif info[0] == 'LPAR Number':
> + s390x_sysinfo[LPAR_NUMBER] = int(info[1].strip())
> + elif info[0] == 'LPAR Name':
> + s390x_sysinfo[LPAR_NAME] = info[1].strip()
> + elif info[0] == 'LPAR CPUs Dedicated':
> + s390x_sysinfo[CPUS_DEDICATED] =\
> + int(info[1].strip())
> + elif info[0] == 'LPAR CPUs Shared':
> + s390x_sysinfo[CPUS_SHARED] = int(info[1].strip())
> + except Exception as e:
> + wok_log.error("Failed to retrieve information from %s file. "
> + "Error: %s", PROC_SYSINFO, e.__str__())
> +
> + return s390x_sysinfo
> +
> + def _get_memory(self):
> + """
> + method to retrieve memory information for all architecture
> + :return: dictionary with keys "online" and "offline"
> + """
> + memory = {}
> + online_memory = 0
> + offline_memory = 0
> + if ARCH.startswith('s390x'):
> + online_mem_pat = r'^Total online memory :\s+(\d+)\s+MB$'
> + offline_mem_pat = r'^Total offline memory:\s+(\d+)\s+MB$'
> + out, err, rc = run_command(LSMEM)
> + # output of lsmem in s390x architecture is expected to be
> + # Address Range Size (MB) State\
> + # Removable Device
> + # ========================================================\
> + # =======================
> + # 0x0000000000000000-0x000000000fffffff 256 online\
> + # no 0
> + # 0x0000000010000000-0x000000002fffffff 512 online\
> + # yes 1-2
> + # 0x0000000030000000-0x000000007fffffff 1280 online\
> + # no 3-7
> + # 0x0000000080000000-0x00000000ffffffff 2048 offline\
> + # - 8-15
> + #
> + # Memory device size : 256 MB
> + # Memory block size : 256 MB
> + # Total online memory : 2048 MB
> + # Total offline memory: 2048 MB
> + if not rc:
> + online_mem =\
> + re.search(online_mem_pat, out.strip(), re.M | re.I)
> + offline_mem =\
> + re.search(offline_mem_pat, out.strip(), re.M | re.I)
> + if online_mem and len(online_mem.groups()) == 1:
> + online_memory = int(online_mem.group(1)) * 1024 * 1024
> + # converting MB to bytes
> + # lsmem always returns memory in MB
> + if offline_mem and len(offline_mem.groups()) == 1:
> + offline_memory = int(offline_mem.group(1)) * 1024 * 1024
> + else:
> + wok_log.error('Failed to retrieve memory information with'
> + ' command %s. Error: %s' % (LSMEM, err))
> + else:
> + if hasattr(psutil, 'phymem_usage'):
> + online_memory = psutil.phymem_usage().total
> + elif hasattr(psutil, 'virtual_memory'):
> + online_memory = psutil.virtual_memory().total
> +
> + memory['online'] = online_memory
> + memory['offline'] = offline_memory
> + return memory
> +
> + def _get_cpus_(self):
> + """
> + method to retrieve online cpus count and offline cpus
> + count for all architecture
> + :return: dictionary with keys "online" and "offline"
> + """
> + cpus = {}
> + total_cpus = int(self.lscpu.get_total_cpus())
> + offline_cpus = 0
> +
> + online_cpus = psutil.NUM_CPUS
> # psutil is unstable on how to get the number of
> # cpus, different versions call it differently
> if hasattr(psutil, 'cpu_count'):
> - cpus = psutil.cpu_count()
> + online_cpus = psutil.cpu_count()
>
> elif hasattr(psutil, 'NUM_CPUS'):
> - cpus = psutil.NUM_CPUS
> + online_cpus = psutil.NUM_CPUS
>
> elif hasattr(psutil, '_psplatform'):
> for method_name in ['_get_num_cpus', 'get_num_cpus']:
>
> method = getattr(psutil._psplatform, method_name, None)
> if method is not None:
> - cpus = method()
> + online_cpus = method()
> break
> + if total_cpus >= online_cpus:
> + offline_cpus = total_cpus - online_cpus
> + cpus['online'] = online_cpus
> + cpus['offline'] = offline_cpus
> + return cpus
> +
> + def _get_base_info(self):
> + """
> + method to retrieve common host information for all architectures
> + :return: dictionary with keys 'os_distro', 'os_version', 'os_codename'
> + 'architecture', 'host', memory
> + """
> + common_info = {}
> + # Include IBM PowerKVM name to supported distro names
> + _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
> + # 'fedora' '17' 'Beefy Miracle'
> + distro, version, codename = platform.linux_distribution(
> + supported_dists=_sup_distros)
> + common_info['os_distro'] = distro
> + common_info['os_version'] = version
> + common_info['os_codename'] = unicode(codename, "utf-8")
> + common_info['architecture'] = ARCH
> + common_info['host'] = platform.node()
> + common_info['memory'] = self._get_memory()
>
> - self.host_info['cpus'] = cpus
> - if hasattr(psutil, 'phymem_usage'):
> - self.host_info['memory'] = psutil.phymem_usage().total
> - elif hasattr(psutil, 'virtual_memory'):
> - self.host_info['memory'] = psutil.virtual_memory().total
> - return self.host_info
> + return common_info
> +
> + def lookup(self, *name):
> + """
> + method to get basic information for host
> + """
> + host_info = self._get_base_info()
> + if ARCH.startswith('s390x'):
> + host_info.update(self._get_s390x_host_info())
> + elif ARCH.startswith('ppc'):
> + host_info['cpus'] = self._get_cpus_()
> + host_info['cpu_model'] = self._get_ppc_cpu_model()
> + else:
> + host_info['cpus'] = self._get_cpus_()
> + host_info['cpu_model'] = self._get_x86_cpu_model()
> + return host_info
>
> def swupdate(self, *name):
> try:
> diff --git a/src/wok/plugins/gingerbase/tests/test_host.py b/src/wok/plugins/gingerbase/tests/test_host.py
> index 5d158bb..5a61285 100644
> --- a/src/wok/plugins/gingerbase/tests/test_host.py
> +++ b/src/wok/plugins/gingerbase/tests/test_host.py
> @@ -68,15 +68,21 @@ class HostTests(unittest.TestCase):
> def test_hostinfo(self):
> resp = self.request('/plugins/gingerbase/host').read()
> info = json.loads(resp)
> - keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
> - 'memory', 'cpus']
> + if platform.machine().startswith('s390x'):
> + keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
> + 'memory', 'cpus', 'architecture', 'host', 'virtualization']
> + else:
> + keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
> + 'memory', 'cpus', 'architecture', 'host']
> + self.assertEquals(psutil.TOTAL_PHYMEM, info['memory']['online'])
> self.assertEquals(sorted(keys), sorted(info.keys()))
>
> distro, version, codename = platform.linux_distribution()
> self.assertEquals(distro, info['os_distro'])
> self.assertEquals(version, info['os_version'])
> self.assertEquals(unicode(codename, "utf-8"), info['os_codename'])
> - self.assertEquals(psutil.TOTAL_PHYMEM, info['memory'])
> + self.assertEqual(platform.node(), info['host'])
> + self.assertEqual(platform.machine(), info['architecture'])
>
> def test_hoststats(self):
> time.sleep(1)
More information about the Kimchi-devel
mailing list