[Kimchi-devel] [PATCH V2] [Ginger-Base 1/1] Issue 728 - Processor Info in s390 architecture

Chandra Shekhar Reddy Potula chandra at linux.vnet.ibm.com
Wed Nov 25 15:04:50 UTC 2015


Reviewed-by: Chandra Shekhar Reddy Potula <chandra at linux.vnet.ibm.com>

On 24/11/15 5:29 PM, sureshab at linux.vnet.ibm.com wrote:
> From: Suresh Babu Angadi <sureshab at in.ibm.com>
>
> Fix for issue 728: processor info displays blank for system z
> this patch set also adds additional capability:
> retrieving architecture and host name (for all architecture)
> split CPUs to show online and offline cpus also
>    includes shared and dedicated cpus for s390x
> split memory to show online and offline
> additional virtualization details(for s390x):
>    virtualization will have hypervisor details and lpar details
>
> Signed-off-by: Suresh Babu Angadi <sureshab at in.ibm.com>
> ---
>   src/wok/plugins/gingerbase/docs/API.md        |  18 +-
>   src/wok/plugins/gingerbase/i18n.py            |   1 +
>   src/wok/plugins/gingerbase/lscpu.py           |  60 ++++++
>   src/wok/plugins/gingerbase/model/host.py      | 259 ++++++++++++++++++++++----
>   src/wok/plugins/gingerbase/tests/test_host.py |  12 +-
>   5 files changed, 307 insertions(+), 43 deletions(-)
>
> diff --git a/src/wok/plugins/gingerbase/docs/API.md b/src/wok/plugins/gingerbase/docs/API.md
> index 8f37ddc..30b0c8f 100644
> --- a/src/wok/plugins/gingerbase/docs/API.md
> +++ b/src/wok/plugins/gingerbase/docs/API.md
> @@ -95,13 +95,25 @@ Contains information of host.
>   **Methods:**
>
>   * **GET**: Retrieve host static information
> -    * memory: Total size of host physical memory
> -              The unit is Bytes
> +    * memory: Memory details of the host. The unit is Bytes
> +                * online: Total size of physical memory currently available to host
> +                * offline: Total offline memory(for s390x), zero for other architectures.
>       * cpu_model: The model name of host CPU
> -    * cpus: The number of online CPUs available on host
> +    * cpus: The host cpus information
> +                * online: Number of online CPUs
> +                * offline: Number of offline CPUs
> +                * shared *(s390x only)*: Number of shared CPUs
> +                * dedicated *(s390x only)*: Number of dedicated  CPUs
>       * os_distro: The OS distribution that runs on host
>       * os_version: The version of OS distribution
>       * os_codename: The code name of OS distribution
> +    * host: Host name of the network node
> +    * architecture: Architecture of the host
> +    * virtualization *(s390x only)*: This is a json with following keys
> +                * hypervisor: Hypervisor name of the host
> +                * hypervisor_vendor: Hypervisor Vendor name
> +                * lpar_number: LPAR Number of host
> +                * lpar_name: Name of host LPAR
>
>   * **POST**: *See Host Actions*
>
> diff --git a/src/wok/plugins/gingerbase/i18n.py b/src/wok/plugins/gingerbase/i18n.py
> index af75c70..a0a8c36 100644
> --- a/src/wok/plugins/gingerbase/i18n.py
> +++ b/src/wok/plugins/gingerbase/i18n.py
> @@ -88,5 +88,6 @@ messages = {
>       "GGBCPUINF0005E": _("This host (or current configuration) does not provide Socket(s) information."),
>       "GGBCPUINF0006E": _("This host (or current configuration) does not provide Core(s) per socket information."),
>       "GGBCPUINF0007E": _("This host (or current configuration) does not provide Thread(s) per core information."),
> +    "GGBCPUINF0008E": _("This host (or current configuration) does not provide CPU(s) information."),
>
>   }
> diff --git a/src/wok/plugins/gingerbase/lscpu.py b/src/wok/plugins/gingerbase/lscpu.py
> index 0fc1e72..4686049 100644
> --- a/src/wok/plugins/gingerbase/lscpu.py
> +++ b/src/wok/plugins/gingerbase/lscpu.py
> @@ -17,10 +17,13 @@
>   # License along with this library; if not, write to the Free Software
>   # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
>   import logging
> +import platform
>
>   from wok.utils import run_command
>   from wok.exception import NotFoundError
>
> +ARCH = platform.machine()
> +
>
>   class LsCpu(object):
>       """
> @@ -64,6 +67,28 @@ class LsCpu(object):
>               # L2 cache:              256K
>               # L3 cache:              3072K
>               # NUMA node0 CPU(s):     0-3
> +            #
> +            # Output of lscpu in s390x is expected to be
> +            # Architecture:          s390x
> +            # CPU op-mode(s):        32-bit, 64-bit
> +            # Byte Order:            Big Endian
> +            # CPU(s):                4
> +            # On-line CPU(s) list:   0,1
> +            # Off-line CPU(s) list:  2,3
> +            # Thread(s) per core:    1
> +            # Core(s) per socket:    6
> +            # Socket(s) per book:    6
> +            # Book(s):               4
> +            # Vendor ID:             IBM/S390
> +            # BogoMIPS:              18115.00
> +            # Hypervisor:            PR/SM
> +            # Hypervisor vendor:     IBM
> +            # Virtualization type:   full
> +            # Dispatching mode:      horizontal
> +            # L1d cache:             96K
> +            # L1i cache:             64K
> +            # L2d cache:             1024K
> +            # L2i cache:             1024K
>
>               if not rc and (not out.isspace()):
>                   lscpuout = out.split('\n')
> @@ -85,6 +110,8 @@ class LsCpu(object):
>           """
>           try:
>               sockets = "Socket(s)"
> +            if ARCH.startswith('s390x'):
> +                sockets = "Socket(s) per book"
>               if len(self.lsCpuInfo) > 0 and sockets in self.lsCpuInfo.keys():
>                   return int(self.lsCpuInfo[sockets])
>               else:
> @@ -124,3 +151,36 @@ class LsCpu(object):
>           except IndexError, e:
>               self.log_error(e)
>               raise NotFoundError("GGBCPUINF0007E")
> +
> +    def get_total_cpus(self):
> +        """
> +
> +        :return:
> +        """
> +        total_cpus = 'CPU(s)'
> +        if len(self.lsCpuInfo) > 0 and total_cpus in self.lsCpuInfo.keys():
> +            return int(self.lsCpuInfo[total_cpus])
> +        else:
> +            self.log_error("Failed to fetch total cpus count in lscpu output")
> +            raise NotFoundError("GGBCPUINF0008E")
> +
> +    def get_hypervisor(self):
> +        """
> +        method to get hypervisor name if present in lscpu o/p
> +        :return: Hypervisor Name
> +        """
> +        hypervisor = 'Hypervisor'
> +        if len(self.lsCpuInfo) > 0 and hypervisor in self.lsCpuInfo.keys():
> +            return self.lsCpuInfo[hypervisor]
> +        return None
> +
> +    def get_hypervisor_vendor(self):
> +        """
> +        method to get hypervisor vendor if present in lscpu o/p
> +        :return: Hypervisor Vendor
> +        """
> +        hypervisor_vendor = 'Hypervisor vendor'
> +        if len(self.lsCpuInfo) > 0 and hypervisor_vendor in \
> +                self.lsCpuInfo.keys():
> +            return self.lsCpuInfo[hypervisor_vendor]
> +        return None
> diff --git a/src/wok/plugins/gingerbase/model/host.py b/src/wok/plugins/gingerbase/model/host.py
> index a58bada..fd4db29 100644
> --- a/src/wok/plugins/gingerbase/model/host.py
> +++ b/src/wok/plugins/gingerbase/model/host.py
> @@ -22,6 +22,7 @@
>   import os
>   import platform
>   import psutil
> +import re
>   import time
>   from cherrypy.process.plugins import BackgroundTask
>   from collections import defaultdict
> @@ -31,9 +32,10 @@ from wok.basemodel import Singleton
>   from wok.config import config as kconfig
>   from wok.exception import InvalidOperation
>   from wok.exception import OperationFailed
> -from wok.utils import add_task, wok_log
> +from wok.utils import add_task, run_command, wok_log
>   from wok.model.tasks import TaskModel
>
> +from wok.plugins.gingerbase.lscpu import LsCpu
>   from wok.plugins.gingerbase.model.debugreports import DebugReportsModel
>   from wok.plugins.gingerbase.repositories import Repositories
>   from wok.plugins.gingerbase.swupdate import SoftwareUpdate
> @@ -48,17 +50,29 @@ DOM_STATE_MAP = {0: 'nostate',
>                    6: 'crashed',
>                    7: 'pmsuspended'}
>
> +ARCH = platform.machine()
> +PROC_CPUINFO = '/proc/cpuinfo'
> +PROC_SYSINFO = '/proc/sysinfo'
> +LSMEM = 'lsmem'
> +CPUS_DEDICATED = 'cpus_dedicated'
> +CPUS_SHARED = 'cpus_shared'
> +LPAR_NAME = 'lpar_name'
> +LPAR_NUMBER = 'lpar_number'
> +
>
>   class HostModel(object):
>       def __init__(self, **kargs):
>           # self.conn = kargs['conn']
>           self.objstore = kargs['objstore']
>           self.task = TaskModel(**kargs)
> -        self.host_info = self._get_host_info()
> +        self.lscpu = LsCpu()
>
> -    def _get_ppc_cpu_info(self):
> +    def _get_ppc_cpu_model(self):
> +        """
> +        method to get cpu_model for ppc architecture
> +        """
>           res = {}
> -        with open('/proc/cpuinfo') as f:
> +        with open(PROC_CPUINFO) as f:
>               for line in f.xreadlines():
>                   # Parse CPU, CPU's revision and CPU's clock information
>                   for key in ['cpu', 'revision', 'clock']:
> @@ -81,56 +95,227 @@ class HostModel(object):
>
>           return ""
>
> -    def _get_host_info(self):
> -        res = {}
> -        if platform.machine().startswith('ppc'):
> -            res['cpu_model'] = self._get_ppc_cpu_info()
> -        else:
> -            with open('/proc/cpuinfo') as f:
> +    def _get_x86_cpu_model(self):
> +        """
> +        method to get cpu_model for x86 architecture
> +        """
> +        try:
> +            with open(PROC_CPUINFO) as f:
>                   for line in f.xreadlines():
>                       if "model name" in line:
> -                        res['cpu_model'] = line.split(':')[1].strip()
> +                        return line.split(':')[1].strip()
>                           break
> +        except Exception as e:
> +            wok_log.error("Failed to retrive cpu_model for "
> +                          "%s. Error: %s", ARCH, e.__str__())
> +        return ""
>
> -        res['cpus'] = 0
> -        res['memory'] = 0L
> -
> -        # Include IBM PowerKVM name to supported distro names
> -        _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
> -        # 'fedora' '17' 'Beefy Miracle'
> -        distro, version, codename = platform.linux_distribution(
> -            supported_dists=_sup_distros)
> -        res['os_distro'] = distro
> -        res['os_version'] = version
> -        res['os_codename'] = unicode(codename, "utf-8")
> -
> -        return res
> -
> -    def lookup(self, *name):
> -        cpus = psutil.NUM_CPUS
> -
> +    def _get_s390x_host_info(self):
> +        """
> +        method to get additional host details
> +        specific to s390x architecture
> +        :return: dictionary
> +        """
> +        host_info = {}
> +        host_info['cpus'] = self._get_cpus()
> +        host_info['cpus']['dedicated'] = 0
> +        host_info['cpus']['shared'] = 0
> +        host_info['cpu_model'] = ""
> +        host_info['virtualization'] = {}
> +        s390x_sysinfo = self._get_s390x_sysinfo()
> +        if 'manufacturer' in s390x_sysinfo.keys():
> +            host_info['cpu_model'] = s390x_sysinfo['manufacturer']
> +        if 'type' in s390x_sysinfo.keys():
> +            host_info['cpu_model'] = \
> +                host_info['cpu_model'] + "/" + s390x_sysinfo['type']
> +        if 'model' in s390x_sysinfo.keys():
> +            host_info['cpu_model'] = \
> +                host_info['cpu_model'] + "/" + s390x_sysinfo['model']
> +        if CPUS_DEDICATED in s390x_sysinfo.keys():
> +            host_info['cpus']['dedicated'] = s390x_sysinfo[CPUS_DEDICATED]
> +        if CPUS_SHARED in s390x_sysinfo.keys():
> +            host_info['cpus']['shared'] = s390x_sysinfo[CPUS_SHARED]
> +        host_info['virtualization']['hypervisor'] = \
> +            self.lscpu.get_hypervisor()
> +        host_info['virtualization']['hypervisor_vendor'] = \
> +            self.lscpu.get_hypervisor_vendor()
> +        host_info['virtualization'][LPAR_NAME] = ''
> +        host_info['virtualization'][LPAR_NUMBER] = ''
> +        if LPAR_NAME in s390x_sysinfo.keys():
> +            host_info['virtualization'][LPAR_NAME] = s390x_sysinfo[LPAR_NAME]
> +        if LPAR_NUMBER in s390x_sysinfo.keys():
> +            host_info['virtualization'][LPAR_NUMBER] = \
> +                s390x_sysinfo[LPAR_NUMBER]
> +
> +        return host_info
> +
> +    def _get_s390x_sysinfo(self):
> +        """
> +        This method retrieves following system information
> +        for s390 architecture
> +        * manufacturer: Manufacturer of host machine
> +        * type: Type of the host machine
> +        * model:Model of host machine
> +        * LPAR_NUMBER: LPAR Number of host
> +        * LPAR_NAME: Name of host LPAR
> +        * CPUS_DEDICATED: LPAR CPUs Dedicated
> +        * CPUS_SHARED: LPAR CPUs Shared
> +
> +        :param self: object of the class self
> +        :return: dictionary with following keys -
> +                 'manufacturer', 'type', 'model', CPUS_SHARED,
> +                 CPUS_DEDICATED, LPAR_NUMBER, LPAR_NAME
> +        """
> +        s390x_sysinfo = {}
> +        try:
> +            with open(PROC_SYSINFO) as f:
> +                for line in f.xreadlines():
> +                    if ":" in line and (len(line.split(':')) == 2):
> +                        info = line.split(':')
> +                        if info[0] == 'Model' and (len(info[1].split()) == 2):
> +                            s390x_sysinfo['model'] = \
> +                                info[1].split()[0].strip() +\
> +                                " "+info[1].split()[1].strip()
> +                        elif info[0] == 'Manufacturer':
> +                            s390x_sysinfo['manufacturer'] = info[1].strip()
> +                        elif info[0] == 'Type':
> +                            s390x_sysinfo['type'] = info[1].strip()
> +                        elif info[0] == 'LPAR Number':
> +                            s390x_sysinfo[LPAR_NUMBER] = int(info[1].strip())
> +                        elif info[0] == 'LPAR Name':
> +                            s390x_sysinfo[LPAR_NAME] = info[1].strip()
> +                        elif info[0] == 'LPAR CPUs Dedicated':
> +                            s390x_sysinfo[CPUS_DEDICATED] =\
> +                                int(info[1].strip())
> +                        elif info[0] == 'LPAR CPUs Shared':
> +                            s390x_sysinfo[CPUS_SHARED] = int(info[1].strip())
> +        except Exception as e:
> +            wok_log.error("Failed to retrieve information from %s file. "
> +                          "Error: %s", PROC_SYSINFO, e.__str__())
> +
> +        return s390x_sysinfo
> +
> +    def _get_memory(self):
> +        """
> +        method to retrieve memory information for all architecture
> +        :return: dictionary with keys "online" and "offline"
> +        """
> +        memory = {}
> +        online_memory = 0
> +        offline_memory = 0
> +        if ARCH.startswith('s390x'):
> +            online_mem_pat = r'^Total online memory :\s+(\d+)\s+MB$'
> +            offline_mem_pat = r'^Total offline memory:\s+(\d+)\s+MB$'
> +            out, err, rc = run_command(LSMEM)
> +            # output of lsmem in s390x architecture is expected to be
> +            # Address Range                          Size (MB)  State\
> +            #     Removable  Device
> +            # ========================================================\
> +            # =======================
> +            # 0x0000000000000000-0x000000000fffffff        256  online\
> +            #    no         0
> +            # 0x0000000010000000-0x000000002fffffff        512  online\
> +            #    yes        1-2
> +            # 0x0000000030000000-0x000000007fffffff       1280  online\
> +            #    no         3-7
> +            # 0x0000000080000000-0x00000000ffffffff       2048  offline\
> +            #   -          8-15
> +            #
> +            # Memory device size  : 256 MB
> +            # Memory block size   : 256 MB
> +            # Total online memory : 2048 MB
> +            # Total offline memory: 2048 MB
> +            if not rc:
> +                online_mem =\
> +                    re.search(online_mem_pat, out.strip(), re.M | re.I)
> +                offline_mem =\
> +                    re.search(offline_mem_pat, out.strip(), re.M | re.I)
> +                if online_mem and len(online_mem.groups()) == 1:
> +                    online_memory = int(online_mem.group(1)) * 1024 * 1024
> +                    # converting MB to bytes
> +                    # lsmem always returns memory in MB
> +                if offline_mem and len(offline_mem.groups()) == 1:
> +                    offline_memory = int(offline_mem.group(1)) * 1024 * 1024
> +            else:
> +                wok_log.error('Failed to retrieve memory information with'
> +                              ' command %s. Error: %s' % (LSMEM, err))
> +        else:
> +            if hasattr(psutil, 'phymem_usage'):
> +                online_memory = psutil.phymem_usage().total
> +            elif hasattr(psutil, 'virtual_memory'):
> +                online_memory = psutil.virtual_memory().total
> +
> +        memory['online'] = online_memory
> +        memory['offline'] = offline_memory
> +        return memory
> +
> +    def _get_cpus_(self):
> +        """
> +        method to retrieve online cpus count and offline cpus
> +        count for all architecture
> +        :return: dictionary with keys "online" and "offline"
> +        """
> +        cpus = {}
> +        total_cpus = int(self.lscpu.get_total_cpus())
> +        offline_cpus = 0
> +
> +        online_cpus = psutil.NUM_CPUS
>           # psutil is unstable on how to get the number of
>           # cpus, different versions call it differently
>           if hasattr(psutil, 'cpu_count'):
> -            cpus = psutil.cpu_count()
> +            online_cpus = psutil.cpu_count()
>
>           elif hasattr(psutil, 'NUM_CPUS'):
> -            cpus = psutil.NUM_CPUS
> +            online_cpus = psutil.NUM_CPUS
>
>           elif hasattr(psutil, '_psplatform'):
>               for method_name in ['_get_num_cpus', 'get_num_cpus']:
>
>                   method = getattr(psutil._psplatform, method_name, None)
>                   if method is not None:
> -                    cpus = method()
> +                    online_cpus = method()
>                       break
> +        if total_cpus >= online_cpus:
> +            offline_cpus = total_cpus - online_cpus
> +        cpus['online'] = online_cpus
> +        cpus['offline'] = offline_cpus
> +        return cpus
> +
> +    def _get_base_info(self):
> +        """
> +        method to retrieve common host information for all architectures
> +        :return: dictionary with keys 'os_distro', 'os_version', 'os_codename'
> +                 'architecture', 'host', memory
> +        """
> +        common_info = {}
> +        # Include IBM PowerKVM name to supported distro names
> +        _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
> +        # 'fedora' '17' 'Beefy Miracle'
> +        distro, version, codename = platform.linux_distribution(
> +            supported_dists=_sup_distros)
> +        common_info['os_distro'] = distro
> +        common_info['os_version'] = version
> +        common_info['os_codename'] = unicode(codename, "utf-8")
> +        common_info['architecture'] = ARCH
> +        common_info['host'] = platform.node()
> +        common_info['memory'] = self._get_memory()
>
> -        self.host_info['cpus'] = cpus
> -        if hasattr(psutil, 'phymem_usage'):
> -            self.host_info['memory'] = psutil.phymem_usage().total
> -        elif hasattr(psutil, 'virtual_memory'):
> -            self.host_info['memory'] = psutil.virtual_memory().total
> -        return self.host_info
> +        return common_info
> +
> +    def lookup(self, *name):
> +        """
> +        method to get basic information for host
> +        """
> +        host_info = self._get_base_info()
> +        if ARCH.startswith('s390x'):
> +            host_info.update(self._get_s390x_host_info())
> +        elif ARCH.startswith('ppc'):
> +            host_info['cpus'] = self._get_cpus_()
> +            host_info['cpu_model'] = self._get_ppc_cpu_model()
> +        else:
> +            host_info['cpus'] = self._get_cpus_()
> +            host_info['cpu_model'] = self._get_x86_cpu_model()
> +        return host_info
>
>       def swupdate(self, *name):
>           try:
> diff --git a/src/wok/plugins/gingerbase/tests/test_host.py b/src/wok/plugins/gingerbase/tests/test_host.py
> index 5d158bb..a994177 100644
> --- a/src/wok/plugins/gingerbase/tests/test_host.py
> +++ b/src/wok/plugins/gingerbase/tests/test_host.py
> @@ -68,15 +68,21 @@ class HostTests(unittest.TestCase):
>       def test_hostinfo(self):
>           resp = self.request('/plugins/gingerbase/host').read()
>           info = json.loads(resp)
> -        keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
> -                'memory', 'cpus']
> +        if platform.machine().startswith('s390x'):
> +            keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
> +                    'memory', 'cpus', 'architecture', 'host', 'virtualization']
> +        else:
> +            keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
> +                    'memory', 'cpus', 'architecture', 'host']
> +            self.assertEquals(psutil.TOTAL_PHYMEM, info['memory'])
>           self.assertEquals(sorted(keys), sorted(info.keys()))
>
>           distro, version, codename = platform.linux_distribution()
>           self.assertEquals(distro, info['os_distro'])
>           self.assertEquals(version, info['os_version'])
>           self.assertEquals(unicode(codename, "utf-8"), info['os_codename'])
> -        self.assertEquals(psutil.TOTAL_PHYMEM, info['memory'])
> +        self.assertEqual(platform.node(), info['host'])
> +        self.assertEqual(platform.machine(), info['architecture'])
>
>       def test_hoststats(self):
>           time.sleep(1)




More information about the Kimchi-devel mailing list