[Kimchi-devel] [PATCH V3] [Ginger-Base 1/1] Issue 728 - Processor Info in s390 architecture
Aline Manera
alinefm at linux.vnet.ibm.com
Thu Nov 26 12:20:51 UTC 2015
On 26/11/2015 10:20, Suresh Babu Angadi wrote:
> Please ignore this patch set, it has minor issue on s390x, i will send
> v4.
OK =)
>
> On 11/26/2015 05:33 PM, Aline Manera wrote:
>>
>> Reviewed-by: Aline Manera <alinefm at linux.vnet.ibm.com>
>>
>> On 26/11/2015 09:01, sureshab at linux.vnet.ibm.com wrote:
>>> From: Suresh Babu Angadi <sureshab at in.ibm.com>
>>>
>>> Fix for issue 728: processor info displays blank for system z
>>> this patch set also adds additional capability:
>>> retrieving architecture and host name (for all architecture)
>>> split CPUs to show online and offline cpus also
>>> includes shared and dedicated cpus for s390x
>>> split memory to show online and offline
>>> additional virtualization details(for s390x):
>>> virtualization will have hypervisor details and lpar details
>>>
>>> Signed-off-by: Suresh Babu Angadi <sureshab at in.ibm.com>
>>> ---
>>> src/wok/plugins/gingerbase/docs/API.md | 18 +-
>>> src/wok/plugins/gingerbase/i18n.py | 1 +
>>> src/wok/plugins/gingerbase/lscpu.py | 60 ++++++
>>> src/wok/plugins/gingerbase/model/host.py | 259
>>> ++++++++++++++++++++++----
>>> src/wok/plugins/gingerbase/tests/test_host.py | 12 +-
>>> 5 files changed, 307 insertions(+), 43 deletions(-)
>>>
>>> diff --git a/src/wok/plugins/gingerbase/docs/API.md
>>> b/src/wok/plugins/gingerbase/docs/API.md
>>> index 8f37ddc..30b0c8f 100644
>>> --- a/src/wok/plugins/gingerbase/docs/API.md
>>> +++ b/src/wok/plugins/gingerbase/docs/API.md
>>> @@ -95,13 +95,25 @@ Contains information of host.
>>> **Methods:**
>>>
>>> * **GET**: Retrieve host static information
>>> - * memory: Total size of host physical memory
>>> - The unit is Bytes
>>> + * memory: Memory details of the host. The unit is Bytes
>>> + * online: Total size of physical memory currently
>>> available to host
>>> + * offline: Total offline memory(for s390x), zero
>>> for other architectures.
>>> * cpu_model: The model name of host CPU
>>> - * cpus: The number of online CPUs available on host
>>> + * cpus: The host cpus information
>>> + * online: Number of online CPUs
>>> + * offline: Number of offline CPUs
>>> + * shared *(s390x only)*: Number of shared CPUs
>>> + * dedicated *(s390x only)*: Number of dedicated CPUs
>>> * os_distro: The OS distribution that runs on host
>>> * os_version: The version of OS distribution
>>> * os_codename: The code name of OS distribution
>>> + * host: Host name of the network node
>>> + * architecture: Architecture of the host
>>> + * virtualization *(s390x only)*: This is a json with following
>>> keys
>>> + * hypervisor: Hypervisor name of the host
>>> + * hypervisor_vendor: Hypervisor Vendor name
>>> + * lpar_number: LPAR Number of host
>>> + * lpar_name: Name of host LPAR
>>>
>>> * **POST**: *See Host Actions*
>>>
>>> diff --git a/src/wok/plugins/gingerbase/i18n.py
>>> b/src/wok/plugins/gingerbase/i18n.py
>>> index af75c70..a0a8c36 100644
>>> --- a/src/wok/plugins/gingerbase/i18n.py
>>> +++ b/src/wok/plugins/gingerbase/i18n.py
>>> @@ -88,5 +88,6 @@ messages = {
>>> "GGBCPUINF0005E": _("This host (or current configuration) does
>>> not provide Socket(s) information."),
>>> "GGBCPUINF0006E": _("This host (or current configuration) does
>>> not provide Core(s) per socket information."),
>>> "GGBCPUINF0007E": _("This host (or current configuration) does
>>> not provide Thread(s) per core information."),
>>> + "GGBCPUINF0008E": _("This host (or current configuration) does
>>> not provide CPU(s) information."),
>>>
>>> }
>>> diff --git a/src/wok/plugins/gingerbase/lscpu.py
>>> b/src/wok/plugins/gingerbase/lscpu.py
>>> index 0fc1e72..a822a15 100644
>>> --- a/src/wok/plugins/gingerbase/lscpu.py
>>> +++ b/src/wok/plugins/gingerbase/lscpu.py
>>> @@ -17,10 +17,13 @@
>>> # License along with this library; if not, write to the Free Software
>>> # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>>> 02110-1301 USA
>>> import logging
>>> +import platform
>>>
>>> from wok.utils import run_command
>>> from wok.exception import NotFoundError
>>>
>>> +ARCH = platform.machine()
>>> +
>>>
>>> class LsCpu(object):
>>> """
>>> @@ -64,6 +67,28 @@ class LsCpu(object):
>>> # L2 cache: 256K
>>> # L3 cache: 3072K
>>> # NUMA node0 CPU(s): 0-3
>>> + #
>>> + # Output of lscpu in s390x is expected to be
>>> + # Architecture: s390x
>>> + # CPU op-mode(s): 32-bit, 64-bit
>>> + # Byte Order: Big Endian
>>> + # CPU(s): 4
>>> + # On-line CPU(s) list: 0,1
>>> + # Off-line CPU(s) list: 2,3
>>> + # Thread(s) per core: 1
>>> + # Core(s) per socket: 6
>>> + # Socket(s) per book: 6
>>> + # Book(s): 4
>>> + # Vendor ID: IBM/S390
>>> + # BogoMIPS: 18115.00
>>> + # Hypervisor: PR/SM
>>> + # Hypervisor vendor: IBM
>>> + # Virtualization type: full
>>> + # Dispatching mode: horizontal
>>> + # L1d cache: 96K
>>> + # L1i cache: 64K
>>> + # L2d cache: 1024K
>>> + # L2i cache: 1024K
>>>
>>> if not rc and (not out.isspace()):
>>> lscpuout = out.split('\n')
>>> @@ -85,6 +110,8 @@ class LsCpu(object):
>>> """
>>> try:
>>> sockets = "Socket(s)"
>>> + if ARCH.startswith('s390x'):
>>> + sockets = "Socket(s) per book"
>>> if len(self.lsCpuInfo) > 0 and sockets in
>>> self.lsCpuInfo.keys():
>>> return int(self.lsCpuInfo[sockets])
>>> else:
>>> @@ -124,3 +151,36 @@ class LsCpu(object):
>>> except IndexError, e:
>>> self.log_error(e)
>>> raise NotFoundError("GGBCPUINF0007E")
>>> +
>>> + def get_total_cpus(self):
>>> + """
>>> + method to get total cpus retrieved from CPU(s) field of lscpu
>>> + :return: total cpus
>>> + """
>>> + total_cpus = 'CPU(s)'
>>> + if len(self.lsCpuInfo) > 0 and total_cpus in
>>> self.lsCpuInfo.keys():
>>> + return int(self.lsCpuInfo[total_cpus])
>>> + else:
>>> + self.log_error("Failed to fetch total cpus count in
>>> lscpu output")
>>> + raise NotFoundError("GGBCPUINF0008E")
>>> +
>>> + def get_hypervisor(self):
>>> + """
>>> + method to get hypervisor name if present in lscpu o/p
>>> + :return: Hypervisor Name
>>> + """
>>> + hypervisor = 'Hypervisor'
>>> + if len(self.lsCpuInfo) > 0 and hypervisor in
>>> self.lsCpuInfo.keys():
>>> + return self.lsCpuInfo[hypervisor]
>>> + return None
>>> +
>>> + def get_hypervisor_vendor(self):
>>> + """
>>> + method to get hypervisor vendor if present in lscpu o/p
>>> + :return: Hypervisor Vendor
>>> + """
>>> + hypervisor_vendor = 'Hypervisor vendor'
>>> + if len(self.lsCpuInfo) > 0 and hypervisor_vendor in \
>>> + self.lsCpuInfo.keys():
>>> + return self.lsCpuInfo[hypervisor_vendor]
>>> + return None
>>> diff --git a/src/wok/plugins/gingerbase/model/host.py
>>> b/src/wok/plugins/gingerbase/model/host.py
>>> index a58bada..fd4db29 100644
>>> --- a/src/wok/plugins/gingerbase/model/host.py
>>> +++ b/src/wok/plugins/gingerbase/model/host.py
>>> @@ -22,6 +22,7 @@
>>> import os
>>> import platform
>>> import psutil
>>> +import re
>>> import time
>>> from cherrypy.process.plugins import BackgroundTask
>>> from collections import defaultdict
>>> @@ -31,9 +32,10 @@ from wok.basemodel import Singleton
>>> from wok.config import config as kconfig
>>> from wok.exception import InvalidOperation
>>> from wok.exception import OperationFailed
>>> -from wok.utils import add_task, wok_log
>>> +from wok.utils import add_task, run_command, wok_log
>>> from wok.model.tasks import TaskModel
>>>
>>> +from wok.plugins.gingerbase.lscpu import LsCpu
>>> from wok.plugins.gingerbase.model.debugreports import
>>> DebugReportsModel
>>> from wok.plugins.gingerbase.repositories import Repositories
>>> from wok.plugins.gingerbase.swupdate import SoftwareUpdate
>>> @@ -48,17 +50,29 @@ DOM_STATE_MAP = {0: 'nostate',
>>> 6: 'crashed',
>>> 7: 'pmsuspended'}
>>>
>>> +ARCH = platform.machine()
>>> +PROC_CPUINFO = '/proc/cpuinfo'
>>> +PROC_SYSINFO = '/proc/sysinfo'
>>> +LSMEM = 'lsmem'
>>> +CPUS_DEDICATED = 'cpus_dedicated'
>>> +CPUS_SHARED = 'cpus_shared'
>>> +LPAR_NAME = 'lpar_name'
>>> +LPAR_NUMBER = 'lpar_number'
>>> +
>>>
>>> class HostModel(object):
>>> def __init__(self, **kargs):
>>> # self.conn = kargs['conn']
>>> self.objstore = kargs['objstore']
>>> self.task = TaskModel(**kargs)
>>> - self.host_info = self._get_host_info()
>>> + self.lscpu = LsCpu()
>>>
>>> - def _get_ppc_cpu_info(self):
>>> + def _get_ppc_cpu_model(self):
>>> + """
>>> + method to get cpu_model for ppc architecture
>>> + """
>>> res = {}
>>> - with open('/proc/cpuinfo') as f:
>>> + with open(PROC_CPUINFO) as f:
>>> for line in f.xreadlines():
>>> # Parse CPU, CPU's revision and CPU's clock
>>> information
>>> for key in ['cpu', 'revision', 'clock']:
>>> @@ -81,56 +95,227 @@ class HostModel(object):
>>>
>>> return ""
>>>
>>> - def _get_host_info(self):
>>> - res = {}
>>> - if platform.machine().startswith('ppc'):
>>> - res['cpu_model'] = self._get_ppc_cpu_info()
>>> - else:
>>> - with open('/proc/cpuinfo') as f:
>>> + def _get_x86_cpu_model(self):
>>> + """
>>> + method to get cpu_model for x86 architecture
>>> + """
>>> + try:
>>> + with open(PROC_CPUINFO) as f:
>>> for line in f.xreadlines():
>>> if "model name" in line:
>>> - res['cpu_model'] = line.split(':')[1].strip()
>>> + return line.split(':')[1].strip()
>>> break
>>> + except Exception as e:
>>> + wok_log.error("Failed to retrive cpu_model for "
>>> + "%s. Error: %s", ARCH, e.__str__())
>>> + return ""
>>>
>>> - res['cpus'] = 0
>>> - res['memory'] = 0L
>>> -
>>> - # Include IBM PowerKVM name to supported distro names
>>> - _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
>>> - # 'fedora' '17' 'Beefy Miracle'
>>> - distro, version, codename = platform.linux_distribution(
>>> - supported_dists=_sup_distros)
>>> - res['os_distro'] = distro
>>> - res['os_version'] = version
>>> - res['os_codename'] = unicode(codename, "utf-8")
>>> -
>>> - return res
>>> -
>>> - def lookup(self, *name):
>>> - cpus = psutil.NUM_CPUS
>>> -
>>> + def _get_s390x_host_info(self):
>>> + """
>>> + method to get additional host details
>>> + specific to s390x architecture
>>> + :return: dictionary
>>> + """
>>> + host_info = {}
>>> + host_info['cpus'] = self._get_cpus()
>>> + host_info['cpus']['dedicated'] = 0
>>> + host_info['cpus']['shared'] = 0
>>> + host_info['cpu_model'] = ""
>>> + host_info['virtualization'] = {}
>>> + s390x_sysinfo = self._get_s390x_sysinfo()
>>> + if 'manufacturer' in s390x_sysinfo.keys():
>>> + host_info['cpu_model'] = s390x_sysinfo['manufacturer']
>>> + if 'type' in s390x_sysinfo.keys():
>>> + host_info['cpu_model'] = \
>>> + host_info['cpu_model'] + "/" + s390x_sysinfo['type']
>>> + if 'model' in s390x_sysinfo.keys():
>>> + host_info['cpu_model'] = \
>>> + host_info['cpu_model'] + "/" + s390x_sysinfo['model']
>>> + if CPUS_DEDICATED in s390x_sysinfo.keys():
>>> + host_info['cpus']['dedicated'] =
>>> s390x_sysinfo[CPUS_DEDICATED]
>>> + if CPUS_SHARED in s390x_sysinfo.keys():
>>> + host_info['cpus']['shared'] = s390x_sysinfo[CPUS_SHARED]
>>> + host_info['virtualization']['hypervisor'] = \
>>> + self.lscpu.get_hypervisor()
>>> + host_info['virtualization']['hypervisor_vendor'] = \
>>> + self.lscpu.get_hypervisor_vendor()
>>> + host_info['virtualization'][LPAR_NAME] = ''
>>> + host_info['virtualization'][LPAR_NUMBER] = ''
>>> + if LPAR_NAME in s390x_sysinfo.keys():
>>> + host_info['virtualization'][LPAR_NAME] =
>>> s390x_sysinfo[LPAR_NAME]
>>> + if LPAR_NUMBER in s390x_sysinfo.keys():
>>> + host_info['virtualization'][LPAR_NUMBER] = \
>>> + s390x_sysinfo[LPAR_NUMBER]
>>> +
>>> + return host_info
>>> +
>>> + def _get_s390x_sysinfo(self):
>>> + """
>>> + This method retrieves following system information
>>> + for s390 architecture
>>> + * manufacturer: Manufacturer of host machine
>>> + * type: Type of the host machine
>>> + * model:Model of host machine
>>> + * LPAR_NUMBER: LPAR Number of host
>>> + * LPAR_NAME: Name of host LPAR
>>> + * CPUS_DEDICATED: LPAR CPUs Dedicated
>>> + * CPUS_SHARED: LPAR CPUs Shared
>>> +
>>> + :param self: object of the class self
>>> + :return: dictionary with following keys -
>>> + 'manufacturer', 'type', 'model', CPUS_SHARED,
>>> + CPUS_DEDICATED, LPAR_NUMBER, LPAR_NAME
>>> + """
>>> + s390x_sysinfo = {}
>>> + try:
>>> + with open(PROC_SYSINFO) as f:
>>> + for line in f.xreadlines():
>>> + if ":" in line and (len(line.split(':')) == 2):
>>> + info = line.split(':')
>>> + if info[0] == 'Model' and
>>> (len(info[1].split()) == 2):
>>> + s390x_sysinfo['model'] = \
>>> + info[1].split()[0].strip() +\
>>> + " "+info[1].split()[1].strip()
>>> + elif info[0] == 'Manufacturer':
>>> + s390x_sysinfo['manufacturer'] =
>>> info[1].strip()
>>> + elif info[0] == 'Type':
>>> + s390x_sysinfo['type'] = info[1].strip()
>>> + elif info[0] == 'LPAR Number':
>>> + s390x_sysinfo[LPAR_NUMBER] =
>>> int(info[1].strip())
>>> + elif info[0] == 'LPAR Name':
>>> + s390x_sysinfo[LPAR_NAME] = info[1].strip()
>>> + elif info[0] == 'LPAR CPUs Dedicated':
>>> + s390x_sysinfo[CPUS_DEDICATED] =\
>>> + int(info[1].strip())
>>> + elif info[0] == 'LPAR CPUs Shared':
>>> + s390x_sysinfo[CPUS_SHARED] =
>>> int(info[1].strip())
>>> + except Exception as e:
>>> + wok_log.error("Failed to retrieve information from %s
>>> file. "
>>> + "Error: %s", PROC_SYSINFO, e.__str__())
>>> +
>>> + return s390x_sysinfo
>>> +
>>> + def _get_memory(self):
>>> + """
>>> + method to retrieve memory information for all architecture
>>> + :return: dictionary with keys "online" and "offline"
>>> + """
>>> + memory = {}
>>> + online_memory = 0
>>> + offline_memory = 0
>>> + if ARCH.startswith('s390x'):
>>> + online_mem_pat = r'^Total online memory :\s+(\d+)\s+MB$'
>>> + offline_mem_pat = r'^Total offline memory:\s+(\d+)\s+MB$'
>>> + out, err, rc = run_command(LSMEM)
>>> + # output of lsmem in s390x architecture is expected to be
>>> + # Address Range Size (MB) State\
>>> + # Removable Device
>>> + #
>>> ========================================================\
>>> + # =======================
>>> + # 0x0000000000000000-0x000000000fffffff 256 online\
>>> + # no 0
>>> + # 0x0000000010000000-0x000000002fffffff 512 online\
>>> + # yes 1-2
>>> + # 0x0000000030000000-0x000000007fffffff 1280 online\
>>> + # no 3-7
>>> + # 0x0000000080000000-0x00000000ffffffff 2048 offline\
>>> + # - 8-15
>>> + #
>>> + # Memory device size : 256 MB
>>> + # Memory block size : 256 MB
>>> + # Total online memory : 2048 MB
>>> + # Total offline memory: 2048 MB
>>> + if not rc:
>>> + online_mem =\
>>> + re.search(online_mem_pat, out.strip(), re.M |
>>> re.I)
>>> + offline_mem =\
>>> + re.search(offline_mem_pat, out.strip(), re.M |
>>> re.I)
>>> + if online_mem and len(online_mem.groups()) == 1:
>>> + online_memory = int(online_mem.group(1)) * 1024
>>> * 1024
>>> + # converting MB to bytes
>>> + # lsmem always returns memory in MB
>>> + if offline_mem and len(offline_mem.groups()) == 1:
>>> + offline_memory = int(offline_mem.group(1)) *
>>> 1024 * 1024
>>> + else:
>>> + wok_log.error('Failed to retrieve memory
>>> information with'
>>> + ' command %s. Error: %s' % (LSMEM, err))
>>> + else:
>>> + if hasattr(psutil, 'phymem_usage'):
>>> + online_memory = psutil.phymem_usage().total
>>> + elif hasattr(psutil, 'virtual_memory'):
>>> + online_memory = psutil.virtual_memory().total
>>> +
>>> + memory['online'] = online_memory
>>> + memory['offline'] = offline_memory
>>> + return memory
>>> +
>>> + def _get_cpus_(self):
>>> + """
>>> + method to retrieve online cpus count and offline cpus
>>> + count for all architecture
>>> + :return: dictionary with keys "online" and "offline"
>>> + """
>>> + cpus = {}
>>> + total_cpus = int(self.lscpu.get_total_cpus())
>>> + offline_cpus = 0
>>> +
>>> + online_cpus = psutil.NUM_CPUS
>>> # psutil is unstable on how to get the number of
>>> # cpus, different versions call it differently
>>> if hasattr(psutil, 'cpu_count'):
>>> - cpus = psutil.cpu_count()
>>> + online_cpus = psutil.cpu_count()
>>>
>>> elif hasattr(psutil, 'NUM_CPUS'):
>>> - cpus = psutil.NUM_CPUS
>>> + online_cpus = psutil.NUM_CPUS
>>>
>>> elif hasattr(psutil, '_psplatform'):
>>> for method_name in ['_get_num_cpus', 'get_num_cpus']:
>>>
>>> method = getattr(psutil._psplatform, method_name,
>>> None)
>>> if method is not None:
>>> - cpus = method()
>>> + online_cpus = method()
>>> break
>>> + if total_cpus >= online_cpus:
>>> + offline_cpus = total_cpus - online_cpus
>>> + cpus['online'] = online_cpus
>>> + cpus['offline'] = offline_cpus
>>> + return cpus
>>> +
>>> + def _get_base_info(self):
>>> + """
>>> + method to retrieve common host information for all
>>> architectures
>>> + :return: dictionary with keys 'os_distro', 'os_version',
>>> 'os_codename'
>>> + 'architecture', 'host', memory
>>> + """
>>> + common_info = {}
>>> + # Include IBM PowerKVM name to supported distro names
>>> + _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
>>> + # 'fedora' '17' 'Beefy Miracle'
>>> + distro, version, codename = platform.linux_distribution(
>>> + supported_dists=_sup_distros)
>>> + common_info['os_distro'] = distro
>>> + common_info['os_version'] = version
>>> + common_info['os_codename'] = unicode(codename, "utf-8")
>>> + common_info['architecture'] = ARCH
>>> + common_info['host'] = platform.node()
>>> + common_info['memory'] = self._get_memory()
>>>
>>> - self.host_info['cpus'] = cpus
>>> - if hasattr(psutil, 'phymem_usage'):
>>> - self.host_info['memory'] = psutil.phymem_usage().total
>>> - elif hasattr(psutil, 'virtual_memory'):
>>> - self.host_info['memory'] = psutil.virtual_memory().total
>>> - return self.host_info
>>> + return common_info
>>> +
>>> + def lookup(self, *name):
>>> + """
>>> + method to get basic information for host
>>> + """
>>> + host_info = self._get_base_info()
>>> + if ARCH.startswith('s390x'):
>>> + host_info.update(self._get_s390x_host_info())
>>> + elif ARCH.startswith('ppc'):
>>> + host_info['cpus'] = self._get_cpus_()
>>> + host_info['cpu_model'] = self._get_ppc_cpu_model()
>>> + else:
>>> + host_info['cpus'] = self._get_cpus_()
>>> + host_info['cpu_model'] = self._get_x86_cpu_model()
>>> + return host_info
>>>
>>> def swupdate(self, *name):
>>> try:
>>> diff --git a/src/wok/plugins/gingerbase/tests/test_host.py
>>> b/src/wok/plugins/gingerbase/tests/test_host.py
>>> index 5d158bb..5a61285 100644
>>> --- a/src/wok/plugins/gingerbase/tests/test_host.py
>>> +++ b/src/wok/plugins/gingerbase/tests/test_host.py
>>> @@ -68,15 +68,21 @@ class HostTests(unittest.TestCase):
>>> def test_hostinfo(self):
>>> resp = self.request('/plugins/gingerbase/host').read()
>>> info = json.loads(resp)
>>> - keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
>>> - 'memory', 'cpus']
>>> + if platform.machine().startswith('s390x'):
>>> + keys = ['os_distro', 'os_version', 'os_codename',
>>> 'cpu_model',
>>> + 'memory', 'cpus', 'architecture', 'host',
>>> 'virtualization']
>>> + else:
>>> + keys = ['os_distro', 'os_version', 'os_codename',
>>> 'cpu_model',
>>> + 'memory', 'cpus', 'architecture', 'host']
>>> + self.assertEquals(psutil.TOTAL_PHYMEM,
>>> info['memory']['online'])
>>> self.assertEquals(sorted(keys), sorted(info.keys()))
>>>
>>> distro, version, codename = platform.linux_distribution()
>>> self.assertEquals(distro, info['os_distro'])
>>> self.assertEquals(version, info['os_version'])
>>> self.assertEquals(unicode(codename, "utf-8"),
>>> info['os_codename'])
>>> - self.assertEquals(psutil.TOTAL_PHYMEM, info['memory'])
>>> + self.assertEqual(platform.node(), info['host'])
>>> + self.assertEqual(platform.machine(), info['architecture'])
>>>
>>> def test_hoststats(self):
>>> time.sleep(1)
>>
>
More information about the Kimchi-devel
mailing list