[Kimchi-devel] [PATCH V3] [Ginger-Base 1/1] Issue 728 - Processor Info in s390 architecture
Suresh Babu Angadi
sureshab at linux.vnet.ibm.com
Thu Nov 26 12:20:10 UTC 2015
Please ignore this patch set, it has minor issue on s390x, i will send v4.
On 11/26/2015 05:33 PM, Aline Manera wrote:
>
> Reviewed-by: Aline Manera <alinefm at linux.vnet.ibm.com>
>
> On 26/11/2015 09:01, sureshab at linux.vnet.ibm.com wrote:
>> From: Suresh Babu Angadi <sureshab at in.ibm.com>
>>
>> Fix for issue 728: processor info displays blank for system z
>> this patch set also adds additional capability:
>> retrieving architecture and host name (for all architecture)
>> split CPUs to show online and offline cpus also
>> includes shared and dedicated cpus for s390x
>> split memory to show online and offline
>> additional virtualization details(for s390x):
>> virtualization will have hypervisor details and lpar details
>>
>> Signed-off-by: Suresh Babu Angadi <sureshab at in.ibm.com>
>> ---
>> src/wok/plugins/gingerbase/docs/API.md | 18 +-
>> src/wok/plugins/gingerbase/i18n.py | 1 +
>> src/wok/plugins/gingerbase/lscpu.py | 60 ++++++
>> src/wok/plugins/gingerbase/model/host.py | 259
>> ++++++++++++++++++++++----
>> src/wok/plugins/gingerbase/tests/test_host.py | 12 +-
>> 5 files changed, 307 insertions(+), 43 deletions(-)
>>
>> diff --git a/src/wok/plugins/gingerbase/docs/API.md
>> b/src/wok/plugins/gingerbase/docs/API.md
>> index 8f37ddc..30b0c8f 100644
>> --- a/src/wok/plugins/gingerbase/docs/API.md
>> +++ b/src/wok/plugins/gingerbase/docs/API.md
>> @@ -95,13 +95,25 @@ Contains information of host.
>> **Methods:**
>>
>> * **GET**: Retrieve host static information
>> - * memory: Total size of host physical memory
>> - The unit is Bytes
>> + * memory: Memory details of the host. The unit is Bytes
>> + * online: Total size of physical memory currently
>> available to host
>> + * offline: Total offline memory(for s390x), zero for
>> other architectures.
>> * cpu_model: The model name of host CPU
>> - * cpus: The number of online CPUs available on host
>> + * cpus: The host cpus information
>> + * online: Number of online CPUs
>> + * offline: Number of offline CPUs
>> + * shared *(s390x only)*: Number of shared CPUs
>> + * dedicated *(s390x only)*: Number of dedicated CPUs
>> * os_distro: The OS distribution that runs on host
>> * os_version: The version of OS distribution
>> * os_codename: The code name of OS distribution
>> + * host: Host name of the network node
>> + * architecture: Architecture of the host
>> + * virtualization *(s390x only)*: This is a json with following keys
>> + * hypervisor: Hypervisor name of the host
>> + * hypervisor_vendor: Hypervisor Vendor name
>> + * lpar_number: LPAR Number of host
>> + * lpar_name: Name of host LPAR
>>
>> * **POST**: *See Host Actions*
>>
>> diff --git a/src/wok/plugins/gingerbase/i18n.py
>> b/src/wok/plugins/gingerbase/i18n.py
>> index af75c70..a0a8c36 100644
>> --- a/src/wok/plugins/gingerbase/i18n.py
>> +++ b/src/wok/plugins/gingerbase/i18n.py
>> @@ -88,5 +88,6 @@ messages = {
>> "GGBCPUINF0005E": _("This host (or current configuration) does
>> not provide Socket(s) information."),
>> "GGBCPUINF0006E": _("This host (or current configuration) does
>> not provide Core(s) per socket information."),
>> "GGBCPUINF0007E": _("This host (or current configuration) does
>> not provide Thread(s) per core information."),
>> + "GGBCPUINF0008E": _("This host (or current configuration) does
>> not provide CPU(s) information."),
>>
>> }
>> diff --git a/src/wok/plugins/gingerbase/lscpu.py
>> b/src/wok/plugins/gingerbase/lscpu.py
>> index 0fc1e72..a822a15 100644
>> --- a/src/wok/plugins/gingerbase/lscpu.py
>> +++ b/src/wok/plugins/gingerbase/lscpu.py
>> @@ -17,10 +17,13 @@
>> # License along with this library; if not, write to the Free Software
>> # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> import logging
>> +import platform
>>
>> from wok.utils import run_command
>> from wok.exception import NotFoundError
>>
>> +ARCH = platform.machine()
>> +
>>
>> class LsCpu(object):
>> """
>> @@ -64,6 +67,28 @@ class LsCpu(object):
>> # L2 cache: 256K
>> # L3 cache: 3072K
>> # NUMA node0 CPU(s): 0-3
>> + #
>> + # Output of lscpu in s390x is expected to be
>> + # Architecture: s390x
>> + # CPU op-mode(s): 32-bit, 64-bit
>> + # Byte Order: Big Endian
>> + # CPU(s): 4
>> + # On-line CPU(s) list: 0,1
>> + # Off-line CPU(s) list: 2,3
>> + # Thread(s) per core: 1
>> + # Core(s) per socket: 6
>> + # Socket(s) per book: 6
>> + # Book(s): 4
>> + # Vendor ID: IBM/S390
>> + # BogoMIPS: 18115.00
>> + # Hypervisor: PR/SM
>> + # Hypervisor vendor: IBM
>> + # Virtualization type: full
>> + # Dispatching mode: horizontal
>> + # L1d cache: 96K
>> + # L1i cache: 64K
>> + # L2d cache: 1024K
>> + # L2i cache: 1024K
>>
>> if not rc and (not out.isspace()):
>> lscpuout = out.split('\n')
>> @@ -85,6 +110,8 @@ class LsCpu(object):
>> """
>> try:
>> sockets = "Socket(s)"
>> + if ARCH.startswith('s390x'):
>> + sockets = "Socket(s) per book"
>> if len(self.lsCpuInfo) > 0 and sockets in
>> self.lsCpuInfo.keys():
>> return int(self.lsCpuInfo[sockets])
>> else:
>> @@ -124,3 +151,36 @@ class LsCpu(object):
>> except IndexError, e:
>> self.log_error(e)
>> raise NotFoundError("GGBCPUINF0007E")
>> +
>> + def get_total_cpus(self):
>> + """
>> + method to get total cpus retrieved from CPU(s) field of lscpu
>> + :return: total cpus
>> + """
>> + total_cpus = 'CPU(s)'
>> + if len(self.lsCpuInfo) > 0 and total_cpus in
>> self.lsCpuInfo.keys():
>> + return int(self.lsCpuInfo[total_cpus])
>> + else:
>> + self.log_error("Failed to fetch total cpus count in
>> lscpu output")
>> + raise NotFoundError("GGBCPUINF0008E")
>> +
>> + def get_hypervisor(self):
>> + """
>> + method to get hypervisor name if present in lscpu o/p
>> + :return: Hypervisor Name
>> + """
>> + hypervisor = 'Hypervisor'
>> + if len(self.lsCpuInfo) > 0 and hypervisor in
>> self.lsCpuInfo.keys():
>> + return self.lsCpuInfo[hypervisor]
>> + return None
>> +
>> + def get_hypervisor_vendor(self):
>> + """
>> + method to get hypervisor vendor if present in lscpu o/p
>> + :return: Hypervisor Vendor
>> + """
>> + hypervisor_vendor = 'Hypervisor vendor'
>> + if len(self.lsCpuInfo) > 0 and hypervisor_vendor in \
>> + self.lsCpuInfo.keys():
>> + return self.lsCpuInfo[hypervisor_vendor]
>> + return None
>> diff --git a/src/wok/plugins/gingerbase/model/host.py
>> b/src/wok/plugins/gingerbase/model/host.py
>> index a58bada..fd4db29 100644
>> --- a/src/wok/plugins/gingerbase/model/host.py
>> +++ b/src/wok/plugins/gingerbase/model/host.py
>> @@ -22,6 +22,7 @@
>> import os
>> import platform
>> import psutil
>> +import re
>> import time
>> from cherrypy.process.plugins import BackgroundTask
>> from collections import defaultdict
>> @@ -31,9 +32,10 @@ from wok.basemodel import Singleton
>> from wok.config import config as kconfig
>> from wok.exception import InvalidOperation
>> from wok.exception import OperationFailed
>> -from wok.utils import add_task, wok_log
>> +from wok.utils import add_task, run_command, wok_log
>> from wok.model.tasks import TaskModel
>>
>> +from wok.plugins.gingerbase.lscpu import LsCpu
>> from wok.plugins.gingerbase.model.debugreports import
>> DebugReportsModel
>> from wok.plugins.gingerbase.repositories import Repositories
>> from wok.plugins.gingerbase.swupdate import SoftwareUpdate
>> @@ -48,17 +50,29 @@ DOM_STATE_MAP = {0: 'nostate',
>> 6: 'crashed',
>> 7: 'pmsuspended'}
>>
>> +ARCH = platform.machine()
>> +PROC_CPUINFO = '/proc/cpuinfo'
>> +PROC_SYSINFO = '/proc/sysinfo'
>> +LSMEM = 'lsmem'
>> +CPUS_DEDICATED = 'cpus_dedicated'
>> +CPUS_SHARED = 'cpus_shared'
>> +LPAR_NAME = 'lpar_name'
>> +LPAR_NUMBER = 'lpar_number'
>> +
>>
>> class HostModel(object):
>> def __init__(self, **kargs):
>> # self.conn = kargs['conn']
>> self.objstore = kargs['objstore']
>> self.task = TaskModel(**kargs)
>> - self.host_info = self._get_host_info()
>> + self.lscpu = LsCpu()
>>
>> - def _get_ppc_cpu_info(self):
>> + def _get_ppc_cpu_model(self):
>> + """
>> + method to get cpu_model for ppc architecture
>> + """
>> res = {}
>> - with open('/proc/cpuinfo') as f:
>> + with open(PROC_CPUINFO) as f:
>> for line in f.xreadlines():
>> # Parse CPU, CPU's revision and CPU's clock
>> information
>> for key in ['cpu', 'revision', 'clock']:
>> @@ -81,56 +95,227 @@ class HostModel(object):
>>
>> return ""
>>
>> - def _get_host_info(self):
>> - res = {}
>> - if platform.machine().startswith('ppc'):
>> - res['cpu_model'] = self._get_ppc_cpu_info()
>> - else:
>> - with open('/proc/cpuinfo') as f:
>> + def _get_x86_cpu_model(self):
>> + """
>> + method to get cpu_model for x86 architecture
>> + """
>> + try:
>> + with open(PROC_CPUINFO) as f:
>> for line in f.xreadlines():
>> if "model name" in line:
>> - res['cpu_model'] = line.split(':')[1].strip()
>> + return line.split(':')[1].strip()
>> break
>> + except Exception as e:
>> + wok_log.error("Failed to retrive cpu_model for "
>> + "%s. Error: %s", ARCH, e.__str__())
>> + return ""
>>
>> - res['cpus'] = 0
>> - res['memory'] = 0L
>> -
>> - # Include IBM PowerKVM name to supported distro names
>> - _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
>> - # 'fedora' '17' 'Beefy Miracle'
>> - distro, version, codename = platform.linux_distribution(
>> - supported_dists=_sup_distros)
>> - res['os_distro'] = distro
>> - res['os_version'] = version
>> - res['os_codename'] = unicode(codename, "utf-8")
>> -
>> - return res
>> -
>> - def lookup(self, *name):
>> - cpus = psutil.NUM_CPUS
>> -
>> + def _get_s390x_host_info(self):
>> + """
>> + method to get additional host details
>> + specific to s390x architecture
>> + :return: dictionary
>> + """
>> + host_info = {}
>> + host_info['cpus'] = self._get_cpus()
>> + host_info['cpus']['dedicated'] = 0
>> + host_info['cpus']['shared'] = 0
>> + host_info['cpu_model'] = ""
>> + host_info['virtualization'] = {}
>> + s390x_sysinfo = self._get_s390x_sysinfo()
>> + if 'manufacturer' in s390x_sysinfo.keys():
>> + host_info['cpu_model'] = s390x_sysinfo['manufacturer']
>> + if 'type' in s390x_sysinfo.keys():
>> + host_info['cpu_model'] = \
>> + host_info['cpu_model'] + "/" + s390x_sysinfo['type']
>> + if 'model' in s390x_sysinfo.keys():
>> + host_info['cpu_model'] = \
>> + host_info['cpu_model'] + "/" + s390x_sysinfo['model']
>> + if CPUS_DEDICATED in s390x_sysinfo.keys():
>> + host_info['cpus']['dedicated'] =
>> s390x_sysinfo[CPUS_DEDICATED]
>> + if CPUS_SHARED in s390x_sysinfo.keys():
>> + host_info['cpus']['shared'] = s390x_sysinfo[CPUS_SHARED]
>> + host_info['virtualization']['hypervisor'] = \
>> + self.lscpu.get_hypervisor()
>> + host_info['virtualization']['hypervisor_vendor'] = \
>> + self.lscpu.get_hypervisor_vendor()
>> + host_info['virtualization'][LPAR_NAME] = ''
>> + host_info['virtualization'][LPAR_NUMBER] = ''
>> + if LPAR_NAME in s390x_sysinfo.keys():
>> + host_info['virtualization'][LPAR_NAME] =
>> s390x_sysinfo[LPAR_NAME]
>> + if LPAR_NUMBER in s390x_sysinfo.keys():
>> + host_info['virtualization'][LPAR_NUMBER] = \
>> + s390x_sysinfo[LPAR_NUMBER]
>> +
>> + return host_info
>> +
>> + def _get_s390x_sysinfo(self):
>> + """
>> + This method retrieves following system information
>> + for s390 architecture
>> + * manufacturer: Manufacturer of host machine
>> + * type: Type of the host machine
>> + * model:Model of host machine
>> + * LPAR_NUMBER: LPAR Number of host
>> + * LPAR_NAME: Name of host LPAR
>> + * CPUS_DEDICATED: LPAR CPUs Dedicated
>> + * CPUS_SHARED: LPAR CPUs Shared
>> +
>> + :param self: object of the class self
>> + :return: dictionary with following keys -
>> + 'manufacturer', 'type', 'model', CPUS_SHARED,
>> + CPUS_DEDICATED, LPAR_NUMBER, LPAR_NAME
>> + """
>> + s390x_sysinfo = {}
>> + try:
>> + with open(PROC_SYSINFO) as f:
>> + for line in f.xreadlines():
>> + if ":" in line and (len(line.split(':')) == 2):
>> + info = line.split(':')
>> + if info[0] == 'Model' and
>> (len(info[1].split()) == 2):
>> + s390x_sysinfo['model'] = \
>> + info[1].split()[0].strip() +\
>> + " "+info[1].split()[1].strip()
>> + elif info[0] == 'Manufacturer':
>> + s390x_sysinfo['manufacturer'] =
>> info[1].strip()
>> + elif info[0] == 'Type':
>> + s390x_sysinfo['type'] = info[1].strip()
>> + elif info[0] == 'LPAR Number':
>> + s390x_sysinfo[LPAR_NUMBER] =
>> int(info[1].strip())
>> + elif info[0] == 'LPAR Name':
>> + s390x_sysinfo[LPAR_NAME] = info[1].strip()
>> + elif info[0] == 'LPAR CPUs Dedicated':
>> + s390x_sysinfo[CPUS_DEDICATED] =\
>> + int(info[1].strip())
>> + elif info[0] == 'LPAR CPUs Shared':
>> + s390x_sysinfo[CPUS_SHARED] =
>> int(info[1].strip())
>> + except Exception as e:
>> + wok_log.error("Failed to retrieve information from %s
>> file. "
>> + "Error: %s", PROC_SYSINFO, e.__str__())
>> +
>> + return s390x_sysinfo
>> +
>> + def _get_memory(self):
>> + """
>> + method to retrieve memory information for all architecture
>> + :return: dictionary with keys "online" and "offline"
>> + """
>> + memory = {}
>> + online_memory = 0
>> + offline_memory = 0
>> + if ARCH.startswith('s390x'):
>> + online_mem_pat = r'^Total online memory :\s+(\d+)\s+MB$'
>> + offline_mem_pat = r'^Total offline memory:\s+(\d+)\s+MB$'
>> + out, err, rc = run_command(LSMEM)
>> + # output of lsmem in s390x architecture is expected to be
>> + # Address Range Size (MB) State\
>> + # Removable Device
>> + # ========================================================\
>> + # =======================
>> + # 0x0000000000000000-0x000000000fffffff 256 online\
>> + # no 0
>> + # 0x0000000010000000-0x000000002fffffff 512 online\
>> + # yes 1-2
>> + # 0x0000000030000000-0x000000007fffffff 1280 online\
>> + # no 3-7
>> + # 0x0000000080000000-0x00000000ffffffff 2048 offline\
>> + # - 8-15
>> + #
>> + # Memory device size : 256 MB
>> + # Memory block size : 256 MB
>> + # Total online memory : 2048 MB
>> + # Total offline memory: 2048 MB
>> + if not rc:
>> + online_mem =\
>> + re.search(online_mem_pat, out.strip(), re.M | re.I)
>> + offline_mem =\
>> + re.search(offline_mem_pat, out.strip(), re.M |
>> re.I)
>> + if online_mem and len(online_mem.groups()) == 1:
>> + online_memory = int(online_mem.group(1)) * 1024
>> * 1024
>> + # converting MB to bytes
>> + # lsmem always returns memory in MB
>> + if offline_mem and len(offline_mem.groups()) == 1:
>> + offline_memory = int(offline_mem.group(1)) *
>> 1024 * 1024
>> + else:
>> + wok_log.error('Failed to retrieve memory information
>> with'
>> + ' command %s. Error: %s' % (LSMEM, err))
>> + else:
>> + if hasattr(psutil, 'phymem_usage'):
>> + online_memory = psutil.phymem_usage().total
>> + elif hasattr(psutil, 'virtual_memory'):
>> + online_memory = psutil.virtual_memory().total
>> +
>> + memory['online'] = online_memory
>> + memory['offline'] = offline_memory
>> + return memory
>> +
>> + def _get_cpus_(self):
>> + """
>> + method to retrieve online cpus count and offline cpus
>> + count for all architecture
>> + :return: dictionary with keys "online" and "offline"
>> + """
>> + cpus = {}
>> + total_cpus = int(self.lscpu.get_total_cpus())
>> + offline_cpus = 0
>> +
>> + online_cpus = psutil.NUM_CPUS
>> # psutil is unstable on how to get the number of
>> # cpus, different versions call it differently
>> if hasattr(psutil, 'cpu_count'):
>> - cpus = psutil.cpu_count()
>> + online_cpus = psutil.cpu_count()
>>
>> elif hasattr(psutil, 'NUM_CPUS'):
>> - cpus = psutil.NUM_CPUS
>> + online_cpus = psutil.NUM_CPUS
>>
>> elif hasattr(psutil, '_psplatform'):
>> for method_name in ['_get_num_cpus', 'get_num_cpus']:
>>
>> method = getattr(psutil._psplatform, method_name,
>> None)
>> if method is not None:
>> - cpus = method()
>> + online_cpus = method()
>> break
>> + if total_cpus >= online_cpus:
>> + offline_cpus = total_cpus - online_cpus
>> + cpus['online'] = online_cpus
>> + cpus['offline'] = offline_cpus
>> + return cpus
>> +
>> + def _get_base_info(self):
>> + """
>> + method to retrieve common host information for all
>> architectures
>> + :return: dictionary with keys 'os_distro', 'os_version',
>> 'os_codename'
>> + 'architecture', 'host', memory
>> + """
>> + common_info = {}
>> + # Include IBM PowerKVM name to supported distro names
>> + _sup_distros = platform._supported_dists + ('ibm_powerkvm',)
>> + # 'fedora' '17' 'Beefy Miracle'
>> + distro, version, codename = platform.linux_distribution(
>> + supported_dists=_sup_distros)
>> + common_info['os_distro'] = distro
>> + common_info['os_version'] = version
>> + common_info['os_codename'] = unicode(codename, "utf-8")
>> + common_info['architecture'] = ARCH
>> + common_info['host'] = platform.node()
>> + common_info['memory'] = self._get_memory()
>>
>> - self.host_info['cpus'] = cpus
>> - if hasattr(psutil, 'phymem_usage'):
>> - self.host_info['memory'] = psutil.phymem_usage().total
>> - elif hasattr(psutil, 'virtual_memory'):
>> - self.host_info['memory'] = psutil.virtual_memory().total
>> - return self.host_info
>> + return common_info
>> +
>> + def lookup(self, *name):
>> + """
>> + method to get basic information for host
>> + """
>> + host_info = self._get_base_info()
>> + if ARCH.startswith('s390x'):
>> + host_info.update(self._get_s390x_host_info())
>> + elif ARCH.startswith('ppc'):
>> + host_info['cpus'] = self._get_cpus_()
>> + host_info['cpu_model'] = self._get_ppc_cpu_model()
>> + else:
>> + host_info['cpus'] = self._get_cpus_()
>> + host_info['cpu_model'] = self._get_x86_cpu_model()
>> + return host_info
>>
>> def swupdate(self, *name):
>> try:
>> diff --git a/src/wok/plugins/gingerbase/tests/test_host.py
>> b/src/wok/plugins/gingerbase/tests/test_host.py
>> index 5d158bb..5a61285 100644
>> --- a/src/wok/plugins/gingerbase/tests/test_host.py
>> +++ b/src/wok/plugins/gingerbase/tests/test_host.py
>> @@ -68,15 +68,21 @@ class HostTests(unittest.TestCase):
>> def test_hostinfo(self):
>> resp = self.request('/plugins/gingerbase/host').read()
>> info = json.loads(resp)
>> - keys = ['os_distro', 'os_version', 'os_codename', 'cpu_model',
>> - 'memory', 'cpus']
>> + if platform.machine().startswith('s390x'):
>> + keys = ['os_distro', 'os_version', 'os_codename',
>> 'cpu_model',
>> + 'memory', 'cpus', 'architecture', 'host',
>> 'virtualization']
>> + else:
>> + keys = ['os_distro', 'os_version', 'os_codename',
>> 'cpu_model',
>> + 'memory', 'cpus', 'architecture', 'host']
>> + self.assertEquals(psutil.TOTAL_PHYMEM,
>> info['memory']['online'])
>> self.assertEquals(sorted(keys), sorted(info.keys()))
>>
>> distro, version, codename = platform.linux_distribution()
>> self.assertEquals(distro, info['os_distro'])
>> self.assertEquals(version, info['os_version'])
>> self.assertEquals(unicode(codename, "utf-8"),
>> info['os_codename'])
>> - self.assertEquals(psutil.TOTAL_PHYMEM, info['memory'])
>> + self.assertEqual(platform.node(), info['host'])
>> + self.assertEqual(platform.machine(), info['architecture'])
>>
>> def test_hoststats(self):
>> time.sleep(1)
>
--
Regards,
Suresh Babu Angadi
More information about the Kimchi-devel
mailing list