[PATCH 0/3] New backend API: get available devices from host
From: Daniel Henrique Barboza <dhbarboza82@gmail.com> This patch set adds a new API to retrieve only the available devices from the host. 'available' means only the devices that weren't attached to any other VM. The backend is generic and can work with all devices, but the API provided here is made for 'PCI' devices only as we do not provide official support for USB and SCSI passthrough yet. Usage: get all available devices: $ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true" get all available PCI devices: $ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true&_cap=pci" Daniel Henrique Barboza (3): Get available host passthrough devices: model changes Adding new API in the UI APIs and docs Mockmodel and test changes for the new API docs/API.md | 2 + src/kimchi/API.json | 6 +++ src/kimchi/i18n.py | 1 + src/kimchi/mockmodel.py | 3 +- src/kimchi/model/host.py | 105 ++++++++++++++++++++++++++++++++++++++++- src/kimchi/model/vmhostdevs.py | 75 ++--------------------------- tests/test_host.py | 11 +++++ ui/js/src/kimchi.api.js | 14 ++++++ 8 files changed, 144 insertions(+), 73 deletions(-) -- 2.4.3
From: Daniel Henrique Barboza <dhbarboza82@gmail.com> - a new method was introduced in model/host.py that returns all the unavailable devices that are being used by all the VMs in the host. This method is used by the get_list() call when the parameter _available_only is set to 'true'. - to avoid problems with circular dependencies, the deduce_dev_name method was moved from model/vmhostdevs to model/host. Signed-off-by: Daniel Henrique Barboza <dhbarboza82@gmail.com> --- src/kimchi/model/host.py | 105 ++++++++++++++++++++++++++++++++++++++++- src/kimchi/model/vmhostdevs.py | 75 ++--------------------------- 2 files changed, 108 insertions(+), 72 deletions(-) diff --git a/src/kimchi/model/host.py b/src/kimchi/model/host.py index b2fa379..e51ede4 100644 --- a/src/kimchi/model/host.py +++ b/src/kimchi/model/host.py @@ -22,6 +22,7 @@ import os import time import platform from collections import defaultdict +from lxml import objectify import psutil from cherrypy.process.plugins import BackgroundTask @@ -34,7 +35,7 @@ from kimchi.exception import InvalidOperation, InvalidParameter from kimchi.exception import NotFoundError, OperationFailed from kimchi.model.config import CapabilitiesModel from kimchi.model.tasks import TaskModel -from kimchi.model.vms import DOM_STATE_MAP +from kimchi.model.vms import DOM_STATE_MAP, VMModel, VMsModel from kimchi.repositories import Repositories from kimchi.swupdate import SoftwareUpdate from kimchi.utils import add_task, kimchi_log @@ -318,8 +319,29 @@ class DevicesModel(object): except AttributeError: self.cap_map['fc_host'] = None + def _get_unavailable_devices(self): + vm_list = VMsModel.get_vms(self.conn) + unavailable_devs = [] + for vm in vm_list: + dom = VMModel.get_vm(vm, self.conn) + xmlstr = dom.XMLDesc(0) + root = objectify.fromstring(xmlstr) + try: + hostdev = root.devices.hostdev + except AttributeError: + continue + + vm_devs = [DeviceModel.deduce_dev_name(e, self.conn) + for e in hostdev] + + for dev in vm_devs: + unavailable_devs.append(dev) + + return unavailable_devs + def get_list(self, _cap=None, _passthrough=None, - _passthrough_affected_by=None): + _passthrough_affected_by=None, + _available_only=None): if _passthrough_affected_by is not None: # _passthrough_affected_by conflicts with _cap and _passthrough if (_cap, _passthrough) != (None, None): @@ -336,8 +358,15 @@ class DevicesModel(object): conn = self.conn.get() passthrough_names = [ dev['name'] for dev in hostdev.get_passthrough_dev_infos(conn)] + dev_names = list(set(dev_names) & set(passthrough_names)) + if _available_only is not None and _available_only.lower() \ + == 'true': + unavailable_devs = self._get_unavailable_devices() + dev_names = [dev for dev in dev_names + if dev not in unavailable_devs] + dev_names.sort() return dev_names @@ -390,6 +419,78 @@ class DeviceModel(object): raise NotFoundError('KCHHOST0003E', {'name': nodedev_name}) return hostdev.get_dev_info(dev) + @staticmethod + def _toint(num_str): + if num_str.startswith('0x'): + return int(num_str, 16) + elif num_str.startswith('0'): + return int(num_str, 8) + else: + return int(num_str) + + @staticmethod + def deduce_dev_name(e, conn): + if e.attrib['type'] == 'pci': + return DeviceModel._deduce_dev_name_pci(e) + elif e.attrib['type'] == 'scsi': + return DeviceModel._deduce_dev_name_scsi(e) + elif e.attrib['type'] == 'usb': + return DeviceModel._deduce_dev_name_usb(e, conn) + return None + + @staticmethod + def _deduce_dev_name_pci(e): + attrib = {} + for field in ('domain', 'bus', 'slot', 'function'): + attrib[field] = DeviceModel._toint(e.source.address.attrib[field]) + return 'pci_%(domain)04x_%(bus)02x_%(slot)02x_%(function)x' % attrib + + @staticmethod + def _deduce_dev_name_scsi(e): + attrib = {} + for field in ('bus', 'target', 'unit'): + attrib[field] = DeviceModel._toint(e.source.address.attrib[field]) + attrib['host'] = DeviceModel._toint( + e.source.adapter.attrib['name'][len('scsi_host'):]) + return 'scsi_%(host)d_%(bus)d_%(target)d_%(unit)d' % attrib + + @staticmethod + def _deduce_dev_name_usb(e, conn): + dev_names = DevicesModel(conn=conn).get_list(_cap='usb_device') + usb_infos = [DeviceModel(conn=conn).lookup(dev_name) + for dev_name in dev_names] + + unknown_dev = None + + try: + evendor = DeviceModel._toint(e.source.vendor.attrib['id']) + eproduct = DeviceModel._toint(e.source.product.attrib['id']) + except AttributeError: + evendor = 0 + eproduct = 0 + else: + unknown_dev = 'usb_vendor_%s_product_%s' % (evendor, eproduct) + + try: + ebus = DeviceModel._toint(e.source.address.attrib['bus']) + edevice = DeviceModel._toint(e.source.address.attrib['device']) + except AttributeError: + ebus = -1 + edevice = -1 + else: + unknown_dev = 'usb_bus_%s_device_%s' % (ebus, edevice) + + for usb_info in usb_infos: + ivendor = DeviceModel._toint(usb_info['vendor']['id']) + iproduct = DeviceModel._toint(usb_info['product']['id']) + if evendor == ivendor and eproduct == iproduct: + return usb_info['name'] + ibus = usb_info['bus'] + idevice = usb_info['device'] + if ebus == ibus and edevice == idevice: + return usb_info['name'] + return unknown_dev + class PackagesUpdateModel(object): def __init__(self, **kargs): diff --git a/src/kimchi/model/vmhostdevs.py b/src/kimchi/model/vmhostdevs.py index 12226ca..d668223 100644 --- a/src/kimchi/model/vmhostdevs.py +++ b/src/kimchi/model/vmhostdevs.py @@ -49,69 +49,7 @@ class VMHostDevsModel(object): except AttributeError: return [] - return [self._deduce_dev_name(e) for e in hostdev] - - @staticmethod - def _toint(num_str): - if num_str.startswith('0x'): - return int(num_str, 16) - elif num_str.startswith('0'): - return int(num_str, 8) - else: - return int(num_str) - - def _deduce_dev_name(self, e): - return getattr(self, '_deduce_dev_name_%s' % e.attrib['type'])(e) - - def _deduce_dev_name_pci(self, e): - attrib = {} - for field in ('domain', 'bus', 'slot', 'function'): - attrib[field] = self._toint(e.source.address.attrib[field]) - return 'pci_%(domain)04x_%(bus)02x_%(slot)02x_%(function)x' % attrib - - def _deduce_dev_name_scsi(self, e): - attrib = {} - for field in ('bus', 'target', 'unit'): - attrib[field] = self._toint(e.source.address.attrib[field]) - attrib['host'] = self._toint( - e.source.adapter.attrib['name'][len('scsi_host'):]) - return 'scsi_%(host)d_%(bus)d_%(target)d_%(unit)d' % attrib - - def _deduce_dev_name_usb(self, e): - dev_names = DevicesModel(conn=self.conn).get_list(_cap='usb_device') - usb_infos = [DeviceModel(conn=self.conn).lookup(dev_name) - for dev_name in dev_names] - - unknown_dev = None - - try: - evendor = self._toint(e.source.vendor.attrib['id']) - eproduct = self._toint(e.source.product.attrib['id']) - except AttributeError: - evendor = 0 - eproduct = 0 - else: - unknown_dev = 'usb_vendor_%s_product_%s' % (evendor, eproduct) - - try: - ebus = self._toint(e.source.address.attrib['bus']) - edevice = self._toint(e.source.address.attrib['device']) - except AttributeError: - ebus = -1 - edevice = -1 - else: - unknown_dev = 'usb_bus_%s_device_%s' % (ebus, edevice) - - for usb_info in usb_infos: - ivendor = self._toint(usb_info['vendor']['id']) - iproduct = self._toint(usb_info['product']['id']) - if evendor == ivendor and eproduct == iproduct: - return usb_info['name'] - ibus = usb_info['bus'] - idevice = usb_info['device'] - if ebus == ibus and edevice == idevice: - return usb_info['name'] - return unknown_dev + return [DeviceModel.deduce_dev_name(e, self.conn) for e in hostdev] def _passthrough_device_validate(self, dev_name): eligible_dev_names = \ @@ -290,10 +228,8 @@ class VMHostDevModel(object): raise NotFoundError('KCHVMHDEV0001E', {'vmid': vmid, 'dev_name': dev_name}) - devsmodel = VMHostDevsModel(conn=self.conn) - for e in hostdev: - deduced_name = devsmodel._deduce_dev_name(e) + deduced_name = DeviceModel.deduce_dev_name(e, self.conn) if deduced_name == dev_name: return {'name': dev_name, 'type': e.attrib['type']} @@ -311,12 +247,11 @@ class VMHostDevModel(object): raise NotFoundError('KCHVMHDEV0001E', {'vmid': vmid, 'dev_name': dev_name}) - devsmodel = VMHostDevsModel(conn=self.conn) - pci_devs = [(devsmodel._deduce_dev_name(e), e) for e in hostdev - if e.attrib['type'] == 'pci'] + pci_devs = [(DeviceModel.deduce_dev_name(e, self.conn), e) + for e in hostdev if e.attrib['type'] == 'pci'] for e in hostdev: - if devsmodel._deduce_dev_name(e) == dev_name: + if DeviceModel.deduce_dev_name(e, self.conn) == dev_name: xmlstr = etree.tostring(e) dom.detachDeviceFlags( xmlstr, get_vm_config_flag(dom, mode='all')) -- 2.4.3
From: Daniel Henrique Barboza <dhbarboza82@gmail.com> Documentating the new API 'getAvailableHostPCIDevices', which uses the existing backend host/devices with a new option. Signed-off-by: Daniel Henrique Barboza <dhbarboza82@gmail.com> --- docs/API.md | 2 ++ src/kimchi/API.json | 6 ++++++ src/kimchi/i18n.py | 1 + ui/js/src/kimchi.api.js | 14 ++++++++++++++ 4 files changed, 23 insertions(+) diff --git a/docs/API.md b/docs/API.md index e022c9e..8b7e9ca 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1013,6 +1013,8 @@ stats history * _passthrough_affected_by: Filter the affected devices in the same group of a certain directly assigned device. The value should be the name of a device. + * _available_only: Filter to list only the host devices that are not + attached to a VM. ### Resource: Device diff --git a/src/kimchi/API.json b/src/kimchi/API.json index e8da7d9..c7ea660 100644 --- a/src/kimchi/API.json +++ b/src/kimchi/API.json @@ -814,6 +814,12 @@ "type": "string", "pattern": "^[_A-Za-z0-9-]+$", "error": "KCHDEVS0003E" + }, + "_available_only": { + "description": "List only devices that are not being used by any other VM", + "type": "string", + "pattern": "^true|false$", + "error": "KCHDEVS0004E" } }, "additionalProperties": false, diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index 54750d9..0a3ebfa 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -45,6 +45,7 @@ messages = { "KCHDEVS0001E": _('Unknown "_cap" specified'), "KCHDEVS0002E": _('"_passthrough" should be "true" or "false"'), "KCHDEVS0003E": _('"_passthrough_affected_by" should be a device name string'), + "KCHDEVS0004E": _('"_available" should be "true" or "false"'), "KCHDISKS0001E": _("Error while getting block devices. Details: %(err)s"), "KCHDISKS0002E": _("Error while getting block device information for %(device)s."), diff --git a/ui/js/src/kimchi.api.js b/ui/js/src/kimchi.api.js index a7c3b2d..44f58d1 100644 --- a/ui/js/src/kimchi.api.js +++ b/ui/js/src/kimchi.api.js @@ -1179,6 +1179,20 @@ var kimchi = { }); }, + getAvailableHostPCIDevices : function(suc, err) { + kimchi.requestJSON({ + url : 'host/devices?_passthrough=true&_cap=pci&_available_only=true', + type : 'GET', + contentType : 'application/json', + dataType : 'json', + resend : true, + success : suc, + error : err ? err : function(data) { + kimchi.message.error(data.responseJSON.reason); + } + }); + }, + getPCIDeviceCompanions : function(pcidev, suc, err) { kimchi.requestJSON({ url : 'host/devices?_passthrough_affected_by=' + pcidev, -- 2.4.3
From: Daniel Henrique Barboza <dhbarboza82@gmail.com> API added: getAvailableHostPCIDevices, option _available_only of host/devices backend. Signed-off-by: Daniel Henrique Barboza <dhbarboza82@gmail.com> --- src/kimchi/mockmodel.py | 3 ++- tests/test_host.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py index aaf1af2..34bcfc9 100644 --- a/src/kimchi/mockmodel.py +++ b/src/kimchi/mockmodel.py @@ -335,7 +335,8 @@ class MockModel(Model): return self._mock_partitions.partitions[name] def _mock_devices_get_list(self, _cap=None, _passthrough=None, - _passthrough_affected_by=None): + _passthrough_affected_by=None, + _available_only=None): if _cap is None: return self._mock_devices.devices.keys() diff --git a/tests/test_host.py b/tests/test_host.py index 1273457..fe6124e 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -190,3 +190,14 @@ class HostTests(unittest.TestCase): dev) affected_devs = [dev['name'] for dev in json.loads(resp.read())] self.assertTrue(set(affected_devs) <= set(dev_names)) + + def test_get_available_passthrough_devices(self): + resp = self.request('/host/devices?_passthrough=true') + all_devs = [dev['name'] for dev in json.loads(resp.read())] + + resp = self.request( + '/host/devices?_passthrough=true&_available_only=true' + ) + available_devs = [dev['name'] for dev in json.loads(resp.read())] + + self.assertLessEqual(len(available_devs), len(all_devs)) -- 2.4.3
Tested-by: Socorro Stoppler <socorro@linux.vnet.ibm.com> On 07/14/2015 08:00 AM, dhbarboza82@gmail.com wrote:
From: Daniel Henrique Barboza <dhbarboza82@gmail.com>
This patch set adds a new API to retrieve only the available devices from the host. 'available' means only the devices that weren't attached to any other VM.
The backend is generic and can work with all devices, but the API provided here is made for 'PCI' devices only as we do not provide official support for USB and SCSI passthrough yet.
Usage:
get all available devices:
$ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true"
get all available PCI devices:
$ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true&_cap=pci"
Daniel Henrique Barboza (3): Get available host passthrough devices: model changes Adding new API in the UI APIs and docs Mockmodel and test changes for the new API
docs/API.md | 2 + src/kimchi/API.json | 6 +++ src/kimchi/i18n.py | 1 + src/kimchi/mockmodel.py | 3 +- src/kimchi/model/host.py | 105 ++++++++++++++++++++++++++++++++++++++++- src/kimchi/model/vmhostdevs.py | 75 ++--------------------------- tests/test_host.py | 11 +++++ ui/js/src/kimchi.api.js | 14 ++++++ 8 files changed, 144 insertions(+), 73 deletions(-)
Reviewed-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com> On 14-07-2015 12:00, dhbarboza82@gmail.com wrote:
From: Daniel Henrique Barboza <dhbarboza82@gmail.com>
This patch set adds a new API to retrieve only the available devices from the host. 'available' means only the devices that weren't attached to any other VM.
The backend is generic and can work with all devices, but the API provided here is made for 'PCI' devices only as we do not provide official support for USB and SCSI passthrough yet.
Usage:
get all available devices:
$ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true"
get all available PCI devices:
$ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true&_cap=pci"
Daniel Henrique Barboza (3): Get available host passthrough devices: model changes Adding new API in the UI APIs and docs Mockmodel and test changes for the new API
docs/API.md | 2 + src/kimchi/API.json | 6 +++ src/kimchi/i18n.py | 1 + src/kimchi/mockmodel.py | 3 +- src/kimchi/model/host.py | 105 ++++++++++++++++++++++++++++++++++++++++- src/kimchi/model/vmhostdevs.py | 75 ++--------------------------- tests/test_host.py | 11 +++++ ui/js/src/kimchi.api.js | 14 ++++++ 8 files changed, 144 insertions(+), 73 deletions(-)
-- Jose Ricardo Ziviani ----------------------------- Software Engineer Linux Technology Center - IBM
Patch set applied to master branch of https://github.com/danielhb/kimchi Thanks for the reviews/tests! ps: this patch set fixes github bug #690 On 07/15/2015 11:08 AM, Jose Ricardo Ziviani wrote:
Reviewed-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com>
On 14-07-2015 12:00, dhbarboza82@gmail.com wrote:
From: Daniel Henrique Barboza <dhbarboza82@gmail.com>
This patch set adds a new API to retrieve only the available devices from the host. 'available' means only the devices that weren't attached to any other VM.
The backend is generic and can work with all devices, but the API provided here is made for 'PCI' devices only as we do not provide official support for USB and SCSI passthrough yet.
Usage:
get all available devices:
$ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true"
get all available PCI devices:
$ curl -u danielhb -H "Content-Type: application/json" -H "Accept: application/json" -X GET "http://localhost:8010/host/devices?_passthrough=true&_available_only=true&_cap=pci"
Daniel Henrique Barboza (3): Get available host passthrough devices: model changes Adding new API in the UI APIs and docs Mockmodel and test changes for the new API
docs/API.md | 2 + src/kimchi/API.json | 6 +++ src/kimchi/i18n.py | 1 + src/kimchi/mockmodel.py | 3 +- src/kimchi/model/host.py | 105 ++++++++++++++++++++++++++++++++++++++++- src/kimchi/model/vmhostdevs.py | 75 ++--------------------------- tests/test_host.py | 11 +++++ ui/js/src/kimchi.api.js | 14 ++++++ 8 files changed, 144 insertions(+), 73 deletions(-)
participants (5)
-
Aline Manera -
Daniel Henrique Barboza -
dhbarboza82＠gmail.com
-
Jose Ricardo Ziviani -
Socorro Stoppler