This patch adds a new feature test to check if the host libvirt version supports memory devices attachment. This support is provided since libvirt 1.2.14 and is the base to make memory hotplug work. When libvirt does not support memory devices, it is going to fail when we try to attach it, raising the libvirt.libvirtError: "unsupported configuration: unknown device type 'memory'" Signed-off-by: Rodrigo Trujillo <rodrigo.trujillo@linux.vnet.ibm.com> --- src/kimchi/model/config.py | 3 +++ src/kimchi/model/featuretests.py | 50 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/kimchi/model/config.py b/src/kimchi/model/config.py index 687531b..8735beb 100644 --- a/src/kimchi/model/config.py +++ b/src/kimchi/model/config.py @@ -55,6 +55,7 @@ class CapabilitiesModel(object): self.fc_host_support = False self.metadata_support = False self.kernel_vfio = False + self.mem_hotplug_support = False # Subscribe function to set host capabilities to be run when cherrypy # server is up @@ -91,6 +92,7 @@ class CapabilitiesModel(object): self.fc_host_support = FeatureTests.libvirt_support_fc_host(conn) self.metadata_support = FeatureTests.has_metadata_support(conn) self.kernel_vfio = FeatureTests.kernel_support_vfio() + self.mem_hotplug_support = FeatureTests.has_mem_hotplug_support(conn) self.libvirt_stream_protocols = [] for p in ['http', 'https', 'ftp', 'ftps', 'tftp']: @@ -139,6 +141,7 @@ class CapabilitiesModel(object): 'auth': kconfig.get("authentication", "method"), 'kernel_vfio': self.kernel_vfio, 'nm_running': FeatureTests.is_nm_running(), + 'mem_hotplug_support': self.mem_hotplug_support } diff --git a/src/kimchi/model/featuretests.py b/src/kimchi/model/featuretests.py index 9400151..047108f 100644 --- a/src/kimchi/model/featuretests.py +++ b/src/kimchi/model/featuretests.py @@ -63,6 +63,25 @@ SIMPLE_VM_XML = """ </os> </domain>""" +MAXMEM_VM_XML = """ +<domain type='%(domain)s'> + <name>%(name)s</name> + <maxMemory slots='1' unit='KiB'>20480</maxMemory> + <memory unit='KiB'>10240</memory> + <os> + <type arch='%(arch)s'>hvm</type> + <boot dev='hd'/> + </os> +</domain>""" + +DEV_MEM_XML = """ +<memory model='dimm'> + <target> + <size unit='KiB'>10240</size> + <node>0</node> + </target> +</memory>""" + SCSI_FC_XML = """ <pool type='scsi'> <name>%(name)s</name> @@ -207,3 +226,34 @@ class FeatureTests(object): return False return True + + @staticmethod + def has_mem_hotplug_support(conn): + ''' + A memory device can be hot-plugged or hot-unplugged since libvirt + version 1.2.14. + ''' + # Libvirt < 1.2.14 does not support memory devices, so firstly, check + # its version, then try to attach a device. These steps avoid errors + # with Libvirt 'test' driver for KVM + version = 1000000*1 + 1000*2 + 14 + if libvirt.getVersion() < version: + return False + + with RollbackContext() as rollback: + FeatureTests.disable_libvirt_error_logging() + rollback.prependDefer(FeatureTests.enable_libvirt_error_logging) + conn_type = conn.getType().lower() + domain_type = 'test' if conn_type == 'test' else 'kvm' + arch = 'i686' if conn_type == 'test' else platform.machine() + arch = 'ppc64' if arch == 'ppc64le' else arch + dom = conn.defineXML(MAXMEM_VM_XML % {'name': FEATURETEST_VM_NAME, + 'domain': domain_type, + 'arch': arch}) + rollback.prependDefer(dom.undefine) + try: + dom.attachDeviceFlags(DEV_MEM_XML, + libvirt.VIR_DOMAIN_MEM_CONFIG) + return True + except libvirt.libvirtError: + return False -- 2.1.0

In order to support memory hotplug, guest must have maxMemory and NUMA configured in the xml. For maxMemory, this patch changes template with this information and the xml generation at guest creation time, adding maxMemory equals to host total memory and memory slots as the integer number of GiB that fit inside maxMemory (by design users will be allowed to add only memory chunks of 1GB). For NUMA, this patch adds the simplest configuration possible, creating only one node with all vcpus and memory set in the template. VM update function was changed in order to properly update memory, numa and cpu informations. This patch also provides a mechanism to set these parameters in old vm xml, user just have to update the memory offline once, then the memory hotplug is going to be set and enabled. Signed-off-by: Rodrigo Trujillo <rodrigo.trujillo@linux.vnet.ibm.com> --- src/kimchi/i18n.py | 7 +++ src/kimchi/model/vms.py | 149 +++++++++++++++++++++++++++++++++++++++++---- src/kimchi/vmtemplate.py | 28 ++++++--- src/kimchi/xmlutils/cpu.py | 60 ++++++++++++++++++ 4 files changed, 222 insertions(+), 22 deletions(-) create mode 100644 src/kimchi/xmlutils/cpu.py diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index d5e93fa..00a6c7c 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -111,6 +111,13 @@ messages = { "KCHVM0038E": _("Unable to suspend VM '%(name)s'. Details: %(err)s"), "KCHVM0039E": _("Cannot resume VM '%(name)s' because it is not paused."), "KCHVM0040E": _("Unable to resume VM '%(name)s'. Details: %(err)s"), + "KCHVM0041E": _("Memory assigned is higher then the maximum allowed in the host."), + "KCHVM0042E": _("VM '%(name)s' does not support live memory update. Update the memory with the machine offline to enable this feature."), + "KCHVM0043E": _("Only increase memory is allowed in active VMs"), + "KCHVM0044E": _("For live memory update, new memory value must be equal old memory value plus multiples of 1024 Mib"), + "KCHVM0045E": _("There are not enough free slots of 1024 Mib in the guest."), + "KCHVM0046E": _("Host's libvirt version does not support memory devices. Libvirt must be >= 1.2.14"), + "KCHVM0047E": _("Error attaching memory device. Details: %(error)s"), "KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), diff --git a/src/kimchi/model/vms.py b/src/kimchi/model/vms.py index ed1500e..90e522d 100644 --- a/src/kimchi/model/vms.py +++ b/src/kimchi/model/vms.py @@ -45,6 +45,7 @@ from kimchi.screenshot import VMScreenshot from kimchi.utils import add_task, convert_data_size, get_next_clone_name from kimchi.utils import import_class, kimchi_log, run_setfacl_set_attr from kimchi.utils import template_name_from_uri +from kimchi.xmlutils.cpu import get_cpu_xml, get_numa_xml from kimchi.xmlutils.utils import xpath_get_text, xml_item_update from kimchi.xmlutils.utils import dictize @@ -59,8 +60,7 @@ DOM_STATE_MAP = {0: 'nostate', 7: 'pmsuspended'} VM_STATIC_UPDATE_PARAMS = {'name': './name', - 'cpus': './vcpu', - 'memory': './memory'} + 'cpus': './vcpu'} VM_LIVE_UPDATE_PARAMS = {} XPATH_DOMAIN_DISK = "/domain/devices/disk[@device='disk']/source/@file" @@ -73,6 +73,8 @@ XPATH_DOMAIN_MEMORY = '/domain/memory' XPATH_DOMAIN_MEMORY_UNIT = '/domain/memory/@unit' XPATH_DOMAIN_UUID = '/domain/uuid' +XPATH_NUMA_CELL = './cpu/numa/cell' + class VMsModel(object): def __init__(self, **kargs): @@ -653,15 +655,24 @@ class VMModel(object): for key, val in params.items(): if key in VM_STATIC_UPDATE_PARAMS: - if key == 'memory': - # Libvirt saves memory in KiB. Retrieved xml has memory - # in KiB too, so new valeu must be in KiB here - val = val * 1024 if type(val) == int: val = str(val) xpath = VM_STATIC_UPDATE_PARAMS[key] new_xml = xml_item_update(new_xml, xpath, val) + # Updating memory and NUMA if necessary, if vm is offline + if not dom.isActive(): + if 'memory' in params: + new_xml = self._update_memory_config(new_xml, params) + elif 'cpus' in params and \ + (xpath_get_text(new_xml, XPATH_NUMA_CELL + '/@memory') != []): + vcpus = params['cpus'] + new_xml = xml_item_update( + new_xml, + XPATH_NUMA_CELL, + value='0-' + str(vcpus - 1) if vcpus > 1 else '0', + attr='cpus') + if 'graphics' in params: new_xml = self._update_graphics(dom, new_xml, params) @@ -689,12 +700,7 @@ class VMModel(object): # Undefine old vm, only if name is going to change dom.undefine() - root = ET.fromstring(new_xml) - currentMem = root.find('.currentMemory') - if currentMem is not None: - root.remove(currentMem) - - dom = conn.defineXML(ET.tostring(root, encoding="utf-8")) + dom = conn.defineXML(new_xml) if 'name' in params: self._redefine_snapshots(dom, snapshots_info) except libvirt.libvirtError as e: @@ -706,8 +712,127 @@ class VMModel(object): 'err': e.get_error_message()}) return dom + def _update_memory_config(self, xml, params): + # Checks if NUMA memory is already configured, if not, checks if CPU + # element is already configured (topology). Then add NUMA element as + # apropriated + root = ET.fromstring(xml) + numa_mem = xpath_get_text(xml, XPATH_NUMA_CELL + '/@memory') + vcpus = params.get('cpus') + if numa_mem == []: + if vcpus is None: + vcpus = int(xpath_get_text(xml, + VM_STATIC_UPDATE_PARAMS['cpus'])[0]) + cpu = root.find('./cpu') + if cpu is None: + cpu = get_cpu_xml(vcpus, params['memory'] << 10) + root.insert(0, ET.fromstring(cpu)) + else: + numa_element = get_numa_xml(vcpus, params['memory'] << 10) + cpu.insert(0, ET.fromstring(numa_element)) + else: + if vcpus is not None: + xml = xml_item_update( + xml, + XPATH_NUMA_CELL, + value='0-' + str(vcpus - 1) if vcpus > 1 else '0', + attr='cpus') + root = ET.fromstring(xml_item_update(xml, XPATH_NUMA_CELL, + str(params['memory'] << 10), + attr='memory')) + + # Remove currentMemory, automatically set later by libvirt + currentMem = root.find('.currentMemory') + if currentMem is not None: + root.remove(currentMem) + + memory = root.find('.memory') + # Update/Adds maxMemory accordingly + if not self.caps.mem_hotplug_support: + if memory is not None: + memory.text = str(params['memory'] << 10) + else: + if memory is not None: + root.remove(memory) + maxMem = root.find('.maxMemory') + host_mem = self.conn.get().getInfo()[1] + slots = (host_mem - params['memory']) >> 10 + # Libvirt does not accepts slots <= 1 + if slots < 0: + raise OperationFailed("KCHVM0041E") + elif slots == 0: + slots = 1 + if maxMem is None: + max_mem_xml = E.maxMemory( + str(host_mem * 1024), + unit='Kib', + slots=str(slots)) + root.insert(0, max_mem_xml) + new_xml = ET.tostring(root, encoding="utf-8") + else: + # Update slots only + new_xml = xml_item_update(ET.tostring(root, encoding="utf-8"), + './maxMemory', + str(slots), + attr='slots') + return new_xml + return ET.tostring(root, encoding="utf-8") + def _live_vm_update(self, dom, params): self._vm_update_access_metadata(dom, params) + if 'memory' in params and dom.isActive(): + self._update_memory_live(dom, params) + + def _update_memory_live(self, dom, params): + # Check if host supports memory device + if not self.caps.mem_hotplug_support: + raise InvalidOperation("KCHVM0046E") + + # Check if the vm xml supports memory hotplug, if not, static update + # must be done firstly, then Kimchi is going to update the xml + xml = dom.XMLDesc(0) + numa_mem = xpath_get_text(xml, XPATH_NUMA_CELL + '/@memory') + max_mem = xpath_get_text(xml, './maxMemory') + if numa_mem == [] or max_mem == []: + raise OperationFailed('KCHVM0042E', {'name': dom.name()}) + + # Memory live update must be done in chunks of 1024 Mib or 1Gib + new_mem = params['memory'] + old_mem = int(xpath_get_text(xml, XPATH_DOMAIN_MEMORY)[0]) >> 10 + if new_mem < old_mem: + raise OperationFailed('KCHVM0043E') + if (new_mem - old_mem) % 1024 != 0: + raise OperationFailed('KCHVM0044E') + + # Check slot spaces: + total_slots = int(xpath_get_text(xml, './maxMemory/@slots')[0]) + needed_slots = (new_mem - old_mem) / 1024 + used_slots = len(xpath_get_text(xml, './devices/memory')) + if needed_slots > (total_slots - used_slots): + raise OperationFailed('KCHVM0045E') + elif needed_slots == 0: + # New memory value is same that current memory set + return + + # Finally, we are ok to hot add the memory devices + try: + self._hot_add_memory_devices(dom, needed_slots) + except Exception as e: + raise OperationFailed("KCHVM0047E", {'error': e.message}) + + def _hot_add_memory_devices(self, dom, amount): + # Hot add given number of memory devices in the guest + flags = libvirt.VIR_DOMAIN_MEM_CONFIG | libvirt.VIR_DOMAIN_MEM_LIVE + # Create memory device xml + mem_dev_xml = etree.tostring( + E.memory( + E.target( + E.size('1', unit='GiB'), + E.node('0')), + model='dimm')) + # Add chunks of 1G of memory + for i in range(amount): + dom.attachDeviceFlags(mem_dev_xml, flags) def _has_video(self, dom): dom = ElementTree.fromstring(dom.XMLDesc(0)) diff --git a/src/kimchi/vmtemplate.py b/src/kimchi/vmtemplate.py index e047228..4143839 100644 --- a/src/kimchi/vmtemplate.py +++ b/src/kimchi/vmtemplate.py @@ -32,6 +32,7 @@ from kimchi.exception import InvalidParameter, IsoFormatError, MissingParameter from kimchi.exception import ImageFormatError, OperationFailed from kimchi.isoinfo import IsoImage from kimchi.utils import check_url_path, pool_name_from_uri +from kimchi.xmlutils.cpu import get_cpu_xml from kimchi.xmlutils.disk import get_disk_xml from kimchi.xmlutils.graphics import get_graphics_xml from kimchi.xmlutils.interface import get_iface_xml @@ -270,17 +271,13 @@ class VMTemplate(object): return input_output def _get_cpu_xml(self): - + # Include CPU topology, if provided cpu_info = self.info.get('cpu_info') - if cpu_info is None: - return "" - cpu_topo = cpu_info.get('topology') - if cpu_topo is None: - return "" - return etree.tostring(E.cpu(E.topology( - sockets=str(cpu_topo['sockets']), - cores=str(cpu_topo['cores']), - threads=str(cpu_topo['threads'])))) + if cpu_info is not None: + cpu_topo = cpu_info.get('topology') + return get_cpu_xml(self.info.get('cpus'), + self.info.get('memory') << 10, + cpu_topo) def to_vm_xml(self, vm_name, vm_uuid, **kwargs): params = dict(self.info) @@ -308,11 +305,22 @@ class VMTemplate(object): else: params['cdroms'] = cdrom_xml + # Setting maximum number of slots to avoid errors when hotplug memory + # Number of slots are the numbers of chunks of 1GB that fit inside + # the max_memory of the host minus memory assigned to the VM + params['slots'] = ((params['max_memory'] >> 10) - + params['memory']) >> 10 + if params['slots'] < 0: + raise OperationFailed("KCHVM0041E") + elif params['slots'] == 0: + params['slots'] = 1 + xml = """ <domain type='%(domain)s'> %(qemu-stream-cmdline)s <name>%(name)s</name> <uuid>%(uuid)s</uuid> + <maxMemory slots='%(slots)s' unit='KiB'>%(max_memory)s</maxMemory> <memory unit='MiB'>%(memory)s</memory> <vcpu>%(cpus)s</vcpu> %(cpu_info)s diff --git a/src/kimchi/xmlutils/cpu.py b/src/kimchi/xmlutils/cpu.py new file mode 100644 index 0000000..32c01a4 --- /dev/null +++ b/src/kimchi/xmlutils/cpu.py @@ -0,0 +1,60 @@ +# +# Project Kimchi +# +# Copyright IBM, Corp. 2015 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import lxml.etree as ET +from lxml.builder import E + + +def get_numa_xml(cpus, memory): + # Returns the NUMA xml to be add into CPU element + # Currently, supports only one node/cell + # <numa> + # <cell id='0' cpus='0-3' memory='512000' unit='KiB'/> + # </numa> + xml = E.numa(E.cell( + id='0', + cpus='0-' + str(cpus - 1) if cpus > 1 else '0', + memory=str(memory), + unit='KiB')) + return ET.tostring(xml) + + +def get_topology_xml(cpu_topo): + # Return the cpu TOPOLOGY element + # <topology sockets='1' cores='2' threads='1'/> + xml = E.topology( + sockets=str(cpu_topo['sockets']), + cores=str(cpu_topo['cores']), + threads=str(cpu_topo['threads'])) + return ET.tostring(xml) + + +def get_cpu_xml(cpus, memory, cpu_topo=None): + # Returns the libvirt CPU element based on given numa and topology + # CPU element will always have numa element + # <cpu> + # <numa> + # <cell id='0' cpus='0-3' memory='512000' unit='KiB'/> + # </numa> + # <topology sockets='1' cores='2' threads='1'/> + # </cpu> + xml = E.cpu(ET.fromstring(get_numa_xml(cpus, memory))) + if cpu_topo is not None: + xml.insert(0, ET.fromstring(get_topology_xml(cpu_topo))) + return ET.tostring(xml) -- 2.1.0