On Wed, Oct 21, 2020 at 10:44 PM Edward Berger <edwberger(a)gmail.com&gt; wrote: > I'm' trying to bring up a test ovirt 4.4 cluster with sr-iov virtual > infiniband interfaces on Mellanox ConnectX-6 cards in some older Opteron > servers while we await delivery of new Epyc servers. > > I have the hypervisors configured for sr-iov and showing the interfaces, > but when I try to bring up a VM with the host device selected it fails to > boot. "Failed Launching ..." shows up on the engine UI. > [image: sriov-hostdev.JPG] > The /var/log/libvirt/qemu log for the VM seems to think something is > wrong with the IOMMU group. If I scroll over to the right it says N/A > under the IOMMU column. Is there some step I need to 'create IOMMU > groups' or any other suggestions? > > Any chance to try another PCI slot? > > [root@n001 qemu]# more C*log > 2020-10-21 20:26:39.266+0000: starting up libvirt version: 6.0.0, > package: 25.2. > el8 (CBS <cbs(a)centos.org&gt;, 2020-09-10-13:20:10, ), qemu version: > 4.2.0qemu-kvm-4 > .2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname: > n001.bil.psc.edu > LC_ALL=C \ > PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \ > HOME=/var/lib/libvirt/qemu/domain-1-C8.2test \ > XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.local/share \ > XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.cache \ > XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.config \ > QEMU_AUDIO_DRV=spice \ > /usr/libexec/qemu-kvm \ > -name guest=C8.2test,debug-threads=on \ > -S \ > -object > secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-C8.2 > test/master-key.aes \ > -machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \ > -cpu Opteron_G5 \ > -m size=16777216k,slots=16,maxmem=67108864k \ > -overcommit mem-lock=off \ > -smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \ > -numa node,nodeid=0,cpus=0-15,mem=16384 \ > -uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \ > -smbios > type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial > > =47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703, > family=oVirt \ > -no-user-config \ > -nodefaults \ > -chardev socket,id=charmonitor,fd=39,server,nowait \ > -mon chardev=charmonitor,id=monitor,mode=control \ > -rtc base=2020-10-21T20:26:38,driftfix=slew \ > -global kvm-pit.lost_tick_policy=delay \ > -no-hpet \ > -no-shutdown \ > -global ICH9-LPC.disable_s3=1 \ > -global ICH9-LPC.disable_s4=1 \ > -boot strict=on \ > -device > pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on, > addr=0x2 \ > -device > pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ > -device > pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \ > -device > pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \ > -device > pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \ > -device > pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \ > -device > pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \ > -device > pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \ > -device > pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on, > addr=0x3 \ > -device > pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \ > -device > pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \ > -device > pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \ > -device > pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \ > -device > pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \ > -device > pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \ > -device > pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \ > -device > qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2 > ,addr=0x0 \ > -device > virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add > r=0x0 \ > -device > virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \ > -device > ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor > t,rerror=report \ > -blockdev > '{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var > > _nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22 > > -879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n > > ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl > y":true,"discard":"unmap"}' \ > -blockdev > '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t > rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \ > -device > scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id= > > 0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id > > =ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332 > ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \ > -chardev socket,id=charchannel0,fd=41,server,nowait \ > -device > virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel > 0,name=ovirt-guest-agent.0 \ > -chardev socket,id=charchannel1,fd=42,server,nowait \ > -device > virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel > 1,name=org.qemu.guest_agent.0 \ > -chardev spicevmc,id=charchannel2,name=vdagent \ > -device > virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel > 2,name=com.redhat.spice.0 \ > -device > usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port= > 1 \ > -vnc 128.182.82.72:0,password \ > -k en-us \ > -spice > port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt > > -spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso > > r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr > edir,seamless-migration=on \ > -device > qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra > > m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 > \ > -device > vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu > s=pci.3,addr=0x0 \ > -object > rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u > random \ > -device > virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb > 82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \ > -sandbox > on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny > \ > -msg timestamp=on > 2020-10-21 20:26:39.267+0000: Domain id=1 is tainted: > custom-hypervisor-feature > libvirt: error : internal error: Invalid device 0000:04:00.1 iommu_group > file / > sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink > 2020-10-21 20:26:39.298+0000: shutting down, reason=failed > 2020-10-21 20:27:41.683+0000: starting up libvirt version: 6.0.0, > package: 25.2. > el8 (CBS <cbs(a)centos.org&gt;, 2020-09-10-13:20:10, ), qemu version: > 4.2.0qemu-kvm-4 > .2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname: > n001.bil.psc.edu > LC_ALL=C \ > PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \ > HOME=/var/lib/libvirt/qemu/domain-2-C8.2test \ > XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.local/share \ > XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.cache \ > XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.config \ > QEMU_AUDIO_DRV=spice \ > /usr/libexec/qemu-kvm \ > -name guest=C8.2test,debug-threads=on \ > -S \ > -object > secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-2-C8.2 > test/master-key.aes \ > -machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \ > -cpu Opteron_G5 \ > -m size=16777216k,slots=16,maxmem=67108864k \ > -overcommit mem-lock=off \ > -smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \ > -numa node,nodeid=0,cpus=0-15,mem=16384 \ > -uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \ > -smbios > type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial > > =47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703, > family=oVirt \ > -no-user-config \ > -nodefaults \ > -chardev socket,id=charmonitor,fd=38,server,nowait \ > -mon chardev=charmonitor,id=monitor,mode=control \ > -rtc base=2020-10-21T20:27:40,driftfix=slew \ > -global kvm-pit.lost_tick_policy=delay \ > -no-hpet \ > -no-shutdown \ > -global ICH9-LPC.disable_s3=1 \ > -global ICH9-LPC.disable_s4=1 \ > -boot strict=on \ > -device > pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on, > addr=0x2 \ > -device > pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ > -device > pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \ > -device > pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \ > -device > pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \ > -device > pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \ > -device > pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \ > -device > pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \ > -device > pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on, > addr=0x3 \ > -device > pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \ > -device > pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \ > -device > pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \ > -device > pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \ > -device > pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \ > -device > pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \ > -device > pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \ > -device > qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2 > ,addr=0x0 \ > -device > virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add > r=0x0 \ > -device > virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \ > -device > ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor > t,rerror=report \ > -blockdev > '{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var > > _nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22 > > -879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n > > ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl > y":true,"discard":"unmap"}' \ > -blockdev > '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t > rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \ > -device > scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id= > > 0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id > > =ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332 > ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \ > -chardev socket,id=charchannel0,fd=40,server,nowait \ > -device > virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel > 0,name=ovirt-guest-agent.0 \ > -chardev socket,id=charchannel1,fd=41,server,nowait \ > -device > virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel > 1,name=org.qemu.guest_agent.0 \ > -chardev spicevmc,id=charchannel2,name=vdagent \ > -device > virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel > 2,name=com.redhat.spice.0 \ > -device > usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port= > 1 \ > -vnc 128.182.82.72:0,password \ > -k en-us \ > -spice > port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt > > -spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso > > r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr > edir,seamless-migration=on \ > -device > qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra > > m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 > \ > -device > vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu > s=pci.3,addr=0x0 \ > -object > rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u > random \ > -device > virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb > 82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \ > -sandbox > on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny > \ > -msg timestamp=on > 2020-10-21 20:27:41.684+0000: Domain id=2 is tainted: > custom-hypervisor-feature > libvirt: error : internal error: Invalid device 0000:04:00.1 iommu_group > file / > sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink > 2020-10-21 20:27:41.715+0000: shutting down, reason=failed > > > > _______________________________________________ > Users mailing list -- users(a)ovirt.org > To unsubscribe send an email to users-leave(a)ovirt.org > Privacy Statement: https://www.ovirt.org/privacy-policy.html > oVirt Code of Conduct: > https://www.ovirt.org/community/about/community-guidelines/ > List Archives: > https://lists.ovirt.org/archives/list/users@ovirt.org/message/4ITEBVKY635... >