Not really. The guy who installed the cards said there's only one slot in these test nodes. On Mon, Oct 26, 2020 at 4:51 AM Dominik Holler <dholler@redhat.com> wrote:
On Wed, Oct 21, 2020 at 10:44 PM Edward Berger <edwberger@gmail.com> wrote:
I'm' trying to bring up a test ovirt 4.4 cluster with sr-iov virtual infiniband interfaces on Mellanox ConnectX-6 cards in some older Opteron servers while we await delivery of new Epyc servers.
I have the hypervisors configured for sr-iov and showing the interfaces, but when I try to bring up a VM with the host device selected it fails to boot. "Failed Launching ..." shows up on the engine UI. [image: sriov-hostdev.JPG] The /var/log/libvirt/qemu log for the VM seems to think something is wrong with the IOMMU group. If I scroll over to the right it says N/A under the IOMMU column. Is there some step I need to 'create IOMMU groups' or any other suggestions?
Any chance to try another PCI slot?
[root@n001 qemu]# more C*log 2020-10-21 20:26:39.266+0000: starting up libvirt version: 6.0.0, package: 25.2. el8 (CBS <cbs@centos.org>, 2020-09-10-13:20:10, ), qemu version: 4.2.0qemu-kvm-4 .2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname: n001.bil.psc.edu LC_ALL=C \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \ HOME=/var/lib/libvirt/qemu/domain-1-C8.2test \ XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.local/share \ XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.cache \ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.config \ QEMU_AUDIO_DRV=spice \ /usr/libexec/qemu-kvm \ -name guest=C8.2test,debug-threads=on \ -S \ -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-C8.2 test/master-key.aes \ -machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \ -cpu Opteron_G5 \ -m size=16777216k,slots=16,maxmem=67108864k \ -overcommit mem-lock=off \ -smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \ -numa node,nodeid=0,cpus=0-15,mem=16384 \ -uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \ -smbios type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial
=47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703, family=oVirt \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=39,server,nowait \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=2020-10-21T20:26:38,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-hpet \ -no-shutdown \ -global ICH9-LPC.disable_s3=1 \ -global ICH9-LPC.disable_s4=1 \ -boot strict=on \ -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on, addr=0x2 \ -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \ -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \ -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \ -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \ -device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \ -device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \ -device pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on, addr=0x3 \ -device pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \ -device pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \ -device pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \ -device pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \ -device pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \ -device pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \ -device pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \ -device qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2 ,addr=0x0 \ -device virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add r=0x0 \ -device virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \ -device ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor t,rerror=report \ -blockdev '{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var
_nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22
-879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n
ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl y":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \ -device scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id=
0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id
=ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332 ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \ -chardev socket,id=charchannel0,fd=41,server,nowait \ -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel 0,name=ovirt-guest-agent.0 \ -chardev socket,id=charchannel1,fd=42,server,nowait \ -device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel 1,name=org.qemu.guest_agent.0 \ -chardev spicevmc,id=charchannel2,name=vdagent \ -device virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel 2,name=com.redhat.spice.0 \ -device usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port= 1 \ -vnc 128.182.82.72:0,password \ -k en-us \ -spice port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt
-spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso
r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr edir,seamless-migration=on \ -device qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra
m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 \ -device vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu s=pci.3,addr=0x0 \ -object rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u random \ -device virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb 82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on 2020-10-21 20:26:39.267+0000: Domain id=1 is tainted: custom-hypervisor-feature libvirt: error : internal error: Invalid device 0000:04:00.1 iommu_group file / sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink 2020-10-21 20:26:39.298+0000: shutting down, reason=failed 2020-10-21 20:27:41.683+0000: starting up libvirt version: 6.0.0, package: 25.2. el8 (CBS <cbs@centos.org>, 2020-09-10-13:20:10, ), qemu version: 4.2.0qemu-kvm-4 .2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname: n001.bil.psc.edu LC_ALL=C \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \ HOME=/var/lib/libvirt/qemu/domain-2-C8.2test \ XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.local/share \ XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.cache \ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.config \ QEMU_AUDIO_DRV=spice \ /usr/libexec/qemu-kvm \ -name guest=C8.2test,debug-threads=on \ -S \ -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-2-C8.2 test/master-key.aes \ -machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \ -cpu Opteron_G5 \ -m size=16777216k,slots=16,maxmem=67108864k \ -overcommit mem-lock=off \ -smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \ -numa node,nodeid=0,cpus=0-15,mem=16384 \ -uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \ -smbios type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial
=47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703, family=oVirt \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=38,server,nowait \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=2020-10-21T20:27:40,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-hpet \ -no-shutdown \ -global ICH9-LPC.disable_s3=1 \ -global ICH9-LPC.disable_s4=1 \ -boot strict=on \ -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on, addr=0x2 \ -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \ -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \ -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \ -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \ -device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \ -device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \ -device pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on, addr=0x3 \ -device pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \ -device pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \ -device pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \ -device pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \ -device pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \ -device pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \ -device pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \ -device qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2 ,addr=0x0 \ -device virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add r=0x0 \ -device virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \ -device ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor t,rerror=report \ -blockdev '{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var
_nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22
-879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n
ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl y":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \ -device scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id=
0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id
=ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332 ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \ -chardev socket,id=charchannel0,fd=40,server,nowait \ -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel 0,name=ovirt-guest-agent.0 \ -chardev socket,id=charchannel1,fd=41,server,nowait \ -device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel 1,name=org.qemu.guest_agent.0 \ -chardev spicevmc,id=charchannel2,name=vdagent \ -device virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel 2,name=com.redhat.spice.0 \ -device usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port= 1 \ -vnc 128.182.82.72:0,password \ -k en-us \ -spice port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt
-spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso
r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr edir,seamless-migration=on \ -device qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra
m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 \ -device vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu s=pci.3,addr=0x0 \ -object rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u random \ -device virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb 82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on 2020-10-21 20:27:41.684+0000: Domain id=2 is tainted: custom-hypervisor-feature libvirt: error : internal error: Invalid device 0000:04:00.1 iommu_group file / sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink 2020-10-21 20:27:41.715+0000: shutting down, reason=failed
_______________________________________________ Users mailing list -- users@ovirt.org To unsubscribe send an email to users-leave@ovirt.org Privacy Statement: https://www.ovirt.org/privacy-policy.html oVirt Code of Conduct: https://www.ovirt.org/community/about/community-guidelines/ List Archives: https://lists.ovirt.org/archives/list/users@ovirt.org/message/4ITEBVKY635NVU...
