Not really.
The guy who installed the cards said there's only one slot in these test
nodes.
On Mon, Oct 26, 2020 at 4:51 AM Dominik Holler <dholler(a)redhat.com> wrote:
On Wed, Oct 21, 2020 at 10:44 PM Edward Berger <edwberger(a)gmail.com>
wrote:
> I'm' trying to bring up a test ovirt 4.4 cluster with sr-iov virtual
> infiniband interfaces on Mellanox ConnectX-6 cards in some older Opteron
> servers while we await delivery of new Epyc servers.
>
> I have the hypervisors configured for sr-iov and showing the interfaces,
> but when I try to bring up a VM with the host device selected it fails to
> boot. "Failed Launching ..." shows up on the engine UI.
> [image: sriov-hostdev.JPG]
> The /var/log/libvirt/qemu log for the VM seems to think something is
> wrong with the IOMMU group. If I scroll over to the right it says N/A
> under the IOMMU column. Is there some step I need to 'create IOMMU
> groups' or any other suggestions?
>
>
Any chance to try another PCI slot?
>
> [root@n001 qemu]# more C*log
> 2020-10-21 20:26:39.266+0000: starting up libvirt version: 6.0.0,
> package: 25.2.
> el8 (CBS <cbs(a)centos.org>, 2020-09-10-13:20:10, ), qemu version:
> 4.2.0qemu-kvm-4
> .2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname:
>
n001.bil.psc.edu
> LC_ALL=C \
> PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
> HOME=/var/lib/libvirt/qemu/domain-1-C8.2test \
> XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.local/share \
> XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.cache \
> XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.config \
> QEMU_AUDIO_DRV=spice \
> /usr/libexec/qemu-kvm \
> -name guest=C8.2test,debug-threads=on \
> -S \
> -object
> secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-C8.2
> test/master-key.aes \
> -machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \
> -cpu Opteron_G5 \
> -m size=16777216k,slots=16,maxmem=67108864k \
> -overcommit mem-lock=off \
> -smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \
> -numa node,nodeid=0,cpus=0-15,mem=16384 \
> -uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \
> -smbios
> type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial
>
> =47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703,
> family=oVirt \
> -no-user-config \
> -nodefaults \
> -chardev socket,id=charmonitor,fd=39,server,nowait \
> -mon chardev=charmonitor,id=monitor,mode=control \
> -rtc base=2020-10-21T20:26:38,driftfix=slew \
> -global kvm-pit.lost_tick_policy=delay \
> -no-hpet \
> -no-shutdown \
> -global ICH9-LPC.disable_s3=1 \
> -global ICH9-LPC.disable_s4=1 \
> -boot strict=on \
> -device
> pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,
> addr=0x2 \
> -device
> pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
> -device
> pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
> -device
> pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
> -device
> pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
> -device
> pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \
> -device
> pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \
> -device
> pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \
> -device
> pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,
> addr=0x3 \
> -device
> pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \
> -device
> pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \
> -device
> pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \
> -device
> pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \
> -device
> pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \
> -device
> pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \
> -device
> pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \
> -device
> qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2
> ,addr=0x0 \
> -device
> virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add
> r=0x0 \
> -device
> virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \
> -device
> ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor
> t,rerror=report \
> -blockdev
>
'{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var
>
> _nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22
>
>
-879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n
>
>
ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl
> y":true,"discard":"unmap"}' \
> -blockdev
>
'{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t
>
rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}'
\
> -device
> scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id=
>
> 0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id
>
> =ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332
> ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \
> -chardev socket,id=charchannel0,fd=41,server,nowait \
> -device
> virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel
> 0,name=ovirt-guest-agent.0 \
> -chardev socket,id=charchannel1,fd=42,server,nowait \
> -device
> virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel
> 1,name=org.qemu.guest_agent.0 \
> -chardev spicevmc,id=charchannel2,name=vdagent \
> -device
> virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel
> 2,name=com.redhat.spice.0 \
> -device
> usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port=
> 1 \
> -vnc 128.182.82.72:0,password \
> -k en-us \
> -spice
> port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt
>
> -spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso
>
> r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr
> edir,seamless-migration=on \
> -device
> qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra
>
> m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
> \
> -device
> vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu
> s=pci.3,addr=0x0 \
> -object
> rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u
> random \
> -device
> virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb
> 82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \
> -sandbox
> on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
> \
> -msg timestamp=on
> 2020-10-21 20:26:39.267+0000: Domain id=1 is tainted:
> custom-hypervisor-feature
> libvirt: error : internal error: Invalid device 0000:04:00.1 iommu_group
> file /
> sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink
> 2020-10-21 20:26:39.298+0000: shutting down, reason=failed
> 2020-10-21 20:27:41.683+0000: starting up libvirt version: 6.0.0,
> package: 25.2.
> el8 (CBS <cbs(a)centos.org>, 2020-09-10-13:20:10, ), qemu version:
> 4.2.0qemu-kvm-4
> .2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname:
>
n001.bil.psc.edu
> LC_ALL=C \
> PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
> HOME=/var/lib/libvirt/qemu/domain-2-C8.2test \
> XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.local/share \
> XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.cache \
> XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.config \
> QEMU_AUDIO_DRV=spice \
> /usr/libexec/qemu-kvm \
> -name guest=C8.2test,debug-threads=on \
> -S \
> -object
> secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-2-C8.2
> test/master-key.aes \
> -machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \
> -cpu Opteron_G5 \
> -m size=16777216k,slots=16,maxmem=67108864k \
> -overcommit mem-lock=off \
> -smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \
> -numa node,nodeid=0,cpus=0-15,mem=16384 \
> -uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \
> -smbios
> type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial
>
> =47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703,
> family=oVirt \
> -no-user-config \
> -nodefaults \
> -chardev socket,id=charmonitor,fd=38,server,nowait \
> -mon chardev=charmonitor,id=monitor,mode=control \
> -rtc base=2020-10-21T20:27:40,driftfix=slew \
> -global kvm-pit.lost_tick_policy=delay \
> -no-hpet \
> -no-shutdown \
> -global ICH9-LPC.disable_s3=1 \
> -global ICH9-LPC.disable_s4=1 \
> -boot strict=on \
> -device
> pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,
> addr=0x2 \
> -device
> pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
> -device
> pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
> -device
> pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
> -device
> pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
> -device
> pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \
> -device
> pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \
> -device
> pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \
> -device
> pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,
> addr=0x3 \
> -device
> pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \
> -device
> pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \
> -device
> pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \
> -device
> pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \
> -device
> pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \
> -device
> pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \
> -device
> pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \
> -device
> qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2
> ,addr=0x0 \
> -device
> virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add
> r=0x0 \
> -device
> virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \
> -device
> ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor
> t,rerror=report \
> -blockdev
>
'{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var
>
> _nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22
>
>
-879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n
>
>
ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl
> y":true,"discard":"unmap"}' \
> -blockdev
>
'{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t
>
rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}'
\
> -device
> scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id=
>
> 0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id
>
> =ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332
> ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \
> -chardev socket,id=charchannel0,fd=40,server,nowait \
> -device
> virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel
> 0,name=ovirt-guest-agent.0 \
> -chardev socket,id=charchannel1,fd=41,server,nowait \
> -device
> virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel
> 1,name=org.qemu.guest_agent.0 \
> -chardev spicevmc,id=charchannel2,name=vdagent \
> -device
> virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel
> 2,name=com.redhat.spice.0 \
> -device
> usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port=
> 1 \
> -vnc 128.182.82.72:0,password \
> -k en-us \
> -spice
> port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt
>
> -spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso
>
> r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr
> edir,seamless-migration=on \
> -device
> qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra
>
> m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
> \
> -device
> vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu
> s=pci.3,addr=0x0 \
> -object
> rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u
> random \
> -device
> virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb
> 82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \
> -sandbox
> on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
> \
> -msg timestamp=on
> 2020-10-21 20:27:41.684+0000: Domain id=2 is tainted:
> custom-hypervisor-feature
> libvirt: error : internal error: Invalid device 0000:04:00.1 iommu_group
> file /
> sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink
> 2020-10-21 20:27:41.715+0000: shutting down, reason=failed
>
>
>
> _______________________________________________
> Users mailing list -- users(a)ovirt.org
> To unsubscribe send an email to users-leave(a)ovirt.org
> Privacy Statement:
https://www.ovirt.org/privacy-policy.html
> oVirt Code of Conduct:
>
https://www.ovirt.org/community/about/community-guidelines/
> List Archives:
>
https://lists.ovirt.org/archives/list/users@ovirt.org/message/4ITEBVKY635...
>