I'm' trying to bring up a test ovirt 4.4 cluster with sr-iov virtual infiniband interfaces on Mellanox ConnectX-6 cards in some older Opteron servers while we await delivery of new Epyc servers.

I have the hypervisors configured for sr-iov and showing the interfaces, but when I try to bring up a VM with the host device selected it fails to boot.  "Failed Launching ..." shows up on the engine UI.

The /var/log/libvirt/qemu log  for the VM seems to think something is wrong with the IOMMU group.  If I scroll over to the right it says N/A under the IOMMU column.  Is there some step I need to 'create IOMMU groups'  or any other suggestions?

[root@n001 qemu]# more C*log
2020-10-21 20:26:39.266+0000: starting up libvirt version: 6.0.0, package: 25.2.
el8 (CBS <cbs@centos.org>, 2020-09-10-13:20:10, ), qemu version: 4.2.0qemu-kvm-4
.2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname: n001.bil.psc.edu
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-1-C8.2test \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-C8.2test/.config \
QEMU_AUDIO_DRV=spice \
/usr/libexec/qemu-kvm \
-name guest=C8.2test,debug-threads=on \
-S \
-object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-C8.2
test/master-key.aes \
-machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \
-cpu Opteron_G5 \
-m size=16777216k,slots=16,maxmem=67108864k \
-overcommit mem-lock=off \
-smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \
-numa node,nodeid=0,cpus=0-15,mem=16384 \
-uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \
-smbios type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial
=47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703,
family=oVirt \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=39,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=2020-10-21T20:26:38,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,
addr=0x2 \
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \
-device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \
-device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \
-device pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,
addr=0x3 \
-device pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \
-device pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \
-device pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \
-device pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \
-device pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \
-device pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \
-device pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \
-device qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2
,addr=0x0 \
-device virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add
r=0x0 \
-device virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \
-device ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor
t,rerror=report \
-blockdev '{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var
_nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22
-879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n
ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl
y":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t
rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \
-device scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id=
0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id
=ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332
ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \
-chardev socket,id=charchannel0,fd=41,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel
0,name=ovirt-guest-agent.0 \
-chardev socket,id=charchannel1,fd=42,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel
1,name=org.qemu.guest_agent.0 \
-chardev spicevmc,id=charchannel2,name=vdagent \
-device virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel
2,name=com.redhat.spice.0 \
-device usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port=
1 \
-vnc 128.182.82.72:0,password \
-k en-us \
-spice port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt
-spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso
r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr
edir,seamless-migration=on \
-device qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra
m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
\
-device vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu
s=pci.3,addr=0x0 \
-object rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u
random \
-device virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb
82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
 \
-msg timestamp=on
2020-10-21 20:26:39.267+0000: Domain id=1 is tainted: custom-hypervisor-feature
libvirt:  error : internal error: Invalid device 0000:04:00.1 iommu_group file /
sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink
2020-10-21 20:26:39.298+0000: shutting down, reason=failed
2020-10-21 20:27:41.683+0000: starting up libvirt version: 6.0.0, package: 25.2.
el8 (CBS <cbs@centos.org>, 2020-09-10-13:20:10, ), qemu version: 4.2.0qemu-kvm-4
.2.0-29.el8.3, kernel: 4.18.0-193.19.1.el8_2.x86_64, hostname: n001.bil.psc.edu
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-2-C8.2test \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-2-C8.2test/.config \
QEMU_AUDIO_DRV=spice \
/usr/libexec/qemu-kvm \
-name guest=C8.2test,debug-threads=on \
-S \
-object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-2-C8.2
test/master-key.aes \
-machine pc-q35-rhel8.1.0,accel=kvm,usb=off,dump-guest-core=off \
-cpu Opteron_G5 \
-m size=16777216k,slots=16,maxmem=67108864k \
-overcommit mem-lock=off \
-smp 4,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 \
-numa node,nodeid=0,cpus=0-15,mem=16384 \
-uuid 07eb98af-0d9f-4f9c-967a-8af302c68703 \
-smbios type=1,manufacturer=oVirt,product=RHEL,version=8.2-2.2004.0.2.el8,serial
=47513848-0036-2500-905b-0025905bff22,uuid=07eb98af-0d9f-4f9c-967a-8af302c68703,
family=oVirt \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=38,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=2020-10-21T20:27:40,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,
addr=0x2 \
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \
-device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \
-device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 \
-device pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,
addr=0x3 \
-device pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 \
-device pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 \
-device pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 \
-device pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 \
-device pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 \
-device pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 \
-device pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 \
-device qemu-xhci,p2=8,p3=8,id=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702,bus=pci.2
,addr=0x0 \
-device virtio-scsi-pci,id=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc,bus=pci.1,add
r=0x0 \
-device virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.4,addr=0x0 \
-device ide-cd,bus=ide.2,id=ua-271c6d06-1f3c-4e45-99cc-a39f39a6dd1b,werror=repor
t,rerror=report \
-blockdev '{"driver":"file","filename":"/rhev/data-center/mnt/10.200.16.152:_var
_nfs__shares_data/662d0028-ceaf-4641-87fa-839254e15209/images/4c332ece-f6db-4b22
-879c-b6912c5b27c0/2918a5f1-c801-4a34-8cf7-89045265f6f3","aio":"threads","node-n
ame":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-onl
y":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":t
rue,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \
-device scsi-hd,bus=ua-64b19c77-98d3-4946-a0f8-c43c1347a5dc.0,channel=0,scsi-id=
0,lun=0,device_id=4c332ece-f6db-4b22-879c-b6912c5b27c0,drive=libvirt-1-format,id
=ua-4c332ece-f6db-4b22-879c-b6912c5b27c0,bootindex=1,write-cache=on,serial=4c332
ece-f6db-4b22-879c-b6912c5b27c0,werror=stop,rerror=stop \
-chardev socket,id=charchannel0,fd=40,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel
0,name=ovirt-guest-agent.0 \
-chardev socket,id=charchannel1,fd=41,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel
1,name=org.qemu.guest_agent.0 \
-chardev spicevmc,id=charchannel2,name=vdagent \
-device virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel
2,name=com.redhat.spice.0 \
-device usb-tablet,id=input0,bus=ua-cd3a4abd-57e9-4f2e-a61f-01128ead8702.0,port=
1 \
-vnc 128.182.82.72:0,password \
-k en-us \
-spice port=5901,tls-port=5902,addr=128.182.82.72,x509-dir=/etc/pki/vdsm/libvirt
-spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=curso
r,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbr
edir,seamless-migration=on \
-device qxl-vga,id=ua-bd6ad9d1-2d02-40fa-b087-4fc4786a2f44,ram_size=67108864,vra
m_size=33554432,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
\
-device vfio-pci,host=0000:04:00.1,id=ua-7fce29ba-fcf7-4bd3-a680-21b200d44e42,bu
s=pci.3,addr=0x0 \
-object rng-random,id=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,filename=/dev/u
random \
-device virtio-rng-pci,rng=objua-e1fb82d8-2455-4763-a5d9-22d7a9903806,id=ua-e1fb
82d8-2455-4763-a5d9-22d7a9903806,bus=pci.5,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
 \
-msg timestamp=on
2020-10-21 20:27:41.684+0000: Domain id=2 is tainted: custom-hypervisor-feature
libvirt:  error : internal error: Invalid device 0000:04:00.1 iommu_group file /
sys/bus/pci/devices/0000:04:00.1/iommu_group is not a symlink
2020-10-21 20:27:41.715+0000: shutting down, reason=failed