Didi - The vdsmd service error is what led me to the vdsm-tool restore-nets concern. See the chain of errors below (along with the other details you requested.)

Thanks! -js -- [jsherman@ovirt01 vdsm]$ ls -la /var/log/vdsm total 48 drwxr-xr-x. 3 vdsm kvm 4096 Mar 5 18:50 . drwxr-xr-x. 13 root root 4096 Mar 6 03:08 .. drwxr-xr-x. 2 vdsm kvm 6 Mar 1 09:18 backup -rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 connectivity.log -rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 mom.log -rw-r--r--. 1 root root 34437 Mar 6 22:18 supervdsm.log -rw-r--r--. 1 root root 1875 Mar 5 17:59 upgrade.log -rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 vdsm.log -- [root@ovirt01 log]# sestatus SELinux status: enabled SELinuxfs mount: /sys/fs/selinux SELinux root directory: /etc/selinux Loaded policy name: targeted Current mode: permissive Mode from config file: enforcing Policy MLS status: enabled Policy deny_unknown status: allowed Max kernel policy version: 28 -- [root@ovirt01 audit]# tail -20 audit.log type=USER_AUTH msg=audit(1457355177.617:519): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=pam_unix acct="jsherman" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_ACCT msg=audit(1457355177.623:520): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:accounting grantors=pam_unix,pam_localuser acct="jsherman" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_CMD msg=audit(1457355177.625:521): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log/vdsm" cmd=73797374656D63746C207374617274207664736D64 terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355177.626:522): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_START msg=audit(1457355177.628:523): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_END msg=audit(1457355179.095:524): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_close grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=CRED_DISP msg=audit(1457355179.095:525): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=SERVICE_START msg=audit(1457355179.099:526): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=vdsm-network comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=failed' type=USER_CMD msg=audit(1457355237.531:527): pid=20768 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log" cmd=6C73202D616C206175646974 terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355237.533:528): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_START msg=audit(1457355237.535:529): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_END msg=audit(1457355237.554:530): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_close grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=CRED_DISP msg=audit(1457355237.555:531): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_CMD msg=audit(1457355242.882:532): pid=20770 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log" cmd=7375202D terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355242.885:533): pid=20770 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_START msg=audit(1457355242.887:534): pid=20770 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_AUTH msg=audit(1457355242.925:535): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=pam_rootok acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' type=USER_ACCT msg=audit(1457355242.925:536): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:accounting grantors=pam_succeed_if acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355242.928:537): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_rootok acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' type=USER_START msg=audit(1457355242.954:538): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_keyinit,pam_limits,pam_systemd,pam_unix,pam_xauth acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' -- [jsherman@ovirt01 vdsm]$ journalctl -xe -- Unit vdsm-network.service has failed. -- -- The result is failed. Mar 07 07:52:59 ovirt01 systemd[1]: Dependency failed for Virtual Desktop Server Manager. -- Subject: Unit vdsmd.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit vdsmd.service has failed. -- -- The result is dependency. Mar 07 07:52:59 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20737:13641695 (system bus name :1 Mar 07 07:52:59 ovirt01 systemd[1]: Dependency failed for MOM instance configured for VDSM purposes. -- Subject: Unit mom-vdsm.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit mom-vdsm.service has failed. -- -- The result is dependency. Mar 07 07:52:59 ovirt01 systemd[1]: Job mom-vdsm.service/start failed with result 'dependency'. Mar 07 07:52:59 ovirt01 systemd[1]: Job vdsmd.service/start failed with result 'dependency'. Mar 07 07:52:59 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state. Mar 07 07:52:59 ovirt01 systemd[1]: vdsm-network.service failed. -- [root@ovirt01 log]# systemctl start mom-vdsm A dependency job for mom-vdsm.service failed. See 'journalctl -xe' for details. [root@ovirt01 log]# journalctl -xe -- Unit vdsm-network.service has failed. -- -- The result is failed. Mar 07 07:57:34 ovirt01 systemd[1]: Dependency failed for Virtual Desktop Server Manager. -- Subject: Unit vdsmd.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit vdsmd.service has failed. -- -- The result is dependency. Mar 07 07:57:34 ovirt01 systemd[1]: Dependency failed for MOM instance configured for VDSM purposes. -- Subject: Unit mom-vdsm.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit mom-vdsm.service has failed. -- -- The result is dependency. Mar 07 07:57:34 ovirt01 systemd[1]: Job mom-vdsm.service/start failed with result 'dependency'. Mar 07 07:57:34 ovirt01 systemd[1]: Job vdsmd.service/start failed with result 'dependency'. Mar 07 07:57:34 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state. Mar 07 07:57:34 ovirt01 systemd[1]: vdsm-network.service failed. Mar 07 07:57:34 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20832:13669245 (system bus name :1 -- [root@ovirt01 log]# systemctl start vdsm-network Job for vdsm-network.service failed because the control process exited with error code. See "systemctl status vdsm-network.service" and "journalctl -xe" for details. [root@ovirt01 log]# journalctl -xe Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: self._raiseerror(v) Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: raise err Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13 Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: Traceback (most recent call last): Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/bin/vdsm-tool", line 219, in main Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: return tool_command[cmd]["command"](*args) Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_com Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: exec_restore(cmd) Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restor Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: raise EnvironmentError('Failed to restore the persisted networks') Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: EnvironmentError: Failed to restore the persisted networks Mar 07 07:58:32 ovirt01 systemd[1]: vdsm-network.service: main process exited, code=exited, status=1/FAILURE Mar 07 07:58:32 ovirt01 systemd[1]: Failed to start Virtual Desktop Server Manager network restoration. -- Subject: Unit vdsm-network.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit vdsm-network.service has failed. -- -- The result is failed. Mar 07 07:58:32 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state. Mar 07 07:58:32 ovirt01 systemd[1]: vdsm-network.service failed. Mar 07 07:58:32 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20860:13674993 (system bus name :1.115, o -- [root@ovirt01 log]# vdsm-tool restore-nets Traceback (most recent call last): File "/usr/share/vdsm/vdsm-restore-net-config", line 429, in <module> restore(args) File "/usr/share/vdsm/vdsm-restore-net-config", line 387, in restore _restore_sriov_numvfs() File "/usr/share/vdsm/vdsm-restore-net-config", line 82, in _restore_sriov_numvfs sriov_devices = _get_sriov_devices() File "/usr/share/vdsm/vdsm-restore-net-config", line 54, in _get_sriov_devices devices = hostdev.list_by_caps() File "/usr/share/vdsm/hostdev.py", line 175, in list_by_caps libvirt_devices = _get_devices_from_libvirt() File "/usr/share/vdsm/hostdev.py", line 160, in _get_devices_from_libvirt for device in libvirtconnection.get().listAllDevices(0)) File "/usr/share/vdsm/hostdev.py", line 160, in <genexpr> for device in libvirtconnection.get().listAllDevices(0)) File "/usr/share/vdsm/hostdev.py", line 107, in _parse_device_params devXML = etree.fromstring(device_xml) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1300, in XML parser.feed(text) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1642, in feed self._raiseerror(v) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror raise err xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13 Traceback (most recent call last): File "/bin/vdsm-tool", line 219, in main return tool_command[cmd]["command"](*args) File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_command exec_restore(cmd) File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restore raise EnvironmentError('Failed to restore the persisted networks') EnvironmentError: Failed to restore the persisted networks On Mon, Mar 7, 2016 at 2:56 AM, Yedidyah Bar David <didi(a)redhat.com&gt; wrote: > > On Sun, Mar 6, 2016 at 7:21 PM, Jonathan Sherman <haviland(a)gmail.com&gt; > wrote: > > Didi, > > > > NetworkManager is off, via "systemctl disable NetworkManager && > > systemctl > > stop NetworkManager" (and rebooted after), and network allowing remote > > ssh > > access and yum commands functional. > > > > Log files with sizes >0 from /var/log/vdsm: > > https://www.dropbox.com/s/iobee7toeyc2jda/supervdsm.log?dl=0 > > https://www.dropbox.com/s/rdxi4ujwqhmekgh/upgrade.log?dl=0 > > So this means you also have an empty (size 0) vdsm.log? Can you please > 'ls -la /var/log/vdsm'? > > Can you check selinux/audit logs? > > > > > Most recent log file from /var/log/ovirt-hosted-engine-setup: > > > > https://www.dropbox.com/s/brbbujmqhhbsx4o/ovirt-hosted-engine-setup-20160... > > This has: > > 2016-03-06 10:41:43 DEBUG otopi.plugins.otopi.services.systemd > plugin.execute:941 execute-output: ('/bin/systemctl', 'start', > 'vdsmd.service') stderr: > A dependency job for vdsmd.service failed. See 'journalctl -xe' for > details. > > Can you please check/post output of 'journalctl -xe'? > > Thanks, > > > > > Thank you for your time! > > -js > > > > > > On Sun, Mar 6, 2016 at 5:23 AM, Yedidyah Bar David <didi(a)redhat.com&gt; > > wrote: > >> > >> On Sun, Mar 6, 2016 at 2:50 AM, Jonathan Sherman <haviland(a)gmail.com&gt; > >> wrote: > >> > I want to extend my lab and am trying an install using the hosted > >> > engine, > >> > but the install is failing for me: > >> > > >> > References used for install: > >> > https://glennsnead.wordpress.com/2016/02/28/ovirt-3-6-installation/ > >> > > >> > http://community.redhat.com/blog/2014/10/up-and-running-with-ovirt-3-5/ > >> > > >> > Setup: > >> > Minimal install of CentOS Linux release 7.2.1511 (Core) > >> > Using DHCP with static address (tried without DHCP with same issue) > >> > yum upgrade > >> > > >> > Installed packages: > >> > yum localinstall -y > >> > http://resources.ovirt.org/pub/yum-repo/ovirt-release36.rpm > >> > yum install -y ovirt-hosted-engine-setup screen glusterfs-server > >> > nfs-utils > >> > vdsm-gluster system-storage-manager mailx > >> > > >> > rebooted > >> > > >> > Tried hosted-engine --deploy, but it errors out. I followed various > >> > logs > >> > and I _think_ it's failing related to vdsm-tool restore-nets. I > >> > could > >> > use > >> > some guidance on this to help track down what's happening and how to > >> > fix. > >> > > >> > (Note: I'd very much prefer using the hosted engine vs. another > >> > option > >> > for > >> > my lab, so I'm willing to put in the time to figure this out. Thanks > >> > for > >> > any assistance!) > >> > > >> > Install error: > >> > [root@ovirt01 network-scripts]# hosted-engine --deploy > >> > [ INFO ] Stage: Initializing > >> > [ INFO ] Generating a temporary VNC password. > >> > [ INFO ] Stage: Environment setup > >> > Continuing will configure this host for serving as > >> > hypervisor > >> > and > >> > create a VM where you have to install the engine afterwards. > >> > Are you sure you want to continue? (Yes, No)[Yes]: > >> > Configuration files: [] > >> > Log file: > >> > > >> > > >> > /var/log/ovirt-hosted-engine-setup/ovirt-hosted-engine-setup-20160305194655-o29y86.log > >> > Version: otopi-1.4.1 (otopi-1.4.1-1.el7.centos) > >> > [ INFO ] Hardware supports virtualization > >> > [ INFO ] Stage: Environment packages setup > >> > [ INFO ] Stage: Programs detection > >> > [ INFO ] Stage: Environment setup > >> > [ ERROR ] Failed to execute stage 'Environment setup': Failed to > >> > start > >> > service 'vdsmd' > >> > [ INFO ] Stage: Clean up > >> > [ INFO ] Generating answer file > >> > > >> > '/var/lib/ovirt-hosted-engine-setup/answers/answers-20160305194703.conf' > >> > [ INFO ] Stage: Pre-termination > >> > [ INFO ] Stage: Termination > >> > [ ERROR ] Hosted Engine deployment failed: this system is not > >> > reliable, > >> > please check the issue, fix and redeploy > >> > Log file is located at > >> > > >> > > >> > /var/log/ovirt-hosted-engine-setup/ovirt-hosted-engine-setup-20160305194655-o29y86.log > >> > > >> > -- > >> > vdsm-tool restore-nets output: > >> > > >> > [root@ovirt01 etc]# /usr/bin/vdsm-tool restore-nets > >> > Traceback (most recent call last): > >> > File "/usr/share/vdsm/vdsm-restore-net-config", line 429, in > >> > <module> > >> > restore(args) > >> > File "/usr/share/vdsm/vdsm-restore-net-config", line 387, in > >> > restore > >> > _restore_sriov_numvfs() > >> > File "/usr/share/vdsm/vdsm-restore-net-config", line 82, in > >> > _restore_sriov_numvfs > >> > sriov_devices = _get_sriov_devices() > >> > File "/usr/share/vdsm/vdsm-restore-net-config", line 54, in > >> > _get_sriov_devices > >> > devices = hostdev.list_by_caps() > >> > File "/usr/share/vdsm/hostdev.py", line 175, in list_by_caps > >> > libvirt_devices = _get_devices_from_libvirt() > >> > File "/usr/share/vdsm/hostdev.py", line 160, in > >> > _get_devices_from_libvirt > >> > for device in libvirtconnection.get().listAllDevices(0)) > >> > File "/usr/share/vdsm/hostdev.py", line 160, in <genexpr> > >> > for device in libvirtconnection.get().listAllDevices(0)) > >> > File "/usr/share/vdsm/hostdev.py", line 107, in > >> > _parse_device_params > >> > devXML = etree.fromstring(device_xml) > >> > File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1300, in > >> > XML > >> > parser.feed(text) > >> > File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1642, in > >> > feed > >> > self._raiseerror(v) > >> > File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in > >> > _raiseerror > >> > raise err > >> > xml.etree.ElementTree.ParseError: not well-formed (invalid token): > >> > line > >> > 4, > >> > column 13 > >> > Traceback (most recent call last): > >> > File "/usr/bin/vdsm-tool", line 219, in main > >> > return tool_command[cmd]["command"](*args) > >> > File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", > >> > line > >> > 41, in restore_command > >> > exec_restore(cmd) > >> > File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", > >> > line > >> > 54, in exec_restore > >> > raise EnvironmentError('Failed to restore the persisted > >> > networks') > >> > EnvironmentError: Failed to restore the persisted networks > >> > >> Please check/post full ovirt-hosted-engine-setup logs, vdsm logs. > >> > >> Do you have NetworkManager? > >> > >> Thanks for the report! > >> > >> Best, > >> -- > >> Didi > > > > > > > > -- > Didi