Didi -

The vdsmd service error is what led me to the vdsm-tool restore-nets concern. See the chain of errors below (along with the other details you requested.)

Thanks!

-js

[jsherman@ovirt01 vdsm]$ ls -la /var/log/vdsm

total 48

drwxr-xr-x. 3 vdsm kvm 4096 Mar 5 18:50 .

drwxr-xr-x. 13 root root 4096 Mar 6 03:08 ..

drwxr-xr-x. 2 vdsm kvm 6 Mar 1 09:18 backup

-rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 connectivity.log

-rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 mom.log

-rw-r--r--. 1 root root 34437 Mar 6 22:18 supervdsm.log

-rw-r--r--. 1 root root 1875 Mar 5 17:59 upgrade.log

-rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 vdsm.log

[root@ovirt01 log]# sestatus

SELinux status: enabled

SELinuxfs mount: /sys/fs/selinux

SELinux root directory: /etc/selinux

Loaded policy name: targeted

Current mode: permissive

Mode from config file: enforcing

Policy MLS status: enabled

Policy deny_unknown status: allowed

Max kernel policy version: 28

[root@ovirt01 audit]# tail -20 audit.log

type=USER_AUTH msg=audit(1457355177.617:519): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=pam_unix acct="jsherman" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_ACCT msg=audit(1457355177.623:520): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:accounting grantors=pam_unix,pam_localuser acct="jsherman" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_CMD msg=audit(1457355177.625:521): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log/vdsm" cmd=73797374656D63746C207374617274207664736D64 terminal=pts/0 res=success'

type=CRED_ACQ msg=audit(1457355177.626:522): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_START msg=audit(1457355177.628:523): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_END msg=audit(1457355179.095:524): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_close grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=CRED_DISP msg=audit(1457355179.095:525): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=SERVICE_START msg=audit(1457355179.099:526): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=vdsm-network comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=failed'

type=USER_CMD msg=audit(1457355237.531:527): pid=20768 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log" cmd=6C73202D616C206175646974 terminal=pts/0 res=success'

type=CRED_ACQ msg=audit(1457355237.533:528): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_START msg=audit(1457355237.535:529): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_END msg=audit(1457355237.554:530): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_close grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=CRED_DISP msg=audit(1457355237.555:531): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_CMD msg=audit(1457355242.882:532): pid=20770 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log" cmd=7375202D terminal=pts/0 res=success'

type=CRED_ACQ msg=audit(1457355242.885:533): pid=20770 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_START msg=audit(1457355242.887:534): pid=20770 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success'

type=USER_AUTH msg=audit(1457355242.925:535): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=pam_rootok acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success'

type=USER_ACCT msg=audit(1457355242.925:536): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:accounting grantors=pam_succeed_if acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success'

type=CRED_ACQ msg=audit(1457355242.928:537): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_rootok acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success'

type=USER_START msg=audit(1457355242.954:538): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_keyinit,pam_limits,pam_systemd,pam_unix,pam_xauth acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success'

[jsherman@ovirt01 vdsm]$ journalctl -xe

-- Unit vdsm-network.service has failed.

-- The result is failed.

Mar 07 07:52:59 ovirt01 systemd[1]: Dependency failed for Virtual Desktop Server Manager.

-- Subject: Unit vdsmd.service has failed

-- Defined-By: systemd

-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel

-- Unit vdsmd.service has failed.

-- The result is dependency.

Mar 07 07:52:59 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20737:13641695 (system bus name :1

Mar 07 07:52:59 ovirt01 systemd[1]: Dependency failed for MOM instance configured for VDSM purposes.

-- Subject: Unit mom-vdsm.service has failed

-- Defined-By: systemd

-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel

-- Unit mom-vdsm.service has failed.

-- The result is dependency.

Mar 07 07:52:59 ovirt01 systemd[1]: Job mom-vdsm.service/start failed with result 'dependency'.

Mar 07 07:52:59 ovirt01 systemd[1]: Job vdsmd.service/start failed with result 'dependency'.

Mar 07 07:52:59 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state.

Mar 07 07:52:59 ovirt01 systemd[1]: vdsm-network.service failed.

[root@ovirt01 log]# systemctl start mom-vdsm

A dependency job for mom-vdsm.service failed. See 'journalctl -xe' for details.

[root@ovirt01 log]# journalctl -xe

-- Unit vdsm-network.service has failed.

-- The result is failed.

Mar 07 07:57:34 ovirt01 systemd[1]: Dependency failed for Virtual Desktop Server Manager.

-- Subject: Unit vdsmd.service has failed

-- Defined-By: systemd

-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel

-- Unit vdsmd.service has failed.

-- The result is dependency.

Mar 07 07:57:34 ovirt01 systemd[1]: Dependency failed for MOM instance configured for VDSM purposes.

-- Subject: Unit mom-vdsm.service has failed

-- Defined-By: systemd

-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel

-- Unit mom-vdsm.service has failed.

-- The result is dependency.

Mar 07 07:57:34 ovirt01 systemd[1]: Job mom-vdsm.service/start failed with result 'dependency'.

Mar 07 07:57:34 ovirt01 systemd[1]: Job vdsmd.service/start failed with result 'dependency'.

Mar 07 07:57:34 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state.

Mar 07 07:57:34 ovirt01 systemd[1]: vdsm-network.service failed.

Mar 07 07:57:34 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20832:13669245 (system bus name :1

[root@ovirt01 log]# systemctl start vdsm-network

Job for vdsm-network.service failed because the control process exited with error code. See "systemctl status vdsm-network.service" and "journalctl -xe" for details.

[root@ovirt01 log]# journalctl -xe

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: self._raiseerror(v)

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: raise err

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: Traceback (most recent call last):

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/bin/vdsm-tool", line 219, in main

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: return tool_command[cmd]["command"](*args)

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_com

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: exec_restore(cmd)

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restor

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: raise EnvironmentError('Failed to restore the persisted networks')

Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: EnvironmentError: Failed to restore the persisted networks

Mar 07 07:58:32 ovirt01 systemd[1]: vdsm-network.service: main process exited, code=exited, status=1/FAILURE

Mar 07 07:58:32 ovirt01 systemd[1]: Failed to start Virtual Desktop Server Manager network restoration.

-- Subject: Unit vdsm-network.service has failed

-- Defined-By: systemd

-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel

-- Unit vdsm-network.service has failed.

-- The result is failed.

Mar 07 07:58:32 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state.

Mar 07 07:58:32 ovirt01 systemd[1]: vdsm-network.service failed.

Mar 07 07:58:32 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20860:13674993 (system bus name :1.115, o

[root@ovirt01 log]# vdsm-tool restore-nets

Traceback (most recent call last):

File "/usr/share/vdsm/vdsm-restore-net-config", line 429, in <module>

restore(args)

File "/usr/share/vdsm/vdsm-restore-net-config", line 387, in restore

_restore_sriov_numvfs()

File "/usr/share/vdsm/vdsm-restore-net-config", line 82, in _restore_sriov_numvfs

sriov_devices = _get_sriov_devices()

File "/usr/share/vdsm/vdsm-restore-net-config", line 54, in _get_sriov_devices

devices = hostdev.list_by_caps()

File "/usr/share/vdsm/hostdev.py", line 175, in list_by_caps

libvirt_devices = _get_devices_from_libvirt()

File "/usr/share/vdsm/hostdev.py", line 160, in _get_devices_from_libvirt

for device in libvirtconnection.get().listAllDevices(0))

File "/usr/share/vdsm/hostdev.py", line 160, in <genexpr>

for device in libvirtconnection.get().listAllDevices(0))

File "/usr/share/vdsm/hostdev.py", line 107, in _parse_device_params

devXML = etree.fromstring(device_xml)

File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1300, in XML

parser.feed(text)

File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1642, in feed

self._raiseerror(v)

File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror

raise err

xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13

Traceback (most recent call last):

File "/bin/vdsm-tool", line 219, in main

return tool_command[cmd]["command"](*args)

File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_command

exec_restore(cmd)

File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restore

raise EnvironmentError('Failed to restore the persisted networks')

EnvironmentError: Failed to restore the persisted networks

On Mon, Mar 7, 2016 at 2:56 AM, Yedidyah Bar David <didi@redhat.com> wrote:

On Sun, Mar 6, 2016 at 7:21 PM, Jonathan Sherman <haviland@gmail.com> wrote:
> Didi,
>
> NetworkManager is off, via "systemctl disable NetworkManager && systemctl
> stop NetworkManager" (and rebooted after), and network allowing remote ssh
> access and yum commands functional.
>
> Log files with sizes >0 from /var/log/vdsm:
> https://www.dropbox.com/s/iobee7toeyc2jda/supervdsm.log?dl=0
> https://www.dropbox.com/s/rdxi4ujwqhmekgh/upgrade.log?dl=0

So this means you also have an empty (size 0) vdsm.log? Can you please
'ls -la /var/log/vdsm'?

Can you check selinux/audit logs?

>
> Most recent log file from /var/log/ovirt-hosted-engine-setup:
> https://www.dropbox.com/s/brbbujmqhhbsx4o/ovirt-hosted-engine-setup-20160306104135-zibehr.log?dl=0

This has:

2016-03-06 10:41:43 DEBUG otopi.plugins.otopi.services.systemd
plugin.execute:941 execute-output: ('/bin/systemctl', 'start',
'vdsmd.service') stderr:
A dependency job for vdsmd.service failed. See 'journalctl -xe' for details.

Can you please check/post output of 'journalctl -xe'?

Thanks,

>
> Thank you for your time!
> -js
>
>
> On Sun, Mar 6, 2016 at 5:23 AM, Yedidyah Bar David <didi@redhat.com> wrote:
>>
>> On Sun, Mar 6, 2016 at 2:50 AM, Jonathan Sherman <haviland@gmail.com>
>> wrote:
>> > I want to extend my lab and am trying an install using the hosted
>> > engine,
>> > but the install is failing for me:
>> >
>> > References used for install:
>> > https://glennsnead.wordpress.com/2016/02/28/ovirt-3-6-installation/
>> > http://community.redhat.com/blog/2014/10/up-and-running-with-ovirt-3-5/
>> >
>> > Setup:
>> > Minimal install of CentOS Linux release 7.2.1511 (Core)
>> > Using DHCP with static address (tried without DHCP with same issue)
>> > yum upgrade
>> >
>> > Installed packages:
>> > yum localinstall -y
>> > http://resources.ovirt.org/pub/yum-repo/ovirt-release36.rpm
>> > yum install -y ovirt-hosted-engine-setup screen glusterfs-server
>> > nfs-utils
>> > vdsm-gluster system-storage-manager mailx
>> >
>> > rebooted
>> >
>> > Tried hosted-engine --deploy, but it errors out. I followed various
>> > logs
>> > and I _think_ it's failing related to vdsm-tool restore-nets. I could
>> > use
>> > some guidance on this to help track down what's happening and how to
>> > fix.
>> >
>> > (Note: I'd very much prefer using the hosted engine vs. another option
>> > for
>> > my lab, so I'm willing to put in the time to figure this out. Thanks
>> > for
>> > any assistance!)
>> >
>> > Install error:
>> > [root@ovirt01 network-scripts]# hosted-engine --deploy
>> > [ INFO ] Stage: Initializing
>> > [ INFO ] Generating a temporary VNC password.
>> > [ INFO ] Stage: Environment setup
>> > Continuing will configure this host for serving as hypervisor
>> > and
>> > create a VM where you have to install the engine afterwards.
>> > Are you sure you want to continue? (Yes, No)[Yes]:
>> > Configuration files: []
>> > Log file:
>> >
>> > /var/log/ovirt-hosted-engine-setup/ovirt-hosted-engine-setup-20160305194655-o29y86.log
>> > Version: otopi-1.4.1 (otopi-1.4.1-1.el7.centos)
>> > [ INFO ] Hardware supports virtualization
>> > [ INFO ] Stage: Environment packages setup
>> > [ INFO ] Stage: Programs detection
>> > [ INFO ] Stage: Environment setup
>> > [ ERROR ] Failed to execute stage 'Environment setup': Failed to start
>> > service 'vdsmd'
>> > [ INFO ] Stage: Clean up
>> > [ INFO ] Generating answer file
>> > '/var/lib/ovirt-hosted-engine-setup/answers/answers-20160305194703.conf'
>> > [ INFO ] Stage: Pre-termination
>> > [ INFO ] Stage: Termination
>> > [ ERROR ] Hosted Engine deployment failed: this system is not reliable,
>> > please check the issue, fix and redeploy
>> > Log file is located at
>> >
>> > /var/log/ovirt-hosted-engine-setup/ovirt-hosted-engine-setup-20160305194655-o29y86.log
>> >
>> > --
>> > vdsm-tool restore-nets output:
>> >
>> > [root@ovirt01 etc]# /usr/bin/vdsm-tool restore-nets
>> > Traceback (most recent call last):
>> > File "/usr/share/vdsm/vdsm-restore-net-config", line 429, in <module>
>> > restore(args)
>> > File "/usr/share/vdsm/vdsm-restore-net-config", line 387, in restore
>> > _restore_sriov_numvfs()
>> > File "/usr/share/vdsm/vdsm-restore-net-config", line 82, in
>> > _restore_sriov_numvfs
>> > sriov_devices = _get_sriov_devices()
>> > File "/usr/share/vdsm/vdsm-restore-net-config", line 54, in
>> > _get_sriov_devices
>> > devices = hostdev.list_by_caps()
>> > File "/usr/share/vdsm/hostdev.py", line 175, in list_by_caps
>> > libvirt_devices = _get_devices_from_libvirt()
>> > File "/usr/share/vdsm/hostdev.py", line 160, in
>> > _get_devices_from_libvirt
>> > for device in libvirtconnection.get().listAllDevices(0))
>> > File "/usr/share/vdsm/hostdev.py", line 160, in <genexpr>
>> > for device in libvirtconnection.get().listAllDevices(0))
>> > File "/usr/share/vdsm/hostdev.py", line 107, in _parse_device_params
>> > devXML = etree.fromstring(device_xml)
>> > File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1300, in
>> > XML
>> > parser.feed(text)
>> > File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1642, in
>> > feed
>> > self._raiseerror(v)
>> > File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in
>> > _raiseerror
>> > raise err
>> > xml.etree.ElementTree.ParseError: not well-formed (invalid token): line
>> > 4,
>> > column 13
>> > Traceback (most recent call last):
>> > File "/usr/bin/vdsm-tool", line 219, in main
>> > return tool_command[cmd]["command"](*args)
>> > File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py",
>> > line
>> > 41, in restore_command
>> > exec_restore(cmd)
>> > File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py",
>> > line
>> > 54, in exec_restore
>> > raise EnvironmentError('Failed to restore the persisted networks')
>> > EnvironmentError: Failed to restore the persisted networks
>>
>> Please check/post full ovirt-hosted-engine-setup logs, vdsm logs.
>>
>> Do you have NetworkManager?
>>
>> Thanks for the report!
>>
>> Best,
>> --
>> Didi
>
>

--
Didi