On Mon, Mar 7, 2016 at 3:01 PM, Jonathan Sherman <haviland@gmail.com> wrote:
Didi -
The vdsmd service error is what led me to the vdsm-tool restore-nets concern. See the chain of errors below (along with the other details you requested.)
Thanks. Changing subject and adding Dan.
Thanks! -js
--
[jsherman@ovirt01 vdsm]$ ls -la /var/log/vdsm total 48 drwxr-xr-x. 3 vdsm kvm 4096 Mar 5 18:50 . drwxr-xr-x. 13 root root 4096 Mar 6 03:08 .. drwxr-xr-x. 2 vdsm kvm 6 Mar 1 09:18 backup -rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 connectivity.log -rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 mom.log -rw-r--r--. 1 root root 34437 Mar 6 22:18 supervdsm.log -rw-r--r--. 1 root root 1875 Mar 5 17:59 upgrade.log -rw-r--r--. 1 vdsm kvm 0 Mar 5 17:53 vdsm.log
--
[root@ovirt01 log]# sestatus SELinux status: enabled SELinuxfs mount: /sys/fs/selinux SELinux root directory: /etc/selinux Loaded policy name: targeted Current mode: permissive Mode from config file: enforcing Policy MLS status: enabled Policy deny_unknown status: allowed Max kernel policy version: 28
--
[root@ovirt01 audit]# tail -20 audit.log type=USER_AUTH msg=audit(1457355177.617:519): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=pam_unix acct="jsherman" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_ACCT msg=audit(1457355177.623:520): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:accounting grantors=pam_unix,pam_localuser acct="jsherman" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_CMD msg=audit(1457355177.625:521): pid=20733 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log/vdsm" cmd=73797374656D63746C207374617274207664736D64 terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355177.626:522): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_START msg=audit(1457355177.628:523): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_END msg=audit(1457355179.095:524): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_close grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=CRED_DISP msg=audit(1457355179.095:525): pid=20733 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=SERVICE_START msg=audit(1457355179.099:526): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=vdsm-network comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=failed' type=USER_CMD msg=audit(1457355237.531:527): pid=20768 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log" cmd=6C73202D616C206175646974 terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355237.533:528): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_START msg=audit(1457355237.535:529): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_END msg=audit(1457355237.554:530): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_close grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=CRED_DISP msg=audit(1457355237.555:531): pid=20768 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_CMD msg=audit(1457355242.882:532): pid=20770 uid=1000 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='cwd="/var/log" cmd=7375202D terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355242.885:533): pid=20770 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_env,pam_unix acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_START msg=audit(1457355242.887:534): pid=20770 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_limits acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/0 res=success' type=USER_AUTH msg=audit(1457355242.925:535): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=pam_rootok acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' type=USER_ACCT msg=audit(1457355242.925:536): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:accounting grantors=pam_succeed_if acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' type=CRED_ACQ msg=audit(1457355242.928:537): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:setcred grantors=pam_rootok acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success' type=USER_START msg=audit(1457355242.954:538): pid=20771 uid=0 auid=1000 ses=19 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 msg='op=PAM:session_open grantors=pam_keyinit,pam_keyinit,pam_limits,pam_systemd,pam_unix,pam_xauth acct="root" exe="/usr/bin/su" hostname=? addr=? terminal=pts/0 res=success'
--
[jsherman@ovirt01 vdsm]$ journalctl -xe -- Unit vdsm-network.service has failed. -- -- The result is failed. Mar 07 07:52:59 ovirt01 systemd[1]: Dependency failed for Virtual Desktop Server Manager. -- Subject: Unit vdsmd.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit vdsmd.service has failed. -- -- The result is dependency. Mar 07 07:52:59 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20737:13641695 (system bus name :1 Mar 07 07:52:59 ovirt01 systemd[1]: Dependency failed for MOM instance configured for VDSM purposes. -- Subject: Unit mom-vdsm.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit mom-vdsm.service has failed. -- -- The result is dependency. Mar 07 07:52:59 ovirt01 systemd[1]: Job mom-vdsm.service/start failed with result 'dependency'. Mar 07 07:52:59 ovirt01 systemd[1]: Job vdsmd.service/start failed with result 'dependency'. Mar 07 07:52:59 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state. Mar 07 07:52:59 ovirt01 systemd[1]: vdsm-network.service failed.
--
[root@ovirt01 log]# systemctl start mom-vdsm A dependency job for mom-vdsm.service failed. See 'journalctl -xe' for details. [root@ovirt01 log]# journalctl -xe -- Unit vdsm-network.service has failed. -- -- The result is failed. Mar 07 07:57:34 ovirt01 systemd[1]: Dependency failed for Virtual Desktop Server Manager. -- Subject: Unit vdsmd.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit vdsmd.service has failed. -- -- The result is dependency. Mar 07 07:57:34 ovirt01 systemd[1]: Dependency failed for MOM instance configured for VDSM purposes. -- Subject: Unit mom-vdsm.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit mom-vdsm.service has failed. -- -- The result is dependency. Mar 07 07:57:34 ovirt01 systemd[1]: Job mom-vdsm.service/start failed with result 'dependency'. Mar 07 07:57:34 ovirt01 systemd[1]: Job vdsmd.service/start failed with result 'dependency'. Mar 07 07:57:34 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state. Mar 07 07:57:34 ovirt01 systemd[1]: vdsm-network.service failed. Mar 07 07:57:34 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20832:13669245 (system bus name :1
--
[root@ovirt01 log]# systemctl start vdsm-network Job for vdsm-network.service failed because the control process exited with error code. See "systemctl status vdsm-network.service" and "journalctl -xe" for details. [root@ovirt01 log]# journalctl -xe Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: self._raiseerror(v) Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: raise err Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13 Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: Traceback (most recent call last): Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/bin/vdsm-tool", line 219, in main Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: return tool_command[cmd]["command"](*args) Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_com Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: exec_restore(cmd) Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restor Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: raise EnvironmentError('Failed to restore the persisted networks') Mar 07 07:58:32 ovirt01 vdsm-tool[20871]: EnvironmentError: Failed to restore the persisted networks Mar 07 07:58:32 ovirt01 systemd[1]: vdsm-network.service: main process exited, code=exited, status=1/FAILURE Mar 07 07:58:32 ovirt01 systemd[1]: Failed to start Virtual Desktop Server Manager network restoration. -- Subject: Unit vdsm-network.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit vdsm-network.service has failed. -- -- The result is failed. Mar 07 07:58:32 ovirt01 systemd[1]: Unit vdsm-network.service entered failed state. Mar 07 07:58:32 ovirt01 systemd[1]: vdsm-network.service failed. Mar 07 07:58:32 ovirt01 polkitd[2395]: Unregistered Authentication Agent for unix-process:20860:13674993 (system bus name :1.115, o
--
[root@ovirt01 log]# vdsm-tool restore-nets Traceback (most recent call last): File "/usr/share/vdsm/vdsm-restore-net-config", line 429, in <module> restore(args) File "/usr/share/vdsm/vdsm-restore-net-config", line 387, in restore _restore_sriov_numvfs() File "/usr/share/vdsm/vdsm-restore-net-config", line 82, in _restore_sriov_numvfs sriov_devices = _get_sriov_devices() File "/usr/share/vdsm/vdsm-restore-net-config", line 54, in _get_sriov_devices devices = hostdev.list_by_caps() File "/usr/share/vdsm/hostdev.py", line 175, in list_by_caps libvirt_devices = _get_devices_from_libvirt() File "/usr/share/vdsm/hostdev.py", line 160, in _get_devices_from_libvirt for device in libvirtconnection.get().listAllDevices(0)) File "/usr/share/vdsm/hostdev.py", line 160, in <genexpr> for device in libvirtconnection.get().listAllDevices(0)) File "/usr/share/vdsm/hostdev.py", line 107, in _parse_device_params devXML = etree.fromstring(device_xml) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1300, in XML parser.feed(text) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1642, in feed self._raiseerror(v) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror raise err xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13 Traceback (most recent call last): File "/bin/vdsm-tool", line 219, in main return tool_command[cmd]["command"](*args) File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_command exec_restore(cmd) File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restore raise EnvironmentError('Failed to restore the persisted networks') EnvironmentError: Failed to restore the persisted networks
On Mon, Mar 7, 2016 at 2:56 AM, Yedidyah Bar David <didi@redhat.com> wrote:
On Sun, Mar 6, 2016 at 7:21 PM, Jonathan Sherman <haviland@gmail.com> wrote:
Didi,
NetworkManager is off, via "systemctl disable NetworkManager && systemctl stop NetworkManager" (and rebooted after), and network allowing remote ssh access and yum commands functional.
Log files with sizes >0 from /var/log/vdsm: https://www.dropbox.com/s/iobee7toeyc2jda/supervdsm.log?dl=0 https://www.dropbox.com/s/rdxi4ujwqhmekgh/upgrade.log?dl=0
So this means you also have an empty (size 0) vdsm.log? Can you please 'ls -la /var/log/vdsm'?
Can you check selinux/audit logs?
Most recent log file from /var/log/ovirt-hosted-engine-setup:
https://www.dropbox.com/s/brbbujmqhhbsx4o/ovirt-hosted-engine-setup-20160306...
This has:
2016-03-06 10:41:43 DEBUG otopi.plugins.otopi.services.systemd plugin.execute:941 execute-output: ('/bin/systemctl', 'start', 'vdsmd.service') stderr: A dependency job for vdsmd.service failed. See 'journalctl -xe' for details.
Can you please check/post output of 'journalctl -xe'?
Thanks,
Thank you for your time! -js
On Sun, Mar 6, 2016 at 5:23 AM, Yedidyah Bar David <didi@redhat.com> wrote:
On Sun, Mar 6, 2016 at 2:50 AM, Jonathan Sherman <haviland@gmail.com> wrote:
I want to extend my lab and am trying an install using the hosted engine, but the install is failing for me:
References used for install: https://glennsnead.wordpress.com/2016/02/28/ovirt-3-6-installation/
http://community.redhat.com/blog/2014/10/up-and-running-with-ovirt-3-5/
Setup: Minimal install of CentOS Linux release 7.2.1511 (Core) Using DHCP with static address (tried without DHCP with same issue) yum upgrade
Installed packages: yum localinstall -y http://resources.ovirt.org/pub/yum-repo/ovirt-release36.rpm yum install -y ovirt-hosted-engine-setup screen glusterfs-server nfs-utils vdsm-gluster system-storage-manager mailx
rebooted
Tried hosted-engine --deploy, but it errors out. I followed various logs and I _think_ it's failing related to vdsm-tool restore-nets. I could use some guidance on this to help track down what's happening and how to fix.
(Note: I'd very much prefer using the hosted engine vs. another option for my lab, so I'm willing to put in the time to figure this out. Thanks for any assistance!)
Install error: [root@ovirt01 network-scripts]# hosted-engine --deploy [ INFO ] Stage: Initializing [ INFO ] Generating a temporary VNC password. [ INFO ] Stage: Environment setup Continuing will configure this host for serving as hypervisor and create a VM where you have to install the engine afterwards. Are you sure you want to continue? (Yes, No)[Yes]: Configuration files: [] Log file:
/var/log/ovirt-hosted-engine-setup/ovirt-hosted-engine-setup-20160305194655-o29y86.log Version: otopi-1.4.1 (otopi-1.4.1-1.el7.centos) [ INFO ] Hardware supports virtualization [ INFO ] Stage: Environment packages setup [ INFO ] Stage: Programs detection [ INFO ] Stage: Environment setup [ ERROR ] Failed to execute stage 'Environment setup': Failed to start service 'vdsmd' [ INFO ] Stage: Clean up [ INFO ] Generating answer file
'/var/lib/ovirt-hosted-engine-setup/answers/answers-20160305194703.conf' [ INFO ] Stage: Pre-termination [ INFO ] Stage: Termination [ ERROR ] Hosted Engine deployment failed: this system is not reliable, please check the issue, fix and redeploy Log file is located at
/var/log/ovirt-hosted-engine-setup/ovirt-hosted-engine-setup-20160305194655-o29y86.log
-- vdsm-tool restore-nets output:
[root@ovirt01 etc]# /usr/bin/vdsm-tool restore-nets Traceback (most recent call last): File "/usr/share/vdsm/vdsm-restore-net-config", line 429, in <module> restore(args) File "/usr/share/vdsm/vdsm-restore-net-config", line 387, in restore _restore_sriov_numvfs() File "/usr/share/vdsm/vdsm-restore-net-config", line 82, in _restore_sriov_numvfs sriov_devices = _get_sriov_devices() File "/usr/share/vdsm/vdsm-restore-net-config", line 54, in _get_sriov_devices devices = hostdev.list_by_caps() File "/usr/share/vdsm/hostdev.py", line 175, in list_by_caps libvirt_devices = _get_devices_from_libvirt() File "/usr/share/vdsm/hostdev.py", line 160, in _get_devices_from_libvirt for device in libvirtconnection.get().listAllDevices(0)) File "/usr/share/vdsm/hostdev.py", line 160, in <genexpr> for device in libvirtconnection.get().listAllDevices(0)) File "/usr/share/vdsm/hostdev.py", line 107, in _parse_device_params devXML = etree.fromstring(device_xml) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1300, in XML parser.feed(text) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1642, in feed self._raiseerror(v) File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror raise err xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4, column 13 Traceback (most recent call last): File "/usr/bin/vdsm-tool", line 219, in main return tool_command[cmd]["command"](*args) File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 41, in restore_command exec_restore(cmd) File "/usr/lib/python2.7/site-packages/vdsm/tool/restore_nets.py", line 54, in exec_restore raise EnvironmentError('Failed to restore the persisted networks') EnvironmentError: Failed to restore the persisted networks
Please check/post full ovirt-hosted-engine-setup logs, vdsm logs.
Do you have NetworkManager?
Thanks for the report!
Best, -- Didi
-- Didi
-- Didi