Having trouble setting up Ovirt
I am trying to setup Ovirt with a self hosted engine and NFS storage for said engine. The storage appears to mounting OK, but when it gets to the point that it is initializing the lockspace it fails spectacularly and shows a Python traceback which I cleaned up and have included below: Traceback (most recent call last): File "/usr/lib64/python2.7/runpy.py", line 162, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/lib64/python2.7/runpy.py", line 72, in _run_code exec code in run_globals File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_setup/reinitialize_lockspace.py", line 30, in <module> ha_cli.reset_lockspace(force) File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/client/client.py", line 270, in reset_lockspace stats = broker.get_stats_from_storage() File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/brokerlink.py", line 135, in get_stats_from_storage result = self._proxy.get_stats() File "/usr/lib64/python2.7/xmlrpclib.py", line 1233, in __call__ return self.__send(self.__name, args) File "/usr/lib64/python2.7/xmlrpclib.py", line 1587, in __request verbose=self.__verbose File "/usr/lib64/python2.7/xmlrpclib.py", line 1273, in request return self.single_request(host, handler, request_body, verbose) File "/usr/lib64/python2.7/xmlrpclib.py", line 1301, in single_request self.send_content(h, request_body) File "/usr/lib64/python2.7/xmlrpclib.py", line 1448, in send_content connection.endheaders(request_body) File "/usr/lib64/python2.7/httplib.py", line 1013, in endheaders self._send_output(message_body) File "/usr/lib64/python2.7/httplib.py", line 864, in _send_output self.send(msg) File "/usr/lib64/python2.7/httplib.py", line 826, in send self.connect() File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/unixrpc.py", line 52, in connect self.sock.connect(base64.b16decode(self.host)) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory The messier ansible output is below: [ INFO ] TASK [Initialize lockspace volume] [ ERROR ] fatal: [localhost]: FAILED! => {"attempts": 5, "changed": true, "cmd": ["hosted-engine", "--reinitialize-lockspace", "--force"], " delta": "0:00:01.007879", "end": "2018-03-05 14:03:00.474295", "msg": "non-zero return code", "rc": 1, "start": "2018-03-05 14:02:59.466416" , "stderr": "Traceback (most recent call last):\n File \"/usr/lib64/python2.7/runpy.py\", line 162, in _run_module_as_main\n \"__main__\ ", fname, loader, pkg_name)\n File \"/usr/lib64/python2.7/runpy.py\", line 72, in _run_code\n exec code in run_globals\n File \"/usr/li b/python2.7/site-packages/ovirt_hosted_engine_setup/reinitialize_lockspace.py\", line 30, in <module>\n ha_cli.reset_lockspace(force)\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/client/client.py\", line 270, in reset_lockspace\n stats = broker.get_stat s_from_storage()\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/brokerlink.py\", line 135, in get_stats_from_storage\ n result = self._proxy.get_stats()\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1233, in __call__\n return self.__send(self.__n ame, args)\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1587, in __request\n verbose=self.__verbose\n File \"/usr/lib64/python2.7 /xmlrpclib.py\", line 1273, in request\n return self.single_request(host, handler, request_body, verbose)\n File \"/usr/lib64/python2.7/ xmlrpclib.py\", line 1301, in single_request\n self.send_content(h, request_body)\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 144 8, in send_content\n connection.endheaders(request_body)\n File \"/usr/lib64/python2.7/httplib.py\", line 1013, in endheaders\n self. _send_output(message_body)\n File \"/usr/lib64/python2.7/httplib.py\", line 864, in _send_output\n self.send(msg)\n File \"/usr/lib64/p ython2.7/httplib.py\", line 826, in send\n self.connect()\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/unixrpc.p y\", line 52, in connect\n self.sock.connect(base64.b16decode(self.host))\n File \"/usr/lib64/python2.7/socket.py\", line 224, in meth\n return getattr(self._sock,name)(*args)\nsocket.error: [Errno 2] No such file or directory", "stderr_lines": ["Traceback (most recent cal l last):", " File \"/usr/lib64/python2.7/runpy.py\", line 162, in _run_module_as_main", " \"__main__\", fname, loader, pkg_name)", " Fi le \"/usr/lib64/python2.7/runpy.py\", line 72, in _run_code", " exec code in run_globals", " File \"/usr/lib/python2.7/site-packages/ovi rt_hosted_engine_setup/reinitialize_lockspace.py\", line 30, in <module>", " ha_cli.reset_lockspace(force)", " File \"/usr/lib/python2.7 /site-packages/ovirt_hosted_engine_ha/client/client.py\", line 270, in reset_lockspace", " stats = broker.get_stats_from_storage()", " F ile \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/brokerlink.py\", line 135, in get_stats_from_storage", " result = self. _proxy.get_stats()", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1233, in __call__", " return self.__send(self.__name, args)", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1587, in __request", " verbose=self.__verbose", " File \"/usr/lib64/python2.7/xmlrpcli b.py\", line 1273, in request", " return self.single_request(host, handler, request_body, verbose)", " File \"/usr/lib64/python2.7/xmlrp clib.py\", line 1301, in single_request", " self.send_content(h, request_body)", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1448 , in send_content", " connection.endheaders(request_body)", " File \"/usr/lib64/python2.7/httplib.py\", line 1013, in endheaders", " self._send_output(message_body)", " File \"/usr/lib64/python2.7/httplib.py\", line 864, in _send_output", " self.send(msg)", " File \"/ usr/lib64/python2.7/httplib.py\", line 826, in send", " self.connect()", " File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_h a/lib/unixrpc.py\", line 52, in connect", " self.sock.connect(base64.b16decode(self.host))", " File \"/usr/lib64/python2.7/socket.py\", line 224, in meth", " return getattr(self._sock,name)(*args)", "socket.error: [Errno 2] No such file or directory"], "stdout": "", "stdou t_lines": []} [ ERROR ] Failed to execute stage 'Closing up': Failed executing ansible-playbook I am a bit lost on how to proceed as I haven't implemented a self-hosted engine before and I haven't found anything that helps online so far. Thanks in advance for any assistance. -- Kristian Petersen System Administrator Dept. of Chemistry and Biochemistry
On Tue, Mar 6, 2018 at 6:42 PM, Kristian Petersen <nesretep@chem.byu.edu> wrote:
I am trying to setup Ovirt with a self hosted engine and NFS storage for said engine. The storage appears to mounting OK, but when it gets to the point that it is initializing the lockspace it fails spectacularly and shows a Python traceback which I cleaned up and have included below:
Traceback (most recent call last): File "/usr/lib64/python2.7/runpy.py", line 162, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/lib64/python2.7/runpy.py", line 72, in _run_code exec code in run_globals File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_setup/reinitialize_lockspace.py", line 30, in <module> ha_cli.reset_lockspace(force) File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/client/client.py", line 270, in reset_lockspace stats = broker.get_stats_from_storage() File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/brokerlink.py", line 135, in get_stats_from_storage result = self._proxy.get_stats() File "/usr/lib64/python2.7/xmlrpclib.py", line 1233, in __call__ return self.__send(self.__name, args) File "/usr/lib64/python2.7/xmlrpclib.py", line 1587, in __request verbose=self.__verbose File "/usr/lib64/python2.7/xmlrpclib.py", line 1273, in request return self.single_request(host, handler, request_body, verbose) File "/usr/lib64/python2.7/xmlrpclib.py", line 1301, in single_request self.send_content(h, request_body) File "/usr/lib64/python2.7/xmlrpclib.py", line 1448, in send_content connection.endheaders(request_body) File "/usr/lib64/python2.7/httplib.py", line 1013, in endheaders self._send_output(message_body) File "/usr/lib64/python2.7/httplib.py", line 864, in _send_output self.send(msg) File "/usr/lib64/python2.7/httplib.py", line 826, in send self.connect() File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/unixrpc.py", line 52, in connect self.sock.connect(base64.b16decode(self.host)) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory
The messier ansible output is below: [ INFO ] TASK [Initialize lockspace volume] [ ERROR ] fatal: [localhost]: FAILED! => {"attempts": 5, "changed": true, "cmd": ["hosted-engine", "--reinitialize-lockspace", "--force"], " delta": "0:00:01.007879", "end": "2018-03-05 14:03:00.474295", "msg": "non-zero return code", "rc": 1, "start": "2018-03-05 14:02:59.466416" , "stderr": "Traceback (most recent call last):\n File \"/usr/lib64/python2.7/runpy.py\", line 162, in _run_module_as_main\n \"__main__\ ", fname, loader, pkg_name)\n File \"/usr/lib64/python2.7/runpy.py\", line 72, in _run_code\n exec code in run_globals\n File \"/usr/li b/python2.7/site-packages/ovirt_hosted_engine_setup/reinitialize_lockspace.py\", line 30, in <module>\n ha_cli.reset_lockspace(force)\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/client/client.py\", line 270, in reset_lockspace\n stats = broker.get_stat s_from_storage()\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/brokerlink.py\", line 135, in get_stats_from_storage\ n result = self._proxy.get_stats()\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1233, in __call__\n return self.__send(self.__n ame, args)\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1587, in __request\n verbose=self.__verbose\n File \"/usr/lib64/python2.7 /xmlrpclib.py\", line 1273, in request\n return self.single_request(host, handler, request_body, verbose)\n File \"/usr/lib64/python2.7/ xmlrpclib.py\", line 1301, in single_request\n self.send_content(h, request_body)\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 144 8, in send_content\n connection.endheaders(request_body)\n File \"/usr/lib64/python2.7/httplib.py\", line 1013, in endheaders\n self. _send_output(message_body)\n File \"/usr/lib64/python2.7/httplib.py\", line 864, in _send_output\n self.send(msg)\n File \"/usr/lib64/p ython2.7/httplib.py\", line 826, in send\n self.connect()\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/unixrpc.p y\", line 52, in connect\n self.sock.connect(base64.b16decode(self.host))\n File \"/usr/lib64/python2.7/socket.py\", line 224, in meth\n return getattr(self._sock,name)(*args)\nsocket.error: [Errno 2] No such file or directory", "stderr_lines": ["Traceback (most recent cal l last):", " File \"/usr/lib64/python2.7/runpy.py\", line 162, in _run_module_as_main", " \"__main__\", fname, loader, pkg_name)", " Fi le \"/usr/lib64/python2.7/runpy.py\", line 72, in _run_code", " exec code in run_globals", " File \"/usr/lib/python2.7/site-packages/ovi rt_hosted_engine_setup/reinitialize_lockspace.py\", line 30, in <module>", " ha_cli.reset_lockspace(force)", " File \"/usr/lib/python2.7 /site-packages/ovirt_hosted_engine_ha/client/client.py\", line 270, in reset_lockspace", " stats = broker.get_stats_from_storage()", " F ile \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/brokerlink.py\", line 135, in get_stats_from_storage", " result = self. _proxy.get_stats()", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1233, in __call__", " return self.__send(self.__name, args)", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1587, in __request", " verbose=self.__verbose", " File \"/usr/lib64/python2.7/xmlrpcli b.py\", line 1273, in request", " return self.single_request(host, handler, request_body, verbose)", " File \"/usr/lib64/python2.7/xmlrp clib.py\", line 1301, in single_request", " self.send_content(h, request_body)", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1448 , in send_content", " connection.endheaders(request_body)", " File \"/usr/lib64/python2.7/httplib.py\", line 1013, in endheaders", " self._send_output(message_body)", " File \"/usr/lib64/python2.7/httplib.py\", line 864, in _send_output", " self.send(msg)", " File \"/ usr/lib64/python2.7/httplib.py\", line 826, in send", " self.connect()", " File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_h a/lib/unixrpc.py\", line 52, in connect", " self.sock.connect(base64.b16decode(self.host))", " File \"/usr/lib64/python2.7/socket.py\", line 224, in meth", " return getattr(self._sock,name)(*args)", "socket.error: [Errno 2] No such file or directory"], "stdout": "", "stdou t_lines": []} [ ERROR ] Failed to execute stage 'Closing up': Failed executing ansible-playbook
Adding Simone. Can you please check/share these logs: /var/log/ovirt-hosted-engine-setup/* /var/log/ovirt-hosted-engine-ha/* /var/log/vdsm/*
I am a bit lost on how to proceed as I haven't implemented a self-hosted engine before and I haven't found anything that helps online so far. Thanks in advance for any assistance.
The new ansible-based setup was enabled by default only very recently, so there is not much experience with it, nor can you find much online. You can try the old behavior by running: hosted-engine --deploy --noansible However, if the failure is due to some misconfiguration on the storage, such as wrong permissions or whatever, this won't help much. Best regards, -- Didi
The problem changed a little since I initially posted this to the list. It is getting past the lockspace task now. I posted again with the new situation but did it separate from this. Simone has responded to that one and if you want I can add you to that loop. Using the --noansible flag seems to cause it to fail even sooner. It fails almost right off the bat, which is interesting to say the least. On Mon, Mar 12, 2018 at 6:39 AM, Yedidyah Bar David <didi@redhat.com> wrote:
I am trying to setup Ovirt with a self hosted engine and NFS storage for said engine. The storage appears to mounting OK, but when it gets to the point that it is initializing the lockspace it fails spectacularly and shows a Python traceback which I cleaned up and have included below:
Traceback (most recent call last): File "/usr/lib64/python2.7/runpy.py", line 162, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/lib64/python2.7/runpy.py", line 72, in _run_code exec code in run_globals File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ setup/reinitialize_lockspace.py", line 30, in <module> ha_cli.reset_lockspace(force) File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ ha/client/client.py", line 270, in reset_lockspace stats = broker.get_stats_from_storage() File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ ha/lib/brokerlink.py", line 135, in get_stats_from_storage result = self._proxy.get_stats() File "/usr/lib64/python2.7/xmlrpclib.py", line 1233, in __call__ return self.__send(self.__name, args) File "/usr/lib64/python2.7/xmlrpclib.py", line 1587, in __request verbose=self.__verbose File "/usr/lib64/python2.7/xmlrpclib.py", line 1273, in request return self.single_request(host, handler, request_body, verbose) File "/usr/lib64/python2.7/xmlrpclib.py", line 1301, in single_request self.send_content(h, request_body) File "/usr/lib64/python2.7/xmlrpclib.py", line 1448, in send_content connection.endheaders(request_body) File "/usr/lib64/python2.7/httplib.py", line 1013, in endheaders self._send_output(message_body) File "/usr/lib64/python2.7/httplib.py", line 864, in _send_output self.send(msg) File "/usr/lib64/python2.7/httplib.py", line 826, in send self.connect() File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ ha/lib/unixrpc.py", line 52, in connect self.sock.connect(base64.b16decode(self.host)) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory
The messier ansible output is below: [ INFO ] TASK [Initialize lockspace volume] [ ERROR ] fatal: [localhost]: FAILED! => {"attempts": 5, "changed": true, "cmd": ["hosted-engine", "--reinitialize-lockspace", "--force"], " delta": "0:00:01.007879", "end": "2018-03-05 14:03:00.474295", "msg": "non-zero return code", "rc": 1, "start": "2018-03-05 14:02:59.466416" , "stderr": "Traceback (most recent call last):\n File \"/usr/lib64/python2.7/runpy.py\", line 162, in _run_module_as_main\n \"__main__\ ", fname, loader, pkg_name)\n File \"/usr/lib64/python2.7/runpy.py\",
72, in _run_code\n exec code in run_globals\n File \"/usr/li b/python2.7/site-packages/ovirt_hosted_engine_setup/ reinitialize_lockspace.py\", line 30, in <module>\n ha_cli.reset_lockspace(force)\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ ha/client/client.py\", line 270, in reset_lockspace\n stats = broker.get_stat s_from_storage()\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ ha/lib/brokerlink.py\", line 135, in get_stats_from_storage\ n result = self._proxy.get_stats()\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1233, in __call__\n return self.__send(self.__n ame, args)\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1587, in __request\n verbose=self.__verbose\n File \"/usr/lib64/python2.7 /xmlrpclib.py\", line 1273, in request\n return self.single_request(host, handler, request_body, verbose)\n File \"/usr/lib64/python2.7/ xmlrpclib.py\", line 1301, in single_request\n self.send_content(h, request_body)\n File \"/usr/lib64/python2.7/xmlrpclib.py\", line 144 8, in send_content\n connection.endheaders(request_body)\n File \"/usr/lib64/python2.7/httplib.py\", line 1013, in endheaders\n self. _send_output(message_body)\n File \"/usr/lib64/python2.7/httplib.py\",
On Tue, Mar 6, 2018 at 6:42 PM, Kristian Petersen <nesretep@chem.byu.edu> wrote: line line
864, in _send_output\n self.send(msg)\n File \"/usr/lib64/p ython2.7/httplib.py\", line 826, in send\n self.connect()\n File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/unixrpc.p y\", line 52, in connect\n self.sock.connect(base64.b16decode(self.host))\n File \"/usr/lib64/python2.7/socket.py\", line 224, in meth\n return getattr(self._sock,name)(*args)\nsocket.error: [Errno 2] No such file or directory", "stderr_lines": ["Traceback (most recent cal l last):", " File \"/usr/lib64/python2.7/runpy.py\", line 162, in _run_module_as_main", " \"__main__\", fname, loader, pkg_name)", " Fi le \"/usr/lib64/python2.7/runpy.py\", line 72, in _run_code", " exec code in run_globals", " File \"/usr/lib/python2.7/site-packages/ovi rt_hosted_engine_setup/reinitialize_lockspace.py\", line 30, in <module>", " ha_cli.reset_lockspace(force)", " File \"/usr/lib/python2.7 /site-packages/ovirt_hosted_engine_ha/client/client.py\", line 270, in reset_lockspace", " stats = broker.get_stats_from_storage()", " F ile \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ ha/lib/brokerlink.py\", line 135, in get_stats_from_storage", " result = self. _proxy.get_stats()", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1233, in __call__", " return self.__send(self.__name, args)", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1587, in __request", " verbose=self.__verbose", " File \"/usr/lib64/python2.7/xmlrpcli b.py\", line 1273, in request", " return self.single_request(host, handler, request_body, verbose)", " File \"/usr/lib64/python2.7/xmlrp clib.py\", line 1301, in single_request", " self.send_content(h, request_body)", " File \"/usr/lib64/python2.7/xmlrpclib.py\", line 1448 , in send_content", " connection.endheaders(request_body)", " File \"/usr/lib64/python2.7/httplib.py\", line 1013, in endheaders", " self._send_output(message_body)", " File \"/usr/lib64/python2.7/httplib.py\", line 864, in _send_output", " self.send(msg)", " File \"/ usr/lib64/python2.7/httplib.py\", line 826, in send", " self.connect()", " File \"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_h a/lib/unixrpc.py\", line 52, in connect", " self.sock.connect(base64.b16decode(self.host))", " File \"/usr/lib64/python2.7/socket.py\", line 224, in meth", " return getattr(self._sock,name)(*args)", "socket.error: [Errno 2] No such file or directory"], "stdout": "", "stdou t_lines": []} [ ERROR ] Failed to execute stage 'Closing up': Failed executing ansible-playbook
Adding Simone.
Can you please check/share these logs:
/var/log/ovirt-hosted-engine-setup/* /var/log/ovirt-hosted-engine-ha/* /var/log/vdsm/*
I am a bit lost on how to proceed as I haven't implemented a self-hosted engine before and I haven't found anything that helps online so far.
Thanks
in advance for any assistance.
The new ansible-based setup was enabled by default only very recently, so there is not much experience with it, nor can you find much online.
You can try the old behavior by running:
hosted-engine --deploy --noansible
However, if the failure is due to some misconfiguration on the storage, such as wrong permissions or whatever, this won't help much.
Best regards, -- Didi
-- Kristian Petersen System Administrator BYU Dept. of Chemistry and Biochemistry
participants (2)
-
Kristian Petersen -
Yedidyah Bar David