Marcin, could you please investigate?


On Fri, 6 Dec 2019, 19:50 Milan Zamazal, <mzamazal@redhat.com> wrote:
[Sorry, sent an unfinished mail by mistake.]

Milan Zamazal <mzamazal@redhat.com> writes:

> Hi, this seems to be a frequent error in Jenkins run recently:
>
>
> =================================== FAILURES ===================================
> ______________________ VdsmClientTests.test_failing_call _______________________
>
> self = <stomprpcclient_test.VdsmClientTests testMethod=test_failing_call>
>
>     def test_failing_call(self):
>         with self._create_client() as client:
>             with self.assertRaises(ServerError) as ex:
>>               client.Test.failingCall()
>
> lib/yajsonrpc/stomprpcclient_test.py:144:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
>
>     def _call(self, namespace, method_name, **kwargs):
>         """
>         Client call method, executes a given command
>     
>         Args:
>             namespace (string): namespace name
>             method_name (string): method name
>             **kwargs: Arbitrary keyword arguments
>     
>         Returns:
>             method result
>     
>         Raises:
>             ClientError: in case of an error in the protocol.
>             TimeoutError: if there is no response after a pre configured time.
>             ServerError: in case of an error while executing the command
>         """
>         method = namespace + "." + method_name
>         timeout = kwargs.pop("_timeout", self._default_timeout)
>     
>         req = yajsonrpc.JsonRpcRequest(
>             method, kwargs, reqId=str(uuid.uuid4()))
>     
>         try:
>             responses = self._client.call(
>                 req, timeout=timeout, flow_id=self._flow_id)
>         except EnvironmentError as e:
>             raise ClientError(method, kwargs, e)
>     
>         if not responses:
>>           raise TimeoutError(method, kwargs, timeout)
> E           vdsm.client.TimeoutError: Request Test.failingCall with args {} timed out after 3 seconds
>
> ../lib/vdsm/client.py:294: TimeoutError
> ------------------------------ Captured log call -------------------------------
> ERROR    vds.dispatcher:betterAsyncore.py:179 uncaptured python exception, closing channel <yajsonrpc.betterAsyncore.Dispatcher ('::1', 47428, 0, 0) at 0x7f48ddc47d10> (<class 'ValueError'>:'b'ept-version:1.2'' contains illegal character ':' [/usr/lib64/python3.7/asyncore.py|readwrite|108] [/usr/lib64/python3.7/asyncore.py|handle_read_event|422] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|handle_read|71] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|_delegate_call|168] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/protocoldetector.py|handle_read|129] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|handle_socket|413] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/rpc/bindingjsonrpc.py|add_socket|54] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|createListener|379] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|StompListener|345] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|__init__|47] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|switch_implementation|86] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|init|363] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/rpc/bindingjsonrpc.py|_onAccept|57] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|set_message_handler|645] [/usr/lib64/python3.7/asyncore.py|handle_read_event|422] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|handle_read|71] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|_delegate_call|168] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|handle_read|421] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|parse|323] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|_parse_command|245] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|decode_value|167])
> WARNING  vds.dispatcher:betterAsyncore.py:179 unhandled close event
> ERROR    root:concurrent.py:267 FINISH thread <Thread(JsonRpc (StompReactor), started daemon 139950158767872)> failed
> Traceback (most recent call last):
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/common/concurrent.py", line 260, in run
>     ret = func(*args, **kwargs)
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py", line 393, in process_requests
>     self._reactor.process_requests()
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 238, in process_requests
>     timeout=self._get_timeout(self._map),
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 253, in _get_timeout
>     interval = disp.next_check_interval()
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 99, in next_check_interval
>     return getattr(self.__impl, "next_check_interval", default_func)()
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py", line 486, in next_check_interval
>     self.handle_timeout()
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py", line 451, in handle_timeout
>     self._frame_handler.handle_timeout(self)
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompclient.py", line 130, in handle_timeout
>     dispatcher._on_timeout)
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py", line 630, in reconnect
>     AsyncDispatcher(self, self._async_client, count=count))
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 271, in reconnect
>     dispatcher.create_socket(address, sslctx)
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 112, in create_socket
>     sock = sslctx.wrapSocket(sock)
>   File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/sslutils.py", line 113, in wrapSocket
>     ca_certs=self.ca_certs)
>   File "/usr/lib64/python3.7/ssl.py", line 1230, in wrap_socket
>     context.load_verify_locations(ca_certs)
> FileNotFoundError: [Errno 2] No such file or directory
> _____________________ VdsmClientTests.test_missing_method ______________________
>
> self = <stomprpcclient_test.VdsmClientTests testMethod=test_missing_method>
>
>     def test_missing_method(self):
>         with self._create_client() as client:
>             with self.assertRaises(ServerError) as ex:
>>               client.Test.missingMethod()
>
> lib/yajsonrpc/stomprpcclient_test.py:155:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
>
>     def _call(self, namespace, method_name, **kwargs):
>         """
>         Client call method, executes a given command
>     
>         Args:
>             namespace (string): namespace name
>             method_name (string): method name
>             **kwargs: Arbitrary keyword arguments
>     
>         Returns:
>             method result
>     
>         Raises:
>             ClientError: in case of an error in the protocol.
>             TimeoutError: if there is no response after a pre configured time.
>             ServerError: in case of an error while executing the command
>         """
>         method = namespace + "." + method_name
>         timeout = kwargs.pop("_timeout", self._default_timeout)
>     
>         req = yajsonrpc.JsonRpcRequest(
>             method, kwargs, reqId=str(uuid.uuid4()))
>     
>         try:
>             responses = self._client.call(
>                 req, timeout=timeout, flow_id=self._flow_id)
>         except EnvironmentError as e:
>             raise ClientError(method, kwargs, e)
>     
>         if not responses:
>>           raise TimeoutError(method, kwargs, timeout)
> E           vdsm.client.TimeoutError: Request Test.missingMethod with args {} timed out after 3 seconds
>
> ../lib/vdsm/client.py:294: TimeoutError

See e.g.
https://jenkins.ovirt.org/blue/organizations/jenkins/vdsm_standard-check-patch/detail/vdsm_standard-check-patch/15459/pipeline/139/

If we can't fix the CI, can we disable the test temporarily?
I experience very frequent Vdsm CI failures these days.

Thanks,
Milan
_______________________________________________
Devel mailing list -- devel@ovirt.org
To unsubscribe send an email to devel-leave@ovirt.org
Privacy Statement: https://www.ovirt.org/site/privacy-policy/
oVirt Code of Conduct: https://www.ovirt.org/community/about/community-guidelines/
List Archives: https://lists.ovirt.org/archives/list/devel@ovirt.org/message/BOM7KFS5BFSPPLMTU4H4X4ATYKBQANNM/