Seen this unrelated failure today on: https://gerrit.ovirt.org/#/c/107206/ Build: https://jenkins.ovirt.org/job/vdsm_standard-check-patch/18831//artifact/c... This smells like the known issue with handling connections. I think the test was disabled in the past, and it started to fail since we enabled the stomp tests recently. If you see the same failures, lets mark this test as xfail until we have time to debug this. See failure bellow. =================================== FAILURES =================================== ______________________ VdsmClientTests.test_event_handler ______________________ self = <stomprpcclient_test.VdsmClientTests testMethod=test_event_handler> def test_event_handler(self): with self._create_client() as client: event_queue = queue.Queue() sub_id = client.subscribe(EVENT_TOPIC, event_queue) lib/yajsonrpc/stomprpcclient_test.py:215: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <stomprpcclient_test._MockedClient object at 0x7f87ab72d160> namespace = 'Test', method_name = 'sendEvent', kwargs = {} method = 'Test.sendEvent', timeout = 3 req = {'jsonrpc': '2.0', 'method': 'Test.sendEvent', 'params': {}, 'id': 'aa4919ae-558f-4403-aab8-f64f0d7cd2ff'} responses = None def _call(self, namespace, method_name, **kwargs): """ Client call method, executes a given command Args: namespace (string): namespace name method_name (string): method name **kwargs: Arbitrary keyword arguments Returns: method result Raises: ClientError: in case of an error in the protocol. TimeoutError: if there is no response after a pre configured time. ServerError: in case of an error while executing the command """ method = namespace + "." + method_name timeout = kwargs.pop("_timeout", self._default_timeout) req = yajsonrpc.JsonRpcRequest( method, kwargs, reqId=str(uuid.uuid4())) try: responses = self._client.call( req, timeout=timeout, flow_id=self._flow_id) except EnvironmentError as e: raise ClientError(method, kwargs, e) if not responses: E vdsm.client.TimeoutError: Request Test.sendEvent with args {} timed out after 3 seconds ../lib/vdsm/client.py:294: TimeoutError ------------------------------ Captured log call ------------------------------- WARNING vds.dispatcher:betterAsyncore.py:179 unhandled read event WARNING vds.dispatcher:betterAsyncore.py:179 unhandled write event ERROR vds.dispatcher:betterAsyncore.py:179 uncaptured python exception, closing channel <yajsonrpc.betterAsyncore.Dispatcher ('::1', 33316, 0, 0) at 0x7f87ab72d128> (<class 'ValueError'>:'b'ept-version:1.2'' contains illegal character ':' [/usr/lib64/python3.6/asyncore.py|readwrite|108] [/usr/lib64/python3.6/asyncore.py|handle_read_event|423] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|handle_read|71] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|_delegate_call|168] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/protocoldetector.py|handle_read|129] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|handle_socket|413] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/rpc/bindingjsonrpc.py|add_socket|54] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|createListener|379] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|StompListener|345] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|__init__|47] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|switch_implementation|86] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py|init|363] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/rpc/bindingjsonrpc.py|_onAccept|57] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|set_message_handler|645] [/usr/lib64/python3.6/asyncore.py|handle_read_event|423] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|handle_read|71] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py|_delegate_call|168] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|handle_read|421] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|parse|323] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|_parse_command|245] [/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py|decode_value|167]) WARNING vds.dispatcher:betterAsyncore.py:179 unhandled close event ERROR root:concurrent.py:267 FINISH thread <Thread(JsonRpc (StompReactor), started daemon 140220662679296)> failed Traceback (most recent call last): File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/common/concurrent.py", line 260, in run ret = func(*args, **kwargs) File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompserver.py", line 393, in process_requests self._reactor.process_requests() File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 238, in process_requests timeout=self._get_timeout(self._map), File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 253, in _get_timeout interval = disp.next_check_interval() File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 99, in next_check_interval return getattr(self.__impl, "next_check_interval", default_func)() File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py", line 486, in next_check_interval self.handle_timeout() File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py", line 451, in handle_timeout self._frame_handler.handle_timeout(self) File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stompclient.py", line 130, in handle_timeout dispatcher._on_timeout) File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/stomp.py", line 630, in reconnect AsyncDispatcher(self, self._async_client, count=count)) File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 271, in reconnect dispatcher.create_socket(address, sslctx) File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/yajsonrpc/betterAsyncore.py", line 112, in create_socket sock = sslctx.wrapSocket(sock) File "/home/jenkins/workspace/vdsm_standard-check-patch/vdsm/lib/vdsm/sslutils.py", line 113, in wrapSocket ca_certs=self.ca_certs) File "/usr/lib64/python3.6/ssl.py", line 1114, in wrap_socket ciphers=ciphers) File "/usr/lib64/python3.6/ssl.py", line 704, in __init__ self._context.load_verify_locations(ca_certs) FileNotFoundError: [Errno 2] No such file or directory