Project: http://jenkins.ovirt.org/job/ovirt-system-tests_hc-basic-suite-master/ Build: http://jenkins.ovirt.org/job/ovirt-system-tests_hc-basic-suite-master/614/ Build Number: 614 Build Status: Still Failing Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #613 [Gal Ben Haim] compat: Set "ex" bash flags Changes for Build #614 [Gal Ben Haim] compat: Set "ex" bash flags ----------------- Failed Tests: ----------------- 2 tests failed. FAILED: 002_bootstrap.add_hosts Error Message: Failed to read response: [(<pycurl.Curl object at 0x7f329a714da8>, 7, 'TCP connection reset by peer')] -------------------- >> begin captured logging << -------------------- ovirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f329a6afc08> Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within res = func() File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 253, in _host_is_up_4 host_obj = host_service.get() File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36676, in get return self._internal_get(headers, query, wait) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get return future.wait() if wait else future File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait response = self._connection.wait(self._context) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait return self.__wait(context, failed_auth) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait raise Error("Failed to read response: {}".format(err_list)) Error: Failed to read response: [(<pycurl.Curl object at 0x7f329a714da8>, 7, 'TCP connection reset by peer')] --------------------- >> end captured logging << --------------------- Stack Trace: File "/usr/lib64/python2.7/unittest/case.py", line 369, in run testMethod() File "/usr/lib/python2.7/site-packages/nose/case.py", line 197, in runTest self.test(*self.arg) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 129, in wrapped_test test() File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 59, in wrapper return func(get_test_prefix(), *args, **kwargs) File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 228, in add_hosts add_hosts_4(prefix) File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 275, in add_hosts_4 testlib.assert_true_within(_host_is_up_4, timeout=15*60) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 263, in assert_true_within assert_equals_within(func, True, timeout, allowed_exceptions) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within res = func() File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 253, in _host_is_up_4 host_obj = host_service.get() File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36676, in get return self._internal_get(headers, query, wait) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get return future.wait() if wait else future File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait response = self._connection.wait(self._context) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait return self.__wait(context, failed_auth) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait raise Error("Failed to read response: {}".format(err_list)) 'Failed to read response: [(<pycurl.Curl object at 0x7f329a714da8>, 7, \'TCP connection reset by peer\')]\n-------------------- >> begin captured logging << --------------------\novirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f329a6afc08>\nTraceback (most recent call last):\n File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within\n res = func()\n File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 253, in _host_is_up_4\n host_obj = host_service.get()\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36676, in get\n return self._internal_get(headers, query, wait)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get\n return future.wait() if wait else future\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait\n response = self._connection.wait(self._context)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait\n return self.__wait(context, failed_auth)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait\n raise Error("Failed to read response: {}".format(err_list))\nError: Failed to read response: [(<pycurl.Curl object at 0x7f329a714da8>, 7, \'TCP connection reset by peer\')]\n--------------------- >> end captured logging << ---------------------' FAILED: 002_bootstrap.add_hosts Error Message: Unable to extract paths from lago-hc-basic-suite-master-engine: unreachable with SSH -------------------- >> begin captured logging << -------------------- ovirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f329a6afc08> Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within res = func() File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 253, in _host_is_up_4 host_obj = host_service.get() File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36676, in get return self._internal_get(headers, query, wait) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get return future.wait() if wait else future File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait response = self._connection.wait(self._context) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait return self.__wait(context, failed_auth) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait raise Error("Failed to read response: {}".format(err_list)) Error: Failed to read response: [(<pycurl.Curl object at 0x7f329a714da8>, 7, 'TCP connection reset by peer')] lago.utils: DEBUG: Looking for a workdir lago.workdir: DEBUG: Checking if /dev/shm/ost/deployment-hc-basic-suite-master is a workdir lago.prefix: INFO: * Collect artifacts: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-master-host-1:/var/log to /dev/shm/ost/deployment-hc-basic-suite-master/default/test_logs/002_bootstrap.add_hosts-20180728224550/lago-hc-basic-suite-master-host-1/_var_log lago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-master-host-0:/var/log to /dev/shm/ost/deployment-hc-basic-suite-master/default/test_logs/002_bootstrap.add_hosts-20180728224550/lago-hc-basic-suite-master-host-0/_var_log lago.ssh: DEBUG: start task:ab2c89cd-4a87-48af-9956-2ed214a75afc:Get ssh client for lago-hc-basic-suite-master-host-1: lago.ssh: DEBUG: start task:d24c285a-a17a-40ca-a833-4b51bc4e9c50:Get ssh client for lago-hc-basic-suite-master-engine: lago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-master-host-2:/var/log to /dev/shm/ost/deployment-hc-basic-suite-master/default/test_logs/002_bootstrap.add_hosts-20180728224550/lago-hc-basic-suite-master-host-2/_var_log lago.ssh: DEBUG: start task:0915675c-6b43-477a-bc43-903a87b92f13:Get ssh client for lago-hc-basic-suite-master-host-0: lago.ssh: DEBUG: start task:da6b5dff-ce01-4baf-9bcb-c08ca0893a08:Get ssh client for lago-hc-basic-suite-master-host-2: lago.ssh: DEBUG: Socket error connecting to lago-hc-basic-suite-master-engine: [Errno None] Unable to connect to port 22 on 192.168.200.99 lago.ssh: DEBUG: Still got 0 tries for lago-hc-basic-suite-master-engine lago.ssh: DEBUG: end task:da6b5dff-ce01-4baf-9bcb-c08ca0893a08:Get ssh client for lago-hc-basic-suite-master-host-2: lago.ssh: DEBUG: end task:ab2c89cd-4a87-48af-9956-2ed214a75afc:Get ssh client for lago-hc-basic-suite-master-host-1: lago.ssh: DEBUG: end task:0915675c-6b43-477a-bc43-903a87b92f13:Get ssh client for lago-hc-basic-suite-master-host-0: lago.ssh: DEBUG: end task:d24c285a-a17a-40ca-a833-4b51bc4e9c50:Get ssh client for lago-hc-basic-suite-master-engine: root: ERROR: - [Thread-7] lago-hc-basic-suite-master-engine: ?[31mERROR?[0m (in 0:00:01) lago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) lago.utils: ERROR: Error while running thread Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue queue.put({'return': func()}) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) ExtractPathError: Unable to extract paths from lago-hc-basic-suite-master-engine: unreachable with SSH lago.prefix: INFO: ?[32mSuccess?[0m (in 0:00:01) lago.prefix: INFO: ?[32mSuccess?[0m (in 0:00:01) lago.prefix: INFO: ?[32mSuccess?[0m (in 0:00:04) root: ERROR: * Collect artifacts: ?[31mERROR?[0m (in 0:00:04) lago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 636, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1528, in collect_artifacts self.virt_env.get_vms().values(), File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in invoke_in_parallel return vt.join_all() File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue queue.put({'return': func()}) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) --------------------- >> end captured logging << --------------------- Stack Trace: File "/usr/lib/python2.7/site-packages/nose/case.py", line 133, in run self.runTest(result) File "/usr/lib/python2.7/site-packages/nose/case.py", line 151, in runTest test(result) File "/usr/lib64/python2.7/unittest/case.py", line 433, in __call__ return self.run(*args, **kwds) File "/usr/lib64/python2.7/unittest/case.py", line 393, in run result.addError(self, sys.exc_info()) File "/usr/lib/python2.7/site-packages/nose/proxy.py", line 131, in addError plugins.addError(self.test, err) File "/usr/lib/python2.7/site-packages/nose/plugins/manager.py", line 99, in __call__ return self.call(*arg, **kw) File "/usr/lib/python2.7/site-packages/nose/plugins/manager.py", line 167, in simple result = meth(*arg, **kw) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 150, in addError self._addFault(test, err) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 161, in _addFault no_skip=False File "/usr/lib/python2.7/site-packages/lago/plugins/cli.py", line 190, in __call__ return self._do_run(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/utils.py", line 505, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/utils.py", line 516, in wrapper return func(*args, prefix=prefix, **kwargs) File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 752, in do_collect prefix.collect_artifacts(output, ignore_nopath=not no_skip) File "/usr/lib/python2.7/site-packages/lago/sdk_utils.py", line 50, in wrapped return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 636, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1528, in collect_artifacts self.virt_env.get_vms().values(), File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in invoke_in_parallel return vt.join_all() File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue queue.put({'return': func()}) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) 'Unable to extract paths from lago-hc-basic-suite-master-engine: unreachable with SSH\n-------------------- >> begin captured logging << --------------------\novirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f329a6afc08>\nTraceback (most recent call last):\n File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within\n res = func()\n File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-master/ovirt-system-tests/hc-basic-suite-master/test-scenarios/002_bootstrap.py", line 253, in _host_is_up_4\n host_obj = host_service.get()\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36676, in get\n return self._internal_get(headers, query, wait)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get\n return future.wait() if wait else future\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait\n response = self._connection.wait(self._context)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait\n return self.__wait(context, failed_auth)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait\n raise Error("Failed to read response: {}".format(err_list))\nError: Failed to read response: [(<pycurl.Curl object at 0x7f329a714da8>, 7, \'TCP connection reset by peer\')]\nlago.utils: DEBUG: Looking for a workdir\nlago.workdir: DEBUG: Checking if /dev/shm/ost/deployment-hc-basic-suite-master is a workdir\nlago.prefix: INFO: * Collect artifacts: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-master-host-1:/var/log to /dev/shm/ost/deployment-hc-basic-suite-master/default/test_logs/002_bootstrap.add_hosts-20180728224550/lago-hc-basic-suite-master-host-1/_var_log\nlago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-master-host-0:/var/log to /dev/shm/ost/deployment-hc-basic-suite-master/default/test_logs/002_bootstrap.add_hosts-20180728224550/lago-hc-basic-suite-master-host-0/_var_log\nlago.ssh: DEBUG: start task:ab2c89cd-4a87-48af-9956-2ed214a75afc:Get ssh client for lago-hc-basic-suite-master-host-1:\nlago.ssh: DEBUG: start task:d24c285a-a17a-40ca-a833-4b51bc4e9c50:Get ssh client for lago-hc-basic-suite-master-engine:\nlago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-master-host-2:/var/log to /dev/shm/ost/deployment-hc-basic-suite-master/default/test_logs/002_bootstrap.add_hosts-20180728224550/lago-hc-basic-suite-master-host-2/_var_log\nlago.ssh: DEBUG: start task:0915675c-6b43-477a-bc43-903a87b92f13:Get ssh client for lago-hc-basic-suite-master-host-0:\nlago.ssh: DEBUG: start task:da6b5dff-ce01-4baf-9bcb-c08ca0893a08:Get ssh client for lago-hc-basic-suite-master-host-2:\nlago.ssh: DEBUG: Socket error connecting to lago-hc-basic-suite-master-engine: [Errno None] Unable to connect to port 22 on 192.168.200.99\nlago.ssh: DEBUG: Still got 0 tries for lago-hc-basic-suite-master-engine\nlago.ssh: DEBUG: end task:da6b5dff-ce01-4baf-9bcb-c08ca0893a08:Get ssh client for lago-hc-basic-suite-master-host-2:\nlago.ssh: DEBUG: end task:ab2c89cd-4a87-48af-9956-2ed214a75afc:Get ssh client for lago-hc-basic-suite-master-host-1:\nlago.ssh: DEBUG: end task:0915675c-6b43-477a-bc43-903a87b92f13:Get ssh client for lago-hc-basic-suite-master-host-0:\nlago.ssh: DEBUG: end task:d24c285a-a17a-40ca-a833-4b51bc4e9c50:Get ssh client for lago-hc-basic-suite-master-engine:\nroot: ERROR: - [Thread-7] lago-hc-basic-suite-master-engine: \x1b[31mERROR\x1b[0m (in 0:00:01)\nlago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts\n vm.collect_artifacts(path, ignore_nopath)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts\n ignore_nopath=ignore_nopath\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths\n return self.provider.extract_paths(paths, *args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths\n format(self.vm.name())\n\nlago.utils: ERROR: Error while running thread\nTraceback (most recent call last):\n File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue\n queue.put({\'return\': func()})\n File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts\n vm.collect_artifacts(path, ignore_nopath)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts\n ignore_nopath=ignore_nopath\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths\n return self.provider.extract_paths(paths, *args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths\n format(self.vm.name())\nExtractPathError: Unable to extract paths from lago-hc-basic-suite-master-engine: unreachable with SSH\nlago.prefix: INFO: \x1b[32mSuccess\x1b[0m (in 0:00:01)\nlago.prefix: INFO: \x1b[32mSuccess\x1b[0m (in 0:00:01)\nlago.prefix: INFO: \x1b[32mSuccess\x1b[0m (in 0:00:04)\nroot: ERROR: * Collect artifacts: \x1b[31mERROR\x1b[0m (in 0:00:04)\nlago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 636, in wrapper\n return func(*args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1528, in collect_artifacts\n self.virt_env.get_vms().values(),\n File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in invoke_in_parallel\n return vt.join_all()\n File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue\n queue.put({\'return\': func()})\n File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts\n vm.collect_artifacts(path, ignore_nopath)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts\n ignore_nopath=ignore_nopath\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths\n return self.provider.extract_paths(paths, *args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths\n format(self.vm.name())\n\n--------------------- >> end captured logging << ---------------------'