Project: http://jenkins.ovirt.org/job/ovirt-system-tests_hc-basic-suite-4.2/ Build: http://jenkins.ovirt.org/job/ovirt-system-tests_hc-basic-suite-4.2/324/ Build Number: 324 Build Status: Still Failing Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #323 [Marcin Mirecki] network: Add ping check for networks connected via multiple routers [Barak Korren] Revert "Upgrade JJB to 2.0.10" Changes for Build #324 [Marcin Mirecki] network: Add ping check for networks connected via multiple routers [Barak Korren] Revert "Upgrade JJB to 2.0.0" [Shirly Radco] Add ovirt-engine-metrics and ovirt-dwh to STDCI-v2 ----------------- Failed Tests: ----------------- 2 tests failed. FAILED: 002_bootstrap.add_hosts Error Message: Failed to read response: [(<pycurl.Curl object at 0x7f86b5942da8>, 7, 'TCP connection reset by peer')] -------------------- >> begin captured logging << -------------------- ovirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f86b591d5f0> Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within res = func() File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 254, in _host_is_up_4 host_obj = host_service.get() File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36846, in get return self._internal_get(headers, query, wait) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get return future.wait() if wait else future File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait response = self._connection.wait(self._context) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait return self.__wait(context, failed_auth) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait raise Error("Failed to read response: {}".format(err_list)) Error: Failed to read response: [(<pycurl.Curl object at 0x7f86b5942da8>, 7, 'TCP connection reset by peer')] --------------------- >> end captured logging << --------------------- Stack Trace: File "/usr/lib64/python2.7/unittest/case.py", line 369, in run testMethod() File "/usr/lib/python2.7/site-packages/nose/case.py", line 197, in runTest self.test(*self.arg) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 129, in wrapped_test test() File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 59, in wrapper return func(get_test_prefix(), *args, **kwargs) File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 229, in add_hosts add_hosts_4(prefix) File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 273, in add_hosts_4 testlib.assert_true_within(_host_is_up_4, timeout=15*60) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 263, in assert_true_within assert_equals_within(func, True, timeout, allowed_exceptions) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within res = func() File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 254, in _host_is_up_4 host_obj = host_service.get() File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36846, in get return self._internal_get(headers, query, wait) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get return future.wait() if wait else future File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait response = self._connection.wait(self._context) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait return self.__wait(context, failed_auth) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait raise Error("Failed to read response: {}".format(err_list)) 'Failed to read response: [(<pycurl.Curl object at 0x7f86b5942da8>, 7, \'TCP connection reset by peer\')]\n-------------------- >> begin captured logging << --------------------\novirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f86b591d5f0>\nTraceback (most recent call last):\n File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within\n res = func()\n File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 254, in _host_is_up_4\n host_obj = host_service.get()\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36846, in get\n return self._internal_get(headers, query, wait)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get\n return future.wait() if wait else future\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait\n response = self._connection.wait(self._context)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait\n return self.__wait(context, failed_auth)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait\n raise Error("Failed to read response: {}".format(err_list))\nError: Failed to read response: [(<pycurl.Curl object at 0x7f86b5942da8>, 7, \'TCP connection reset by peer\')]\n--------------------- >> end captured logging << ---------------------' FAILED: 002_bootstrap.add_hosts Error Message: Unable to extract paths from lago-hc-basic-suite-4-2-engine: unreachable with SSH -------------------- >> begin captured logging << -------------------- ovirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f86b591d5f0> Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within res = func() File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 254, in _host_is_up_4 host_obj = host_service.get() File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36846, in get return self._internal_get(headers, query, wait) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get return future.wait() if wait else future File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait response = self._connection.wait(self._context) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait return self.__wait(context, failed_auth) File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait raise Error("Failed to read response: {}".format(err_list)) Error: Failed to read response: [(<pycurl.Curl object at 0x7f86b5942da8>, 7, 'TCP connection reset by peer')] lago.utils: DEBUG: Looking for a workdir lago.workdir: DEBUG: Checking if /dev/shm/ost/deployment-hc-basic-suite-4.2 is a workdir lago.prefix: INFO: * Collect artifacts: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.prefix: INFO: ?[0m?[0m lago.ssh: DEBUG: start task:90ee2a0b-c973-4114-832c-8b6d916bb6e9:Get ssh client for lago-hc-basic-suite-4-2-engine: lago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-4-2-host1:/var/log to /dev/shm/ost/deployment-hc-basic-suite-4.2/default/test_logs/002_bootstrap.add_hosts-20180705094959/lago-hc-basic-suite-4-2-host1/_var_log lago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-4-2-host0:/var/log to /dev/shm/ost/deployment-hc-basic-suite-4.2/default/test_logs/002_bootstrap.add_hosts-20180705094959/lago-hc-basic-suite-4-2-host0/_var_log lago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-4-2-host2:/var/log to /dev/shm/ost/deployment-hc-basic-suite-4.2/default/test_logs/002_bootstrap.add_hosts-20180705094959/lago-hc-basic-suite-4-2-host2/_var_log lago.ssh: DEBUG: start task:81d11234-e3b6-4f45-8804-3c7a50c36968:Get ssh client for lago-hc-basic-suite-4-2-host1: lago.ssh: DEBUG: start task:f291a917-bf87-4147-978f-9dc12842001c:Get ssh client for lago-hc-basic-suite-4-2-host0: lago.ssh: DEBUG: start task:8b68e16c-b76f-4f36-b897-db3e848077d8:Get ssh client for lago-hc-basic-suite-4-2-host2: lago.ssh: DEBUG: Socket error connecting to lago-hc-basic-suite-4-2-engine: [Errno None] Unable to connect to port 22 on 192.168.200.99 lago.ssh: DEBUG: Still got 0 tries for lago-hc-basic-suite-4-2-engine lago.ssh: DEBUG: end task:81d11234-e3b6-4f45-8804-3c7a50c36968:Get ssh client for lago-hc-basic-suite-4-2-host1: lago.ssh: DEBUG: end task:8b68e16c-b76f-4f36-b897-db3e848077d8:Get ssh client for lago-hc-basic-suite-4-2-host2: lago.ssh: DEBUG: end task:f291a917-bf87-4147-978f-9dc12842001c:Get ssh client for lago-hc-basic-suite-4-2-host0: lago.ssh: DEBUG: end task:90ee2a0b-c973-4114-832c-8b6d916bb6e9:Get ssh client for lago-hc-basic-suite-4-2-engine: root: ERROR: - [Thread-7] lago-hc-basic-suite-4-2-engine: ?[31mERROR?[0m (in 0:00:01) lago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) lago.utils: ERROR: Error while running thread Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue queue.put({'return': func()}) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) ExtractPathError: Unable to extract paths from lago-hc-basic-suite-4-2-engine: unreachable with SSH lago.prefix: INFO: ?[32mSuccess?[0m (in 0:00:01) lago.prefix: INFO: ?[32mSuccess?[0m (in 0:00:01) lago.prefix: INFO: ?[32mSuccess?[0m (in 0:00:04) root: ERROR: * Collect artifacts: ?[31mERROR?[0m (in 0:00:04) lago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 636, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1528, in collect_artifacts self.virt_env.get_vms().values(), File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in invoke_in_parallel return vt.join_all() File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue queue.put({'return': func()}) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) --------------------- >> end captured logging << --------------------- Stack Trace: File "/usr/lib/python2.7/site-packages/nose/case.py", line 133, in run self.runTest(result) File "/usr/lib/python2.7/site-packages/nose/case.py", line 151, in runTest test(result) File "/usr/lib64/python2.7/unittest/case.py", line 433, in __call__ return self.run(*args, **kwds) File "/usr/lib64/python2.7/unittest/case.py", line 393, in run result.addError(self, sys.exc_info()) File "/usr/lib/python2.7/site-packages/nose/proxy.py", line 131, in addError plugins.addError(self.test, err) File "/usr/lib/python2.7/site-packages/nose/plugins/manager.py", line 99, in __call__ return self.call(*arg, **kw) File "/usr/lib/python2.7/site-packages/nose/plugins/manager.py", line 167, in simple result = meth(*arg, **kw) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 150, in addError self._addFault(test, err) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 161, in _addFault no_skip=False File "/usr/lib/python2.7/site-packages/lago/plugins/cli.py", line 190, in __call__ return self._do_run(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/utils.py", line 505, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/utils.py", line 516, in wrapper return func(*args, prefix=prefix, **kwargs) File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 752, in do_collect prefix.collect_artifacts(output, ignore_nopath=not no_skip) File "/usr/lib/python2.7/site-packages/lago/sdk_utils.py", line 50, in wrapped return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 636, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1528, in collect_artifacts self.virt_env.get_vms().values(), File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in invoke_in_parallel return vt.join_all() File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue queue.put({'return': func()}) File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts vm.collect_artifacts(path, ignore_nopath) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts ignore_nopath=ignore_nopath File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths return self.provider.extract_paths(paths, *args, **kwargs) File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths format(self.vm.name()) 'Unable to extract paths from lago-hc-basic-suite-4-2-engine: unreachable with SSH\n-------------------- >> begin captured logging << --------------------\novirtlago.testlib: ERROR: * Unhandled exception in <function _host_is_up_4 at 0x7f86b591d5f0>\nTraceback (most recent call last):\n File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 219, in assert_equals_within\n res = func()\n File "/home/jenkins/workspace/ovirt-system-tests_hc-basic-suite-4.2/ovirt-system-tests/hc-basic-suite-4.2/test-scenarios/002_bootstrap.py", line 254, in _host_is_up_4\n host_obj = host_service.get()\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/services.py", line 36846, in get\n return self._internal_get(headers, query, wait)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 211, in _internal_get\n return future.wait() if wait else future\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/service.py", line 54, in wait\n response = self._connection.wait(self._context)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 496, in wait\n return self.__wait(context, failed_auth)\n File "/usr/lib64/python2.7/site-packages/ovirtsdk4/__init__.py", line 510, in __wait\n raise Error("Failed to read response: {}".format(err_list))\nError: Failed to read response: [(<pycurl.Curl object at 0x7f86b5942da8>, 7, \'TCP connection reset by peer\')]\nlago.utils: DEBUG: Looking for a workdir\nlago.workdir: DEBUG: Checking if /dev/shm/ost/deployment-hc-basic-suite-4.2 is a workdir\nlago.prefix: INFO: * Collect artifacts: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.prefix: INFO: \x1b[0m\x1b[0m\nlago.ssh: DEBUG: start task:90ee2a0b-c973-4114-832c-8b6d916bb6e9:Get ssh client for lago-hc-basic-suite-4-2-engine:\nlago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-4-2-host1:/var/log to /dev/shm/ost/deployment-hc-basic-suite-4.2/default/test_logs/002_bootstrap.add_hosts-20180705094959/lago-hc-basic-suite-4-2-host1/_var_log\nlago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-4-2-host0:/var/log to /dev/shm/ost/deployment-hc-basic-suite-4.2/default/test_logs/002_bootstrap.add_hosts-20180705094959/lago-hc-basic-suite-4-2-host0/_var_log\nlago.plugins.vm: DEBUG: Extracting scp://lago-hc-basic-suite-4-2-host2:/var/log to /dev/shm/ost/deployment-hc-basic-suite-4.2/default/test_logs/002_bootstrap.add_hosts-20180705094959/lago-hc-basic-suite-4-2-host2/_var_log\nlago.ssh: DEBUG: start task:81d11234-e3b6-4f45-8804-3c7a50c36968:Get ssh client for lago-hc-basic-suite-4-2-host1:\nlago.ssh: DEBUG: start task:f291a917-bf87-4147-978f-9dc12842001c:Get ssh client for lago-hc-basic-suite-4-2-host0:\nlago.ssh: DEBUG: start task:8b68e16c-b76f-4f36-b897-db3e848077d8:Get ssh client for lago-hc-basic-suite-4-2-host2:\nlago.ssh: DEBUG: Socket error connecting to lago-hc-basic-suite-4-2-engine: [Errno None] Unable to connect to port 22 on 192.168.200.99\nlago.ssh: DEBUG: Still got 0 tries for lago-hc-basic-suite-4-2-engine\nlago.ssh: DEBUG: end task:81d11234-e3b6-4f45-8804-3c7a50c36968:Get ssh client for lago-hc-basic-suite-4-2-host1:\nlago.ssh: DEBUG: end task:8b68e16c-b76f-4f36-b897-db3e848077d8:Get ssh client for lago-hc-basic-suite-4-2-host2:\nlago.ssh: DEBUG: end task:f291a917-bf87-4147-978f-9dc12842001c:Get ssh client for lago-hc-basic-suite-4-2-host0:\nlago.ssh: DEBUG: end task:90ee2a0b-c973-4114-832c-8b6d916bb6e9:Get ssh client for lago-hc-basic-suite-4-2-engine:\nroot: ERROR: - [Thread-7] lago-hc-basic-suite-4-2-engine: \x1b[31mERROR\x1b[0m (in 0:00:01)\nlago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts\n vm.collect_artifacts(path, ignore_nopath)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts\n ignore_nopath=ignore_nopath\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths\n return self.provider.extract_paths(paths, *args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths\n format(self.vm.name())\n\nlago.utils: ERROR: Error while running thread\nTraceback (most recent call last):\n File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue\n queue.put({\'return\': func()})\n File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts\n vm.collect_artifacts(path, ignore_nopath)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts\n ignore_nopath=ignore_nopath\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths\n return self.provider.extract_paths(paths, *args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths\n format(self.vm.name())\nExtractPathError: Unable to extract paths from lago-hc-basic-suite-4-2-engine: unreachable with SSH\nlago.prefix: INFO: \x1b[32mSuccess\x1b[0m (in 0:00:01)\nlago.prefix: INFO: \x1b[32mSuccess\x1b[0m (in 0:00:01)\nlago.prefix: INFO: \x1b[32mSuccess\x1b[0m (in 0:00:04)\nroot: ERROR: * Collect artifacts: \x1b[31mERROR\x1b[0m (in 0:00:04)\nlago.prefix: DEBUG: File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 636, in wrapper\n return func(*args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1528, in collect_artifacts\n self.virt_env.get_vms().values(),\n File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in invoke_in_parallel\n return vt.join_all()\n File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in _ret_via_queue\n queue.put({\'return\': func()})\n File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1524, in _collect_artifacts\n vm.collect_artifacts(path, ignore_nopath)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 644, in collect_artifacts\n ignore_nopath=ignore_nopath\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 384, in extract_paths\n return self.provider.extract_paths(paths, *args, **kwargs)\n File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 253, in extract_paths\n format(self.vm.name())\n\n--------------------- >> end captured logging << ---------------------'