Project: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/ Build: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/... Build Number: 253 Build Status: Still Failing Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #251 [Martin Necas] ansible: disable iscsi Changes for Build #252 [Martin Necas] ansible: disable iscsi Changes for Build #253 [Martin Necas] ansible: disable iscsi ----------------- Failed Tests: ----------------- 1 tests failed. FAILED: basic-suite-master.test-scenarios.002_bootstrap_pytest.test_verify_add_hosts Error Message: RuntimeError: 1 hosts failed installation: lago-basic-suite-master-host-0: install_failed Stack Trace: engine_api = <ovirtsdk4.Connection object at 0x7f266bd745d0> @order_by(_TEST_LIST) def test_verify_add_hosts(engine_api): hosts_service = engine_api.system_service().hosts_service() hosts_status = hosts_service.list(search='datacenter={}'.format(DC_NAME)) total_hosts = len(hosts_status) dump_hosts = _host_status_to_print(hosts_service, hosts_status) LOGGER.debug('Host status, verify_add_hosts:\n {}'.format(dump_hosts)) testlib.assert_true_within( lambda: _single_host_up(hosts_service, total_hosts),

timeout=constants.ADD_HOST_TIMEOUT

) ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:434: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python2.7/site-packages/ovirtlago/testlib.py:278: in assert_true_within assert_equals_within(func, True, timeout, allowed_exceptions) /usr/lib/python2.7/site-packages/ovirtlago/testlib.py:234: in assert_equals_within res = func() ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:433: in <lambda> lambda: _single_host_up(hosts_service, total_hosts), ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:230: in _single_host_up _check_problematic_hosts(hosts_service) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ hosts_service = <ovirtsdk4.services.HostsService object at 0x7f266bf0c050> def _check_problematic_hosts(hosts_service): problematic_hosts = hosts_service.list(search='datacenter={} AND status != installing and status != initializing and status != up'.format(DC_NAME)) if len(problematic_hosts): dump_hosts = '%s hosts failed installation:\n' % len(problematic_hosts) for host in problematic_hosts: host_service = hosts_service.host_service(host.id) dump_hosts += '%s: %s\n' % (host.name, host_service.get().status)

raise RuntimeError(dump_hosts)

E RuntimeError: 1 hosts failed installation: E lago-basic-suite-master-host-0: install_failed ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:239: RuntimeError

Project: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/ Build: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/... Build Number: 255 Build Status: Still Failing Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #251 [Martin Necas] ansible: disable iscsi Changes for Build #252 [Martin Necas] ansible: disable iscsi Changes for Build #253 [Martin Necas] ansible: disable iscsi Changes for Build #254 [Martin Necas] ansible: disable iscsi Changes for Build #255 [Martin Necas] ansible: disable iscsi ----------------- Failed Tests: ----------------- 1 tests failed. FAILED: basic-suite-master.test-scenarios.002_bootstrap_pytest.test_verify_add_hosts Error Message: RuntimeError: 1 hosts failed installation: lago-basic-suite-master-host-0: install_failed Stack Trace: engine_api = <ovirtsdk4.Connection object at 0x7f9786c685d0> @order_by(_TEST_LIST) def test_verify_add_hosts(engine_api): hosts_service = engine_api.system_service().hosts_service() hosts_status = hosts_service.list(search='datacenter={}'.format(DC_NAME)) total_hosts = len(hosts_status) dump_hosts = _host_status_to_print(hosts_service, hosts_status) LOGGER.debug('Host status, verify_add_hosts:\n {}'.format(dump_hosts)) testlib.assert_true_within( lambda: _single_host_up(hosts_service, total_hosts),

timeout=constants.ADD_HOST_TIMEOUT

) ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:434: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python2.7/site-packages/ovirtlago/testlib.py:278: in assert_true_within assert_equals_within(func, True, timeout, allowed_exceptions) /usr/lib/python2.7/site-packages/ovirtlago/testlib.py:234: in assert_equals_within res = func() ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:433: in <lambda> lambda: _single_host_up(hosts_service, total_hosts), ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:230: in _single_host_up _check_problematic_hosts(hosts_service) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ hosts_service = <ovirtsdk4.services.HostsService object at 0x7f9786e000d0> def _check_problematic_hosts(hosts_service): problematic_hosts = hosts_service.list(search='datacenter={} AND status != installing and status != initializing and status != up'.format(DC_NAME)) if len(problematic_hosts): dump_hosts = '%s hosts failed installation:\n' % len(problematic_hosts) for host in problematic_hosts: host_service = hosts_service.host_service(host.id) dump_hosts += '%s: %s\n' % (host.name, host_service.get().status)

raise RuntimeError(dump_hosts)

E RuntimeError: 1 hosts failed installation: E lago-basic-suite-master-host-0: install_failed ../basic-suite-master/test-scenarios/002_bootstrap_pytest.py:239: RuntimeError

Project: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/ Build: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/... Build Number: 257 Build Status: Still Failing Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #251 [Martin Necas] ansible: disable iscsi Changes for Build #252 [Martin Necas] ansible: disable iscsi Changes for Build #253 [Martin Necas] ansible: disable iscsi Changes for Build #254 [Martin Necas] ansible: disable iscsi Changes for Build #255 [Martin Necas] ansible: disable iscsi Changes for Build #256 [Martin Necas] ansible: disable iscsi Changes for Build #257 [Michal Skrivanek] smaller OST VMs ----------------- Failed Tests: ----------------- 1 tests failed. FAILED: basic-suite-master.test-scenarios.004_basic_sanity_pytest.test_suspend_resume_vm0 Error Message: LagoSSHTimeoutException: Timed out (in 117 s) trying to ssh to 192.168.201.142 Stack Trace: prefix = <ovirtlago.prefix.OvirtPrefix object at 0x7f735b212c10> @order_by(_TEST_LIST) def test_suspend_resume_vm0(prefix): vm_host = _vm_host(prefix, VM0_NAME) ret = vm_host.ssh(['tail', '-1', VDSM_LOG]) assert ret.code == EX_OK log_items = ret.out.decode('utf-8').split() global _log_time_before_suspend _log_time_before_suspend = log_items[0] + ' ' + log_items[1] # date + time

assert_vm0_is_alive(prefix)

../basic-suite-master/test-scenarios/004_basic_sanity_pytest.py:1210: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ../basic-suite-master/test-scenarios/004_basic_sanity_pytest.py:191: in assert_vm0_is_alive assert_vm_is_alive(prefix, VM0_NAME) ../basic-suite-master/test-scenarios/004_basic_sanity_pytest.py:199: in assert_vm_is_alive assert _vm_ssh(prefix, vm_hostname, ['true']).code == EX_OK ../basic-suite-master/test-scenarios/004_basic_sanity_pytest.py:186: in _vm_ssh tries=tries, /usr/lib/python2.7/site-packages/lago/ssh.py:46: in ssh password=password, _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ip_addr = '192.168.201.142', ssh_key = None, host_name = '192.168.201.142' ssh_tries = 0, propagate_fail = True, username = 'cirros', password = 'gocubsgo' def get_ssh_client( ip_addr, ssh_key=None, host_name=None, ssh_tries=None, propagate_fail=True, username='root', password='123456', ): """ Get a connected SSH client Args: ip_addr(str): IP address of the endpoint ssh_key(str or list of str): Path to a file which contains the private key hotname(str): The hostname of the endpoint ssh_tries(int): The number of attempts to connect to the endpoint propagate_fail(bool): If set to true, this event will be in the log and fail the outer stage. Otherwise, it will be discarded. username(str): The username to authenticate with password(str): Used for password authentication or for private key decryption Raises: :exc:`~LagoSSHTimeoutException`: If the client failed to connect after "ssh_tries" """ host_name = host_name or ip_addr with LogTask( 'Get ssh client for %s' % host_name, level='debug', propagate_fail=propagate_fail, ): ssh_timeout = int(config.get('ssh_timeout')) if ssh_tries is None: ssh_tries = int(config.get('ssh_tries', 10)) start_time = time.time() client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy(), ) while ssh_tries > 0: try: client.connect( ip_addr, username=username, password=password, key_filename=ssh_key, timeout=ssh_timeout, ) break except (socket.error, socket.timeout) as err: LOGGER.debug( 'Socket error connecting to %s: %s', host_name, err, ) except paramiko.ssh_exception.SSHException as err: LOGGER.debug( 'SSH error connecting to %s: %s', host_name, err, ) except EOFError as err: LOGGER.debug('EOFError connecting to %s: %s', host_name, err) ssh_tries -= 1 LOGGER.debug( 'Still got %d tries for %s', ssh_tries, host_name, ) time.sleep(1) else: end_time = time.time() raise LagoSSHTimeoutException( 'Timed out (in %d s) trying to ssh to %s' %

(end_time - start_time, host_name)

) E LagoSSHTimeoutException: Timed out (in 117 s) trying to ssh to 192.168.201.142 /usr/lib/python2.7/site-packages/lago/ssh.py:388: LagoSSHTimeoutException

Project: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/ Build: https://jenkins.ovirt.org/job/ovirt-system-tests_basic-suite-master_nightly/... Build Number: 259 Build Status: Fixed Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #251 [Martin Necas] ansible: disable iscsi Changes for Build #252 [Martin Necas] ansible: disable iscsi Changes for Build #253 [Martin Necas] ansible: disable iscsi Changes for Build #254 [Martin Necas] ansible: disable iscsi Changes for Build #255 [Martin Necas] ansible: disable iscsi Changes for Build #256 [Martin Necas] ansible: disable iscsi Changes for Build #257 [Michal Skrivanek] smaller OST VMs Changes for Build #258 [Michal Skrivanek] smaller OST VMs Changes for Build #259 [Michal Skrivanek] smaller OST VMs ----------------- Failed Tests: ----------------- All tests passed