Project: http://jenkins.ovirt.org/job/ovirt-system-tests_he-node-ng-suite-4.2/ Build: http://jenkins.ovirt.org/job/ovirt-system-tests_he-node-ng-suite-4.2/402/ Build Number: 402 Build Status: Still Failing Triggered By: Started by timer ------------------------------------- Changes Since Last Success: ------------------------------------- Changes for Build #401 [Eitan Raviv] networking: connect NFS to engine over IPv6 Changes for Build #402 [Eitan Raviv] networking: connect NFS to engine over IPv6 ----------------- Failed Tests: ----------------- 1 tests failed. FAILED: 010_local_maintenance_cli.local_maintenance Error Message: 143 -------------------- >> begin captured logging << -------------------- root: INFO: * Waiting For System Stability... lago.ssh: DEBUG: start task:b68b9515-30f1-44db-bed6-44d2738b2dc0:Get ssh client for lago-he-node-ng-suite-4-2-host-1: lago.ssh: DEBUG: end task:b68b9515-30f1-44db-bed6-44d2738b2dc0:Get ssh client for lago-he-node-ng-suite-4-2-host-1: lago.ssh: DEBUG: Running 4d9989d2 on lago-he-node-ng-suite-4-2-host-1: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 4d9989d2 on lago-he-node-ng-suite-4-2-host-1 returned with 0 lago.ssh: DEBUG: Command 4d9989d2 on lago-he-node-ng-suite-4-2-host-1 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3909 (Mon Oct 15 01:40:22 2018)\nhost-id=1\nscore=3400\nvm_conf_refresh_time=3909 (Mon Oct 15 01:40:22 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineUp\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "7bca6ff6", "local_conf_timestamp": 3909, "host-ts": 3909}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3912 (Mon Oct 15 01:40:25 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3912 (Mon Oct 15 01:40:25 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "b545216f", "local_conf_timestamp": 3912, "host-ts": 3912}, "global_maintenance": false} lago.ssh: DEBUG: start task:bfd3d8d3-2c78-4620-a38c-d200f3e30c71:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:bfd3d8d3-2c78-4620-a38c-d200f3e30c71:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 4ea68cbc on lago-he-node-ng-suite-4-2-host-0: hosted-engine --set-maintenance --mode=local lago.ssh: DEBUG: Command 4ea68cbc on lago-he-node-ng-suite-4-2-host-0 returned with 0 root: INFO: * Waiting for engine to migrate... lago.ssh: DEBUG: start task:23e0748f-ee9b-4a24-8678-bcf6b6fbb52b:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:23e0748f-ee9b-4a24-8678-bcf6b6fbb52b:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 4fc7dd58 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 4fc7dd58 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 4fc7dd58 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3919 (Mon Oct 15 01:40:33 2018)\nhost-id=1\nscore=3400\nvm_conf_refresh_time=3919 (Mon Oct 15 01:40:33 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineUp\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "a7ad2fc9", "local_conf_timestamp": 3919, "host-ts": 3919}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3912 (Mon Oct 15 01:40:25 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3912 (Mon Oct 15 01:40:25 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "b545216f", "local_conf_timestamp": 3912, "host-ts": 3912}, "global_maintenance": false} lago.ssh: DEBUG: start task:ae003eae-0b79-47d3-bd44-a4887bf18f07:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:ae003eae-0b79-47d3-bd44-a4887bf18f07:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 57f5fc26 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 57f5fc26 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 57f5fc26 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3929 (Mon Oct 15 01:40:42 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3929 (Mon Oct 15 01:40:42 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "3b1abee5", "local_conf_timestamp": 3929, "host-ts": 3929}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3922 (Mon Oct 15 01:40:35 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3922 (Mon Oct 15 01:40:35 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "b98e2060", "local_conf_timestamp": 3922, "host-ts": 3922}, "global_maintenance": false} lago.ssh: DEBUG: start task:4fd20f7f-1e7b-47da-81a8-c4ed724935bd:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:4fd20f7f-1e7b-47da-81a8-c4ed724935bd:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 5ba16fd6 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 5ba16fd6 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 5ba16fd6 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3939 (Mon Oct 15 01:40:52 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3939 (Mon Oct 15 01:40:53 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "79b77b76", "local_conf_timestamp": 3939, "host-ts": 3939}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3932 (Mon Oct 15 01:40:45 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3932 (Mon Oct 15 01:40:45 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "bdc8df65", "local_conf_timestamp": 3932, "host-ts": 3932}, "global_maintenance": false} lago.ssh: DEBUG: start task:d1201ae0-4524-4376-a31a-4bfbc3216d89:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:d1201ae0-4524-4376-a31a-4bfbc3216d89:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 5eb69df4 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 5eb69df4 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 5eb69df4 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3939 (Mon Oct 15 01:40:52 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3939 (Mon Oct 15 01:40:53 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "79b77b76", "local_conf_timestamp": 3939, "host-ts": 3939}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3932 (Mon Oct 15 01:40:45 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3932 (Mon Oct 15 01:40:45 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "bdc8df65", "local_conf_timestamp": 3932, "host-ts": 3932}, "global_maintenance": false} lago.ssh: DEBUG: start task:33986491-d513-4366-9ecb-d36cf85ec530:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:33986491-d513-4366-9ecb-d36cf85ec530:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 6257a962 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 6257a962 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 6257a962 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3949 (Mon Oct 15 01:41:03 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3950 (Mon Oct 15 01:41:03 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "e50b465f", "local_conf_timestamp": 3950, "host-ts": 3949}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3942 (Mon Oct 15 01:40:55 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3942 (Mon Oct 15 01:40:55 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "d72517e6", "local_conf_timestamp": 3942, "host-ts": 3942}, "global_maintenance": false} lago.ssh: DEBUG: start task:9f41633d-f4be-4cff-be69-575033c3f84b:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:9f41633d-f4be-4cff-be69-575033c3f84b:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 654d6652 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 654d6652 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 654d6652 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3949 (Mon Oct 15 01:41:03 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3950 (Mon Oct 15 01:41:03 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "e50b465f", "local_conf_timestamp": 3950, "host-ts": 3949}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3942 (Mon Oct 15 01:40:55 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3942 (Mon Oct 15 01:40:55 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "d72517e6", "local_conf_timestamp": 3942, "host-ts": 3942}, "global_maintenance": false} lago.ssh: DEBUG: start task:87d1798f-da36-4bf1-ac0a-c017bcabd8e5:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:87d1798f-da36-4bf1-ac0a-c017bcabd8e5:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 686cc30a on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 686cc30a on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 686cc30a on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3959 (Mon Oct 15 01:41:12 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3959 (Mon Oct 15 01:41:12 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "2bfff178", "local_conf_timestamp": 3959, "host-ts": 3959}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3952 (Mon Oct 15 01:41:05 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3952 (Mon Oct 15 01:41:05 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "d363e8e3", "local_conf_timestamp": 3952, "host-ts": 3952}, "global_maintenance": false} lago.ssh: DEBUG: start task:95fbec13-41a2-4817-a0c0-76cd14dcae82:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:95fbec13-41a2-4817-a0c0-76cd14dcae82:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 6b55ae38 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 6b55ae38 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 6b55ae38 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3959 (Mon Oct 15 01:41:12 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3959 (Mon Oct 15 01:41:12 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "2bfff178", "local_conf_timestamp": 3959, "host-ts": 3959}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3962 (Mon Oct 15 01:41:15 2018)\nhost-id=2\nscore=3000\nvm_conf_refresh_time=3962 (Mon Oct 15 01:41:15 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "6c8c3162", "local_conf_timestamp": 3962, "host-ts": 3962}, "global_maintenance": false} lago.ssh: DEBUG: start task:362674eb-acfb-4586-960b-0bd74a07e9d4:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:362674eb-acfb-4586-960b-0bd74a07e9d4:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 6eaaac8c on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 6eaaac8c on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 6eaaac8c on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3969 (Mon Oct 15 01:41:23 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3970 (Mon Oct 15 01:41:23 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "621c3d9a", "local_conf_timestamp": 3970, "host-ts": 3969}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3962 (Mon Oct 15 01:41:15 2018)\nhost-id=2\nscore=3000\nvm_conf_refresh_time=3962 (Mon Oct 15 01:41:15 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "6c8c3162", "local_conf_timestamp": 3962, "host-ts": 3962}, "global_maintenance": false} lago.ssh: DEBUG: start task:a648568d-8a9b-4b06-bea1-f62b69c71a61:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:a648568d-8a9b-4b06-bea1-f62b69c71a61:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 71b5d1fe on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 71b5d1fe on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 71b5d1fe on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3979 (Mon Oct 15 01:41:32 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3979 (Mon Oct 15 01:41:32 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "ace88abd", "local_conf_timestamp": 3979, "host-ts": 3979}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3972 (Mon Oct 15 01:41:25 2018)\nhost-id=2\nscore=3000\nvm_conf_refresh_time=3972 (Mon Oct 15 01:41:25 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "68cace67", "local_conf_timestamp": 3972, "host-ts": 3972}, "global_maintenance": false} lago.ssh: DEBUG: start task:f2d8227b-a46b-49aa-ada5-cee44bed73f2:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:f2d8227b-a46b-49aa-ada5-cee44bed73f2:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 74e170b8 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 74e170b8 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 74e170b8 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3979 (Mon Oct 15 01:41:32 2018)\nhost-id=1\nscore=3000\nvm_conf_refresh_time=3979 (Mon Oct 15 01:41:32 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineMigratingAway\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "ace88abd", "local_conf_timestamp": 3979, "host-ts": 3979}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3972 (Mon Oct 15 01:41:25 2018)\nhost-id=2\nscore=3000\nvm_conf_refresh_time=3972 (Mon Oct 15 01:41:25 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "68cace67", "local_conf_timestamp": 3972, "host-ts": 3972}, "global_maintenance": false} lago.ssh: DEBUG: start task:9c6da9f4-7403-4ab5-9ead-2396ecce2955:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:9c6da9f4-7403-4ab5-9ead-2396ecce2955:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 780d5680 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 780d5680 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 780d5680 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3989 (Mon Oct 15 01:41:42 2018)\nhost-id=1\nscore=3400\nvm_conf_refresh_time=3989 (Mon Oct 15 01:41:43 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"reason": "VM migrated away successfully", "health": "bad", "vm": "down", "detail": "Down"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "a22b3da5", "local_conf_timestamp": 3989, "host-ts": 3989}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3982 (Mon Oct 15 01:41:34 2018)\nhost-id=2\nscore=3000\nvm_conf_refresh_time=3982 (Mon Oct 15 01:41:35 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "572dcb54", "local_conf_timestamp": 3982, "host-ts": 3982}, "global_maintenance": false} lago.ssh: DEBUG: start task:92836948-f4f4-49cb-a0a0-07c451a9bda0:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:92836948-f4f4-49cb-a0a0-07c451a9bda0:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 79213b90 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 79213b90 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 79213b90 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3989 (Mon Oct 15 01:41:42 2018)\nhost-id=1\nscore=3400\nvm_conf_refresh_time=3989 (Mon Oct 15 01:41:43 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"reason": "VM migrated away successfully", "health": "bad", "vm": "down", "detail": "Down"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "a22b3da5", "local_conf_timestamp": 3989, "host-ts": 3989}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3982 (Mon Oct 15 01:41:34 2018)\nhost-id=2\nscore=3000\nvm_conf_refresh_time=3982 (Mon Oct 15 01:41:35 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineDown\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "572dcb54", "local_conf_timestamp": 3982, "host-ts": 3982}, "global_maintenance": false} lago.ssh: DEBUG: start task:354f9bec-0ba3-443e-b556-7609d0951714:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: end task:354f9bec-0ba3-443e-b556-7609d0951714:Get ssh client for lago-he-node-ng-suite-4-2-host-0: lago.ssh: DEBUG: Running 803f1398 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json lago.ssh: DEBUG: Command 803f1398 on lago-he-node-ng-suite-4-2-host-0 returned with 0 lago.ssh: DEBUG: Command 803f1398 on lago-he-node-ng-suite-4-2-host-0 output: {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3999 (Mon Oct 15 01:41:52 2018)\nhost-id=1\nscore=0\nvm_conf_refresh_time=3999 (Mon Oct 15 01:41:52 2018)\nconf_on_shared_storage=True\nmaintenance=True\nstate=LocalMaintenance\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down_missing", "detail": "unknown"}, "score": 0, "stopped": false, "maintenance": true, "crc32": "ac90fcbc", "local_conf_timestamp": 3999, "host-ts": 3999}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\nmetadata_feature_version=1\ntimestamp=3992 (Mon Oct 15 01:41:45 2018)\nhost-id=2\nscore=3400\nvm_conf_refresh_time=3992 (Mon Oct 15 01:41:45 2018)\nconf_on_shared_storage=True\nmaintenance=False\nstate=EngineUp\nstopped=False\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "7ebbf29f", "local_conf_timestamp": 3992, "host-ts": 3992}, "global_maintenance": false} root: INFO: * Engine has migrated. root: INFO: * Waiting For System Stability... cli: DEBUG: signal 15 was caught --------------------- >> end captured logging << --------------------- Stack Trace: File "/usr/lib64/python2.7/unittest/case.py", line 369, in run testMethod() File "/usr/lib/python2.7/site-packages/nose/case.py", line 197, in runTest self.test(*self.arg) File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 142, in wrapped_test test() File "/usr/lib/python2.7/site-packages/ovirtlago/testlib.py", line 60, in wrapper return func(get_test_prefix(), *args, **kwargs) File "/home/jenkins/workspace/ovirt-system-tests_he-node-ng-suite-4.2/ovirt-system-tests/he-node-ng-suite-4.2/test-scenarios/010_local_maintenance_cli.py", line 125, in local_maintenance _wait_for_engine_migration(host, he_index, "bad", "Migration Destination") File "/home/jenkins/workspace/ovirt-system-tests_he-node-ng-suite-4.2/ovirt-system-tests/he-node-ng-suite-4.2/test-scenarios/010_local_maintenance_cli.py", line 45, in _wait_for_engine_migration time.sleep(wait_value) File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 922, in exit_handler sys.exit(128 + signum) '143\n-------------------- >> begin captured logging << --------------------\nroot: INFO: * Waiting For System Stability...\nlago.ssh: DEBUG: start task:b68b9515-30f1-44db-bed6-44d2738b2dc0:Get ssh client for lago-he-node-ng-suite-4-2-host-1:\nlago.ssh: DEBUG: end task:b68b9515-30f1-44db-bed6-44d2738b2dc0:Get ssh client for lago-he-node-ng-suite-4-2-host-1:\nlago.ssh: DEBUG: Running 4d9989d2 on lago-he-node-ng-suite-4-2-host-1: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 4d9989d2 on lago-he-node-ng-suite-4-2-host-1 returned with 0\nlago.ssh: DEBUG: Command 4d9989d2 on lago-he-node-ng-suite-4-2-host-1 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3909 (Mon Oct 15 01:40:22 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=3909 (Mon Oct 15 01:40:22 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineUp\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "7bca6ff6", "local_conf_timestamp": 3909, "host-ts": 3909}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3912 (Mon Oct 15 01:40:25 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3912 (Mon Oct 15 01:40:25 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "b545216f", "local_conf_timestamp": 3912, "host-ts": 3912}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:bfd3d8d3-2c78-4620-a38c-d200f3e30c71:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:bfd3d8d3-2c78-4620-a38c-d200f3e30c71:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 4ea68cbc on lago-he-node-ng-suite-4-2-host-0: hosted-engine --set-maintenance --mode=local\nlago.ssh: DEBUG: Command 4ea68cbc on lago-he-node-ng-suite-4-2-host-0 returned with 0\nroot: INFO: * Waiting for engine to migrate...\nlago.ssh: DEBUG: start task:23e0748f-ee9b-4a24-8678-bcf6b6fbb52b:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:23e0748f-ee9b-4a24-8678-bcf6b6fbb52b:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 4fc7dd58 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 4fc7dd58 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 4fc7dd58 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3919 (Mon Oct 15 01:40:33 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=3919 (Mon Oct 15 01:40:33 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineUp\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "a7ad2fc9", "local_conf_timestamp": 3919, "host-ts": 3919}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3912 (Mon Oct 15 01:40:25 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3912 (Mon Oct 15 01:40:25 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "b545216f", "local_conf_timestamp": 3912, "host-ts": 3912}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:ae003eae-0b79-47d3-bd44-a4887bf18f07:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:ae003eae-0b79-47d3-bd44-a4887bf18f07:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 57f5fc26 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 57f5fc26 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 57f5fc26 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3929 (Mon Oct 15 01:40:42 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3929 (Mon Oct 15 01:40:42 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "3b1abee5", "local_conf_timestamp": 3929, "host-ts": 3929}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3922 (Mon Oct 15 01:40:35 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3922 (Mon Oct 15 01:40:35 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "b98e2060", "local_conf_timestamp": 3922, "host-ts": 3922}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:4fd20f7f-1e7b-47da-81a8-c4ed724935bd:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:4fd20f7f-1e7b-47da-81a8-c4ed724935bd:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 5ba16fd6 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 5ba16fd6 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 5ba16fd6 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3939 (Mon Oct 15 01:40:52 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3939 (Mon Oct 15 01:40:53 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "79b77b76", "local_conf_timestamp": 3939, "host-ts": 3939}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3932 (Mon Oct 15 01:40:45 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3932 (Mon Oct 15 01:40:45 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "bdc8df65", "local_conf_timestamp": 3932, "host-ts": 3932}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:d1201ae0-4524-4376-a31a-4bfbc3216d89:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:d1201ae0-4524-4376-a31a-4bfbc3216d89:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 5eb69df4 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 5eb69df4 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 5eb69df4 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3939 (Mon Oct 15 01:40:52 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3939 (Mon Oct 15 01:40:53 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "79b77b76", "local_conf_timestamp": 3939, "host-ts": 3939}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3932 (Mon Oct 15 01:40:45 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3932 (Mon Oct 15 01:40:45 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "bdc8df65", "local_conf_timestamp": 3932, "host-ts": 3932}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:33986491-d513-4366-9ecb-d36cf85ec530:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:33986491-d513-4366-9ecb-d36cf85ec530:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 6257a962 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 6257a962 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 6257a962 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3949 (Mon Oct 15 01:41:03 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3950 (Mon Oct 15 01:41:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "e50b465f", "local_conf_timestamp": 3950, "host-ts": 3949}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3942 (Mon Oct 15 01:40:55 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3942 (Mon Oct 15 01:40:55 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "d72517e6", "local_conf_timestamp": 3942, "host-ts": 3942}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:9f41633d-f4be-4cff-be69-575033c3f84b:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:9f41633d-f4be-4cff-be69-575033c3f84b:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 654d6652 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 654d6652 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 654d6652 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3949 (Mon Oct 15 01:41:03 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3950 (Mon Oct 15 01:41:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "e50b465f", "local_conf_timestamp": 3950, "host-ts": 3949}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3942 (Mon Oct 15 01:40:55 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3942 (Mon Oct 15 01:40:55 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "d72517e6", "local_conf_timestamp": 3942, "host-ts": 3942}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:87d1798f-da36-4bf1-ac0a-c017bcabd8e5:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:87d1798f-da36-4bf1-ac0a-c017bcabd8e5:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 686cc30a on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 686cc30a on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 686cc30a on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3959 (Mon Oct 15 01:41:12 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3959 (Mon Oct 15 01:41:12 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "2bfff178", "local_conf_timestamp": 3959, "host-ts": 3959}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3952 (Mon Oct 15 01:41:05 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3952 (Mon Oct 15 01:41:05 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "d363e8e3", "local_conf_timestamp": 3952, "host-ts": 3952}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:95fbec13-41a2-4817-a0c0-76cd14dcae82:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:95fbec13-41a2-4817-a0c0-76cd14dcae82:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 6b55ae38 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 6b55ae38 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 6b55ae38 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3959 (Mon Oct 15 01:41:12 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3959 (Mon Oct 15 01:41:12 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "2bfff178", "local_conf_timestamp": 3959, "host-ts": 3959}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3962 (Mon Oct 15 01:41:15 2018)\\nhost-id=2\\nscore=3000\\nvm_conf_refresh_time=3962 (Mon Oct 15 01:41:15 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "6c8c3162", "local_conf_timestamp": 3962, "host-ts": 3962}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:362674eb-acfb-4586-960b-0bd74a07e9d4:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:362674eb-acfb-4586-960b-0bd74a07e9d4:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 6eaaac8c on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 6eaaac8c on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 6eaaac8c on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3969 (Mon Oct 15 01:41:23 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3970 (Mon Oct 15 01:41:23 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "621c3d9a", "local_conf_timestamp": 3970, "host-ts": 3969}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3962 (Mon Oct 15 01:41:15 2018)\\nhost-id=2\\nscore=3000\\nvm_conf_refresh_time=3962 (Mon Oct 15 01:41:15 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "6c8c3162", "local_conf_timestamp": 3962, "host-ts": 3962}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:a648568d-8a9b-4b06-bea1-f62b69c71a61:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:a648568d-8a9b-4b06-bea1-f62b69c71a61:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 71b5d1fe on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 71b5d1fe on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 71b5d1fe on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3979 (Mon Oct 15 01:41:32 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3979 (Mon Oct 15 01:41:32 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "ace88abd", "local_conf_timestamp": 3979, "host-ts": 3979}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3972 (Mon Oct 15 01:41:25 2018)\\nhost-id=2\\nscore=3000\\nvm_conf_refresh_time=3972 (Mon Oct 15 01:41:25 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "68cace67", "local_conf_timestamp": 3972, "host-ts": 3972}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:f2d8227b-a46b-49aa-ada5-cee44bed73f2:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:f2d8227b-a46b-49aa-ada5-cee44bed73f2:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 74e170b8 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 74e170b8 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 74e170b8 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3979 (Mon Oct 15 01:41:32 2018)\\nhost-id=1\\nscore=3000\\nvm_conf_refresh_time=3979 (Mon Oct 15 01:41:32 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineMigratingAway\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"health": "good", "vm": "up", "detail": "Migration Source"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "ace88abd", "local_conf_timestamp": 3979, "host-ts": 3979}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3972 (Mon Oct 15 01:41:25 2018)\\nhost-id=2\\nscore=3000\\nvm_conf_refresh_time=3972 (Mon Oct 15 01:41:25 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "68cace67", "local_conf_timestamp": 3972, "host-ts": 3972}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:9c6da9f4-7403-4ab5-9ead-2396ecce2955:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:9c6da9f4-7403-4ab5-9ead-2396ecce2955:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 780d5680 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 780d5680 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 780d5680 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3989 (Mon Oct 15 01:41:42 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=3989 (Mon Oct 15 01:41:43 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"reason": "VM migrated away successfully", "health": "bad", "vm": "down", "detail": "Down"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "a22b3da5", "local_conf_timestamp": 3989, "host-ts": 3989}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3982 (Mon Oct 15 01:41:34 2018)\\nhost-id=2\\nscore=3000\\nvm_conf_refresh_time=3982 (Mon Oct 15 01:41:35 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "572dcb54", "local_conf_timestamp": 3982, "host-ts": 3982}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:92836948-f4f4-49cb-a0a0-07c451a9bda0:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:92836948-f4f4-49cb-a0a0-07c451a9bda0:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 79213b90 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 79213b90 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 79213b90 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3989 (Mon Oct 15 01:41:42 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=3989 (Mon Oct 15 01:41:43 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"reason": "VM migrated away successfully", "health": "bad", "vm": "down", "detail": "Down"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "a22b3da5", "local_conf_timestamp": 3989, "host-ts": 3989}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3982 (Mon Oct 15 01:41:34 2018)\\nhost-id=2\\nscore=3000\\nvm_conf_refresh_time=3982 (Mon Oct 15 01:41:35 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineDown\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "Migration Destination"}, "score": 3000, "stopped": false, "maintenance": false, "crc32": "572dcb54", "local_conf_timestamp": 3982, "host-ts": 3982}, "global_maintenance": false}\n\nlago.ssh: DEBUG: start task:354f9bec-0ba3-443e-b556-7609d0951714:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: end task:354f9bec-0ba3-443e-b556-7609d0951714:Get ssh client for lago-he-node-ng-suite-4-2-host-0:\nlago.ssh: DEBUG: Running 803f1398 on lago-he-node-ng-suite-4-2-host-0: hosted-engine --vm-status --json\nlago.ssh: DEBUG: Command 803f1398 on lago-he-node-ng-suite-4-2-host-0 returned with 0\nlago.ssh: DEBUG: Command 803f1398 on lago-he-node-ng-suite-4-2-host-0 output:\n {"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3999 (Mon Oct 15 01:41:52 2018)\\nhost-id=1\\nscore=0\\nvm_conf_refresh_time=3999 (Mon Oct 15 01:41:52 2018)\\nconf_on_shared_storage=True\\nmaintenance=True\\nstate=LocalMaintenance\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-0", "host-id": 1, "engine-status": {"reason": "vm not running on this host", "health": "bad", "vm": "down_missing", "detail": "unknown"}, "score": 0, "stopped": false, "maintenance": true, "crc32": "ac90fcbc", "local_conf_timestamp": 3999, "host-ts": 3999}, "2": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=3992 (Mon Oct 15 01:41:45 2018)\\nhost-id=2\\nscore=3400\\nvm_conf_refresh_time=3992 (Mon Oct 15 01:41:45 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineUp\\nstopped=False\\n", "hostname": "lago-he-node-ng-suite-4-2-host-1", "host-id": 2, "engine-status": {"health": "good", "vm": "up", "detail": "Up"}, "score": 3400, "stopped": false, "maintenance": false, "crc32": "7ebbf29f", "local_conf_timestamp": 3992, "host-ts": 3992}, "global_maintenance": false}\n\nroot: INFO: * Engine has migrated.\nroot: INFO: * Waiting For System Stability...\ncli: DEBUG: signal 15 was caught\n--------------------- >> end captured logging << ---------------------'