[ovirt-users] Re: Diary of hosted engine install woes

9 Oct 2018

      Okay, I went back to using a bond (instead of an individual NIC).  Above network problem is fixed and now proceeds as far as ever.  Hangs for around 10 minutes at:

[ INFO  ] TASK [Check engine VM health]
The hosted-engine-setup-ansible-create_target_vm log has:
2018-10-08 23:42:01,664+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Check engine VM health', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}

Then repeats the following line for around 10 minutes:
2018-10-08 23:42:01,866+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313cd338d0> kwargs

Before eventually, the console outputs the following error:
[ ERROR ] fatal: [localhost]: FAILED! => {"attempts": 120, "changed": true, "cmd": ["hosted-engine", "--vm-status", "--json"], "delta": "0:00:00.167677", "end": "2018-10-08 23:53:11.112436", "rc": 0, "start": "2018-10-08 23:53:10.944759", "stderr": "", "stderr_lines": [], "stdout": "{\"1\": {\"conf_on_shared_storage\": true, \"live-data\": true, \"extra\": \"metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=49491 (Mon Oct  8 23:53:03 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=49491 (Mon Oct  8 23:53:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineStarting\\nstopped=False\\n\", \"hostname\": \"host\", \"host-id\": 1, \"engine-status\": {\"reason\": \"failed liveliness check\", \"health\": \"bad\", \"vm\": \"up\", \"detail\": \"Up\"}, \"score\": 3400, \"stopped\": false, \"maintenance\": false, \"crc32\": \"75452be7\", \"local_conf_timestamp\": 49491, \"host-ts\": 49491}, \"global_maintenance\": false}", "stdout_lines": ["{\"1\": {\"c
 onf_on_shared_storage\": true, \"live-data\": true, \"extra\": \"metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=49491 (Mon Oct  8 23:53:03 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=49491 (Mon Oct  8 23:53:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineStarting\\nstopped=False\\n\", \"hostname\": \"host\", \"host-id\": 1, \"engine-status\": {\"reason\": \"failed liveliness check\", \"health\": \"bad\", \"vm\": \"up\", \"detail\": \"Up\"}, \"score\": 3400, \"stopped\": false, \"maintenance\": false, \"crc32\": \"75452be7\", \"local_conf_timestamp\": 49491, \"host-ts\": 49491}, \"global_maintenance\": false}"]}
[ INFO  ] TASK [Check VM status at virt level]

The hosted-engine-setup-ansible-create_target_vm log shows the following when this error occurs:

2018-10-08 23:53:11,812+0100 DEBUG var changed: host "localhost" var "ansible_failed_result" type "<type 'dict'>" value: "{
    "_ansible_no_log": false, 
    "_ansible_parsed": true, 
    "attempts": 120, 
    "changed": true, 
    "cmd": [
        "hosted-engine", 
        "--vm-status", 
        "--json"
    ], 
    "delta": "0:00:00.167677", 
    "end": "2018-10-08 23:53:11.112436", 
    "failed": true, 
    "invocation": {
        "module_args": {
            "_raw_params": "hosted-engine --vm-status --json", 
            "_uses_shell": false, 
            "argv": null, 
            "chdir": null, 
            "creates": null, 
            "executable": null, 
            "removes": null, 
            "stdin": null, 
            "warn": true
        }
    }, 
    "rc": 0, 
    "start": "2018-10-08 23:53:10.944759", 
    "stderr": "", 
    "stderr_lines": [], 
    "stdout": "{\"1\": {\"conf_on_shared_storage\": true, \"live-data\": true, \"extra\": \"metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=49491 (Mon Oct  8 23:53:03 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=49491 (Mon Oct  8 23:53:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineStarting\\nstopped=False\\n\", \"hostname\": \"host\", \"host-id\": 1, \"engine-status\": {\"reason\": \"failed liveliness check\", \"health\": \"bad\", \"vm\": \"up\", \"detail\": \"Up\"}, \"score\": 3400, \"stopped\": false, \"maintenance\": false, \"crc32\": \"75452be7\", \"local_conf_timestamp\": 49491, \"host-ts\": 49491}, \"global_maintenance\": false}", 
    "stdout_lines": [
        "{\"1\": {\"conf_on_shared_storage\": true, \"live-data\": true, \"extra\": \"metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=49491 (Mon Oct  8 23:53:03 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=49491 (Mon Oct  8 23:53:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineStarting\\nstopped=False\\n\", \"hostname\": \"host\", \"host-id\": 1, \"engine-status\": {\"reason\": \"failed liveliness check\", \"health\": \"bad\", \"vm\": \"up\", \"detail\": \"Up\"}, \"score\": 3400, \"stopped\": false, \"maintenance\": false, \"crc32\": \"75452be7\", \"local_conf_timestamp\": 49491, \"host-ts\": 49491}, \"global_maintenance\": false}"
    ]
}"
2018-10-08 23:53:11,813+0100 DEBUG var changed: host "localhost" var "ansible_failed_task" type "<type 'dict'>" value: "{
    "action": "command", 
    "any_errors_fatal": false, 
    "args": {
        "_ansible_check_mode": false, 
        "_ansible_debug": false, 
        "_ansible_diff": false, 
        "_ansible_keep_remote_files": false, 
        "_ansible_module_name": "command", 
        "_ansible_no_log": false, 
        "_ansible_remote_tmp": "~/.ansible/tmp", 
        "_ansible_selinux_special_fs": [
            "fuse", 
            "nfs", 
            "vboxsf", 
            "ramfs", 
            "9p"
        ], 
        "_ansible_shell_executable": "/bin/sh", 
        "_ansible_socket": null, 
        "_ansible_syslog_facility": "LOG_USER", 
        "_ansible_tmpdir": "/root/.ansible/tmp/ansible-tmp-1539039190.78-59276929025529/", 
        "_ansible_verbosity": 0, 
        "_ansible_version": "2.6.5", 
        "_raw_params": "hosted-engine --vm-status --json", 
        "warn": true
    }, 
    "async": 0, 
    "async_val": 0, 
    "become": null, 
    "become_flags": null, 
    "become_method": null, 
    "become_user": null, 
    "changed_when": [
        true
    ], 
    "check_mode": null, 
    "connection": "local", 
    "debugger": null, 
    "delay": 5, 
    "delegate_facts": false, 
    "delegate_to": null, 
    "diff": null, 
    "environment": [
        {
            "LANG": "en_US.UTF-8", 
            "LC_ALL": "en_US.UTF-8", 
            "LC_MESSAGES": "en_US.UTF-8"
        }
    ], 
    "failed_when": [], 
    "finalized": false, 
    "ignore_errors": null, 
    "loop": null, 
    "loop_control": null, 
    "loop_with": null, 
    "module_defaults": [], 
    "name": "Check engine VM health", 
    "no_log": null, 
    "notify": null, 
    "parent": {
        "any_errors_fatal": null, 
        "become": null, 
        "become_flags": null, 
        "become_method": null, 
        "become_user": null, 
        "check_mode": null, 
        "connection": "local", 
        "debugger": null, 
        "delegate_facts": false, 
        "delegate_to": null, 
        "dep_chain": null, 
        "diff": null, 
        "environment": null, 
        "eor": false, 
        "ignore_errors": null, 
        "module_defaults": null, 
        "name": "Wait for the engine to come up on the target VM", 
        "no_log": null, 
        "port": null, 
        "remote_user": null, 
        "run_once": null, 
        "tags": [], 
        "vars": {}, 
        "when": []
    }, 
    "parent_type": "Block", 
    "poll": 10, 
    "port": null, 
    "register": "health_result", 
    "remote_user": null, 
    "retries": 120, 
    "run_once": null, 
    "squashed": false, 
    "tags": [], 
    "until": [
        "health_result.rc == 0 and 'health' in health_result.stdout and health_result.stdout|from_json|json_query('*.\"engine-status\".\"health\"')|first==\"good\""
    ], 
    "uuid": "525400d0-30bf-04f1-b63b-00000000006b", 
    "vars": {}, 
    "when": []
}"
2018-10-08 23:53:11,813+0100 DEBUG var changed: host "localhost" var "health_result" type "<type 'dict'>" value: "{
    "attempts": 120, 
    "changed": true, 
    "cmd": [
        "hosted-engine", 
        "--vm-status", 
        "--json"
    ], 
    "delta": "0:00:00.167677", 
    "end": "2018-10-08 23:53:11.112436", 
    "failed": true, 
    "rc": 0, 
    "start": "2018-10-08 23:53:10.944759", 
    "stderr": "", 
    "stderr_lines": [], 
    "stdout": "{\"1\": {\"conf_on_shared_storage\": true, \"live-data\": true, \"extra\": \"metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=49491 (Mon Oct  8 23:53:03 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=49491 (Mon Oct  8 23:53:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineStarting\\nstopped=False\\n\", \"hostname\": \"host\", \"host-id\": 1, \"engine-status\": {\"reason\": \"failed liveliness check\", \"health\": \"bad\", \"vm\": \"up\", \"detail\": \"Up\"}, \"score\": 3400, \"stopped\": false, \"maintenance\": false, \"crc32\": \"75452be7\", \"local_conf_timestamp\": 49491, \"host-ts\": 49491}, \"global_maintenance\": false}", 
    "stdout_lines": [
        "{\"1\": {\"conf_on_shared_storage\": true, \"live-data\": true, \"extra\": \"metadata_parse_version=1\\nmetadata_feature_version=1\\ntimestamp=49491 (Mon Oct  8 23:53:03 2018)\\nhost-id=1\\nscore=3400\\nvm_conf_refresh_time=49491 (Mon Oct  8 23:53:03 2018)\\nconf_on_shared_storage=True\\nmaintenance=False\\nstate=EngineStarting\\nstopped=False\\n\", \"hostname\": \"host\", \"host-id\": 1, \"engine-status\": {\"reason\": \"failed liveliness check\", \"health\": \"bad\", \"vm\": \"up\", \"detail\": \"Up\"}, \"score\": 3400, \"stopped\": false, \"maintenance\": false, \"crc32\": \"75452be7\", \"local_conf_timestamp\": 49491, \"host-ts\": 49491}, \"global_maintenance\": false}"
    ]
}"
2018-10-08 23:53:11,813+0100 ERROR ansible failed {'status': 'FAILED', 'ansible_type': 'task', 'ansible_task': u'Check engine VM health', 'ansible_result': u'type: <type \'dict\'>\nstr: {\'_ansible_parsed\': True, \'stderr_lines\': [], u\'changed\': True, u\'end\': u\'2018-10-08 23:53:11.112436\', \'_ansible_no_log\': False, u\'stdout\': u\'{"1": {"conf_on_shared_storage": true, "live-data": true, "extra": "metadata_parse_version=1\\\\nmetadata_feature_version=1\\\\ntimestamp=49491 (Mon Oct  8 23:53:', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml'}
2018-10-08 23:53:11,813+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313cd338d0> kwargs ignore_errors:None 
2018-10-08 23:53:12,517+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Check VM status at virt level', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:12,517+0100 DEBUG ansible on_any args TASK: Check VM status at virt level kwargs is_conditional:False 
2018-10-08 23:53:13,433+0100 DEBUG var changed: host "localhost" var "vm_status_virsh" type "<type 'dict'>" value: "{
    "changed": true, 
    "cmd": "virsh -r list | grep HostedEngine | grep running", 
    "delta": "0:00:00.053603", 
    "end": "2018-10-08 23:53:12.741182", 
    "failed": false, 
    "rc": 0, 
    "start": "2018-10-08 23:53:12.687579", 
    "stderr": "", 
    "stderr_lines": [], 
    "stdout": " 2     HostedEngine                   running", 
    "stdout_lines": [
        " 2     HostedEngine                   running"
    ]
}"
2018-10-08 23:53:13,434+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u'Check VM status at virt level', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:13,434+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313caedc10> kwargs 
2018-10-08 23:53:14,159+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'debug', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:14,159+0100 DEBUG ansible on_any args TASK: debug kwargs is_conditional:False 
2018-10-08 23:53:14,861+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u'', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:14,861+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313c161ad0> kwargs 
2018-10-08 23:53:15,575+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Fail if engine VM is not running', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:15,576+0100 DEBUG ansible on_any args TASK: Fail if engine VM is not running kwargs is_conditional:False 
2018-10-08 23:53:16,261+0100 INFO ansible skipped {'status': 'SKIPPED', 'ansible_task': u'Fail if engine VM is not running', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:16,261+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313ccec510> kwargs 
2018-10-08 23:53:16,967+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Get target engine VM IPv4 address', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:16,968+0100 DEBUG ansible on_any args TASK: Get target engine VM IPv4 address kwargs is_conditional:False 
2018-10-08 23:53:17,819+0100 DEBUG var changed: host "localhost" var "engine_vm_ipv4" type "<type 'dict'>" value: "{
    "changed": true, 
    "cmd": "getent ahostsv4 ovirt-engine.example.com | cut -d' ' -f1 | uniq", 
    "delta": "0:00:00.004903", 
    "end": "2018-10-08 23:53:17.143283", 
    "failed": false, 
    "rc": 0, 
    "start": "2018-10-08 23:53:17.138380", 
    "stderr": "", 
    "stderr_lines": [], 
    "stdout": "10.0.0.109", 
    "stdout_lines": [
        "10.0.0.109"
    ]
}"
2018-10-08 23:53:17,819+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u'Get target engine VM IPv4 address', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:17,819+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313caedc10> kwargs 
2018-10-08 23:53:18,531+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u"Get VDSM's target engine VM stats", 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:18,531+0100 DEBUG ansible on_any args TASK: Get VDSM's target engine VM stats kwargs is_conditional:False 
2018-10-08 23:53:19,620+0100 DEBUG var changed: host "localhost" var "engine_vdsm_stats" type "<type 'dict'>" value: "{
    "changed": true, 
    "cmd": [
        "vdsm-client", 
        "VM", 
        "getStats", 
        "vmID=e1af6b26-9e48-251-940c-7bfadf920f3f"
    ], 
    "delta": "0:00:00.257926", 
    "end": "2018-10-08 23:53:18.961394", 
    "failed": false, 
    "rc": 0, 
    "start": "2018-10-08 23:53:18.703468", 
    "stderr": "", 
    "stderr_lines": [], 
    "stdout": "[\n    {\n        \"displayInfo\": [\n            {\n                \"tlsPort\": \"5901\", \n                \"ipAddress\": \"10.0.0.171\", \n                \"port\": \"5900\", \n                \"type\": \"spice\"\n            }, \n            {\n                \"tlsPort\": \"-1\", \n                \"ipAddress\": \"10.0.0.171\", \n                \"port\": \"5902\", \n                \"type\": \"vnc\"\n            }\n        ], \n        \"memUsage\": \"18\", \n        \"acpiEnable\": \"true\", \n        \"guestFQDN\": \"ovirt-engine.example.com\", \n        \"vmId\": \"e1af6b26-9e48-251-940c-7bfadf920f3f\", \n        \"session\": \"Unknown\", \n        \"netIfaces\": [\n            {\n                \"name\": \"eth0\", \n                \"inet6\": [], \n                \"inet\": [], \n                \"hw\": \"00:18:3d:5b:11:5c\"\n            }\n        ], \n        \"timeOffset\": \"0\", \n        \"memoryStats\": {\n            \"swap_out\": 0, \n            \
 "majflt\": 0, \n            \"minflt\": 198, \n            \"mem_cached\": \"447696\", \n            \"mem_free\": \"13259132\", \n            \"mem_buffers\": \"2104\", \n            \"swap_in\": 0, \n            \"pageflt\": 198, \n            \"mem_total\": \"16258492\", \n            \"mem_unused\": \"13259132\"\n        }, \n        \"balloonInfo\": {\n            \"balloon_max\": \"16777216\", \n            \"balloon_cur\": \"16777216\", \n            \"balloon_target\": \"16777216\", \n            \"balloon_min\": \"1048576\"\n        }, \n        \"pauseCode\": \"NOERR\", \n        \"disksUsage\": [\n            {\n                \"path\": \"/\", \n                \"total\": \"6565134336\", \n                \"used\": \"206525344\", \n                \"fs\": \"xfs\"\n            }, \n            {\n                \"path\": \"/boot\", \n                \"total\": \"1063256064\", \n                \"used\": \"169177088\", \n                \"fs\": \"xfs\"\n            }, \n 
            {\n                \"path\": \"/tmp\", \n                \"total\": \"2136997888\", \n                \"used\": \"33935360\", \n                \"fs\": \"xfs\"\n            }, \n            {\n                \"path\": \"/home\", \n                \"total\": \"1063256064\", \n                \"used\": \"33796096\", \n                \"fs\": \"xfs\"\n            }, \n            {\n                \"path\": \"/var\", \n                \"total\": \"21464350720\", \n                \"used\": \"456179712\", \n                \"fs\": \"xfs\"\n            }, \n            {\n                \"path\": \"/var/log\", \n                \"total\": \"10726932480\", \n                \"used\": \"43315200\", \n                \"fs\": \"xfs\"\n            }, \n            {\n                \"path\": \"/var/log/audit\", \n                \"total\": \"1063256064\", \n                \"used\": \"3252160\", \n                \"fs\": \"xfs\"\n            }\n        ], \n        \"network\":
  {\n            \"vnet0\": {\n                \"macAddr\": \"00:18:3d:5b:11:5c\", \n                \"rxDropped\": \"0\", \n                \"tx\": \"1710\", \n                \"txDropped\": \"0\", \n                \"rxErrors\": \"0\", \n                \"rx\": \"349635\", \n                \"txErrors\": \"0\", \n                \"state\": \"unknown\", \n                \"sampleTime\": 4344173.61, \n                \"speed\": \"1000\", \n                \"name\": \"vnet0\"\n            }\n        }, \n        \"vmType\": \"kvm\", \n        \"cpuUser\": \"3.00\", \n        \"elapsedTime\": \"585\", \n        \"vmJobs\": {}, \n        \"cpuSys\": \"1.27\", \n        \"appsList\": [\n            \"kernel-3.10.0-862.11.6.el7\", \n            \"cloud-init-0.7.9-24.el7.centos.1\", \n            \"ovirt-guest-agent-common-1.0.14-1.el7\"\n        ], \n        \"guestOs\": \"3.10.0-862.11.6.el7.x86_64\", \n        \"vmName\": \"HostedEngine\", \n        \"vcpuCount\": \"4\", \n        \"has
 h\": \"3205592835746233126\", \n        \"lastLogin\": 1539038623.192216, \n        \"cpuUsage\": \"11670000000\", \n        \"vcpuPeriod\": 100000, \n        \"guestIPs\": \"\", \n        \"guestTimezone\": {\n            \"zone\": \"Europe/London\", \n            \"offset\": 0\n        }, \n        \"vcpuQuota\": \"-1\", \n        \"guestContainers\": [], \n        \"kvmEnable\": \"true\", \n        \"disks\": {\n            \"vda\": {\n                \"readLatency\": \"0\", \n                \"writtenBytes\": \"44758528\", \n                \"writeOps\": \"1063\", \n                \"apparentsize\": \"53687091200\", \n                \"readOps\": \"16151\", \n                \"writeLatency\": \"631097\", \n                \"imageID\": \"758f667c-6e9b-43eb-b09b-c983d78a6374\", \n                \"readBytes\": \"475618816\", \n                \"flushLatency\": \"30762\", \n                \"readRate\": \"0.0\", \n                \"truesize\": \"2670891008\", \n                \"wr
 iteRate\": \"3276.8\"\n            }, \n            \"hdc\": {\n                \"readLatency\": \"0\", \n                \"writtenBytes\": \"0\", \n                \"writeOps\": \"0\", \n                \"apparentsize\": \"0\", \n                \"readOps\": \"4\", \n                \"writeLatency\": \"0\", \n                \"readBytes\": \"152\", \n                \"flushLatency\": \"0\", \n                \"readRate\": \"0.0\", \n                \"truesize\": \"0\", \n                \"writeRate\": \"0.0\"\n            }\n        }, \n        \"monitorResponse\": \"0\", \n        \"guestOsInfo\": {\n            \"kernel\": \"3.10.0-862.11.6.el7.x86_64\", \n            \"type\": \"linux\", \n            \"version\": \"7.5.1804\", \n            \"distribution\": \"CentOS Linux\", \n            \"arch\": \"x86_64\", \n            \"codename\": \"Core\"\n        }, \n        \"username\": \"None\", \n        \"guestName\": \"ovirt-engine.example.com\", \n        \"status\": \"Up\", 
 \n        \"guestCPUCount\": 4, \n        \"clientIp\": \"\", \n        \"statusTime\": \"4344173610\"\n    }\n]", 
    "stdout_lines": [
        "[", 
        "    {", 
        "        \"displayInfo\": [", 
        "            {", 
        "                \"tlsPort\": \"5901\", ", 
        "                \"ipAddress\": \"10.0.0.171\", ", 
        "                \"port\": \"5900\", ", 
        "                \"type\": \"spice\"", 
        "            }, ", 
        "            {", 
        "                \"tlsPort\": \"-1\", ", 
        "                \"ipAddress\": \"10.0.0.171\", ", 
        "                \"port\": \"5902\", ", 
        "                \"type\": \"vnc\"", 
        "            }", 
        "        ], ", 
        "        \"memUsage\": \"18\", ", 
        "        \"acpiEnable\": \"true\", ", 
        "        \"guestFQDN\": \"ovirt-engine.example.com\", ", 
        "        \"vmId\": \"e1af6b26-9e48-251-940c-7bfadf920f3f\", ", 
        "        \"session\": \"Unknown\", ", 
        "        \"netIfaces\": [", 
        "            {", 
        "                \"name\": \"eth0\", ", 
        "                \"inet6\": [], ", 
        "                \"inet\": [], ", 
        "                \"hw\": \"00:18:3d:5b:11:5c\"", 
        "            }", 
        "        ], ", 
        "        \"timeOffset\": \"0\", ", 
        "        \"memoryStats\": {", 
        "            \"swap_out\": 0, ", 
        "            \"majflt\": 0, ", 
        "            \"minflt\": 198, ", 
        "            \"mem_cached\": \"447696\", ", 
        "            \"mem_free\": \"13259132\", ", 
        "            \"mem_buffers\": \"2104\", ", 
        "            \"swap_in\": 0, ", 
        "            \"pageflt\": 198, ", 
        "            \"mem_total\": \"16258492\", ", 
        "            \"mem_unused\": \"13259132\"", 
        "        }, ", 
        "        \"balloonInfo\": {", 
        "            \"balloon_max\": \"16777216\", ", 
        "            \"balloon_cur\": \"16777216\", ", 
        "            \"balloon_target\": \"16777216\", ", 
        "            \"balloon_min\": \"1048576\"", 
        "        }, ", 
        "        \"pauseCode\": \"NOERR\", ", 
        "        \"disksUsage\": [", 
        "            {", 
        "                \"path\": \"/\", ", 
        "                \"total\": \"6565134336\", ", 
        "                \"used\": \"206525344\", ", 
        "                \"fs\": \"xfs\"", 
        "            }, ", 
        "            {", 
        "                \"path\": \"/boot\", ", 
        "                \"total\": \"1063256064\", ", 
        "                \"used\": \"169177088\", ", 
        "                \"fs\": \"xfs\"", 
        "            }, ", 
        "            {", 
        "                \"path\": \"/tmp\", ", 
        "                \"total\": \"2136997888\", ", 
        "                \"used\": \"33935360\", ", 
        "                \"fs\": \"xfs\"", 
        "            }, ", 
        "            {", 
        "                \"path\": \"/home\", ", 
        "                \"total\": \"1063256064\", ", 
        "                \"used\": \"33796096\", ", 
        "                \"fs\": \"xfs\"", 
        "            }, ", 
        "            {", 
        "                \"path\": \"/var\", ", 
        "                \"total\": \"21464350720\", ", 
        "                \"used\": \"456179712\", ", 
        "                \"fs\": \"xfs\"", 
        "            }, ", 
        "            {", 
        "                \"path\": \"/var/log\", ", 
        "                \"total\": \"10726932480\", ", 
        "                \"used\": \"43315200\", ", 
        "                \"fs\": \"xfs\"", 
        "            }, ", 
        "            {", 
        "                \"path\": \"/var/log/audit\", ", 
        "                \"total\": \"1063256064\", ", 
        "                \"used\": \"3252160\", ", 
        "                \"fs\": \"xfs\"", 
        "            }", 
        "        ], ", 
        "        \"network\": {", 
        "            \"vnet0\": {", 
        "                \"macAddr\": \"00:18:3d:5b:11:5c\", ", 
        "                \"rxDropped\": \"0\", ", 
        "                \"tx\": \"1710\", ", 
        "                \"txDropped\": \"0\", ", 
        "                \"rxErrors\": \"0\", ", 
        "                \"rx\": \"349635\", ", 
        "                \"txErrors\": \"0\", ", 
        "                \"state\": \"unknown\", ", 
        "                \"sampleTime\": 4344173.61, ", 
        "                \"speed\": \"1000\", ", 
        "                \"name\": \"vnet0\"", 
        "            }", 
        "        }, ", 
        "        \"vmType\": \"kvm\", ", 
        "        \"cpuUser\": \"3.00\", ", 
        "        \"elapsedTime\": \"585\", ", 
        "        \"vmJobs\": {}, ", 
        "        \"cpuSys\": \"1.27\", ", 
        "        \"appsList\": [", 
        "            \"kernel-3.10.0-862.11.6.el7\", ", 
        "            \"cloud-init-0.7.9-24.el7.centos.1\", ", 
        "            \"ovirt-guest-agent-common-1.0.14-1.el7\"", 
        "        ], ", 
        "        \"guestOs\": \"3.10.0-862.11.6.el7.x86_64\", ", 
        "        \"vmName\": \"HostedEngine\", ", 
        "        \"vcpuCount\": \"4\", ", 
        "        \"hash\": \"3205592835746233126\", ", 
        "        \"lastLogin\": 1539038623.192216, ", 
        "        \"cpuUsage\": \"11670000000\", ", 
        "        \"vcpuPeriod\": 100000, ", 
        "        \"guestIPs\": \"\", ", 
        "        \"guestTimezone\": {", 
        "            \"zone\": \"Europe/London\", ", 
        "            \"offset\": 0", 
        "        }, ", 
        "        \"vcpuQuota\": \"-1\", ", 
        "        \"guestContainers\": [], ", 
        "        \"kvmEnable\": \"true\", ", 
        "        \"disks\": {", 
        "            \"vda\": {", 
        "                \"readLatency\": \"0\", ", 
        "                \"writtenBytes\": \"44758528\", ", 
        "                \"writeOps\": \"1063\", ", 
        "                \"apparentsize\": \"53687091200\", ", 
        "                \"readOps\": \"16151\", ", 
        "                \"writeLatency\": \"631097\", ", 
        "                \"imageID\": \"758f667c-6e9b-43eb-b09b-c983d78a6374\", ", 
        "                \"readBytes\": \"475618816\", ", 
        "                \"flushLatency\": \"30762\", ", 
        "                \"readRate\": \"0.0\", ", 
        "                \"truesize\": \"2670891008\", ", 
        "                \"writeRate\": \"3276.8\"", 
        "            }, ", 
        "            \"hdc\": {", 
        "                \"readLatency\": \"0\", ", 
        "                \"writtenBytes\": \"0\", ", 
        "                \"writeOps\": \"0\", ", 
        "                \"apparentsize\": \"0\", ", 
        "                \"readOps\": \"4\", ", 
        "                \"writeLatency\": \"0\", ", 
        "                \"readBytes\": \"152\", ", 
        "                \"flushLatency\": \"0\", ", 
        "                \"readRate\": \"0.0\", ", 
        "                \"truesize\": \"0\", ", 
        "                \"writeRate\": \"0.0\"", 
        "            }", 
        "        }, ", 
        "        \"monitorResponse\": \"0\", ", 
        "        \"guestOsInfo\": {", 
        "            \"kernel\": \"3.10.0-862.11.6.el7.x86_64\", ", 
        "            \"type\": \"linux\", ", 
        "            \"version\": \"7.5.1804\", ", 
        "            \"distribution\": \"CentOS Linux\", ", 
        "            \"arch\": \"x86_64\", ", 
        "            \"codename\": \"Core\"", 
        "        }, ", 
        "        \"username\": \"None\", ", 
        "        \"guestName\": \"ovirt-engine.example.com\", ", 
        "        \"status\": \"Up\", ", 
        "        \"guestCPUCount\": 4, ", 
        "        \"clientIp\": \"\", ", 
        "        \"statusTime\": \"4344173610\"", 
        "    }", 
        "]"
    ]
}"
2018-10-08 23:53:19,620+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u"Get VDSM's target engine VM stats", 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:19,620+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313cc17e10> kwargs 
2018-10-08 23:53:20,321+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Convert stats to JSON format', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:20,322+0100 DEBUG ansible on_any args TASK: Convert stats to JSON format kwargs is_conditional:False 
2018-10-08 23:53:21,008+0100 DEBUG var changed: host "localhost" var "json_stats" type "<type 'list'>" value: "[
    {
        "acpiEnable": "true", 
        "appsList": [
            "kernel-3.10.0-862.11.6.el7", 
            "cloud-init-0.7.9-24.el7.centos.1", 
            "ovirt-guest-agent-common-1.0.14-1.el7"
        ], 
        "balloonInfo": {
            "balloon_cur": "16777216", 
            "balloon_max": "16777216", 
            "balloon_min": "1048576", 
            "balloon_target": "16777216"
        }, 
        "clientIp": "", 
        "cpuSys": "1.27", 
        "cpuUsage": "11670000000", 
        "cpuUser": "3.00", 
        "disks": {
            "hdc": {
                "apparentsize": "0", 
                "flushLatency": "0", 
                "readBytes": "152", 
                "readLatency": "0", 
                "readOps": "4", 
                "readRate": "0.0", 
                "truesize": "0", 
                "writeLatency": "0", 
                "writeOps": "0", 
                "writeRate": "0.0", 
                "writtenBytes": "0"
            }, 
            "vda": {
                "apparentsize": "53687091200", 
                "flushLatency": "30762", 
                "imageID": "758f667c-6e9b-43eb-b09b-c983d78a6374", 
                "readBytes": "475618816", 
                "readLatency": "0", 
                "readOps": "16151", 
                "readRate": "0.0", 
                "truesize": "2670891008", 
                "writeLatency": "631097", 
                "writeOps": "1063", 
                "writeRate": "3276.8", 
                "writtenBytes": "44758528"
            }
        }, 
        "disksUsage": [
            {
                "fs": "xfs", 
                "path": "/", 
                "total": "6565134336", 
                "used": "206525344"
            }, 
            {
                "fs": "xfs", 
                "path": "/boot", 
                "total": "1063256064", 
                "used": "169177088"
            }, 
            {
                "fs": "xfs", 
                "path": "/tmp", 
                "total": "2136997888", 
                "used": "33935360"
            }, 
            {
                "fs": "xfs", 
                "path": "/home", 
                "total": "1063256064", 
                "used": "33796096"
            }, 
            {
                "fs": "xfs", 
                "path": "/var", 
                "total": "21464350720", 
                "used": "456179712"
            }, 
            {
                "fs": "xfs", 
                "path": "/var/log", 
                "total": "10726932480", 
                "used": "43315200"
            }, 
            {
                "fs": "xfs", 
                "path": "/var/log/audit", 
                "total": "1063256064", 
                "used": "3252160"
            }
        ], 
        "displayInfo": [
            {
                "ipAddress": "10.0.0.171", 
                "port": "5900", 
                "tlsPort": "5901", 
                "type": "spice"
            }, 
            {
                "ipAddress": "10.0.0.171", 
                "port": "5902", 
                "tlsPort": "-1", 
                "type": "vnc"
            }
        ], 
        "elapsedTime": "585", 
        "guestCPUCount": 4, 
        "guestContainers": [], 
        "guestFQDN": "ovirt-engine.example.com", 
        "guestIPs": "", 
        "guestName": "ovirt-engine.example.com", 
        "guestOs": "3.10.0-862.11.6.el7.x86_64", 
        "guestOsInfo": {
            "arch": "x86_64", 
            "codename": "Core", 
            "distribution": "CentOS Linux", 
            "kernel": "3.10.0-862.11.6.el7.x86_64", 
            "type": "linux", 
            "version": "7.5.1804"
        }, 
        "guestTimezone": {
            "offset": 0, 
            "zone": "Europe/London"
        }, 
        "hash": "3205592835746233126", 
        "kvmEnable": "true", 
        "lastLogin": 1539038623.192216, 
        "memUsage": "18", 
        "memoryStats": {
            "majflt": 0, 
            "mem_buffers": "2104", 
            "mem_cached": "447696", 
            "mem_free": "13259132", 
            "mem_total": "16258492", 
            "mem_unused": "13259132", 
            "minflt": 198, 
            "pageflt": 198, 
            "swap_in": 0, 
            "swap_out": 0
        }, 
        "monitorResponse": "0", 
        "netIfaces": [
            {
                "hw": "00:18:3d:5b:11:5c", 
                "inet": [], 
                "inet6": [], 
                "name": "eth0"
            }
        ], 
        "network": {
            "vnet0": {
                "macAddr": "00:18:3d:5b:11:5c", 
                "name": "vnet0", 
                "rx": "349635", 
                "rxDropped": "0", 
                "rxErrors": "0", 
                "sampleTime": 4344173.61, 
                "speed": "1000", 
                "state": "unknown", 
                "tx": "1710", 
                "txDropped": "0", 
                "txErrors": "0"
            }
        }, 
        "pauseCode": "NOERR", 
        "session": "Unknown", 
        "status": "Up", 
        "statusTime": "4344173610", 
        "timeOffset": "0", 
        "username": "None", 
        "vcpuCount": "4", 
        "vcpuPeriod": 100000, 
        "vcpuQuota": "-1", 
        "vmId": "e1af6b26-9e48-251-940c-7bfadf920f3f", 
        "vmJobs": {}, 
        "vmName": "HostedEngine", 
        "vmType": "kvm"
    }
]"
2018-10-08 23:53:21,009+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u'Convert stats to JSON format', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:21,009+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313db268d0> kwargs 
2018-10-08 23:53:21,720+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Get target engine VM IPv4 address from VDSM stats', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:21,720+0100 DEBUG ansible on_any args TASK: Get target engine VM IPv4 address from VDSM stats kwargs is_conditional:False 
2018-10-08 23:53:22,397+0100 DEBUG var changed: host "localhost" var "engine_vm_ip_vdsm" type "<class 'ansible.utils.unsafe_proxy.AnsibleUnsafeText'>" value: """"
2018-10-08 23:53:22,398+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u'Get target engine VM IPv4 address from VDSM stats', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:22,398+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313caed510> kwargs 
2018-10-08 23:53:23,124+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'debug', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:23,124+0100 DEBUG ansible on_any args TASK: debug kwargs is_conditional:False 
2018-10-08 23:53:23,807+0100 INFO ansible ok {'status': 'OK', 'ansible_task': u'', 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:23,807+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313ca343d0> kwargs 
2018-10-08 23:53:24,511+0100 INFO ansible task start {'status': 'OK', 'ansible_task': u'Fail if the Engine has no IP address', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'task'}
2018-10-08 23:53:24,511+0100 DEBUG ansible on_any args TASK: Fail if the Engine has no IP address kwargs is_conditional:False 
2018-10-08 23:53:25,152+0100 DEBUG var changed: host "localhost" var "ansible_play_hosts" type "<type 'list'>" value: "[]"
2018-10-08 23:53:25,152+0100 DEBUG var changed: host "localhost" var "play_hosts" type "<type 'list'>" value: "[]"
2018-10-08 23:53:25,152+0100 DEBUG var changed: host "localhost" var "ansible_play_batch" type "<type 'list'>" value: "[]"
2018-10-08 23:53:25,153+0100 ERROR ansible failed {'status': 'FAILED', 'ansible_type': 'task', 'ansible_task': u'Fail if the Engine has no IP address', 'ansible_result': u"type: <type 'dict'>\nstr: {'msg': u'Engine VM has no IP address. Please check your network configuration', 'changed': False, '_ansible_no_log': False}", 'ansible_host': u'localhost', 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml'}
2018-10-08 23:53:25,153+0100 DEBUG ansible on_any args <ansible.executor.task_result.TaskResult object at 0x7f313caed6d0> kwargs ignore_errors:None 
2018-10-08 23:53:25,154+0100 INFO ansible stats {'status': 'FAILED', 'ansible_playbook_duration': 1064.367312, 'ansible_result': u"type: <type 'dict'>\nstr: {u'ovirt-engine.example.com': {'unreachable': 0, 'skipped': 0, 'ok': 11, 'changed': 6, 'failures': 0}, u'localhost': {'unreachable': 0, 'skipped': 5, 'ok': 82, 'changed': 30, 'failures': 2}}", 'ansible_playbook': u'/usr/share/ovirt-hosted-engine-setup/ansible/create_target_vm.yml', 'ansible_type': 'finish'}
2018-10-08 23:53:25,154+0100 DEBUG ansible on_any args <ansible.executor.stats.AggregateStats object at 0x7f313ef6c290> kwargs

I run it again (I clear the shared NFS storage each attempt), then get the following output to console:
[ INFO  ] TASK [Undefine leftover engine VM]
[ ERROR ] fatal: [localhost]: FAILED! => {"changed": true, "cmd": ["virsh", "undefine", "--managed-save", "HostedEngine"], "delta": "0:00:00.050407", "end": "2018-10-09 00:16:58.085198", "msg": "non-zero return code", "rc": 1, "start": "2018-10-09 00:16:58.034791", "stderr": "error: Failed to undefine domain HostedEngine\nerror: Requested operation is not valid: cannot undefine transient domain", "stderr_lines": ["error: Failed to undefine domain HostedEngine", "error: Requested operation is not valid: cannot undefine transient domain"], "stdout": "", "stdout_lines": []}

[ovirt-users] Re: Diary of hosted engine install woes

me＠brendanh.com