I think I found it:
In the vdsm log we need to search by UUID before START and in imageio daemon via image folder UUID:
[root@ovirt-hv-01 vdsm]# xzgrep e935c4cd-c171-40fa-9204-db59f11ac013 vdsm.log.*xz
vdsm.log.22.xz:2019-07-17 13:36:09,219+0000 INFO (jsonrpc/1) [vdsm.api] START prepareImage(sdUUID=u'644aacaa-12e1-4fcd-b3aa-941678cf95bd', spUUID=u'10525aa0-839d-11e9-a016-00163e4f2a6d', imgUUID=u'3452459d-aec6-430e-9509-1d9ca815b2d8',
leafUUID=u'b44659a9-607a-4eeb-a255-99532fd4fce4', allowIllegal=True) from=::ffff:172.17.29.10,52964, flow_id=e935c4cd-c171-40fa-9204-db59f11ac013, task_id=d7eb8ef8-0216-4338-ac5e-07ff41ba5ace (api:48)
vdsm.log.22.xz:2019-07-17 13:36:09,416+0000 INFO (jsonrpc/1) [vdsm.api] FINISH prepareImage return={'info': {'path': u'/rhev/data-center/mnt/172.17.28.5:_ovirt__production/644aacaa-12e1-4fcd-b3aa-941678cf95bd/images/3452459d-aec6-430e-9509-1d9ca815b2d8/b44659a9-607a-4eeb-a255-99532fd4fce4',
'type': 'file'}, 'path': u'/rhev/data-center/mnt/172.17.28.5:_ovirt__production/644aacaa-12e1-4fcd-b3aa-941678cf95bd/images/3452459d-aec6-430e-9509-1d9ca815b2d8/b44659a9-607a-4eeb-a255-99532fd4fce4', 'imgVolumesInfo': [{'domainID': u'644aacaa-12e1-4fcd-b3aa-941678cf95bd',
'leaseOffset': 0, 'path': u'/rhev/data-center/mnt/172.17.28.5:_ovirt__production/644aacaa-12e1-4fcd-b3aa-941678cf95bd/images/3452459d-aec6-430e-9509-1d9ca815b2d8/b44659a9-607a-4eeb-a255-99532fd4fce4', 'volumeID': u'b44659a9-607a-4eeb-a255-99532fd4fce4', 'leasePath':
u'/rhev/data-center/mnt/172.17.28.5:_ovirt__production/644aacaa-12e1-4fcd-b3aa-941678cf95bd/images/3452459d-aec6-430e-9509-1d9ca815b2d8/b44659a9-607a-4eeb-a255-99532fd4fce4.lease', 'imageID': u'3452459d-aec6-430e-9509-1d9ca815b2d8'}]} from=::ffff:172.17.29.10,52964,
flow_id=e935c4cd-c171-40fa-9204-db59f11ac013, task_id=d7eb8ef8-0216-4338-ac5e-07ff41ba5ace (api:54)
vdsm.log.22.xz:2019-07-17 13:36:09,422+0000 INFO (jsonrpc/6) [vdsm.api] START getQemuImageInfo(sdUUID=u'644aacaa-12e1-4fcd-b3aa-941678cf95bd', spUUID=u'10525aa0-839d-11e9-a016-00163e4f2a6d', imgUUID=u'3452459d-aec6-430e-9509-1d9ca815b2d8',
volUUID=u'b44659a9-607a-4eeb-a255-99532fd4fce4', options=None) from=::ffff:172.17.29.10,52964, flow_id=e935c4cd-c171-40fa-9204-db59f11ac013, task_id=aa11989c-d308-491a-ab80-935c642fb758 (api:48)
vdsm.log.22.xz:2019-07-17 13:36:09,522+0000 INFO (jsonrpc/6) [vdsm.api] FINISH getQemuImageInfo return={'info': {'compat': u'1.1', 'clustersize': 65536, 'actualsize': 204800, 'virtualsize': 42949672960,
'format': u'qcow2'}} from=::ffff:172.17.29.10,52964, flow_id=e935c4cd-c171-40fa-9204-db59f11ac013, task_id=aa11989c-d308-491a-ab80-935c642fb758 (api:54)
vdsm.log.22.xz:2019-07-17 13:36:09,528+0000 INFO (jsonrpc/3) [vdsm.api] START teardownImage(sdUUID=u'644aacaa-12e1-4fcd-b3aa-941678cf95bd', spUUID=u'10525aa0-839d-11e9-a016-00163e4f2a6d', imgUUID=u'3452459d-aec6-430e-9509-1d9ca815b2d8',
volUUID=None) from=::ffff:172.17.29.10,52964, flow_id=e935c4cd-c171-40fa-9204-db59f11ac013, task_id=a806250b-875a-42a3-bac8-a58388f86af3 (api:48)
vdsm.log.22.xz:2019-07-17 13:36:09,529+0000 INFO (jsonrpc/3) [vdsm.api] FINISH teardownImage return=None from=::ffff:172.17.29.10,52964, flow_id=e935c4cd-c171-40fa-9204-db59f11ac013, task_id=a806250b-875a-42a3-bac8-a58388f86af3
(api:54)
[root@ovirt-hv-01 ovirt-imageio-daemon]# grep 3452459d-aec6-430e-9509-1d9ca815b2d daemon.log
2019-07-17 13:36:11,407 INFO (Thread-1) [tickets] [local] ADD ticket={u'uuid': u'10096b11-d10d-43aa-ad16-668b29a8c152', u'ops': [u'write'], u'url': u'file:///rhev/data-center/mnt/172.17.28.5:_ovirt__production/644aacaa-12e1-4fcd-b3aa-941678cf95bd/images/3452459d-aec6-430e-9509-1d9ca815b2d8/b44659a9-607a-4eeb-a255-99532fd4fce4',
u'sparse': True, u'timeout': 300, u'transfer_id': u'28eda0c2-e36b-4e70-91ea-2ecf4a030d19', u'size': 47244640256}
From: "Vrgotic, Marko" <M.Vrgotic@activevideo.com>
Date: Thursday, 18 July 2019 at 12:44
To: Nir Soffer <nsoffer@redhat.com>
Cc: users <users@ovirt.org>, Daniel Erez <derez@redhat.com>
Subject: Re: [ovirt-users] Stuck in "Finalizing" disk upload phase
Hi Nir,
Sure: here is the ovirt-engine/engine.log related to transaction:
[root@ovirt-engine ovirt-engine]# zgrep "3452459d-aec6-430e-9509-1d9ca815b2d8" /var/log/ovirt-engine/engine.log-20190718.gz | head -n 40
2019-07-17 13:36:01,634Z INFO [org.ovirt.engine.core.vdsbroker.irsbroker.CreateImageVDSCommand] (default task-376) [e935c4cd-c171-40fa-9204-db59f11ac013] START, CreateImageVDSCommand( CreateImageVDSCommandParameters:{storagePoolId='10525aa0-839d-11e9-a016-00163e4f2a6d',
ignoreFailoverLimit='false', storageDomainId='644aacaa-12e1-4fcd-b3aa-941678cf95bd', imageGroupId='3452459d-aec6-430e-9509-1d9ca815b2d8', imageSizeInBytes='42949672960', volumeFormat='COW', newImageId='b44659a9-607a-4eeb-a255-99532fd4fce4', imageType='Sparse',
newImageDescription='{"DiskAlias":"av-07-centos-65-base","DiskDescription":""}', imageInitialSizeInBytes='0'}), log id: 41539302
2019-07-17 13:36:09,110Z INFO [org.ovirt.engine.core.vdsbroker.irsbroker.GetImageInfoVDSCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-92) [e935c4cd-c171-40fa-9204-db59f11ac013] START,
GetImageInfoVDSCommand( GetImageInfoVDSCommandParameters:{storagePoolId='10525aa0-839d-11e9-a016-00163e4f2a6d', ignoreFailoverLimit='false', storageDomainId='644aacaa-12e1-4fcd-b3aa-941678cf95bd', imageGroupId='3452459d-aec6-430e-9509-1d9ca815b2d8', imageId='b44659a9-607a-4eeb-a255-99532fd4fce4'}),
log id: 3712c16a
2019-07-17 13:36:09,111Z INFO [org.ovirt.engine.core.vdsbroker.vdsbroker.GetVolumeInfoVDSCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-92) [e935c4cd-c171-40fa-9204-db59f11ac013] START,
GetVolumeInfoVDSCommand(HostName = ovirt-hv-03.avinity.tv, GetVolumeInfoVDSCommandParameters:{hostId='1faf7ede-6f81-41c7-8ba8-d146e6617422', storagePoolId='10525aa0-839d-11e9-a016-00163e4f2a6d', storageDomainId='644aacaa-12e1-4fcd-b3aa-941678cf95bd',
imageGroupId='3452459d-aec6-430e-9509-1d9ca815b2d8', imageId='b44659a9-607a-4eeb-a255-99532fd4fce4'}), log id: 1b0d57f3
2019-07-17 13:36:09,420Z INFO [org.ovirt.engine.core.vdsbroker.vdsbroker.GetQemuImageInfoVDSCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-92) [e935c4cd-c171-40fa-9204-db59f11ac013] START,
GetQemuImageInfoVDSCommand(HostName = ovirt-hv-01.avinity.tv, GetVolumeInfoVDSCommandParameters:{hostId='e7e3f1dc-8037-4e74-a44c-442bdb02197d', storagePoolId='10525aa0-839d-11e9-a016-00163e4f2a6d', storageDomainId='644aacaa-12e1-4fcd-b3aa-941678cf95bd',
imageGroupId='3452459d-aec6-430e-9509-1d9ca815b2d8', imageId='b44659a9-607a-4eeb-a255-99532fd4fce4'}), log id: 775faf53
2019-07-17 13:36:11,059Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (default task-376) [a43180ec-afc7-429e-9f30-9e851eaf7ce7]
Lock Acquired to object 'EngineLock:{exclusiveLocks='', sharedLocks='[3452459d-aec6-430e-9509-1d9ca815b2d8=DISK]'}'
2019-07-17 13:36:11,121Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (default task-376) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Running command: TransferDiskImageCommand
internal: false. Entities affected : ID: 3452459d-aec6-430e-9509-1d9ca815b2d8 Type: DiskAction group EDIT_DISK_PROPERTIES with role type USER
2019-07-17 13:36:11,142Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (default task-376) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Successfully added Upload disk 'av-07-centos-65-base'
(disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4') for image transfer command '28eda0c2-e36b-4e70-91ea-2ecf4a030d19'
2019-07-17 13:36:11,356Z INFO [org.ovirt.engine.core.vdsbroker.irsbroker.SetVolumeLegalityVDSCommand] (default task-376) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] START, SetVolumeLegalityVDSCommand(
SetVolumeLegalityVDSCommandParameters:{storagePoolId='10525aa0-839d-11e9-a016-00163e4f2a6d', ignoreFailoverLimit='false', storageDomainId='644aacaa-12e1-4fcd-b3aa-941678cf95bd', imageGroupId='3452459d-aec6-430e-9509-1d9ca815b2d8', imageId='b44659a9-607a-4eeb-a255-99532fd4fce4'}),
log id: 5e0f7654
2019-07-17 13:36:11,394Z INFO [org.ovirt.engine.core.vdsbroker.vdsbroker.AddImageTicketVDSCommand] (default task-376) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] START, AddImageTicketVDSCommand(HostName
= ovirt-hv-01.avinity.tv, AddImageTicketVDSCommandParameters:{hostId='e7e3f1dc-8037-4e74-a44c-442bdb02197d', ticketId='10096b11-d10d-43aa-ad16-668b29a8c152', timeout='300', operations='[write]', size='47244640256', url='file:///rhev/data-center/mnt/172.17.28.5:_ovirt__production/644aacaa-12e1-4fcd-b3aa-941678cf95bd/images/3452459d-aec6-430e-9509-1d9ca815b2d8/b44659a9-607a-4eeb-a255-99532fd4fce4',
filename='null', sparse='true', transferId='28eda0c2-e36b-4e70-91ea-2ecf4a030d19'}), log id: 3bf6b943
2019-07-17 13:36:11,503Z INFO [org.ovirt.engine.core.bll.storage.disk.image.ImageTransferUpdater] (default task-376) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Updating image transfer 28eda0c2-e36b-4e70-91ea-2ecf4a030d19
(image 3452459d-aec6-430e-9509-1d9ca815b2d8) phase to Paused by System
2019-07-17 13:36:11,563Z INFO [org.ovirt.engine.core.bll.storage.disk.image.ImageTransferUpdater] (default task-376) [edda86b2-d78b-45ed-a523-18966af140f4] Updating image transfer 28eda0c2-e36b-4e70-91ea-2ecf4a030d19
(image 3452459d-aec6-430e-9509-1d9ca815b2d8) phase to Finalizing Success
2019-07-17 13:36:11,610Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-60) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Finalizing
successful transfer for Upload disk 'av-07-centos-65-base' (disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4')
2019-07-17 13:36:11,612Z WARN [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-60) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Failed
to stop image transfer session. Ticket does not exist for image '3452459d-aec6-430e-9509-1d9ca815b2d8'
2019-07-17 13:36:13,658Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-69) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Finalizing
successful transfer for Upload disk 'av-07-centos-65-base' (disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4')
2019-07-17 13:36:13,659Z WARN [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-69) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Failed
to stop image transfer session. Ticket does not exist for image '3452459d-aec6-430e-9509-1d9ca815b2d8'
Question regarding transfer UUID – is that one I marked green in the log lines above? Regarding Host that executed action, is in any way related to lines I marked in RED?
Kindly awaiting your reply.
— — —
Met vriendelijke groet / Kind regards,
Marko Vrgotic
From: Nir Soffer <nsoffer@redhat.com>
Date: Thursday, 18 July 2019 at 12:20
To: "Vrgotic, Marko" <M.Vrgotic@activevideo.com>
Cc: users <users@ovirt.org>, Daniel Erez <derez@redhat.com>
Subject: Re: [ovirt-users] Stuck in "Finalizing" disk upload phase
On Thu, Jul 18, 2019 at 10:49 AM Vrgotic, Marko <M.Vrgotic@activevideo.com> wrote:
Dear Nir,
None of my Hosts has any updated left to be added, they are as up to date as they can be.
The imageio packages version installed is:
Ovirt-imageio-common-1.5.1-0.el7.x86_64
Ovirt-imageio-daemon-1.5.1-0.el7.x86_64
We need logs to understand the issue.
Can you share logs from the time the upload was started?
- engine log (/var/log/ovirt-engine/engine.log)
- vdsm log on the host that performed the upload (/var/log/vdsm/vdsm.log)
- daemon logs on that host (/var/log/ovirt-imageio-daemon/daemon.log)
To locate the right host you can grep for the transfer uuid that should be mentioned in engine logs.
Additional software info from Host
OS Version:
RHEL - 7 - 6.1810.2.el7.centos
OS Description:
CentOS Linux 7 (Core)
Kernel Version:
3.10.0 - 957.21.3.el7.x86_64
KVM Version:
2.12.0 - 18.el7_6.5.1
LIBVIRT Version:
libvirt-4.5.0-10.el7_6.12
VDSM Version:
vdsm-4.30.17-1.el7
SPICE Version:
0.14.0 - 6.el7_6.1
GlusterFS Version:
[N/A]
CEPH Version:
librbd1-10.2.5-4.el7
Open vSwitch Version:
openvswitch-2.10.1-3.el7
Kernel Features:
PTI: 1, IBRS: 0, RETP: 1, SSBD: 3
From: "Vrgotic, Marko" <M.Vrgotic@activevideo.com>
Date: Thursday, 18 July 2019 at 08:03
To: Nir Soffer <nsoffer@redhat.com>
Cc: users <users@ovirt.org>, Daniel Erez <derez@redhat.com>
Subject: Re: [ovirt-users] Stuck in "Finalizing" disk upload phase
Hi Nir,
Sure, i will check.
Is there a way for Adminstrator to view tickets or close them by force?
Sent from my iPhone
On 18 Jul 2019, at 00:05, Nir Soffer <nsoffer@redhat.com> wrote:
On Wed, Jul 17, 2019, 19:20 Vrgotic, Marko <M.Vrgotic@activevideo.com> wrote:
Dear oVIrt,
I initiated upload of qcow2 disk image for Centos 6.5:
It reached finalizing phase and than started throwing following errors:
2019-07-17 14:40:51,480Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-86) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Finalizing successful transfer for Upload disk 'av-07-centos-65-base' (disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4')
2019-07-17 14:40:51,480Z WARN [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-86) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Failed to stop image transfer session. Ticket does not exist for image '3452459d-aec6-430e-9509-1d9ca815b2d8'
Old versions of ovirt-imageio-daemon were failing to remove a ticket if the ticket does not exist, and engine did not handle this well.
This was fixed in 4.2. Are you running the latest version on the hosts?
Please update ovirt-imageio-daemon to latest version.
Nir
2019-07-17 14:41:01,572Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-19) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Finalizing successful transfer for Upload disk 'av-07-centos-65-base' (disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4')
2019-07-17 14:41:01,574Z WARN [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-19) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Failed to stop image transfer session. Ticket does not exist for image '3452459d-aec6-430e-9509-1d9ca815b2d8'
2019-07-17 14:41:11,690Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-7) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Finalizing successful transfer for Upload disk 'av-07-centos-65-base' (disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4')
2019-07-17 14:41:11,690Z WARN [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-7) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Failed to stop image transfer session. Ticket does not exist for image '3452459d-aec6-430e-9509-1d9ca815b2d8'
2019-07-17 14:41:21,781Z INFO [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-12) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Finalizing successful transfer for Upload disk 'av-07-centos-65-base' (disk id: '3452459d-aec6-430e-9509-1d9ca815b2d8', image id: 'b44659a9-607a-4eeb-a255-99532fd4fce4')
2019-07-17 14:41:21,782Z WARN [org.ovirt.engine.core.bll.storage.disk.image.TransferDiskImageCommand] (EE-ManagedThreadFactory-engineScheduled-Thread-12) [a43180ec-afc7-429e-9f30-9e851eaf7ce7] Failed to stop image transfer session. Ticket does not exist for image '3452459d-aec6-430e-9509-1d9ca815b2d8'
I can not cancel it, can not stop it, not via UI not via force option using ovirt_disk module.
Help!
oVIrt 4.3.4.3-1 version running with CentOS 7.6 Hosts.
Kindly awaiting your reply.
— — —
Met vriendelijke groet / Kind regards,
Marko VrgoticActiveVideo
_______________________________________________
Users mailing list -- users@ovirt.org
To unsubscribe send an email to users-leave@ovirt.org
Privacy Statement: https://www.ovirt.org/site/privacy-policy/
oVirt Code of Conduct: https://www.ovirt.org/community/about/community-guidelines/
List Archives: https://lists.ovirt.org/archives/list/users@ovirt.org/message/LJPWK5A3346ZCDSWEAG6WU3JLRXEGX22/