7:48 p.m.
Hello,
This is what I could gather from the gluster logs around the time frame of the HE
shutdown.
NODE1:
[root@ov-no1 glusterfs]# more bricks/gluster_bricks-vmstore-vmstore.log-20200830 |egrep
"( W | E )"|more
[2020-08-27 15:35:03.090477] W [glusterfsd.c:1570:cleanup_and_exit]
(-->/lib64/libpthread.so.0(+0x7dd5) [0x7fa6e04a3dd5] -->/usr/sbin/glusterfsd(glus
terfs_sigwaiter+0xe5) [0x55a40138d1b5] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b)
[0x55a40138d01b] ) 0-: received signum (15), shutting down
[2020-08-27 15:35:14.926794] E [MSGID: 100018]
[glusterfsd.c:2333:glusterfs_pidfile_update] 0-glusterfsd: pidfile
/var/run/gluster/vols/vmstore/ov-no
1.ariadne-t.local-gluster_bricks-vmstore-vmstore.pid lock failed [Resource temporarily
unavailable]
[root@ov-no1 glusterfs]# more bricks/gluster_bricks-data-data.log-20200830 |egrep "(
W | E )"|more
[2020-08-27 15:35:01.087875] W [glusterfsd.c:1570:cleanup_and_exit]
(-->/lib64/libpthread.so.0(+0x7dd5) [0x7fc3cbf69dd5] -->/usr/sbin/glusterfsd(glus
terfs_sigwaiter+0xe5) [0x555e313711b5] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b)
[0x555e3137101b] ) 0-: received signum (15), shutting down
[2020-08-27 15:35:14.890471] E [MSGID: 100018]
[glusterfsd.c:2333:glusterfs_pidfile_update] 0-glusterfsd: pidfile
/var/run/gluster/vols/data/ov-no1.a
riadne-t.local-gluster_bricks-data-data.pid lock failed [Resource temporarily
unavailable]
[root@ov-no1 glusterfs]# more bricks/gluster_bricks-engine-engine.log-20200830 |egrep
"( W | E )"|more
[2020-08-27 15:35:02.088732] W [glusterfsd.c:1570:cleanup_and_exit]
(-->/lib64/libpthread.so.0(+0x7dd5) [0x7f70b99cbdd5] -->/usr/sbin/glusterfsd(glus
terfs_sigwaiter+0xe5) [0x55ebd132b1b5] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b)
[0x55ebd132b01b] ) 0-: received signum (15), shutting down
[2020-08-27 15:35:14.907603] E [MSGID: 100018]
[glusterfsd.c:2333:glusterfs_pidfile_update] 0-glusterfsd: pidfile
/var/run/gluster/vols/engine/ov-no1
.ariadne-t.local-gluster_bricks-engine-engine.pid lock failed [Resource temporarily
unavailable]
[root@ov-no1 glusterfs]# more bricks/gluster_bricks-vmstore-vmstore.log |egrep "( W |
E )"|more
[nothing in the output]
[root@ov-no1 glusterfs]# more bricks/gluster_bricks-data-data.log |egrep "( W | E
)"|more
[nothing in the output]
[root@ov-no1 glusterfs]# more bricks/gluster_bricks-engine-engine.log |egrep "( W | E
)"|more
[nothing in the output]
[root@ov-no1 glusterfs]# more cmd_history.log | egrep "(WARN|error|fail)" |more
[2020-09-01 02:00:38.685251] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 03:02:39.094984] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 11:18:32.510224] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 14:24:33.778942] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[root@ov-no1 glusterfs]# cat glusterd.log | egrep "( W | E )" |more
[2020-09-01 07:00:31.326169] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f23d8ac8a1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f23d8ac11be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f23
d8ae806f] ) 0-management: Unable to get transaction opinfo for transaction ID
:435d3780-aa0c-4a64-bc28-56ae394159d0
[2020-09-01 08:02:31.551563] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f23d8ac8a1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f23d8ac11be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f23
d8ae806f] ) 0-management: Unable to get transaction opinfo for transaction ID
:930a8a08-1044-41cf-b921-913b982e0c72
[2020-09-01 09:04:31.786157] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f23d8ac8a1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f23d8ac11be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f23
d8ae806f] ) 0-management: Unable to get transaction opinfo for transaction ID
:9942b579-5240-4fee-bb4c-78b9a1c98da8
[2020-09-01 10:06:32.014362] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f23d8ac8a1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f23d8ac11be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f23
d8ae806f] ) 0-management: Unable to get transaction opinfo for transaction ID
:324fc904-b147-4801-89d5-485120e1fb84
[2020-09-01 11:08:32.240906] W [MSGID: 106061]
[glusterd-geo-rep.c:2388:glusterd_op_gsync_args_get] 0-management: master not found
[2020-09-01 11:18:32.510041] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x812c, unique = 0, s
ent = 2020-09-01 11:08:32.242199, timeout = 600 for 192.168.48.202:24007
[root@ov-no1 glusterfs]# cat events.log-20200828 | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat events.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glfsheal-data.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glfsheal-vmstore.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glfsheal-engine.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glfsheal-data.log-20200830 | egrep "( W | E )"
|more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glfsheal-vmstore.log-20200830 | egrep "( W | E )"
|more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glfsheal-engine.log-20200830 | egrep "( W | E )"
|more
[nothing in the output]
[root@ov-no1 glusterfs]# cat glustershd.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_engine.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_data.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_vmstore.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_data.log
[2020-08-31 00:41:02.169424] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-08-31 00:41:02.170418] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-08-31 08:59:07.694673] I [MSGID: 133022] [shard.c:3673:shard_delete_shards]
0-data-shard: Deleted shards of gfid=d25c9bb6-4a25-4025-8652-686be0a4bdf1 from backend
[2020-08-31 08:59:07.890175] I [MSGID: 133017] [shard.c:6696:shard_seek] 0-data-shard:
seek called on 0e4c3fb8-7617-4e43-879a-cc784198216d. [Operation not supported]
[2020-08-31 08:59:07.892803] I [MSGID: 133017] [shard.c:6696:shard_seek] 0-data-shard:
seek called on 17c220e1-fa2c-42d7-a948-6ea2bf5a93d7. [Operation not supported]
[2020-08-31 08:59:07.897929] I [MSGID: 133017] [shard.c:6696:shard_seek] 0-data-shard:
seek called on f9315204-ec16-4c46-847e-d84f0169ea8f. [Operation not supported]
[2020-08-31 08:59:07.928488] I [MSGID: 133022] [shard.c:3673:shard_delete_shards]
0-data-shard: Deleted shards of gfid=17c220e1-fa2c-42d7-a948-6ea2bf5a93d7 from backend
[2020-09-01 00:36:01.667677] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-01 00:36:01.668986] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-02 00:48:02.278423] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-02 00:48:02.279637] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-04 00:36:01.546102] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-04 00:36:01.547101] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[root@ov-no1 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_vmstore.log
[2020-08-28 00:12:02.424709] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-08-29 00:29:02.375537] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-08-29 00:29:02.377088] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-08-30 00:13:03.449758] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-08-30 00:13:03.451232] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-08-31 00:41:02.169556] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-08-31 00:41:02.171040] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-01 00:36:01.667768] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-01 00:36:01.669572] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-02 00:48:02.278490] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-02 00:48:02.280253] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-04 00:36:01.546134] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-04 00:36:01.547694] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[root@ov-no1 glusterfs]#
[root@ov-no1 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_engine.log
[2020-08-30 00:13:03.447882] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-08-30 00:13:03.449756] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-08-31 00:41:02.167497] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-08-31 00:41:02.169892] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-01 00:36:01.666356] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-01 00:36:01.668276] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-02 00:48:02.277024] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-02 00:48:02.279273] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-04 00:36:01.543799] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-04 00:36:01.545779] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[root@ov-no1 glusterfs]# cat glustershd.log | egrep "( W | E )" |more
[root@ov-no1 glusterfs]# cat glustershd.log
[2020-08-30 20:31:26.884546] I [MSGID: 108026]
[afr-self-heal-common.c:1741:afr_log_selfheal] 0-engine-replicate-0: Completed data
selfheal on 3fafabf3-d0cd-4b9a-8dd7-43145451f7cf. sources=[1] 2 sinks=0
.
.
.
[every ten minutes the same log appears]
.
.
[2020-08-31 15:03:57.893224] I [MSGID: 108026]
[afr-self-heal-common.c:1741:afr_log_selfheal] 0-engine-replicate-0: Completed data
selfheal on 3fafabf3-d0cd-4b9a-8dd7-43145451f7cf. sources=[1] 2 sinks=0
The message "I [MSGID: 108026] [afr-self-heal-common.c:1741:afr_log_selfheal]
0-engine-replicate-0: Completed data selfheal on 3fafabf3-d0cd-4b9a-8dd7-43145451f7cf.
sources=[1] 2 sinks=0 " repeated 15 times between [2020-08-31 15:03:57.893224] and
[2020-08-31 15:04:42.605183]
[2020-08-31 15:14:46.171044] I [MSGID: 108026]
[afr-self-heal-common.c:1741:afr_log_selfheal] 0-engine-replicate-0: Completed data
selfheal on 3fafabf3-d0cd-4b9a-8dd7-43145451f7cf. sources=[1] 2 sinks=0
[2020-09-01 00:36:01.663267] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-01 00:36:01.665109] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[2020-09-02 00:48:02.274383] I [MSGID: 100011] [glusterfsd.c:1641:reincarnate]
0-glusterfsd: Fetching the volume file from server...
[2020-09-02 00:48:02.275562] I [glusterfsd-mgmt.c:2019:mgmt_getspec_cbk] 0-glusterfs: No
change in volfile,continuing
[root@ov-no1 glusterfs]# zcat cli.log-20200902.gz |egrep "( W | E )" |more
[2020-09-01 09:04:39.799900] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:04:39.801917] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:24:41.581140] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:39:43.395156] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:39:43.397147] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:49:44.128540] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:54:44.629071] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 10:04:45.642437] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 10:04:45.904006] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:04:45.905279] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:39:49.183655] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 10:44:49.968219] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:44:49.970487] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:49:50.206005] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 10:59:51.489242] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:59:51.491277] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[root@ov-no1 glusterfs]# cat cli.log |grep "( W | E )" |more
[nothing in the output]
[root@ov-no1 glusterfs]# cat cmd_history.log|egrep "( W| E|fail)" |more
[2020-09-01 00:58:38.265743] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 02:00:38.685251] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 03:02:39.094984] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 11:18:32.510224] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 14:24:33.778942] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 16:28:34.610703] : volume geo-replication status : FAILED : Commit failed on
ov-no2.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
NODE2:
[root@ov-no2 ~]# cd /var/log/glusterfs/
[root@ov-no2 glusterfs]# more bricks/gluster_bricks-data-data.log-20200830 |egrep "(
W | E )"|more
[2020-08-27 15:35:07.532064] W [glusterfsd.c:1570:cleanup_and_exit]
(-->/lib64/libpthread.so.0(+0x7dd5) [0x7f01ef674dd5] -->/usr/sbin/glusterfsd(glus
terfs_sigwaiter+0xe5) [0x55d96c9791b5] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b)
[0x55d96c97901b] ) 0-: received signum (15), shutting down
[2020-08-27 15:35:07.545460] W [rpc-clnt.c:1704:rpc_clnt_submit] 0-glusterfs: failed to
submit rpc-request (unique: 0, XID: 0x33 Program: Gluster Por
tmap, ProgVers: 1, Proc: 5) to rpc-transport (glusterfs)
[2020-08-27 15:35:07.545626] E [timer.c:34:gf_timer_call_after]
(-->/usr/sbin/glusterfsd(mgmt_submit_request+0x13d) [0x55d96c97ecdd]
-->/lib64/libgfr
pc.so.0(rpc_clnt_submit+0x72d) [0x7f01f05dfc1d]
-->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7f01f0844349] ) 0-timer:
Either ctx is NULL
or ctx cleanup started [Invalid argument]
[2020-08-27 15:35:07.545724] E [timer.c:34:gf_timer_call_after]
(-->/usr/sbin/glusterfsd(mgmt_submit_request+0x13d) [0x55d96c97ecdd]
-->/lib64/libgfr
pc.so.0(rpc_clnt_submit+0x72d) [0x7f01f05dfc1d]
-->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7f01f0844349] ) 0-timer:
Either ctx is NULL
or ctx cleanup started [Invalid argument]
[2020-08-27 16:25:42.954545] W [inodelk.c:609:pl_inodelk_log_cleanup] 0-data-server:
releasing lock on 03848819-f37b-4fa7-81d9-3f5dc3eef2da held by {
client=0x7fd84400f350, pid=7725 lk-owner=e8100a64447f0000}
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]# more bricks/gluster_bricks-engine-engine.log-20200830 |egrep
"( W | E )"|more
[2020-08-27 15:35:08.532812] W [glusterfsd.c:1570:cleanup_and_exit]
(-->/lib64/libpthread.so.0(+0x7dd5) [0x7ff67fa1edd5] -->/usr/sbin/glusterfsd(glus
terfs_sigwaiter+0xe5) [0x55d4f15951b5] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b)
[0x55d4f159501b] ) 0-: received signum (15), shutting down
[2020-08-27 15:35:08.536715] W [rpc-clnt.c:1704:rpc_clnt_submit] 0-glusterfs: failed to
submit rpc-request (unique: 0, XID: 0x33 Program: Gluster Por
tmap, ProgVers: 1, Proc: 5) to rpc-transport (glusterfs)
[2020-08-27 15:35:08.536903] E [timer.c:34:gf_timer_call_after]
(-->/usr/sbin/glusterfsd(mgmt_submit_request+0x13d) [0x55d4f159acdd]
-->/lib64/libgfr
pc.so.0(rpc_clnt_submit+0x72d) [0x7ff680989c1d]
-->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7ff680bee349] ) 0-timer:
Either ctx is NULL
or ctx cleanup started [Invalid argument]
[2020-08-27 15:35:08.537018] E [timer.c:34:gf_timer_call_after]
(-->/usr/sbin/glusterfsd(mgmt_submit_request+0x13d) [0x55d4f159acdd]
-->/lib64/libgfr
pc.so.0(rpc_clnt_submit+0x72d) [0x7ff680989c1d]
-->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7ff680bee349] ) 0-timer:
Either ctx is NULL
or ctx cleanup started [Invalid argument]
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]#
[root@ov-no2 glusterfs]# more bricks/gluster_bricks-vmstore-vmstore.log-20200830 |egrep
"( W | E )"|more
[2020-08-27 15:35:09.533389] W [glusterfsd.c:1570:cleanup_and_exit]
(-->/lib64/libpthread.so.0(+0x7dd5) [0x7f2d8f5f8dd5] -->/usr/sbin/glusterfsd(glus
terfs_sigwaiter+0xe5) [0x563cbb1081b5] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b)
[0x563cbb10801b] ) 0-: received signum (15), shutting down
[2020-08-27 15:35:09.540492] W [rpc-clnt.c:1704:rpc_clnt_submit] 0-glusterfs: failed to
submit rpc-request (unique: 0, XID: 0x33 Program: Gluster Por
tmap, ProgVers: 1, Proc: 5) to rpc-transport (glusterfs)
[2020-08-27 15:35:09.540657] E [timer.c:34:gf_timer_call_after]
(-->/usr/sbin/glusterfsd(mgmt_submit_request+0x13d) [0x563cbb10dcdd]
-->/lib64/libgfr
pc.so.0(rpc_clnt_submit+0x72d) [0x7f2d90563c1d]
-->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7f2d907c8349] ) 0-timer:
Either ctx is NULL
or ctx cleanup started [Invalid argument]
[2020-08-27 15:35:09.540762] E [timer.c:34:gf_timer_call_after]
(-->/usr/sbin/glusterfsd(mgmt_submit_request+0x13d) [0x563cbb10dcdd]
-->/lib64/libgfr
pc.so.0(rpc_clnt_submit+0x72d) [0x7f2d90563c1d]
-->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7f2d907c8349] ) 0-timer:
Either ctx is NULL
or ctx cleanup started [Invalid argument]
[root@ov-no2 glusterfs]# more bricks/gluster_bricks-data-data.log |egrep "( W | E
)"|more
[root@ov-no2 glusterfs]# more bricks/gluster_bricks-vmstore-vmstore.log |egrep "( W |
E )"|more
[root@ov-no2 glusterfs]# more bricks/gluster_bricks-engine-engine.log |egrep "( W | E
)"|more
[root@ov-no2 glusterfs]# more cmd_history.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no2 glusterfs]# cat glusterd.log | egrep "( W | E )" |more
[2020-09-01 07:00:31.325533] W [MSGID: 106061]
[glusterd-geo-rep.c:2388:glusterd_op_gsync_args_get] 0-management: master not found
[2020-09-01 07:10:36.677633] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x7f74, unique = 0, s
ent = 2020-09-01 07:00:31.326798, timeout = 600 for 192.168.48.203:24007
[2020-09-01 07:10:36.677697] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no3.ariadne-t.local. Please
check log file for details.
[2020-09-01 07:10:40.678204] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x804d, unique = 0, s
ent = 2020-09-01 07:00:31.326774, timeout = 600 for 192.168.48.201:24007
[2020-09-01 07:10:40.678259] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no1.ariadne-t.local. Please
check log file for details.
[2020-09-01 07:10:40.678395] E [rpcsvc.c:1577:rpcsvc_submit_generic] 0-rpc-service: failed
to submit message (XID: 0x2, Program: GlusterD svc cli, Pr
ogVers: 2, Proc: 22) to rpc-transport (socket.management)
[2020-09-01 07:10:40.678414] E [MSGID: 106430]
[glusterd-utils.c:558:glusterd_submit_reply] 0-glusterd: Reply submission failed
[2020-09-01 08:02:31.553170] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7efc4e68ea1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7efc4e6871be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7efc
4e6ae06f] ) 0-management: Unable to get transaction opinfo for transaction ID
:930a8a08-1044-41cf-b921-913b982e0c72
[2020-09-01 09:04:31.784847] W [MSGID: 106061]
[glusterd-geo-rep.c:2388:glusterd_op_gsync_args_get] 0-management: master not found
[2020-09-01 09:14:31.577944] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x82ac, unique = 0, s
ent = 2020-09-01 09:04:31.786089, timeout = 600 for 192.168.48.201:24007
[2020-09-01 09:14:31.578010] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no1.ariadne-t.local. Please
check log file for details.
[2020-09-01 09:14:37.578887] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x81d3, unique = 0, s
ent = 2020-09-01 09:04:31.786116, timeout = 600 for 192.168.48.203:24007
[2020-09-01 09:14:37.578945] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no3.ariadne-t.local. Please
check log file for details.
[2020-09-01 09:14:37.579189] E [rpcsvc.c:1577:rpcsvc_submit_generic] 0-rpc-service: failed
to submit message (XID: 0x2, Program: GlusterD svc cli, Pr
ogVers: 2, Proc: 22) to rpc-transport (socket.management)
[2020-09-01 09:14:37.579210] E [MSGID: 106430]
[glusterd-utils.c:558:glusterd_submit_reply] 0-glusterd: Reply submission failed
[2020-09-01 10:06:32.014210] W [MSGID: 106061]
[glusterd-geo-rep.c:2388:glusterd_op_gsync_args_get] 0-management: master not found
[2020-09-01 10:16:32.031372] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x83d7, unique = 0, s
ent = 2020-09-01 10:06:32.015520, timeout = 600 for 192.168.48.201:24007
[2020-09-01 10:16:32.031415] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no1.ariadne-t.local. Please
check log file for details.
[2020-09-01 10:16:38.032229] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x82fe, unique = 0, s
ent = 2020-09-01 10:06:32.015546, timeout = 600 for 192.168.48.203:24007
[2020-09-01 10:16:38.032281] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no3.ariadne-t.local. Please
check log file for details.
[2020-09-01 10:16:38.032491] E [rpcsvc.c:1577:rpcsvc_submit_generic] 0-rpc-service: failed
to submit message (XID: 0x2, Program: GlusterD svc cli, Pr
ogVers: 2, Proc: 22) to rpc-transport (socket.management)
[2020-09-01 10:16:38.032519] E [MSGID: 106430]
[glusterd-utils.c:558:glusterd_submit_reply] 0-glusterd: Reply submission failed
[root@ov-no2 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_engine.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no2 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_data.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no2 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_vmstore.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no2 glusterfs]# cat cli.log | egrep "( W | E )" |more
[2020-09-01 08:49:38.289905] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:49:38.291840] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:04:39.528219] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:09:40.049281] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:09:40.318414] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:09:40.319842] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:14:40.557140] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:19:41.064083] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:19:41.338070] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:19:41.340016] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:24:41.860173] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:24:41.862096] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:29:42.110479] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:29:42.380124] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:29:42.381580] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:34:42.628085] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:44:43.895269] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:44:43.897282] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:54:44.902854] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:54:44.904765] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[root@ov-no2 glusterfs]# cat cmd_history.log | egrep "( W | E |fail)" |more
[2020-09-01 04:04:39.342142] : volume geo-replication status : FAILED : Commit failed on
ov-no3.ariadne-t.local. Please check log file for details.
Commit failed on ov-no1.ariadne-t.local. Please check log file for details.
[2020-09-01 05:06:39.783310] : volume geo-replication status : FAILED : Commit failed on
ov-no3.ariadne-t.local. Please check log file for details.
Commit failed on ov-no1.ariadne-t.local. Please check log file for details.
[2020-09-01 07:10:40.678333] : volume geo-replication status : FAILED : Commit failed on
ov-no3.ariadne-t.local. Please check log file for details.
Commit failed on ov-no1.ariadne-t.local. Please check log file for details.
[2020-09-01 09:14:37.579120] : volume geo-replication status : FAILED : Commit failed on
ov-no1.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
[2020-09-01 10:16:38.032427] : volume geo-replication status : FAILED : Commit failed on
ov-no1.ariadne-t.local. Please check log file for details.
Commit failed on ov-no3.ariadne-t.local. Please check log file for details.
NODE3:
[root@ov-no3 glusterfs]# more bricks/gluster_bricks-engine-engine.log | egrep "( W |
E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# more bricks/gluster_bricks-vmstore-vmstore.log | egrep "( W
| E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# more bricks/gluster_bricks-data-data.log | egrep "( W | E
)" |more
[nothing in the output]
[root@ov-no3 glusterfs]# zcat cli.log |egrep "( W | E )" |more
[2020-09-01 07:29:30.165436] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 07:29:30.166677] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 07:34:30.666721] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 07:34:30.668801] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 07:44:31.431679] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 07:49:31.936985] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 07:54:32.447922] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 08:09:34.242293] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:09:34.244149] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:29:36.002322] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 08:34:36.773092] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:34:36.775476] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:44:37.512700] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 08:54:38.528131] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 08:54:38.797499] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:54:38.798780] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:59:39.030389] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 08:59:39.293311] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 08:59:39.294612] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:14:40.823633] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:14:40.825547] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:34:42.890897] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:34:42.893012] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:39:43.129137] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:44:43.629375] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:49:44.395649] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:49:44.397886] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:59:45.148111] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 09:59:45.409361] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 09:59:45.410701] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:14:46.647785] W [cli-rpc-ops.c:10753:gf_cli_generate_snapshot_event] 0-cli:
Cannot generate event for unknown type.
[2020-09-01 10:14:46.908152] E [cli-xml-output.c:5838:cli_xml_snapshot_begin_composite_op]
0-cli: Failed to get sub-cmd
[2020-09-01 10:14:46.909401] E [cli-xml-output.c:5884:cli_xml_snapshot_end_composite_op]
0-cli: Failed to get sub-cmd
[root@ov-no3 glusterfs]# more cmd_history.log | egrep "( W | E )" |more
[root@ov-no3 glusterfs]# cat glusterd.log | egrep "( W | E )" |more
[2020-09-01 08:02:31.550060] W [MSGID: 106061]
[glusterd-geo-rep.c:2388:glusterd_op_gsync_args_get] 0-management: master not found
[2020-09-01 08:12:31.582984] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x7fe0, unique = 0, s
ent = 2020-09-01 08:02:31.551222, timeout = 600 for 192.168.48.202:24007
[2020-09-01 08:12:31.583040] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no2.ariadne-t.local. Please
check log file for details.
[2020-09-01 08:12:31.583086] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x7fd9, unique = 0, s
ent = 2020-09-01 08:02:31.551255, timeout = 600 for 192.168.48.201:24007
[2020-09-01 08:12:31.583100] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no1.ariadne-t.local. Please
check log file for details.
[2020-09-01 08:12:31.583253] E [rpcsvc.c:1577:rpcsvc_submit_generic] 0-rpc-service: failed
to submit message (XID: 0x2, Program: GlusterD svc cli, Pr
ogVers: 2, Proc: 22) to rpc-transport (socket.management)
[2020-09-01 08:12:31.583269] E [MSGID: 106430]
[glusterd-utils.c:558:glusterd_submit_reply] 0-glusterd: Reply submission failed
[2020-09-01 09:04:31.786592] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f56fecfba1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f56fecf41be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f56
fed1b06f] ) 0-management: Unable to get transaction opinfo for transaction ID
:9942b579-5240-4fee-bb4c-78b9a1c98da8
[2020-09-01 10:06:32.014980] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f56fecfba1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f56fecf41be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f56
fed1b06f] ) 0-management: Unable to get transaction opinfo for transaction ID
:324fc904-b147-4801-89d5-485120e1fb84
[2020-09-01 11:08:32.243189] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f56fecfba1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f56fecf41be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f56
fed1b06f] ) 0-management: Unable to get transaction opinfo for transaction ID
:3b7b79b3-344c-4a67-b66e-c8fb8ddb2a2a
[2020-09-01 12:10:32.473993] E [glusterd-op-sm.c:8132:glusterd_op_sm]
(-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x23a1e) [0x7f56fecfba1e]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x1c1be) [0x7f56fecf41be]
-->/usr/lib64/glusterfs/6.4/xlator/mgmt/glusterd.so(+0x4306f) [0x7f56
fed1b06f] ) 0-management: Unable to get transaction opinfo for transaction ID
:2521f708-6bc4-4be4-b37f-41d67839e941
[2020-09-01 13:12:32.707217] W [MSGID: 106061]
[glusterd-geo-rep.c:2388:glusterd_op_gsync_args_get] 0-management: master not found
[2020-09-01 13:22:33.887995] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x866e, unique = 0, s
ent = 2020-09-01 13:12:32.708475, timeout = 600 for 192.168.48.202:24007
[2020-09-01 13:22:33.888057] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no2.ariadne-t.local. Please
check log file for details.
[2020-09-01 13:22:33.888095] E [rpc-clnt.c:183:call_bail] 0-management: bailing out frame
type(glusterd mgmt), op(--(4)), xid = 0x8667, unique = 0, s
ent = 2020-09-01 13:12:32.708500, timeout = 600 for 192.168.48.201:24007
[2020-09-01 13:22:33.888113] E [MSGID: 106152] [glusterd-syncop.c:104:gd_collate_errors]
0-glusterd: Commit failed on ov-no1.ariadne-t.local. Please
check log file for details.
[2020-09-01 13:22:33.888234] E [rpcsvc.c:1577:rpcsvc_submit_generic] 0-rpc-service: failed
to submit message (XID: 0x2, Program: GlusterD svc cli, Pr
ogVers: 2, Proc: 22) to rpc-transport (socket.management)
[2020-09-01 13:22:33.888269] E [MSGID: 106430]
[glusterd-utils.c:558:glusterd_submit_reply] 0-glusterd: Reply submission failed
[root@ov-no3 glusterfs]# cat events.log-20200828 | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# cat glfsheal-data.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# cat glfsheal-vmstore.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# cat glfsheal-engine.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# cat glustershd.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_engine.log | egrep "( W | E
)" |more
[2020-08-31 14:49:13.805467] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 14:51:13.882683] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 14:53:12.691884] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 14:55:12.713320] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 14:57:13.146570] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 14:59:13.502830] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 15:01:13.039274] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 15:03:13.401571] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 15:05:13.764951] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 15:07:14.140156] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 15:09:13.061657] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[2020-08-31 15:11:14.616511] W [MSGID: 114061] [client-common.c:2644:client_pre_fsync_v2]
0-engine-client-0: (3fafabf3-d0cd-4b9a-8dd7-43145451f7cf)
remote_fd is -1. EBADFD [File descriptor in bad state]
[root@ov-no3 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_data.log | egrep "( W | E
)" |more
[nothing recent in the output]
[root@ov-no3 glusterfs]# cat
rhev-data-center-mnt-glusterSD-ov-no1.ariadne-t.local\:_vmstore.log | egrep "( W | E
)" |more
[nothing recent in the output]
[root@ov-no3 glusterfs]# cat glustershd.log | egrep "( W | E )" |more
[nothing in the output]
[root@ov-no3 glusterfs]# cat events.log | egrep "( W | E )" |more
[nothing in the output]
Journalctl results on the nodes about the gluster service:
NODE1:
[root@ov-no1 ~]# journalctl --unit glusterd -l
-- Logs begin at Thu 2020-08-27 19:30:14 EEST, end at Fri 2020-09-04 16:34:42 EEST. --
Aug 27 19:30:52 ov-no1.ariadne-t.local systemd[1]: Starting GlusterFS, a clustered
file-system server...
Aug 27 19:30:56 ov-no1.ariadne-t.local systemd[1]: Started GlusterFS, a clustered
file-system server.
Aug 27 19:30:56 ov-no1.ariadne-t.local glusterd[5591]: [2020-08-27 16:30:56.544861] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:30:56 ov-no1.ariadne-t.local glusterd[5591]: [2020-08-27 16:30:56.558812] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:30:56 ov-no1.ariadne-t.local glusterd[5591]: [2020-08-27 16:30:56.571473] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
NODE2:
[root@ov-no2 ~]# journalctl --unit glusterd -l
-- Logs begin at Wed 2019-09-11 17:40:13 EEST, end at Fri 2020-09-04 16:34:27 EEST. --
Aug 27 18:35:01 ov-no2.ariadne-t.local systemd[1]: Stopping GlusterFS, a clustered
file-system server...
Aug 27 18:35:01 ov-no2.ariadne-t.local systemd[1]: Stopped GlusterFS, a clustered
file-system server.
Aug 27 18:35:01 ov-no2.ariadne-t.local systemd[1]: Starting GlusterFS, a clustered
file-system server...
Aug 27 18:35:05 ov-no2.ariadne-t.local systemd[1]: Started GlusterFS, a clustered
file-system server.
Aug 27 18:35:07 ov-no2.ariadne-t.local glusterd[28722]: [2020-08-27 15:35:07.519155] C
[MSGID: 106002] [glusterd-server-quorum.c:355:glusterd_do_volu
Aug 27 18:35:08 ov-no2.ariadne-t.local glusterd[28722]: [2020-08-27 15:35:08.532018] C
[MSGID: 106002] [glusterd-server-quorum.c:355:glusterd_do_volu
Aug 27 18:35:09 ov-no2.ariadne-t.local glusterd[28722]: [2020-08-27 15:35:09.532867] C
[MSGID: 106002] [glusterd-server-quorum.c:355:glusterd_do_volu
Aug 27 18:35:10 ov-no2.ariadne-t.local glusterd[28722]: [2020-08-27 15:35:10.535976] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volu
Aug 27 18:35:10 ov-no2.ariadne-t.local glusterd[28722]: [2020-08-27 15:35:10.550171] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volu
Aug 27 18:35:10 ov-no2.ariadne-t.local glusterd[28722]: [2020-08-27 15:35:10.563481] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volu
Aug 27 19:25:23 ov-no2.ariadne-t.local glustershd[22187]: [2020-08-27 16:25:23.640530] C
[rpc-clnt-ping.c:155:rpc_clnt_ping_timer_expired] 0-data-cli
NODE3:
[root@ov-no3 glusterfs]# journalctl --unit glusterd -l
-- Logs begin at Thu 2020-08-27 19:09:30 EEST, end at Fri 2020-09-04 16:34:02 EEST. --
Aug 27 19:10:08 ov-no3.ariadne-t.local systemd[1]: Starting GlusterFS, a clustered
file-system server...
Aug 27 19:10:13 ov-no3.ariadne-t.local systemd[1]: Started GlusterFS, a clustered
file-system server.
Aug 27 19:10:13 ov-no3.ariadne-t.local glusterd[5233]: [2020-08-27 16:10:13.274001] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:10:13 ov-no3.ariadne-t.local glusterd[5233]: [2020-08-27 16:10:13.288391] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:10:13 ov-no3.ariadne-t.local glusterd[5233]: [2020-08-27 16:10:13.300866] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:14:38 ov-no3.ariadne-t.local systemd[1]: Stopping GlusterFS, a clustered
file-system server...
Aug 27 19:14:38 ov-no3.ariadne-t.local systemd[1]: Stopped GlusterFS, a clustered
file-system server.
Aug 27 19:14:38 ov-no3.ariadne-t.local systemd[1]: Starting GlusterFS, a clustered
file-system server...
Aug 27 19:14:41 ov-no3.ariadne-t.local systemd[1]: Started GlusterFS, a clustered
file-system server.
Aug 27 19:14:48 ov-no3.ariadne-t.local glusterd[9843]: [2020-08-27 16:14:48.117530] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:14:48 ov-no3.ariadne-t.local glusterd[9843]: [2020-08-27 16:14:48.131054] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:14:48 ov-no3.ariadne-t.local glusterd[9843]: [2020-08-27 16:14:48.141323] C
[MSGID: 106003] [glusterd-server-quorum.c:348:glusterd_do_volum
Aug 27 19:25:25 ov-no3.ariadne-t.local glustershd[10081]: [2020-08-27 16:25:25.286189] C
[rpc-clnt-ping.c:155:rpc_clnt_ping_timer_expired] 0-data-cli
Thanks for your help,
Maria Souvalioti