more rhev-data-center-mnt-glusterSD-glustermount2:data2.log-20161106
[2016-11-04 22:54:32.118587] I [MSGID: 100030] [glusterfsd.c:2338:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.7.16 (args: /usr/sbin/glusterfs --volfile-server=glustermou
nt2 --volfile-server=fai-kvm-1-vmn.ravnalaska.net --volfile-server=fai-kvm-2-vmn.ravnalaska.net --volfile-server=fai-kvm-3-vmn.ravnalaska.net --volfile-server=fai-kvm-4-vmn.ravnalaska.net --volfile-id
=data2 /rhev/data-center/mnt/glusterSD/glustermount2:data2)
[2016-11-04 22:54:32.128807] I [MSGID: 101190] [event-epoll.c:632:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2016-11-04 22:54:32.138959] I [MSGID: 101190] [event-epoll.c:632:event_dispatch_epoll_worker] 0-epoll: Started thread with index 2
[2016-11-04 22:54:32.139878] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-0: parent translators are ready, attempting connect on transport
[2016-11-04 22:54:32.142057] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-1: parent translators are ready, attempting connect on transport
[2016-11-04 22:54:32.142331] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-0: changing port to 49156 (from 0)
[2016-11-04 22:54:32.143961] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-2: parent translators are ready, attempting connect on transport
[2016-11-04 22:54:32.146123] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-04 22:54:32.146308] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-1: changing port to 49156 (from 0)
[2016-11-04 22:54:32.147618] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-3: parent translators are ready, attempting connect on transport
[2016-11-04 22:54:32.149874] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-0: Connected to data2-client-0, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-04 22:54:32.149891] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-04 22:54:32.149941] I [MSGID: 108005] [afr-common.c:4299:afr_notify] 0-data2-replicate-0: Subvolume 'data2-client-0' came back up; going online.
[2016-11-04 22:54:32.150000] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-0: Server lk version = 1
[2016-11-04 22:54:32.150080] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-04 22:54:32.150274] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-2: changing port to 49156 (from 0)
Final graph:
+------------------------------------------------------------------------------+
  1: volume data2-client-0
  2:     type protocol/client
  3:     option clnt-lk-version 1
  4:     option volfile-checksum 0
  5:     option volfile-key data2
  6:     option client-version 3.7.16
  7:     option process-uuid fai-kvm-1-gfs.ravnalaska.net-105264-2016/11/04-22:54:32:114962-data2-client-0-0-0
  8:     option fops-version 1298437
  9:     option ping-timeout 42
 10:     option remote-host fai-kvm-1-vmn.ravnalaska.net
 11:     option remote-subvolume /kvm2/gluster/data2/brick
 12:     option transport-type socket
 13:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 14:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 15:     option filter-O_DIRECT enable
 16:     option send-gids true
 17: end-volume
 18:
 19: volume data2-client-1
 20:     type protocol/client
 21:     option clnt-lk-version 1
 22:     option volfile-checksum 0
 23:     option volfile-key data2
 24:     option client-version 3.7.16
 25:     option process-uuid fai-kvm-1-gfs.ravnalaska.net-105264-2016/11/04-22:54:32:114962-data2-client-1-0-0
 26:     option fops-version 1298437
 27:     option ping-timeout 42
 28:     option remote-host fai-kvm-2-vmn.ravnalaska.net
 29:     option remote-subvolume /kvm2/gluster/data2/brick
 30:     option transport-type socket
 31:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 32:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 33:     option filter-O_DIRECT enable
 34:     option send-gids true
 35: end-volume
 36:
 37: volume data2-replicate-0
 38:     type cluster/replicate
 39:     option eager-lock enable
 40:     option quorum-type auto
 41:     subvolumes data2-client-0 data2-client-1
 42: end-volume
 43:
 44: volume data2-client-2
 45:     type protocol/client
 46:     option ping-timeout 42
 47:     option remote-host fai-kvm-3-vmn.ravnalaska.net
 48:     option remote-subvolume /kvm2/gluster/data2/brick
 49:     option transport-type socket
 50:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 51:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 52:     option filter-O_DIRECT enable
 53:     option send-gids true
 54: end-volume
 55:
 56: volume data2-client-3
 57:     type protocol/client
 58:     option ping-timeout 42
 59:     option remote-host fai-kvm-4-vmn.ravnalaska.net
 60:     option remote-subvolume /kvm2/gluster/data2/brick
 61:     option transport-type socket
 62:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 63:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 64:     option filter-O_DIRECT enable
 65:     option send-gids true
 66: end-volume
 67:
 68: volume data2-replicate-1
 69:     type cluster/replicate
 70:     option eager-lock enable
 71:     option quorum-type auto
 72:     subvolumes data2-client-2 data2-client-3
 73: end-volume
 74:
 75: volume data2-stripe-0
 76:     type cluster/stripe
 77:     subvolumes data2-replicate-0 data2-replicate-1
 78: end-volume
 79:
 80: volume data2-dht
 81:     type cluster/distribute
 82:     subvolumes data2-stripe-0
 83: end-volume
 84:
 85: volume data2-write-behind
 86:     type performance/write-behind
 87:     subvolumes data2-dht
 88: end-volume
 89:
 90: volume data2-readdir-ahead
 91:     type performance/readdir-ahead
 92:     subvolumes data2-write-behind
 93: end-volume
 94:
 95: volume data2-open-behind
 96:     type performance/open-behind
 97:     subvolumes data2-readdir-ahead
 98: end-volume
 99:
100: volume data2
101:     type debug/io-stats
102:     option log-level INFO
103:     option latency-measurement off
104:     option count-fop-hits off
105:     subvolumes data2-open-behind
106: end-volume
107:
108: volume meta-autoload
109:     type meta
110:     subvolumes data2
111: end-volume
112:
+------------------------------------------------------------------------------+
[2016-11-04 22:54:32.151927] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-1: Connected to data2-client-1, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-04 22:54:32.151939] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-04 22:54:32.152083] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-1: Server lk version = 1
[2016-11-04 22:54:32.152341] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-3: changing port to 49152 (from 0)
[2016-11-04 22:54:32.154296] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-2: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-04 22:54:32.154774] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-2: Connected to data2-client-2, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-04 22:54:32.154789] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-2: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-04 22:54:32.154821] I [MSGID: 108005] [afr-common.c:4299:afr_notify] 0-data2-replicate-1: Subvolume 'data2-client-2' came back up; going online.
[2016-11-04 22:54:32.154916] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-2: Server lk version = 1
[2016-11-04 22:54:32.156348] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-3: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-04 22:54:32.156860] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-3: Connected to data2-client-3, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-04 22:54:32.156874] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-3: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-04 22:54:32.160382] I [fuse-bridge.c:5171:fuse_graph_setup] 0-fuse: switched to graph 0
[2016-11-04 22:54:32.160451] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-3: Server lk version = 1
[2016-11-04 22:54:32.160524] I [fuse-bridge.c:4083:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.22 kernel 7.22
[2016-11-04 22:54:32.161171] I [MSGID: 108031] [afr-common.c:2071:afr_local_discovery_cbk] 0-data2-replicate-0: selecting local read_child data2-client-0
[2016-11-05 00:05:37.796730] W [glusterfsd.c:1251:cleanup_and_exit] (-->/lib64/libpthread.so.0(+0x7dc5) [0x7fe90527adc5] -->/usr/sbin/glusterfs(glusterfs_sigwaiter+0xe5) [0x7fe9068f5915] -->/usr/sbin/
glusterfs(cleanup_and_exit+0x6b) [0x7fe9068f578b] ) 0-: received signum (15), shutting down
[2016-11-05 00:05:37.796770] I [fuse-bridge.c:5719:fini] 0-fuse: Unmounting '/rhev/data-center/mnt/glusterSD/glustermount2:data2'.
[2016-11-05 00:13:04.440431] I [MSGID: 100030] [glusterfsd.c:2338:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.7.16 (args: /usr/sbin/glusterfs --volfile-server=glustermou
nt2 --volfile-server=fai-kvm-1-vmn.ravnalaska.net --volfile-server=fai-kvm-2-vmn.ravnalaska.net --volfile-server=fai-kvm-3-vmn.ravnalaska.net --volfile-server=fai-kvm-4-vmn.ravnalaska.net --volfile-id
=data2 /rhev/data-center/mnt/glusterSD/glustermount2:data2)
[2016-11-05 00:13:04.452151] I [MSGID: 101190] [event-epoll.c:632:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2016-11-05 00:13:04.463593] I [MSGID: 101190] [event-epoll.c:632:event_dispatch_epoll_worker] 0-epoll: Started thread with index 2
[2016-11-05 00:13:04.464394] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-0: parent translators are ready, attempting connect on transport
[2016-11-05 00:13:04.466698] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-1: parent translators are ready, attempting connect on transport
[2016-11-05 00:13:04.466885] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-0: changing port to 49156 (from 0)
[2016-11-05 00:13:04.468672] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-2: parent translators are ready, attempting connect on transport
[2016-11-05 00:13:04.470686] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:13:04.470849] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-1: changing port to 49156 (from 0)
[2016-11-05 00:13:04.472375] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-3: parent translators are ready, attempting connect on transport
[2016-11-05 00:13:04.475429] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-0: Connected to data2-client-0, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:13:04.475448] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:13:04.475503] I [MSGID: 108005] [afr-common.c:4299:afr_notify] 0-data2-replicate-0: Subvolume 'data2-client-0' came back up; going online.
[2016-11-05 00:13:04.475615] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-0: Server lk version = 1
[2016-11-05 00:13:04.475687] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:13:04.475857] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-2: changing port to 49156 (from 0)
Final graph:
+------------------------------------------------------------------------------+
  1: volume data2-client-0
  2:     type protocol/client
  3:     option clnt-lk-version 1
  4:     option volfile-checksum 0
  5:     option volfile-key data2
  6:     option client-version 3.7.16
  7:     option process-uuid fai-kvm-1-gfs.ravnalaska.net-13785-2016/11/05-00:13:04:437247-data2-client-0-0-0
  8:     option fops-version 1298437
  9:     option ping-timeout 42
 10:     option remote-host fai-kvm-1-vmn.ravnalaska.net
 11:     option remote-subvolume /kvm2/gluster/data2/brick
 12:     option transport-type socket
 13:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 14:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 15:     option filter-O_DIRECT enable
 16:     option send-gids true
 17: end-volume
 18:
 19: volume data2-client-1
 20:     type protocol/client
 21:     option clnt-lk-version 1
 22:     option volfile-checksum 0
 23:     option volfile-key data2
 24:     option client-version 3.7.16
 25:     option process-uuid fai-kvm-1-gfs.ravnalaska.net-13785-2016/11/05-00:13:04:437247-data2-client-1-0-0
 26:     option fops-version 1298437
 27:     option ping-timeout 42
 28:     option remote-host fai-kvm-2-vmn.ravnalaska.net
 29:     option remote-subvolume /kvm2/gluster/data2/brick
 30:     option transport-type socket
 31:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 32:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 33:     option filter-O_DIRECT enable
 34:     option send-gids true
 35: end-volume
 36:
 37: volume data2-replicate-0
 38:     type cluster/replicate
 39:     option eager-lock enable
 40:     option quorum-type auto
 41:     subvolumes data2-client-0 data2-client-1
 42: end-volume
 43:
 44: volume data2-client-2
 45:     type protocol/client
 46:     option ping-timeout 42
 47:     option remote-host fai-kvm-3-vmn.ravnalaska.net
 48:     option remote-subvolume /kvm2/gluster/data2/brick
 49:     option transport-type socket
 50:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 51:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 52:     option filter-O_DIRECT enable
 53:     option send-gids true
 54: end-volume
 55:
 56: volume data2-client-3
 57:     type protocol/client
 58:     option ping-timeout 42
 59:     option remote-host fai-kvm-4-vmn.ravnalaska.net
 60:     option remote-subvolume /kvm2/gluster/data2/brick
 61:     option transport-type socket
 62:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 63:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 64:     option filter-O_DIRECT enable
 65:     option send-gids true
 66: end-volume
 67:
 68: volume data2-replicate-1
 69:     type cluster/replicate
 70:     option eager-lock enable
 71:     option quorum-type auto
 72:     subvolumes data2-client-2 data2-client-3
 73: end-volume
 74:
 75: volume data2-stripe-0
 76:     type cluster/stripe
 77:     subvolumes data2-replicate-0 data2-replicate-1
 78: end-volume
 79:
 80: volume data2-dht
 81:     type cluster/distribute
 82:     subvolumes data2-stripe-0
 83: end-volume
 84:
 85: volume data2-write-behind
 86:     type performance/write-behind
 87:     subvolumes data2-dht
 88: end-volume
 89:
 90: volume data2-readdir-ahead
 91:     type performance/readdir-ahead
 92:     subvolumes data2-write-behind
 93: end-volume
 94:
 95: volume data2-open-behind
 96:     type performance/open-behind
 97:     subvolumes data2-readdir-ahead
 98: end-volume
 99:
100: volume data2
101:     type debug/io-stats
102:     option log-level INFO
103:     option latency-measurement off
104:     option count-fop-hits off
105:     subvolumes data2-open-behind
106: end-volume
107:
108: volume meta-autoload
109:     type meta
110:     subvolumes data2
111: end-volume
112:
+------------------------------------------------------------------------------+
[2016-11-05 00:13:04.477682] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-1: Connected to data2-client-1, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:13:04.477702] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:13:04.477834] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-1: Server lk version = 1
[2016-11-05 00:13:04.478162] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-3: changing port to 49152 (from 0)
[2016-11-05 00:13:04.480593] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-2: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:13:04.480961] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-2: Connected to data2-client-2, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:13:04.480982] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-2: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:13:04.481020] I [MSGID: 108005] [afr-common.c:4299:afr_notify] 0-data2-replicate-1: Subvolume 'data2-client-2' came back up; going online.
[2016-11-05 00:13:04.481120] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-2: Server lk version = 1
[2016-11-05 00:13:04.483500] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-3: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:13:04.483956] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-3: Connected to data2-client-3, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:13:04.483973] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-3: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:13:04.488332] I [fuse-bridge.c:5171:fuse_graph_setup] 0-fuse: switched to graph 0
[2016-11-05 00:13:04.488889] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-3: Server lk version = 1
[2016-11-05 00:13:04.488933] I [fuse-bridge.c:4083:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.22 kernel 7.22
[2016-11-05 00:13:04.489627] I [MSGID: 108031] [afr-common.c:2071:afr_local_discovery_cbk] 0-data2-replicate-0: selecting local read_child data2-client-0
[2016-11-05 00:52:07.549997] I [fuse-bridge.c:5012:fuse_thread_proc] 0-fuse: unmounting /rhev/data-center/mnt/glusterSD/glustermount2:data2
[2016-11-05 00:52:07.550544] W [glusterfsd.c:1251:cleanup_and_exit] (-->/lib64/libpthread.so.0(+0x7dc5) [0x7f40b1c24dc5] -->/usr/sbin/glusterfs(glusterfs_sigwaiter+0xe5) [0x7f40b329f915] -->/usr/sbin/
glusterfs(cleanup_and_exit+0x6b) [0x7f40b329f78b] ) 0-: received signum (15), shutting down
[2016-11-05 00:52:07.550583] I [fuse-bridge.c:5719:fini] 0-fuse: Unmounting '/rhev/data-center/mnt/glusterSD/glustermount2:data2'.
[2016-11-05 00:52:39.557750] I [MSGID: 100030] [glusterfsd.c:2338:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.7.16 (args: /usr/sbin/glusterfs --volfile-server=glustermou
nt2 --volfile-server=fai-kvm-1-vmn.ravnalaska.net --volfile-server=fai-kvm-2-vmn.ravnalaska.net --volfile-server=fai-kvm-3-vmn.ravnalaska.net --volfile-server=fai-kvm-4-vmn.ravnalaska.net --volfile-id
=data2 /rhev/data-center/mnt/glusterSD/glustermount2:data2)
[2016-11-05 00:52:39.567614] I [MSGID: 101190] [event-epoll.c:632:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2016-11-05 00:52:39.578148] I [MSGID: 101190] [event-epoll.c:632:event_dispatch_epoll_worker] 0-epoll: Started thread with index 2
[2016-11-05 00:52:39.579039] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-0: parent translators are ready, attempting connect on transport
[2016-11-05 00:52:39.581159] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-1: parent translators are ready, attempting connect on transport
[2016-11-05 00:52:39.581483] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-0: changing port to 49156 (from 0)
[2016-11-05 00:52:39.583109] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-2: parent translators are ready, attempting connect on transport
[2016-11-05 00:52:39.585673] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:52:39.585849] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-1: changing port to 49156 (from 0)
[2016-11-05 00:52:39.587044] I [MSGID: 114020] [client.c:2113:notify] 0-data2-client-3: parent translators are ready, attempting connect on transport
[2016-11-05 00:52:39.589467] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-0: Connected to data2-client-0, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:52:39.589499] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:52:39.589571] I [MSGID: 108005] [afr-common.c:4299:afr_notify] 0-data2-replicate-0: Subvolume 'data2-client-0' came back up; going online.
[2016-11-05 00:52:39.589654] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-0: Server lk version = 1
[2016-11-05 00:52:39.589753] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:52:39.590026] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-2: changing port to 49156 (from 0)
[2016-11-05 00:52:39.590074] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-1: Connected to data2-client-1, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:52:39.590088] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-1: Server and Client lk-version numbers are not same, reopening the fds
Final graph:
+------------------------------------------------------------------------------+
  1: volume data2-client-0
  2:     type protocol/client
  3:     option clnt-lk-version 1
  4:     option volfile-checksum 0
  5:     option volfile-key data2
  6:     option client-version 3.7.16
  7:     option process-uuid fai-kvm-1-gfs.ravnalaska.net-30520-2016/11/05-00:52:39:553646-data2-client-0-0-0
  8:     option fops-version 1298437
  9:     option ping-timeout 42
 10:     option remote-host fai-kvm-1-vmn.ravnalaska.net
 11:     option remote-subvolume /kvm2/gluster/data2/brick
 12:     option transport-type socket
 13:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 14:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 15:     option filter-O_DIRECT enable
 16:     option send-gids true
 17: end-volume
 18:
 19: volume data2-client-1
 20:     type protocol/client
 21:     option clnt-lk-version 1
 22:     option volfile-checksum 0
 23:     option volfile-key data2
 24:     option client-version 3.7.16
 25:     option process-uuid fai-kvm-1-gfs.ravnalaska.net-30520-2016/11/05-00:52:39:553646-data2-client-1-0-0
 26:     option fops-version 1298437
 27:     option ping-timeout 42
 28:     option remote-host fai-kvm-2-vmn.ravnalaska.net
 29:     option remote-subvolume /kvm2/gluster/data2/brick
 30:     option transport-type socket
 31:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 32:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 33:     option filter-O_DIRECT enable
 34:     option send-gids true
 35: end-volume
 36:
 37: volume data2-replicate-0
 38:     type cluster/replicate
 39:     option eager-lock enable
 40:     option quorum-type auto
 41:     subvolumes data2-client-0 data2-client-1
 42: end-volume
 43:
 44: volume data2-client-2
 45:     type protocol/client
 46:     option ping-timeout 42
 47:     option remote-host fai-kvm-3-vmn.ravnalaska.net
 48:     option remote-subvolume /kvm2/gluster/data2/brick
 49:     option transport-type socket
 50:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 51:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 52:     option filter-O_DIRECT enable
 53:     option send-gids true
 54: end-volume
 55:
 56: volume data2-client-3
 57:     type protocol/client
 58:     option ping-timeout 42
 59:     option remote-host fai-kvm-4-vmn.ravnalaska.net
 60:     option remote-subvolume /kvm2/gluster/data2/brick
 61:     option transport-type socket
 62:     option username 0b7a59b1-4aab-4041-a271-bd8a07e5cba5
 63:     option password 6dca1e9d-f6a8-490b-bb60-babc56eb131a
 64:     option filter-O_DIRECT enable
 65:     option send-gids true
 66: end-volume
 67:
 68: volume data2-replicate-1
 69:     type cluster/replicate
 70:     option eager-lock enable
 71:     option quorum-type auto
 72:     subvolumes data2-client-2 data2-client-3
 73: end-volume
 74:
 75: volume data2-stripe-0
 76:     type cluster/stripe
 77:     subvolumes data2-replicate-0 data2-replicate-1
 78: end-volume
 79:
 80: volume data2-dht
 81:     type cluster/distribute
 82:     subvolumes data2-stripe-0
 83: end-volume
 84:
 85: volume data2-write-behind
 86:     type performance/write-behind
 87:     subvolumes data2-dht
 88: end-volume
 89:
 90: volume data2-readdir-ahead
 91:     type performance/readdir-ahead
 92:     subvolumes data2-write-behind
 93: end-volume
 94:
 95: volume data2-open-behind
 96:     type performance/open-behind
 97:     subvolumes data2-readdir-ahead
 98: end-volume
 99:
100: volume data2
101:     type debug/io-stats
102:     option log-level INFO
103:     option latency-measurement off
104:     option count-fop-hits off
105:     subvolumes data2-open-behind
106: end-volume
107:
108: volume meta-autoload
109:     type meta
110:     subvolumes data2
111: end-volume
112:
+------------------------------------------------------------------------------+
[2016-11-05 00:52:39.591502] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-1: Server lk version = 1
[2016-11-05 00:52:39.591817] I [rpc-clnt.c:1960:rpc_clnt_reconfig] 0-data2-client-3: changing port to 49152 (from 0)
[2016-11-05 00:52:39.593714] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-2: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:52:39.594285] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-2: Connected to data2-client-2, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:52:39.594308] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-2: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:52:39.594365] I [MSGID: 108005] [afr-common.c:4299:afr_notify] 0-data2-replicate-1: Subvolume 'data2-client-2' came back up; going online.
[2016-11-05 00:52:39.594552] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-2: Server lk version = 1
[2016-11-05 00:52:39.595616] I [MSGID: 114057] [client-handshake.c:1437:select_server_supported_programs] 0-data2-client-3: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2016-11-05 00:52:39.596115] I [MSGID: 114046] [client-handshake.c:1213:client_setvolume_cbk] 0-data2-client-3: Connected to data2-client-3, attached to remote volume '/kvm2/gluster/data2/brick'.
[2016-11-05 00:52:39.596130] I [MSGID: 114047] [client-handshake.c:1224:client_setvolume_cbk] 0-data2-client-3: Server and Client lk-version numbers are not same, reopening the fds
[2016-11-05 00:52:39.600510] I [fuse-bridge.c:5171:fuse_graph_setup] 0-fuse: switched to graph 0
[2016-11-05 00:52:39.600578] I [MSGID: 114035] [client-handshake.c:193:client_set_lk_version_cbk] 0-data2-client-3: Server lk version = 1
[2016-11-05 00:52:39.600654] I [fuse-bridge.c:4083:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.22 kernel 7.22
[2016-11-05 00:52:39.601452] I [MSGID: 108031] [afr-common.c:2071:afr_local_discovery_cbk] 0-data2-replicate-0: selecting local read_child data2-client-0
[2016-11-05 00:52:41.619852] I [fuse-bridge.c:5012:fuse_thread_proc] 0-fuse: unmounting /rhev/data-center/mnt/glusterSD/glustermount2:data2
[2016-11-05 00:52:41.620095] W [glusterfsd.c:1251:cleanup_and_exit] (-->/lib64/libpthread.so.0(+0x7dc5) [0x7f3444227dc5] -->/usr/sbin/glusterfs(glusterfs_sigwaiter+0xe5) [0x7f34458a2915] -->/usr/sbin/
glusterfs(cleanup_and_exit+0x6b) [0x7f34458a278b] ) 0-: received signum (15), shutting down
[2016-11-05 00:52:41.620110] I [fuse-bridge.c:5719:fini] 0-fuse: Unmounting '/rhev/data-center/mnt/glusterSD/glustermount2:data2'.
[root@fai-kvm-1-gfs glusterfs]#