2:05 p.m.
Hello,
I am trying to setup a single hyperconverged node. The disks that I will be using for the
creation of my engine and all VMs are on a RAID 5. This volume is using hardware raid
with an LSI controller (LSI 9361-4i). I am unsure of the appropriate values to use for
the Gluster config. Here is the info about my environment.
[root@vmh ~]# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime,seclabel)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
devtmpfs on /dev type devtmpfs
(rw,nosuid,seclabel,size=65713924k,nr_inodes=16428481,mode=755)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev,seclabel)
devpts on /dev/pts type devpts
(rw,nosuid,noexec,relatime,seclabel,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,seclabel,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,seclabel,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/freezer type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,freezer)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,pids)
cgroup on /sys/fs/cgroup/cpuset type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,cpuset)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,cpuacct,cpu)
cgroup on /sys/fs/cgroup/blkio type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,blkio)
cgroup on /sys/fs/cgroup/hugetlb type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,net_prio,net_cls)
cgroup on /sys/fs/cgroup/devices type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,devices)
cgroup on /sys/fs/cgroup/perf_event type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,perf_event)
cgroup on /sys/fs/cgroup/memory type cgroup
(rw,nosuid,nodev,noexec,relatime,seclabel,memory)
configfs on /sys/kernel/config type configfs (rw,relatime)
/dev/mapper/onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1 on / type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
selinuxfs on /sys/fs/selinux type selinuxfs (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs
(rw,relatime,fd=22,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=43386)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime,seclabel)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel)
hugetlbfs on /dev/hugepages1G type hugetlbfs (rw,relatime,seclabel,pagesize=1G)
/dev/mapper/onn_vmh-tmp on /tmp type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
/dev/mapper/onn_vmh-home on /home type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
/dev/sdb1 on /boot type ext4 (rw,relatime,seclabel,data=ordered)
/dev/mapper/onn_vmh-var on /var type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
/dev/mapper/onn_vmh-var_log on /var/log type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
/dev/mapper/onn_vmh-var_crash on /var/crash type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
/dev/mapper/onn_vmh-var_log_audit on /var/log/audit type ext4
(rw,relatime,seclabel,discard,stripe=16,data=ordered)
sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)
/dev/mapper/3600605b00a2faca222fb4da81ac9bdb1p1 on /data type ext4
(rw,relatime,seclabel,discard,stripe=64,data=ordered)
tmpfs on /run/user/0 type tmpfs
(rw,nosuid,nodev,relatime,seclabel,size=13149252k,mode=700)
[root@vmh ~]# blkid
/dev/sda1: UUID="ce130131-c457-46a0-b6de-e50cc89a6da3" TYPE="ext4"
PARTUUID="5fff73ae-70d4-4697-a307-5f68a4c00f4c"
/dev/sdb1: UUID="47422043-e5d0-4541-86ff-193f61a779b0" TYPE="ext4"
/dev/sdb2: UUID="2f2cf71b-b68e-985a-6433-ed1889595df0"
UUID_SUB="a3e32abe-109c-b436-7712-6c9d9f1d57c9"
LABEL="vmh.cyber-range.lan:pv00" TYPE="linux_raid_member"
/dev/sdc1: UUID="2f2cf71b-b68e-985a-6433-ed1889595df0"
UUID_SUB="7132acc1-e210-f645-5df2-e1a1eff7f836"
LABEL="vmh.cyber-range.lan:pv00" TYPE="linux_raid_member"
/dev/md127: UUID="utB9xu-zva6-j5Ci-3E49-uDya-g2AJ-MQu5Rd"
TYPE="LVM2_member"
/dev/mapper/onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1:
UUID="8656ff1e-d217-4088-b353-6d2b9f602ce3" TYPE="ext4"
/dev/mapper/onn_vmh-swap: UUID="57a165d1-116e-4e64-a694-2618ffa3a79e"
TYPE="swap"
/dev/mapper/3600605b00a2faca222fb4da81ac9bdb1p1:
UUID="ce130131-c457-46a0-b6de-e50cc89a6da3" TYPE="ext4"
PARTUUID="dac9e1fc-b0d7-43da-b52c-66bb059d8137"
/dev/mapper/onn_vmh-root: UUID="7cc65568-d408-43ab-a793-b6c110d7ba98"
TYPE="ext4"
/dev/mapper/onn_vmh-home: UUID="c7e344d0-f401-4504-b52e-9b5c6023c10e"
TYPE="ext4"
/dev/mapper/onn_vmh-tmp: UUID="3323e010-0dab-4166-b15d-d739a09b4c03"
TYPE="ext4"
/dev/mapper/onn_vmh-var: UUID="c20af647-364d-4a11-8776-8e6aac362425"
TYPE="ext4"
/dev/mapper/onn_vmh-var_log: UUID="eb2e159a-4d50-411d-912c-c8e11f297fea"
TYPE="ext4"
/dev/mapper/onn_vmh-var_log_audit: UUID="65aa85b9-c053-4244-b3b0-c5f91de8ad59"
TYPE="ext4"
/dev/mapper/onn_vmh-var_crash: UUID="579bbef5-bab6-49ce-b8ed-ab92afa85ea8"
TYPE="ext4"
/dev/mapper/3600605b00a2faca222fb4da81ac9bdb1: PTTYPE="gpt"
[root@vmh ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE
MOUNTPOINT
sda 8:0 0 8.2T 0 disk
├─sda1 8:1 0 8.2T 0 part
└─3600605b00a2faca222fb4da81ac9bdb1 253:5 0 8.2T 0 mpath
└─3600605b00a2faca222fb4da81ac9bdb1p1 253:6 0 8.2T 0 part
/data
sdb 8:16 0 223.6G 0 disk
├─sdb1 8:17 0 1G 0 part
/boot
└─sdb2 8:18 0 222.6G 0 part
└─md127 9:127 0 222.5G 0 raid1
├─onn_vmh-pool00_tmeta 253:0 0 1G 0 lvm
│ └─onn_vmh-pool00-tpool 253:2 0 173.6G 0 lvm
│ ├─onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1 253:3 0 146.6G 0 lvm /
│ ├─onn_vmh-pool00 253:7 0 173.6G 0 lvm
│ ├─onn_vmh-root 253:8 0 146.6G 0 lvm
│ ├─onn_vmh-home 253:9 0 1G 0 lvm
/home
│ ├─onn_vmh-tmp 253:10 0 1G 0 lvm
/tmp
│ ├─onn_vmh-var 253:11 0 15G 0 lvm
/var
│ ├─onn_vmh-var_log 253:12 0 8G 0 lvm
/var/log
│ ├─onn_vmh-var_log_audit 253:13 0 2G 0 lvm
/var/log/audit
│ └─onn_vmh-var_crash 253:14 0 10G 0 lvm
/var/crash
├─onn_vmh-pool00_tdata 253:1 0 173.6G 0 lvm
│ └─onn_vmh-pool00-tpool 253:2 0 173.6G 0 lvm
│ ├─onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1 253:3 0 146.6G 0 lvm /
│ ├─onn_vmh-pool00 253:7 0 173.6G 0 lvm
│ ├─onn_vmh-root 253:8 0 146.6G 0 lvm
│ ├─onn_vmh-home 253:9 0 1G 0 lvm
/home
│ ├─onn_vmh-tmp 253:10 0 1G 0 lvm
/tmp
│ ├─onn_vmh-var 253:11 0 15G 0 lvm
/var
│ ├─onn_vmh-var_log 253:12 0 8G 0 lvm
/var/log
│ ├─onn_vmh-var_log_audit 253:13 0 2G 0 lvm
/var/log/audit
│ └─onn_vmh-var_crash 253:14 0 10G 0 lvm
/var/crash
└─onn_vmh-swap 253:4 0 4G 0 lvm
[SWAP]
sdc 8:32 0 223.6G 0 disk
└─sdc1 8:33 0 222.6G 0 part
└─md127 9:127 0 222.5G 0 raid1
├─onn_vmh-pool00_tmeta 253:0 0 1G 0 lvm
│ └─onn_vmh-pool00-tpool 253:2 0 173.6G 0 lvm
│ ├─onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1 253:3 0 146.6G 0 lvm /
│ ├─onn_vmh-pool00 253:7 0 173.6G 0 lvm
│ ├─onn_vmh-root 253:8 0 146.6G 0 lvm
│ ├─onn_vmh-home 253:9 0 1G 0 lvm
/home
│ ├─onn_vmh-tmp 253:10 0 1G 0 lvm
/tmp
│ ├─onn_vmh-var 253:11 0 15G 0 lvm
/var
│ ├─onn_vmh-var_log 253:12 0 8G 0 lvm
/var/log
│ ├─onn_vmh-var_log_audit 253:13 0 2G 0 lvm
/var/log/audit
│ └─onn_vmh-var_crash 253:14 0 10G 0 lvm
/var/crash
├─onn_vmh-pool00_tdata 253:1 0 173.6G 0 lvm
│ └─onn_vmh-pool00-tpool 253:2 0 173.6G 0 lvm
│ ├─onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1 253:3 0 146.6G 0 lvm /
│ ├─onn_vmh-pool00 253:7 0 173.6G 0 lvm
│ ├─onn_vmh-root 253:8 0 146.6G 0 lvm
│ ├─onn_vmh-home 253:9 0 1G 0 lvm
/home
│ ├─onn_vmh-tmp 253:10 0 1G 0 lvm
/tmp
│ ├─onn_vmh-var 253:11 0 15G 0 lvm
/var
│ ├─onn_vmh-var_log 253:12 0 8G 0 lvm
/var/log
│ ├─onn_vmh-var_log_audit 253:13 0 2G 0 lvm
/var/log/audit
│ └─onn_vmh-var_crash 253:14 0 10G 0 lvm
/var/crash
└─onn_vmh-swap 253:4 0 4G 0 lvm
[SWAP]
[root@vmh ~]# df -h
Filesystem Size Used Avail Use%
Mounted on
/dev/mapper/onn_vmh-ovirt--node--ng--4.2.5.1--0.20180821.0+1 145G 1.9G 135G 2% /
devtmpfs 63G 0 63G 0% /dev
tmpfs 63G 4.0K 63G 1%
/dev/shm
tmpfs 63G 19M 63G 1% /run
tmpfs 63G 0 63G 0%
/sys/fs/cgroup
/dev/mapper/onn_vmh-tmp 976M 2.8M 906M 1% /tmp
/dev/mapper/onn_vmh-home 976M 2.6M 907M 1% /home
/dev/sdb1 976M 204M 706M 23% /boot
/dev/mapper/onn_vmh-var 15G 43M 14G 1% /var
/dev/mapper/onn_vmh-var_log 7.8G 58M 7.3G 1%
/var/log
/dev/mapper/onn_vmh-var_crash 9.8G 37M 9.2G 1%
/var/crash
/dev/mapper/onn_vmh-var_log_audit 2.0G 7.7M 1.8G 1%
/var/log/audit
/dev/mapper/3600605b00a2faca222fb4da81ac9bdb1p1 8.2T 90M 7.8T 1% /data
tmpfs 13G 0 13G 0%
/run/user/0
[root@vmh ~]# lspci | grep RAID
3b:00.0 RAID bus controller: LSI Logic / Symbios Logic MegaRAID SAS-3 3108 [Invader] (rev
02)
[root@vmh ~]# lsmod | grep mega
megaraid_sas 145373 2
[root@vmh ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default
qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp96s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group
default qlen 1000
link/ether 0c:c4:7a:f9:b9:88 brd ff:ff:ff:ff:ff:ff
inet 172.30.50.3/24 brd 172.30.50.255 scope global noprefixroute dynamic enp96s0f0
valid_lft 47994sec preferred_lft 47994sec
inet6 fe80::119b:61e4:f9:1dca/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: enp96s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group
default qlen 1000
link/ether 0c:c4:7a:f9:b9:89 brd ff:ff:ff:ff:ff:ff
inet 172.30.50.4/24 brd 172.30.50.255 scope global noprefixroute dynamic enp96s0f1
valid_lft 47994sec preferred_lft 47994sec
inet6 fe80::498f:1c33:7de6:1fd/64 scope link noprefixroute
valid_lft forever preferred_lft forever
19: ;vdsmdummy;: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default
qlen 1000
link/ether 2e:78:2c:51:c8:12 brd ff:ff:ff:ff:ff:ff
In DNS I have defined .3 address as vmh.cyber-range.lan and .4 is defined as
ovirtbe.cyber-range.lan. .3 is the frontend IP and .4 is the backednd IP.
I have done the following so far with my Gluster Config file, I'm sure it needs
further edits.
[hosts]
vmh.cyber-range.lan
[disktype]
jbod #Possible values raid6, raid10, raid5, jbod
[diskcount]
@NUMBER_OF_DATA_DISKS@ #Ignored in case of jbod
[stripesize]
256 #256 in case of jbod
[script1]
action=execute
ignore_script_errors=no
file=/usr/share/gdeploy/scripts/grafton-sanity-check.sh -d @DEVICE@
#[vdo] # Note: Uncomment if dedupe & compression needs to be enabled on device. Needs
kmod-vdo module
#action=create
#names=@VDO_DEVICE_Name@
#devices=@DEVICE@
#logicalsize=@logical_size@T # Note:logicalsize is 10x physical space on disk
##slabsize=32G # Note: used only when the physical size is few TBs
#blockmapcachesize=128M
#readcache=enabled
#readcachesize=20M
#emulate512=enabled
#writepolicy=auto
[pv]
action=create
devices=@DEVICE@ # Change to @VDO_DEVICE_name@ if using vdo
[vg1]
action=create
vgname=gluster_vg1
pvname=@DEVICE@ # Change to @VDO_DEVICE_name@ if using vdo
[lv1]
action=create
vgname=gluster_vg1
lvname=engine_lv
lvtype=thick
size=100GB
mount=/gluster_bricks/engine
[lv2]
action=create
vgname=gluster_vg1
poolname=lvthinpool
lvtype=thinpool
poolmetadatasize=7.7TB
size=7.7TB #For example: 18000GB, depending on device capacity. Units to be specified.
[lv3]
action=create
lvname=lv_vmdisks
poolname=lvthinpool
vgname=gluster_vg1
lvtype=thinlv
mount=/gluster_bricks/vmstore
virtualsize=@SIZE@ # Units to be specified, for instance 5000GB
[lv4]
action=create
lvname=lv_datadisks
poolname=lvthinpool
vgname=gluster_vg1
lvtype=thinlv
mount=/gluster_bricks/data
virtualsize=@SIZE@ # Units to be specified, for instance 5000GB
#[lv5]
#action=setup-cache
#ssd=@SSD_DEVICE@
#vgname=gluster_vg1
#poolname=lvthinpool
#cache_lv=lvcache
#cache_lvsize=5GB # Provide device size
## cachemode=writeback
[shell2]
action=execute
command=vdsm-tool configure --force
[script3]
action=execute
file=/usr/share/gdeploy/scripts/blacklist_all_disks.sh
ignore_script_errors=no
[selinux]
yes
[service3]
action=restart
service=glusterd
slice_setup=yes
[firewalld]
action=add
ports=111/tcp,2049/tcp,54321/tcp,5900/tcp,5900-6923/tcp,5666/tcp,16514/tcp,54322/tcp
services=glusterfs
[script2]
action=execute
file=/usr/share/gdeploy/scripts/disable-gluster-hooks.sh
[shell3]
action=execute
command=usermod -a -G gluster qemu
[volume]
action=create
volname=engine
transport=tcp
key=storage.owner-uid,storage.owner-gid,features.shard,performance.low-prio-threads,performance.strict-o-direct,network.remote-dio,network.ping-timeout,user.cifs,nfs.disable,performance.quick-read,performance.read-ahead,performance.io-cache,cluster.eager-lock
value=36,36,on,32,on,off,30,off,on,off,off,off,enable
brick_dirs=/gluster_bricks/engine/engine
ignore_volume_errors=no
[volume2]
action=create
volname=vmstore
transport=tcp
key=storage.owner-uid,storage.owner-gid,features.shard,performance.low-prio-threads,performance.strict-o-direct,network.remote-dio,network.ping-timeout,user.cifs,nfs.disable,performance.quick-read,performance.read-ahead,performance.io-cache,cluster.eager-lock
value=36,36,on,32,on,off,30,off,on,off,off,off,enable
brick_dirs=/gluster_bricks/vmstore/vmstore
ignore_volume_errors=no
[volume3]
action=create
volname=data
transport=tcp
key=storage.owner-uid,storage.owner-gid,features.shard,performance.low-prio-threads,performance.strict-o-direct,network.remote-dio,network.ping-timeout,user.cifs,nfs.disable,performance.quick-read,performance.read-ahead,performance.io-cache,cluster.eager-lock
value=36,36,on,32,on,off,30,off,on,off,off,off,enable
brick_dirs=/gluster_bricks/data/data
ignore_volume_errors=no
Can anyone kindly provide the correct values and corrections needed? I am not nearly as
good with Linux storage as I'd like to be. Please let me know if any further info is
required. Many thanks in advance!!!
2259
days inactive
2259
days old
0 comments
1 participants
participants (1)
-
Jeremy Tourville