8:48 p.m.
The response was that when I select the oVirt HCI storage volumes to deploy to (with VDO
enabled) which are a single 512GB SSD with only one small IDM VM running. The IPI OCP 4.7
deployment fails. RH closed ticket because "gluster volume is too slow".
I then tried to create with my other 1TB SSD in each server gluster volume without VDO and
see if that worked.. but though I matched all settings / gluster options oVirt set, IPI
OCP would not show disk as deployable option. I then figured I would use GUI to create
bricks vs ansible means (trying to be good and stop doing direct shell but build as
ansible playbooks).. but that test is on hold because servers for next two weeks are being
re-tasked for another POC. So I figured I would do some rethinking if oVirt HCI on RHEV
4.5 with Gluster is a rat hole that will never work.
Below is the output from fio test OCP team asked me to run to show gluster was too slow.
##################
ansible@LT-A0070501:/mnt/c/GitHub/penguinpages_cluster_devops/cluster_devops$ ssh
core(a)172.16.100.184
[core@localhost ~]$ journalctl -b -f -u release-image.service -u bootkube.service
-- Logs begin at Sun 2021-04-11 19:18:07 UTC. --
Apr 13 11:50:23 localhost bootkube.sh[1276476]: [#404] failed to fetch discovery: Get
"https://localhost:6443/api?timeout=32s": x509: certificate has expired or is
not yet valid: current time 2021-04-13T11:50:23Z is after 2021-04-12T19:12:30Z
Apr 13 11:50:23 localhost bootkube.sh[1276476]: [#405] failed to fetch discovery: Get
"https://localhost:6443/api?timeout=32s": x509: certificate has expired or is
not yet valid: current time 2021-04-13T11:50:23Z is after 2021-04-12T19:12:30Z
<snip>
[core@localhost ~]$ su -
Password:
su: Authentication failure
[core@localhost ~]$ sudo podman run --volume /var/lib/etcd:/var/lib/etcd:Z
quay.io/openshift-scale/etcd-perf Trying to pull quay.io/openshift-scale/etcd-perf
Trying to pull quay.io/openshift-scale/etcd-perf...
Getting image source signatures
Copying blob a3ed95caeb02 done
Copying blob a3ed95caeb02 done
Copying blob a3ed95caeb02 done
Copying blob a3ed95caeb02 skipped: already exists
Copying blob fcc022b71ae4 done
Copying blob a93d706457b7 done
Copying blob 763b3f36c462 done
Writing manifest to image destination
Storing signatures
---------------------------------------------------------------- Running fio
---------------------------------------------------------------------------
{
"fio version" : "fio-3.7",
"timestamp" : 1618315279,
"timestamp_ms" : 1618315279798,
"time" : "Tue Apr 13 12:01:19 2021",
"global options" : {
"rw" : "write",
"ioengine" : "sync",
"fdatasync" : "1",
"directory" : "/var/lib/etcd",
"size" : "22m",
"bs" : "2300"
},
"jobs" : [
{
"jobname" : "etcd_perf",
"groupid" : 0,
"error" : 0,
"eta" : 0,
"elapsed" : 507,
"job options" : {
"name" : "etcd_perf"
},
"read" : {
"io_bytes" : 0,
"io_kbytes" : 0,
"bw_bytes" : 0,
"bw" : 0,
"iops" : 0.000000,
"runtime" : 0,
"total_ios" : 0,
"short_ios" : 10029,
"drop_ios" : 0,
"slat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000
},
"clat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"percentile" : {
"1.000000" : 0,
"5.000000" : 0,
"10.000000" : 0,
"20.000000" : 0,
"30.000000" : 0,
"40.000000" : 0,
"50.000000" : 0,
"60.000000" : 0,
"70.000000" : 0,
"80.000000" : 0,
"90.000000" : 0,
"95.000000" : 0,
"99.000000" : 0,
"99.500000" : 0,
"99.900000" : 0,
"99.950000" : 0,
"99.990000" : 0
}
},
"lat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000
},
"bw_min" : 0,
"bw_max" : 0,
"bw_agg" : 0.000000,
"bw_mean" : 0.000000,
"bw_dev" : 0.000000,
"bw_samples" : 0,
"iops_min" : 0,
"iops_max" : 0,
"iops_mean" : 0.000000,
"iops_stddev" : 0.000000,
"iops_samples" : 0
},
"write" : {
"io_bytes" : 23066700,
"io_kbytes" : 22526,
"bw_bytes" : 45589,
"bw" : 44,
"iops" : 19.821372,
"runtime" : 505969,
"total_ios" : 10029,
"short_ios" : 0,
"drop_ios" : 0,
"slat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000
},
"clat_ns" : {
"min" : 12011,
"max" : 680340,
"mean" : 26360.617210,
"stddev" : 15390.749240,
"percentile" : {
"1.000000" : 13632,
"5.000000" : 14528,
"10.000000" : 15808,
"20.000000" : 18304,
"30.000000" : 19584,
"40.000000" : 20864,
"50.000000" : 22656,
"60.000000" : 25472,
"70.000000" : 28800,
"80.000000" : 33536,
"90.000000" : 40192,
"95.000000" : 46848,
"99.000000" : 64768,
"99.500000" : 73216,
"99.900000" : 96768,
"99.950000" : 103936,
"99.990000" : 651264
}
},
"lat_ns" : {
"min" : 13570,
"max" : 682541,
"mean" : 28604.818825,
"stddev" : 15726.819671
},
"bw_min" : 4,
"bw_max" : 89,
"bw_agg" : 100.000000,
"bw_mean" : 46.770285,
"bw_dev" : 15.478852,
"bw_samples" : 949,
"iops_min" : 1,
"iops_max" : 40,
"iops_mean" : 21.048472,
"iops_stddev" : 6.908059,
"iops_samples" : 949
},
"trim" : {
"io_bytes" : 0,
"io_kbytes" : 0,
"bw_bytes" : 0,
"bw" : 0,
"iops" : 0.000000,
"runtime" : 0,
"total_ios" : 0,
"short_ios" : 0,
"drop_ios" : 0,
"slat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000
},
"clat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"percentile" : {
"1.000000" : 0,
"5.000000" : 0,
"10.000000" : 0,
"20.000000" : 0,
"30.000000" : 0,
"40.000000" : 0,
"50.000000" : 0,
"60.000000" : 0,
"70.000000" : 0,
"80.000000" : 0,
"90.000000" : 0,
"95.000000" : 0,
"99.000000" : 0,
"99.500000" : 0,
"99.900000" : 0,
"99.950000" : 0,
"99.990000" : 0
}
},
"lat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000
},
"bw_min" : 0,
"bw_max" : 0,
"bw_agg" : 0.000000,
"bw_mean" : 0.000000,
"bw_dev" : 0.000000,
"bw_samples" : 0,
"iops_min" : 0,
"iops_max" : 0,
"iops_mean" : 0.000000,
"iops_stddev" : 0.000000,
"iops_samples" : 0
},
"sync" : {
"lat_ns" : {
"min" : 10082235,
"max" : 6139419269,
"mean" : 50405768.015057,
"stddev" : 121884279.530146,
"percentile" : {
"1.000000" : 11075584,
"5.000000" : 11993088,
"10.000000" : 12779520,
"20.000000" : 15925248,
"30.000000" : 19267584,
"40.000000" : 30539776,
"50.000000" : 42729472,
"60.000000" : 49545216,
"70.000000" : 55836672,
"80.000000" : 64225280,
"90.000000" : 81264640,
"95.000000" : 113770496,
"99.000000" : 212860928,
"99.500000" : 258998272,
"99.900000" : 1535115264,
"99.950000" : 2868903936,
"99.990000" : 5335154688
}
},
"total_ios" : 0
},
"usr_cpu" : 0.045457,
"sys_cpu" : 0.145266,
"ctx" : 40541,
"majf" : 0,
"minf" : 13,
"iodepth_level" : {
"1" : 200.000000,
"2" : 0.000000,
"4" : 0.000000,
"8" : 0.000000,
"16" : 0.000000,
"32" : 0.000000,
">=64" : 0.000000
},
"latency_ns" : {
"2" : 0.000000,
"4" : 0.000000,
"10" : 0.000000,
"20" : 0.000000,
"50" : 0.000000,
"100" : 0.000000,
"250" : 0.000000,
"500" : 0.000000,
"750" : 0.000000,
"1000" : 0.000000
},
"latency_us" : {
"2" : 0.000000,
"4" : 0.000000,
"10" : 0.000000,
"20" : 33.702263,
"50" : 62.498754,
"100" : 3.709243,
"250" : 0.059827,
"500" : 0.000000,
"750" : 0.029913,
"1000" : 0.000000
},
"latency_ms" : {
"2" : 0.000000,
"4" : 0.000000,
"10" : 0.000000,
"20" : 0.000000,
"50" : 0.000000,
"100" : 0.000000,
"250" : 0.000000,
"500" : 0.000000,
"750" : 0.000000,
"1000" : 0.000000,
"2000" : 0.000000,
">=2000" : 0.000000
},
"latency_depth" : 1,
"latency_target" : 0,
"latency_percentile" : 100.000000,
"latency_window" : 0
}
],
"disk_util" : [
{
"name" : "sda",
"read_ios" : 86,
"write_ios" : 34333,
"read_merges" : 0,
"write_merges" : 4065,
"read_ticks" : 1207,
"write_ticks" : 1389885,
"in_queue" : 1372762,
"util" : 95.134115
}
]
}
--------------------------------------------------------------------------------------------------------------------------------------------------------
99th percentile of fsync is 212860928 ns
99th percentile of the fsync is greater than the recommended value which is 10 ms, faster
disks are recommended to host etcd for better performance
[core@localhost ~]$