Degraded vdev

[nasiryilmaz]

I have really weird issue raidz2-3 keeps failing with drive errors.
I replace all failed drives with spare brand new drives and still fails.
if I move failed drive to (sday) raidz2-0 it becomes online without any errors. I detach sday and attach back to raidz2-3 fails again. I have supermicro storage server with JBOD attached with SAS cable. SAS cable replaced, SAS controller slot is changed, SAS controller port is changed,all Seagate (ST10000NM0096) drive firmware and driver updated, SAS Controller firmware updated,all failed Seagate replaced with the same size WD and no luck keeps failing , ZFS updated to zfs-kmod-0.8.6-1 any idea?

root@rdsredhen01 ~]# uname -a
Linux rdsredhen01.cwru.edu 3.10.0-1160.11.1.el7.x86_64 #1 SMP Mon Nov 30 13:05:31 EST 2020 x86_64 x86_64 x86_64 GNU/Linux

[root@rdsredhen01 ~]# cat /etc/redhat-release
Red Hat Enterprise Linux Server release 7.9 (Maipo)

[root@rdsredhen01 ~]# zfs --version
zfs-0.8.6-1
zfs-kmod-0.8.6-1 '

root@rdsredhen01 ~]# zpool status
pool: pool
state: DEGRADED
status: One or more devices has experienced an error resulting in data corruption. Applications may be affected.
action: Restore the file in question if possible. Otherwise, restore the entire pool from backup.
see: http://zfsonlinux.org/msg/ZFS-8000-8A
scan: scrub in progress since Wed Feb 10 18:52:04 2021 267T scanned at 2.51G/s, 265T issued at 2.49G/s, 269T total
0B repaired, 98.66% done, 0 days 00:24:44 to go
config:

    NAME        STATE     READ WRITE CKSUM
    pool        DEGRADED     0     0     0
      raidz2-0  ONLINE       0     0     0
        sdd     ONLINE       0     0     0
        sdg     ONLINE       0     0     0
        sdp     ONLINE       0     0     0
        sdr     ONLINE       0     0     0
        sdf     ONLINE       0     0     0
        sdj     ONLINE       0     0     0
        sde     ONLINE       0     0     0
        sdc     ONLINE       0     0     0
        sdu     ONLINE       0     0     0
        sdx     ONLINE       0     0     0
        sdo     ONLINE       0     0     0
      raidz2-1  ONLINE       0     0     0
        sdw     ONLINE       0     0     0
        sdi     ONLINE       0     0     0
        sdq     ONLINE       0     0     0
        sdl     ONLINE       0     0     0
        sds     ONLINE       0     0     0
        sdh     ONLINE       0     0     0
        sdm     ONLINE       0     0     0
        sdt     ONLINE       0     0     0
        sdk     ONLINE       0     0     0
        sdb     ONLINE       0     0     0
        sdv     ONLINE       0     0     0
      raidz2-2  ONLINE       0     0     0
        sdaf    ONLINE       0     0     0
        sdz     ONLINE       0     0     0
        sdad    ONLINE       0     0     0
        sdal    ONLINE       0     0     0
        sdae    ONLINE       0     0     0
        sdak    ONLINE       0     0     0
        sdac    ONLINE       0     0     0
        sdai    ONLINE       0     0     0
        sdaj    ONLINE       0     0     0
        sdba    ONLINE       0     0     0
        sdn     ONLINE       0     0     0
      raidz2-3  DEGRADED     0     0     0
        sday    DEGRADED     0     0 2.85M  too many errors
        sdag    DEGRADED     0     0 2.85M  too many errors
        sdas    DEGRADED     0     0 2.85M  too many errors
        sdat    DEGRADED     0     0 2.85M  too many errors
        sdah    DEGRADED     0     0 2.85M  too many errors
        sdau    DEGRADED     0     0 2.85M  too many errors
        sdav    DEGRADED     0     0 2.85M  too many errors
        sdam    DEGRADED     0     0 2.85M  too many errors
        sdax    DEGRADED     0     0 2.85M  too many errors
        sdbb    DEGRADED     0     0 2.85M  too many errors
        sdan    DEGRADED     0     0 2.85M  too many errors
		------------------------------------------------------------

[root@rdsredhen01 ~]# lsscsi -g
[0:0:0:0] disk SEAGATE ST10000NM0096 E005 /dev/sdb /dev/sg1
[0:0:1:0] disk SEAGATE ST10000NM0096 E005 /dev/sdc /dev/sg2
[0:0:2:0] disk SEAGATE ST10000NM0096 E005 /dev/sdd /dev/sg3
[0:0:3:0] disk SEAGATE ST10000NM0096 E005 /dev/sde /dev/sg4
[0:0:4:0] disk SEAGATE ST10000NM0096 E005 /dev/sdf /dev/sg5
[0:0:5:0] disk SEAGATE ST10000NM0096 E005 /dev/sdg /dev/sg6
[0:0:6:0] disk SEAGATE ST10000NM0096 E005 /dev/sdh /dev/sg7
[0:0:7:0] disk SEAGATE ST10000NM0096 E005 /dev/sdi /dev/sg8
[0:0:8:0] disk SEAGATE ST10000NM0096 E005 /dev/sdj /dev/sg9
[0:0:9:0] disk SEAGATE ST10000NM0096 E005 /dev/sdk /dev/sg10
[0:0:10:0] disk SEAGATE ST10000NM0096 E005 /dev/sdl /dev/sg11
[0:0:11:0] disk SEAGATE ST10000NM0096 E005 /dev/sdm /dev/sg12
[0:0:12:0] disk SEAGATE ST10000NM0096 E005 /dev/sdn /dev/sg13
[0:0:13:0] disk SEAGATE ST10000NM0096 E005 /dev/sdo /dev/sg14
[0:0:14:0] disk SEAGATE ST10000NM0096 E005 /dev/sdp /dev/sg15
[0:0:15:0] disk SEAGATE ST10000NM0096 E005 /dev/sdq /dev/sg16
[0:0:16:0] disk SEAGATE ST10000NM0096 E005 /dev/sdr /dev/sg17
[0:0:17:0] disk SEAGATE ST10000NM0096 E005 /dev/sds /dev/sg18
[0:0:18:0] disk SEAGATE ST10000NM0096 E005 /dev/sdt /dev/sg19
[0:0:19:0] disk SEAGATE ST10000NM0096 E005 /dev/sdu /dev/sg20
[0:0:20:0] disk SEAGATE ST10000NM0096 E005 /dev/sdv /dev/sg21
[0:0:21:0] disk SEAGATE ST10000NM0096 E005 /dev/sdw /dev/sg22
[0:0:22:0] disk SEAGATE ST10000NM0096 E005 /dev/sdx /dev/sg23
[0:0:23:0] disk SEAGATE ST10000NM0096 E005 /dev/sdy /dev/sg24
[0:0:24:0] enclosu SMC SC846P 0c1f - /dev/sg25
[0:0:25:0] disk SEAGATE ST10000NM0096 E005 /dev/sdz /dev/sg26
[0:0:26:0] disk SEAGATE ST10000NM0096 E005 /dev/sdaa /dev/sg27
[0:0:27:0] disk SEAGATE ST10000NM0096 E005 /dev/sdab /dev/sg28
[0:0:28:0] disk SEAGATE ST10000NM0096 E005 /dev/sdac /dev/sg29
[0:0:29:0] disk SEAGATE ST10000NM0096 E005 /dev/sdad /dev/sg30
[0:0:30:0] disk SEAGATE ST10000NM0096 E005 /dev/sdae /dev/sg31
[0:0:31:0] disk SEAGATE ST10000NM0096 E005 /dev/sdaf /dev/sg32
[0:0:32:0] disk SEAGATE ST10000NM0096 E005 /dev/sdag /dev/sg33
[0:0:33:0] disk SEAGATE ST10000NM0096 E005 /dev/sdah /dev/sg34
[0:0:34:0] disk SEAGATE ST10000NM0096 E005 /dev/sdai /dev/sg35
[0:0:35:0] disk SEAGATE ST10000NM0096 E005 /dev/sdaj /dev/sg36
[0:0:36:0] disk SEAGATE ST10000NM0096 E005 /dev/sdak /dev/sg37
[0:0:37:0] disk SEAGATE ST10000NM0096 E005 /dev/sdal /dev/sg38
[0:0:38:0] disk SEAGATE ST10000NM0096 E005 /dev/sdam /dev/sg39
[0:0:39:0] disk SEAGATE ST10000NM0096 E005 /dev/sdan /dev/sg40
[0:0:40:0] disk SEAGATE ST10000NM0096 E005 /dev/sdao /dev/sg41
[0:0:41:0] disk SEAGATE ST10000NM0096 E005 /dev/sdap /dev/sg42
[0:0:42:0] enclosu SMC SC846P 0c1f - /dev/sg43
[0:0:43:0] disk HGST HUH721010AL5200 A384 /dev/sdaq /dev/sg44
[0:0:44:0] disk HGST HUH721010AL5200 A384 /dev/sdar /dev/sg45
[0:0:45:0] disk SEAGATE ST10000NM0096 E005 /dev/sdas /dev/sg46
[0:0:46:0] disk SEAGATE ST10000NM0096 E005 /dev/sdat /dev/sg47
[0:0:47:0] disk SEAGATE ST10000NM0096 E005 /dev/sdau /dev/sg48
[0:0:48:0] disk SEAGATE ST10000NM0096 E005 /dev/sdav /dev/sg49
[0:0:49:0] disk HGST HUH721010AL5200 A384 /dev/sdaw /dev/sg50
[0:0:50:0] disk SEAGATE ST10000NM0096 E005 /dev/sdax /dev/sg51
[0:0:51:0] disk SEAGATE ST10000NM0096 E005 /dev/sday /dev/sg52
[0:0:52:0] disk HGST HUH721010AL5200 A384 /dev/sdaz /dev/sg53
[0:0:53:0] disk SEAGATE ST10000NM0096 E005 /dev/sdba /dev/sg54
[0:0:54:0] disk SEAGATE ST10000NM0096 E005 /dev/sdbb /dev/sg55
[0:0:55:0] disk HGST HUH721010AL5200 A384 /dev/sdbc /dev/sg56
[0:0:56:0] disk HGST HUH721010AL5200 A384 /dev/sdbd /dev/sg57
[0:0:57:0] disk HGST HUH721010AL5200 A384 /dev/sdbe /dev/sg58
[0:0:58:0] disk HGST HUH721010AL5200 A384 /dev/sdbf /dev/sg59
[0:0:59:0] disk HGST HUH721010AL5200 A384 /dev/sdbg /dev/sg60
[0:0:60:0] disk HGST HUH721010AL5200 A384 /dev/sdbh /dev/sg61
[0:0:61:0] enclosu LSI SAS3x36 0601 - /dev/sg62
[5:0:0:0] disk ATA INTEL SSDSC2KB24 0100 /dev/sda /dev/sg0
[6:0:0:0] disk ATA INTEL SSDSC2KB24 0100 /dev/sdbi /dev/sg63