From db1b967293764ed602112442a5ca66d1e888e949 Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Tue, 9 Mar 2021 12:58:09 -0800 Subject: [PATCH] fix histogram median calculation (#263) The histogram approximate median is calculated as about double what it should be. When we are on the bucket that goes past the median, the histogram is the *previous* bucket, plus part of this bucket. Signed-off-by: Matthew Ahrens --- sdb/commands/zfs/histograms.py | 2 +- .../data/regression_output/zfs/spa -H | 4 +- .../data/regression_output/zfs/spa -mH | 48 +++++++++---------- .../data/regression_output/zfs/spa -vH | 8 ++-- .../data/regression_output/zfs/spa -vmH | 48 +++++++++---------- ..._normal_class.mc_histogram | zfs_histogram | 2 +- ...member ms_allocatable.rt_histogram | zhist | 2 +- ...member ms_sm.sm_phys.smp_histogram | zhist | 2 +- ...mber ms_sm.sm_phys.smp_histogram | zhist 9 | 2 +- 9 files changed, 59 insertions(+), 59 deletions(-) diff --git a/sdb/commands/zfs/histograms.py b/sdb/commands/zfs/histograms.py index e8a5ba4e..6f9eb48e 100644 --- a/sdb/commands/zfs/histograms.py +++ b/sdb/commands/zfs/histograms.py @@ -90,7 +90,7 @@ def histogram_median(hist: drgn.Object, offset: int = 0) -> int: for (bucket, value) in enumerate(hist): space_in_bucket = int(value) << (bucket + offset) if space_left <= space_in_bucket: - median = 1 << (bucket + offset) + median = 1 << (bucket + offset - 1) # # Size of segments may vary within one bucket thus we # attempt to approximate the median by looking at the diff --git a/tests/integration/data/regression_output/zfs/spa -H b/tests/integration/data/regression_output/zfs/spa -H index cfb05875..bc85223f 100644 --- a/tests/integration/data/regression_output/zfs/spa -H +++ b/tests/integration/data/regression_output/zfs/spa -H @@ -23,7 +23,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa089413b8000 meta-domain seg-size count -------- ----- @@ -45,6 +45,6 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB 0xffffa08955c44000 rpool ** No histogram data available ** diff --git a/tests/integration/data/regression_output/zfs/spa -mH b/tests/integration/data/regression_output/zfs/spa -mH index 2345bf5f..b32dd225 100644 --- a/tests/integration/data/regression_output/zfs/spa -mH +++ b/tests/integration/data/regression_output/zfs/spa -mH @@ -23,7 +23,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB ADDR STATE AUX DESCRIPTION ------------------------------------------------------------ 0xffffa089486fc000 HEALTHY NONE root @@ -50,7 +50,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB ADDR ID OFFSET FREE FRAG UCMU ----------------------------------------------------------------- 0xffffa0894e6e1000 0 0x0 511MB 0% 8KB @@ -76,7 +76,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e4000 1 0x20000000 511MB 0% 8KB seg-size count -------- ----- @@ -100,7 +100,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e2800 2 0x40000000 511MB 0% 8KB seg-size count -------- ----- @@ -124,7 +124,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e3800 3 0x60000000 511MB 0% 8KB seg-size count -------- ----- @@ -148,7 +148,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e3000 4 0x80000000 511MB 0% 8KB seg-size count -------- ----- @@ -172,7 +172,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e4800 5 0xa0000000 511MB 0% 8KB seg-size count -------- ----- @@ -196,7 +196,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e7800 6 0xc0000000 511MB 0% 4KB seg-size count -------- ----- @@ -220,7 +220,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e2000 7 0xe0000000 511MB 0% 8KB seg-size count -------- ----- @@ -244,7 +244,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e637800 8 0x100000000 511MB 0% 8KB seg-size count -------- ----- @@ -268,7 +268,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e634800 9 0x120000000 511MB 0% 4KB seg-size count -------- ----- @@ -291,7 +291,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e633000 10 0x140000000 511MB 0% 4KB seg-size count -------- ----- @@ -314,7 +314,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e633800 11 0x160000000 509MB 0% 4KB seg-size count -------- ----- @@ -338,7 +338,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e632800 12 0x180000000 511MB 0% 4KB seg-size count -------- ----- @@ -362,7 +362,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e634000 13 0x1a0000000 511MB 0% 4KB seg-size count -------- ----- @@ -386,7 +386,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e631000 14 0x1c0000000 511MB 0% 4KB seg-size count -------- ----- @@ -410,7 +410,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa08948af8000 HEALTHY NONE /dev/sdb1 0xffffa08949ff8000 HEALTHY NONE /dev/sdc1 0xffffa08949e58000 HEALTHY NONE /dev/sdd1 @@ -453,7 +453,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB ADDR STATE AUX DESCRIPTION ------------------------------------------------------------ 0xffffa08953aa4000 HEALTHY NONE root @@ -478,7 +478,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB ADDR ID OFFSET FREE FRAG UCMU ----------------------------------------------------------------- 0xffffa0894c701000 0 0x0 256MB 0% 0B @@ -502,7 +502,7 @@ ADDR NAME 16.0MB: 0 32.0MB: 0 64.0MB: 1 * - Approx. Median: 95.9MB + Approx. Median: 48.0MB 0xffffa0894c700800 2 0x20000000 255MB 0% 8KB seg-size count -------- ----- @@ -524,7 +524,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 0 128.0MB: 1 * - Approx. Median: 191.9MB + Approx. Median: 96.0MB 0xffffa0894c706000 3 0x30000000 255MB 0% 8KB seg-size count -------- ----- @@ -546,7 +546,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 0 128.0MB: 1 * - Approx. Median: 192.0MB + Approx. Median: 96.0MB 0xffffa0894c707000 4 0x40000000 223MB 0% 4KB seg-size count -------- ----- @@ -568,12 +568,12 @@ ADDR NAME 32.0MB: 0 64.0MB: 0 128.0MB: 1 * - Approx. Median: 192.0MB + Approx. Median: 96.0MB 0xffffa0894c706800 5 0x50000000 255MB 0% 4KB seg-size count -------- ----- 128.0MB: 1 * - Approx. Median: 192.0MB + Approx. Median: 96.0MB 0xffffa0894c703000 6 0x60000000 256MB 0% 0B 0xffffa0894c703800 7 0x70000000 256MB 0% 0B 0xffffa0894c705000 8 0x80000000 256MB 0% 0B diff --git a/tests/integration/data/regression_output/zfs/spa -vH b/tests/integration/data/regression_output/zfs/spa -vH index 134d3e89..d12388a4 100644 --- a/tests/integration/data/regression_output/zfs/spa -vH +++ b/tests/integration/data/regression_output/zfs/spa -vH @@ -23,7 +23,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB ADDR STATE AUX DESCRIPTION ------------------------------------------------------------ 0xffffa089486fc000 HEALTHY NONE root @@ -50,7 +50,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa08948af8000 HEALTHY NONE /dev/sdb1 0xffffa08949ff8000 HEALTHY NONE /dev/sdc1 0xffffa08949e58000 HEALTHY NONE /dev/sdd1 @@ -76,7 +76,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB ADDR STATE AUX DESCRIPTION ------------------------------------------------------------ 0xffffa08953aa4000 HEALTHY NONE root @@ -101,7 +101,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB 0xffffa08953aac000 HEALTHY NONE /tmp/dks0 0xffffa08953ab0000 HEALTHY NONE /tmp/dks1 0xffffa08953ab4000 HEALTHY NONE /tmp/dks2 diff --git a/tests/integration/data/regression_output/zfs/spa -vmH b/tests/integration/data/regression_output/zfs/spa -vmH index 2345bf5f..b32dd225 100644 --- a/tests/integration/data/regression_output/zfs/spa -vmH +++ b/tests/integration/data/regression_output/zfs/spa -vmH @@ -23,7 +23,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB ADDR STATE AUX DESCRIPTION ------------------------------------------------------------ 0xffffa089486fc000 HEALTHY NONE root @@ -50,7 +50,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** - Approx. Median: 384.0MB + Approx. Median: 192.0MB ADDR ID OFFSET FREE FRAG UCMU ----------------------------------------------------------------- 0xffffa0894e6e1000 0 0x0 511MB 0% 8KB @@ -76,7 +76,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e4000 1 0x20000000 511MB 0% 8KB seg-size count -------- ----- @@ -100,7 +100,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e2800 2 0x40000000 511MB 0% 8KB seg-size count -------- ----- @@ -124,7 +124,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e3800 3 0x60000000 511MB 0% 8KB seg-size count -------- ----- @@ -148,7 +148,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e3000 4 0x80000000 511MB 0% 8KB seg-size count -------- ----- @@ -172,7 +172,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e4800 5 0xa0000000 511MB 0% 8KB seg-size count -------- ----- @@ -196,7 +196,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e7800 6 0xc0000000 511MB 0% 4KB seg-size count -------- ----- @@ -220,7 +220,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e6e2000 7 0xe0000000 511MB 0% 8KB seg-size count -------- ----- @@ -244,7 +244,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e637800 8 0x100000000 511MB 0% 8KB seg-size count -------- ----- @@ -268,7 +268,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e634800 9 0x120000000 511MB 0% 4KB seg-size count -------- ----- @@ -291,7 +291,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e633000 10 0x140000000 511MB 0% 4KB seg-size count -------- ----- @@ -314,7 +314,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e633800 11 0x160000000 509MB 0% 4KB seg-size count -------- ----- @@ -338,7 +338,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e632800 12 0x180000000 511MB 0% 4KB seg-size count -------- ----- @@ -362,7 +362,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e634000 13 0x1a0000000 511MB 0% 4KB seg-size count -------- ----- @@ -386,7 +386,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa0894e631000 14 0x1c0000000 511MB 0% 4KB seg-size count -------- ----- @@ -410,7 +410,7 @@ ADDR NAME 64.0MB: 0 128.0MB: 0 256.0MB: 1 * - Approx. Median: 384.0MB + Approx. Median: 192.0MB 0xffffa08948af8000 HEALTHY NONE /dev/sdb1 0xffffa08949ff8000 HEALTHY NONE /dev/sdc1 0xffffa08949e58000 HEALTHY NONE /dev/sdd1 @@ -453,7 +453,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB ADDR STATE AUX DESCRIPTION ------------------------------------------------------------ 0xffffa08953aa4000 HEALTHY NONE root @@ -478,7 +478,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 1 * 128.0MB: 4 **** - Approx. Median: 184.0MB + Approx. Median: 92.0MB ADDR ID OFFSET FREE FRAG UCMU ----------------------------------------------------------------- 0xffffa0894c701000 0 0x0 256MB 0% 0B @@ -502,7 +502,7 @@ ADDR NAME 16.0MB: 0 32.0MB: 0 64.0MB: 1 * - Approx. Median: 95.9MB + Approx. Median: 48.0MB 0xffffa0894c700800 2 0x20000000 255MB 0% 8KB seg-size count -------- ----- @@ -524,7 +524,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 0 128.0MB: 1 * - Approx. Median: 191.9MB + Approx. Median: 96.0MB 0xffffa0894c706000 3 0x30000000 255MB 0% 8KB seg-size count -------- ----- @@ -546,7 +546,7 @@ ADDR NAME 32.0MB: 0 64.0MB: 0 128.0MB: 1 * - Approx. Median: 192.0MB + Approx. Median: 96.0MB 0xffffa0894c707000 4 0x40000000 223MB 0% 4KB seg-size count -------- ----- @@ -568,12 +568,12 @@ ADDR NAME 32.0MB: 0 64.0MB: 0 128.0MB: 1 * - Approx. Median: 192.0MB + Approx. Median: 96.0MB 0xffffa0894c706800 5 0x50000000 255MB 0% 4KB seg-size count -------- ----- 128.0MB: 1 * - Approx. Median: 192.0MB + Approx. Median: 96.0MB 0xffffa0894c703000 6 0x60000000 256MB 0% 0B 0xffffa0894c703800 7 0x70000000 256MB 0% 0B 0xffffa0894c705000 8 0x80000000 256MB 0% 0B diff --git a/tests/integration/data/regression_output/zfs/spa data | member spa_normal_class.mc_histogram | zfs_histogram b/tests/integration/data/regression_output/zfs/spa data | member spa_normal_class.mc_histogram | zfs_histogram index f3a5db08..54a3ac16 100644 --- a/tests/integration/data/regression_output/zfs/spa data | member spa_normal_class.mc_histogram | zfs_histogram +++ b/tests/integration/data/regression_output/zfs/spa data | member spa_normal_class.mc_histogram | zfs_histogram @@ -20,4 +20,4 @@ seg-size count 64.0MB: 0 128.0MB: 0 256.0MB: 15 *************** -Approx. Median: 384.0MB +Approx. Median: 192.0MB diff --git a/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_allocatable.rt_histogram | zhist b/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_allocatable.rt_histogram | zhist index 3ae1fc30..af035907 100644 --- a/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_allocatable.rt_histogram | zhist +++ b/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_allocatable.rt_histogram | zhist @@ -19,4 +19,4 @@ seg-size count 64.0MB: 0 128.0MB: 0 256.0MB: 1 * -Approx. Median: 384.0MB +Approx. Median: 192.0MB diff --git a/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist b/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist index 1d71d7a5..4a78270f 100644 --- a/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist +++ b/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist @@ -20,4 +20,4 @@ seg-size count 128.0KB: 0 256.0KB: 0 512.0KB: 1 * -Approx. Median: 768.0KB +Approx. Median: 384.0KB diff --git a/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist 9 b/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist 9 index 3c37397f..9538b4c8 100644 --- a/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist 9 +++ b/tests/integration/data/regression_output/zfs/spa data | vdev | metaslab | filter 'obj.ms_loaded == 1' | head 1 | member ms_sm.sm_phys.smp_histogram | zhist 9 @@ -20,4 +20,4 @@ seg-size count 64.0MB: 0 128.0MB: 0 256.0MB: 1 * -Approx. Median: 384.0MB +Approx. Median: 192.0MB