Weighted sem

src/scalarstats.jl

@@ -260,12 +260,37 @@ variation(x) = ((m, s) = mean_and_std(x); s/m)
 realXcY(x::Real, y::Real) = x*y
 realXcY(x::Complex, y::Complex) = real(x)*real(y) + imag(x)*imag(y)
 
+
+"""
+    var_sample_mean(x[, weights::AbstractWeights]; mean=nothing)
+
+Return the estimated variance of the mean for a collection `x`. When using no weights, this is
+equal to the (sample) standard deviation divided by the sample size. If weights are used, 
+the variance of the sample mean is calculated as follows:
+
+* `AnalyticWeights`: Not implemented.
+* `FrequencyWeights`: ``\\frac{\\sum_{i=1}^n w_i (x_i - \\bar{x_i})^2}{(\\sum w_i) (\\sum w_i - 1)}``
+* `ProbabilityWeights`: ``\\frac{n}{n-1} \\frac{\\sum_{i=1}^n w_i^2 (x_i - \\bar{x_i})^2}{\\left( \\sum w_i \\right)^2}``
+
+The standard error is then the square root of the above quantities.
 """
-    sem(x)
 
-Return the standard error of the mean of collection `x`,
-i.e. `sqrt(var(x, corrected=true) / length(x))`.
+
+"""
+    sem(x[, weights::AbstractWeights]; mean=nothing)
+
+Return the standard error of the mean for a collection `x`. When using no weights, this is
+equal to the (sample) standard deviation divided by the sample size. If weights are used, 
+the variance of the sample mean is calculated as follows:
+
+* `AnalyticWeights`: Not implemented.
+* `FrequencyWeights`: ``\\frac{\\sum_{i=1}^n w_i (x_i - \\bar{x_i})^2}{(\\sum w_i) (\\sum w_i - 1)}``
+* `ProbabilityWeights`: ``\\frac{n}{n-1} \\frac{\\sum_{i=1}^n w_i^2 (x_i - \\bar{x_i})^2}{\\left( \\sum w_i \\right)^2}``
+
+The standard error is then the square root of the above quantities.
 """
+sem(x; mean::Number) = sqrt(varm(x, mean) / count)
+sem(x; mean::Nothing) = sem(x)
 function sem(x)
     y = iterate(x)
     if y === nothing
@@ -290,7 +315,27 @@ function sem(x)
         M = new_M
     end
     var = S / (count - 1)
-    return sqrt(var/count)
+    return sqrt(var / count)
+end
+
+function sem(x, weights::UnitWeights; kwargs...)
+    if length(x) == length(weights)
+        T = promote(eltype(x), eltype(weights))
+        return sem(Iterators.map(T, x); kwargs...)
+    else
+        throw(ArgumentError("data and weights must be the same length"))
+    end
+end
+
+# Weighted methods for the above
+sem(x, weights::FrequencyWeights) = sqrt(var(x, weights; corrected=true) / weights.sum)
+
+function sem(x, weights::ProbabilityWeights)
+    μ = mean(x, weights)
+    var = sum(zip(x, weights)) do (x, w)
+        return (w * (x - μ))^2
+    end
+    return sqrt(var * n / (n - 1)) / weights.sum
 end
 
 # Median absolute deviation

test/scalarstats.jl

@@ -109,6 +109,10 @@ z2 = [8. 2. 3. 1.; 24. 10. -1. -1.; 20. 12. 1. -2.]
 
 @test sem([1:5;]) ≈ 0.707106781186548
 @test sem(skipmissing([missing; 1:5; missing])) ≈ 0.707106781186548
+@test sem([1:5;], UnitWeights{Int}(5)) ≈ 0.707106781186548
+@test sem([1:5;], ProbabilityWeights([1:5;])) ≈ 0.6166 rtol=.001
+@test sem([1:100;], ProbabilityWeights([1:100;])) ≈ 2.440215 rtol=.001
+@test sem([1:5;], FrequencyWeights([1:5;])) ≈ sem([1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5])
 @test sem(Int[]) === NaN
 @test sem(skipmissing(Union{Int,Missing}[missing, missing])) === NaN
 @test_throws MethodError sem(Any[])