bootstrap LRT

Project.toml

@@ -7,6 +7,8 @@ authors = ["Phillip Alday <me@phillipalday.com> and contributors"]
 GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d"
@@ -20,6 +22,8 @@ GLM = "1.9"
 LinearAlgebra = "1"
 MixedModels = "5.9"
 MixedModelsDatasets = "0.2"
+ProgressMeter = "1"
+Random = "1"
 RDatasets = "0.7.7, 0.8"
 StableRNGs = "1"
 Statistics = "1"

src/MixedModelsExtras.jl

@@ -2,11 +2,13 @@ module MixedModelsExtras
 
 using LinearAlgebra
 using MixedModels
+using Random
 using Statistics
 using StatsBase
 using StatsModels
 using Tables
 
+using MixedModels: replicate
 using GLM: linkinv, Link
 using StatsModels: termnames, vif, gvif
 export termnames, gvif, vif
@@ -25,6 +27,9 @@ export partial_fitted
 include("shrinkage.jl")
 export shrinkagenorm, shrinkagetables
 
+include("bootstrap.jl")
+export bootstrap_lrt
+
 include("tables.jl")
 export ictable
 

src/bootstrap.jl

@@ -0,0 +1,84 @@
+"""
+    bootstrap_lrt(rng::AbstractRNG, n::Integer, m0::MixedModel, ms::MixedModel...;
+                  optsum_overrides=(;), progress=true)
+
+Bootstrapped likelihood ratio test applied to a set of nested models.
+
+The first model is used to simulate `n` dataset replicates, where the ground truth is that
+specified by the first model. Each of the other models is then refit to those
+null data and the underlying distribution of deviance differences is then captured.
+For final computation of the p-values, the observed difference in deviance  between the
+original models is compared against this null distribution.
+
+!!! note
+    The precision of the resulting p-value cannot exceed ``1/n``.
+
+!!! warn
+    This method is **not** thread safe. For efficiency , the models are modified
+    during bootstrapping and the original fits are only restored at the end.
+
+!!! note
+    The nesting of the models is not checked.  It is incumbent on the user
+    to check this. This differs from `StatsModels.lrtest` as nesting in
+    mixed models, especially in the random effects specification, may be non obvious.
+
+This functionality may be deprecated in the future in favor of `StatsModels.lrtest`.
+"""
+function bootstrap_lrt(rng::AbstractRNG, n::Integer, m0::MixedModel, ms::MixedModel...;
+                       optsum_overrides=(;), progress=true)
+    y0 = copy(response(m0))
+    ys = [copy(response(m)) for m in ms]
+    local models
+    local dofs
+    local formulas
+    local devs
+    local lls
+    local devdiffs
+    local pvals
+    try
+        models = [m0; ms...]
+        models = models[sortperm(dof.(models))]
+        dofs = dof.(models)
+        formulas = string.(formula.(models))
+        devs = deviance.(models)
+        lls = loglikelihood.(models)
+        devdiffs = .-(diff(devs))
+
+        for (key, val) in pairs(optsum_overrides)
+            setfield!(m0.optsum, key, val)
+            for m in ms
+                setfield!(m.optsum, key, val)
+            end
+        end
+        nulldist = replicate(n; progress) do
+            simulate!(rng, m0)
+            refit!(m0; progress=false)
+            for m in ms
+                refit!(m, response(m0); progress=false)
+            end
+            return deviance.(models)
+        end
+        nulldist = stack(nulldist; dims=1)
+        nulldist = .-(diff(nulldist; dims=2))
+        pvals = map(enumerate(devdiffs)) do (idx, dev)
+            if dev > 0
+                mean(>=(dev), view(nulldist, :, idx))
+            else
+                NaN
+            end
+        end
+    finally
+        # restore the original fits
+        if progress
+            @info "Bootstrapping complete, cleaning up..."
+        end
+        refit!(m0, y0; progress=false)
+        for (m, y) in zip(ms, ys)
+            refit!(m, y; progress=false)
+        end
+    end
+    lrt = StatsModels.LRTestResult(Int(nobs(m0)), Tuple(devs), Tuple(lls), Tuple(dofs),
+                                   (NaN, pvals...))
+    return MixedModels.LikelihoodRatioTest(Tuple(formulas), lrt,
+                                           first(models) isa LinearMixedModel)
+end

test/bootstrap.jl

@@ -0,0 +1,46 @@
+sleepstudy = dataset(:sleepstudy)
+fm0 = fit(MixedModel, @formula(reaction ~ 1 + days + (1 | subj)),
+          sleepstudy; progress)
+fm1 = fit(MixedModel, @formula(reaction ~ 1 + days + (1 + days | subj)),
+          sleepstudy; progress)
+fmzc = fit(MixedModel, @formula(reaction ~ 1 + days + zerocorr(1 + days | subj)),
+           sleepstudy; progress)
+
+# fm0 ⊂ fmzc ⊂ fm1, in ascending dof order, so the analytic LRT is well defined
+# and gives us a reference to check the bootstrap version against.
+lrt = likelihoodratiotest(fm0, fmzc, fm1)
+d0, dzc, d1 = deviance(fm0), deviance(fmzc), deviance(fm1)
+
+boot = bootstrap_lrt(StableRNG(42), 200, fm0, fmzc, fm1;
+                     progress, optsum_overrides=(; maxfeval=500))
+
+@testset "bookkeeping matches the analytic LRT" begin
+    @test boot.dof == lrt.dof
+    @test all(isapprox.(boot.deviance, lrt.deviance))
+    @test boot.formulas == lrt.formulas
+    @test boot.linear
+end
+
+@testset "p-values track the sign of the analytic LRT" begin
+    @test isnan(boot.pvalues[1])
+    # fm0 vs fmzc: huge deviance drop, analytic p ≈ 9e-11
+    @test boot.pvalues[2] < 0.05
+    # fmzc vs fm1: negligible deviance drop, analytic p ≈ 0.80
+    @test boot.pvalues[3] > 0.3
+end
+
+@testset "original fits are restored after bootstrapping" begin
+    @test deviance(fm0) ≈ d0
+    @test deviance(fmzc) ≈ dzc
+    @test deviance(fm1) ≈ d1
+end
+
+@testset "bookkeeping is independent of the order of ms..." begin
+    # only the identity of m0 (the data-generating model) should matter;
+    # re-sorting by dof for reporting must not depend on argument order
+    boot2 = bootstrap_lrt(StableRNG(42), 200, fm0, fm1, fmzc; progress)
+    @test boot2.dof == boot.dof
+    @test boot2.formulas == boot.formulas
+    @test boot2.deviance == boot.deviance
+    @test isequal(boot2.pvalues, boot.pvalues)
+end

test/runtests.jl

@@ -28,6 +28,10 @@ end
     include("shrinkage.jl")
 end
 
+@testset "bootstrap" begin
+    include("bootstrap.jl")
+end
+
 @testset "tables" begin
     include("tables.jl")
 end

test/set_up_tests.jl

@@ -13,6 +13,7 @@ using Tables
 using Test
 
 using GLM: linkinv, Link
+using MixedModels: likelihoodratiotest
 using MixedModelsDatasets: dataset
 using MixedModelsExtras: _ranef
 using RDatasets: dataset as rdataset