WEE_Linear_reg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Method definition PO(Float64) in module Main at In[1]:26 overwritten at In[1]:63.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "WEE_cts (generic function with 2 methods)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Implement linear regression in WEE\n",
    "using DataFrames, GLM, Distributions, Optim\n",
    "\n",
    "function WEE_cts(formula::DataFrames.Formula, D, data::DataFrames.DataFrame, pd_pop::Float64, boot::Int64 = 0)\n",
    "    # Input formu, D, data, pd_pop, boot = 0\n",
    "    mf = ModelFrame(formu, data)\n",
    "    responseV = formu.lhs # get y (response varible)\n",
    "    y = mf.df[responseV]\n",
    "    \n",
    "    namesx = DataFrames.allvars(formu)[1:end-1] #get string x1, x2\n",
    "    \n",
    "    xx = ModelMatrix(mf)\n",
    "    \n",
    "    temp_data = convert(DataFrame, xx.m[:,2:end])\n",
    "    temp_data[:D] = D \n",
    "    temp_data[:y] = y\n",
    "    \n",
    "    n1 = sum(D.==1)\n",
    "    n0 = sum(D.==0)\n",
    "    \n",
    "    # compute the weight p(D|X)\n",
    "    \n",
    "    formu.lhs = :D\n",
    "    gamma=coef(glm(formu, temp_data, Binomial()))\n",
    "    \n",
    "    function PO(gamma0::Float64)\n",
    "        gamma[1] = gamma0\n",
    "        (mean(exp(xx.m * gamma)./(1+exp(xx.m * gamma))) - pd_pop)^2\n",
    "    end\n",
    "    \n",
    "    # Get the argument when PO has the minimum value\n",
    "    gamma[1] = optimize(PO, -100, 100).minimum\n",
    "    temp_data[:estpx] = exp(xx.m * gamma)./(1+exp(xx.m * gamma));\n",
    "    \n",
    "    \n",
    "    #estimate py in cases and controls separately\n",
    "    formu.lhs = :y\n",
    "    \n",
    "    pyD1 = lm(formu, temp_data[temp_data[:D] .== 1,:]) # fit the case\n",
    "    pyD0 = lm(formu, temp_data[temp_data[:D] .== 0,:]) # fit the control\n",
    "    \n",
    "    py1 = predict(pyD0, temp_data[temp_data[:D] .== 1,:]) # generate pseudo control\n",
    "    py0 = predict(pyD1, temp_data[temp_data[:D] .== 0,:]) # generate pseudo case\n",
    "    \n",
    "    data1 = DataFrame(D = repmat([0], n1), y = py1)\n",
    "    data1[namesx[1]] = temp_data[temp_data[:D] .== 1,:][1]\n",
    "    data1[namesx[2]] = temp_data[temp_data[:D] .== 1,:][2]\n",
    "    data1[:estpx] = temp_data[temp_data[:D] .== 1,:][:estpx]\n",
    "    \n",
    "    data0 = DataFrame(D = repmat([1], n0), y = py0)\n",
    "    data0[namesx[1]] = temp_data[temp_data[:D] .== 0,:][1]\n",
    "    data0[namesx[2]] = temp_data[temp_data[:D] .== 0,:][2]\n",
    "    data0[:estpx] = temp_data[temp_data[:D] .== 0,:][:estpx]\n",
    "    \n",
    "    alldat = vcat(temp_data, data1, data0)\n",
    "    \n",
    "    alldat[alldat[:D] .== 0, :estpx] = 1 - alldat[alldat[:D] .== 0, :estpx]\n",
    "    \n",
    "    # the point estimate\n",
    "    lm1 = glm(formu, alldat, Normal(), wts = convert(Array, alldat[:,:estpx]))\n",
    "    cf = coef(lm1)[2:end]\n",
    "    \n",
    "    function PO(gamma0::Float64)\n",
    "        gamma[1] = gamma0\n",
    "        (mean(exp(xx.m * gamma)./(1+exp(xx.m * gamma))) - pd_pop)^2\n",
    "    end\n",
    "    \n",
    "    #bootstrap SE\n",
    "    bootcoef = DataFrame() # Build an empty dataframe\n",
    "    for n = 1:size(namesx)[1]\n",
    "        bootcoef[Symbol(namesx[n])] = Float64[]\n",
    "    end\n",
    "    \n",
    "    if boot == 0\n",
    "        push!(bootcoef, cf)\n",
    "        bootcoef\n",
    "    else\n",
    "        sample_cases = temp_data[temp_data[:D] .== 1,:]\n",
    "        sample_controls = temp_data[temp_data[:D] .== 0,:]\n",
    "        \n",
    "        \n",
    "        for iboot in 1:boot\n",
    "            \n",
    "            boot_cases_sample = sample_cases[sample(1:n1, n1, replace = true), :]\n",
    "            boot_controls_sample = sample_controls[sample(1:n0, n0, replace = true), :]\n",
    "            bootsample = vcat(boot_cases_sample, boot_controls_sample)\n",
    "            \n",
    "            bootmf = ModelFrame(formu, bootsample)\n",
    "            bootxx = ModelMatrix(bootmf)\n",
    "            \n",
    "            # compute the weight p(D|X) \n",
    "            formu.lhs = :D\n",
    "            gamma = coef(glm(formu, bootsample, Binomial()))\n",
    "            \n",
    "            function Boot_PO(gamma0::Float64)\n",
    "                gamma[1] = gamma0\n",
    "                (mean(exp(bootxx.m * gamma)./(1+exp(bootxx.m * gamma))) - pd_pop)^2\n",
    "            end\n",
    "            \n",
    "            gamma[1] = optimize(Boot_PO, -100, 100).minimum\n",
    "            bootsample[:estpx] = exp(bootxx.m * gamma)./(1+exp(bootxx.m * gamma))\n",
    "        \n",
    "            formu.lhs = :y\n",
    "            \n",
    "            pyD1 = lm(formu, boot_cases_sample) # fit the case\n",
    "            pyD0 = lm(formu, boot_controls_sample) # fit the control\n",
    "            \n",
    "            py1 = predict(pyD0, boot_cases_sample) # generate pseudo control\n",
    "            py0 = predict(pyD1, boot_controls_sample) # generate pseudo case\n",
    "            \n",
    "            data1 = DataFrame(D = repmat([0], n1), y = py1)\n",
    "            data1[namesx[1]] = boot_cases_sample[1]\n",
    "            data1[namesx[2]] = boot_cases_sample[2]\n",
    "            data1[:estpx] = boot_cases_sample[:estpx]\n",
    "            \n",
    "            data0 = DataFrame(D = repmat([1], n0), y = py0)\n",
    "            data0[namesx[1]] = boot_controls_sample[1]\n",
    "            data0[namesx[2]] = boot_controls_sample[2]\n",
    "            data0[:estpx] = boot_controls_sample[:estpx]\n",
    "            \n",
    "            alldat = vcat(bootsample, data1, data0)\n",
    "            alldat[alldat[:D] .== 0, :estpx] = 1 - alldat[alldat[:D] .== 0, :estpx]\n",
    "            \n",
    "            push!(bootcoef, coef(glm(formu, alldat, Normal(), wts = convert(Array, alldat[:,:estpx])))[2:end])\n",
    "            \n",
    "        end\n",
    "        \n",
    "        var_num = var(Array(bootcoef), 1)\n",
    "        chisq = (cf.^2)./squeeze(var_num, 1)\n",
    "        pvalue = [ccdf(Chisq(1), m) for m in chisq]\n",
    "        TAB = DataFrame(Variable = namesx, Estimate = cf, StdErr = squeeze(sqrt(var_num), 1), Chisq = chisq, p_value = pvalue)\n",
    "    end\n",
    "end\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>Variable</th><th>Estimate</th><th>StdErr</th><th>Chisq</th><th>p_value</th></tr></thead><tbody><tr><th>1</th><td>x1</td><td>0.06809922658172869</td><td>0.048914918638623635</td><td>1.9382138300712572</td><td>0.16386260494360555</td></tr><tr><th>2</th><td>x2</td><td>0.004067321434493954</td><td>0.033690474694431635</td><td>0.01457480424673794</td><td>0.903907942653077</td></tr></tbody></table>"
      ],
      "text/plain": [
       "2×5 DataFrames.DataFrame\n",
       "│ Row │ Variable │ Estimate   │ StdErr    │ Chisq     │ p_value  │\n",
       "├─────┼──────────┼────────────┼───────────┼───────────┼──────────┤\n",
       "│ 1   │ x1       │ 0.0680992  │ 0.0489149 │ 1.93821   │ 0.163863 │\n",
       "│ 2   │ x2       │ 0.00406732 │ 0.0336905 │ 0.0145748 │ 0.903908 │"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test function WEE_cts\n",
    "using DataFrames, GLM, Distributions, Optim\n",
    "\n",
    "# Input Data\n",
    "x = DataFrame(x1 = rand(Binomial(2, 0.3), 3000), x2 = rand(Binomial(2, 0.3), 3000))\n",
    "y1 = rand(Normal(0, 1), 3000)\n",
    "D1 = vcat([0 for i = 1:1000], [1 for i = 1:2000])   #vcat(repmat([0], 1000), repmat([1], 2000)) /// repeat([1, 2, 3, 4], outer=[2])\n",
    "y1d1 = DataFrame(y1 = y1, D1 = D1)\n",
    "\n",
    "data = hcat(x, y1d1)\n",
    "pd = 0.1\n",
    "formu = y1 ~ x1 + x2\n",
    "boot = 10\n",
    "pd_pop = 0.1\n",
    "D = D1;\n",
    "\n",
    "WEE_cts(formu, D, data, pd_pop, boot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Julia 0.5.0",
   "language": "julia",
   "name": "julia-0.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}